resilien
/
terraform
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'master' into stable-website

This commit is contained in:
James Bardin 2018-03-15 20:50:52 +00:00
parent dd3d9e298d 7878d66b38
commit 9578557f8f
566 changed files with 77299 additions and 14664 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.travis.yml
View File

@ -2,7 +2,7 @@ dist: trusty
sudo: false
language: go
go:
- 1.9.1
- "1.10"
# add TF_CONSUL_TEST=1 to run consul tests
# they were causing timouts in travis
@ -34,7 +34,6 @@ script:
- make vendor-status
- make test
- make e2etest
- make vet
- GOOS=windows go build
branches:
only:

31
CHANGELOG.md
View File

@ -1,3 +1,34 @@
## 0.11.4 (March 15, 2018)
IMPROVEMENTS:
* cli: `terraform state list` now accepts a new argument `-id=...` for filtering resources for display by their remote ids ([#17221](https://github.com/hashicorp/terraform/issues/17221))
* cli: `terraform destroy` now uses the option `-auto-approve` instead of `-force`, for consistency with `terraform apply`. The old flag is preserved for backward-compatibility, but is now deprecated; it will be retained for at least one major release. ([#17218](https://github.com/hashicorp/terraform/issues/17218))
* connection/ssh: Add support for host key verifiation ([#17354](https://github.com/hashicorp/terraform/issues/17354))
* backend/s3: add support for the cn-northwest-1 region ([#17216](https://github.com/hashicorp/terraform/issues/17216))
* provisioner/local-exec: Allow setting custom environment variables when running commands ([#13880](https://github.com/hashicorp/terraform/issues/13880))
* provisioner/habitat: Detect if hab user exists and only create if necessary ([#17195](https://github.com/hashicorp/terraform/issues/17195))
* provisioner/habitat: Allow custom service name ([#17196](https://github.com/hashicorp/terraform/issues/17196))
* general: https URLs are now supported in the HTTP_PROXY environment variable for URLs interpreted by Terraform Core. (This will not immediately be true for all Terraform provider plugins, since each must upgrade its own HTTP client.) [go1.10:net/http](https://golang.org/doc/go1.10#net/http)
BUG FIXES:
* core: Make sure state is locked during initial refresh ([#17422](https://github.com/hashicorp/terraform/issues/17422))
* core: Fix interpolation error when count references another interpolated count value ([#17368](https://github.com/hashicorp/terraform/issues/17368))
* core: Halt on fatal provisioner errors, rather than retrying until a timeout ([#17359](https://github.com/hashicorp/terraform/issues/17359))
* core: When handling a forced exit due to multiple interrupts, prevent the process from exiting while the state is being written ([#17323](https://github.com/hashicorp/terraform/issues/17323))
* core: Fix handling of locals and outputs at destroy time ([#17241](https://github.com/hashicorp/terraform/issues/17241))
* core: Fix regression in handling of `count` arguments that refer to `count` attributes from other resources ([#17548](https://github.com/hashicorp/terraform/issues/17548))
* provider/terraform: restore support for the deprecated `environment` argument to the `terraform_remote_state` data source ([#17545](https://github.com/hashicorp/terraform/issues/17545))
* backend/gcs: Report the correct lock ID for GCS state locks ([#17397](https://github.com/hashicorp/terraform/issues/17397))
PROVIDER SDK CHANGES (not user-facing):
* helper/schema: Prevent crash on removal of computed field in CustomizeDiff ([#17261](https://github.com/hashicorp/terraform/issues/17261))
* helper/schema: Allow ResourceDiff.ForceNew on nested fields (avoid crash) ([#17463](https://github.com/hashicorp/terraform/issues/17463))
* helper/schema: Allow `TypeMap` to have a `*schema.Schema` as its `Elem`, for consistency with `TypeSet` and `TypeList` ([#17097](https://github.com/hashicorp/terraform/issues/17097))
* helper/validation: Add ValidateRFC3339TimeString function ([#17484](https://github.com/hashicorp/terraform/issues/17484))
## 0.11.3 (January 31, 2018)
IMPROVEMENTS:

16
Makefile
View File

@ -1,7 +1,7 @@
TEST?=./...
GOFMT_FILES?=$$(find . -name '*.go' | grep -v vendor)
default: test vet
default: test
tools:
go get -u github.com/kardianos/govendor
@ -30,7 +30,6 @@ plugin-dev: generate
# we run this one package at a time here because running the entire suite in
# one command creates memory usage issues when running in Travis-CI.
test: fmtcheck generate
go test -i $(TEST) || exit 1
go list $(TEST) | xargs -t -n4 go test $(TESTARGS) -timeout=60s -parallel=4
# testacc runs acceptance tests
@ -68,17 +67,6 @@ cover:
go tool cover -html=coverage.out
rm coverage.out
# vet runs the Go source code static analysis tool `vet` to find
# any common errors.
vet:
@echo 'go vet ./...'
@go vet ./... ; if [ $$? -eq 1 ]; then \
echo ""; \
echo "Vet found suspicious constructs. Please check the reported constructs"; \
echo "and fix them if necessary before submitting the code for review."; \
exit 1; \
fi
# generate runs `go generate` to build the dynamically generated
# source files.
generate:
@ -102,4 +90,4 @@ vendor-status:
# under parallel conditions.
.NOTPARALLEL:
.PHONY: bin cover default dev e2etest fmt fmtcheck generate plugin-dev quickdev test-compile test testacc testrace tools vendor-status vet
.PHONY: bin cover default dev e2etest fmt fmtcheck generate plugin-dev quickdev test-compile test testacc testrace tools vendor-status

17
backend/backend.go
View File

@ -9,6 +9,7 @@ import (
"errors"
"time"
"github.com/hashicorp/terraform/command/clistate"
"github.com/hashicorp/terraform/config/module"
"github.com/hashicorp/terraform/state"
"github.com/hashicorp/terraform/terraform"
@ -135,6 +136,10 @@ type Operation struct {
// state.Lockers for its duration, and Unlock when complete.
LockState bool
// StateLocker is used to lock the state while providing UI feedback to the
// user. This will be supplied by the Backend itself.
StateLocker clistate.Locker
// The duration to retry obtaining a State lock.
StateLockTimeout time.Duration
@ -145,8 +150,6 @@ type Operation struct {
// RunningOperation is the result of starting an operation.
type RunningOperation struct {
// Context should be used to track Done and Err for errors.
//
// For implementers of a backend, this context should not wrap the
// passed in context. Otherwise, canceling the parent context will
// immediately mark this context as "done" but those aren't the semantics
@ -154,6 +157,16 @@ type RunningOperation struct {
// is fully done.
context.Context
// Stop requests the operation to complete early, by calling Stop on all
// the plugins. If the process needs to terminate immediately, call Cancel.
Stop context.CancelFunc
// Cancel is the context.CancelFunc associated with the embedded context,
// and can be called to terminate the operation early.
// Once Cancel is called, the operation should return as soon as possible
// to avoid running operations during process exit.
Cancel context.CancelFunc
// Err is the error of the operation. This is populated after
// the operation has completed.
Err error

86
backend/local/backend.go
View File

@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"io/ioutil"
"log"
"os"
"path/filepath"
"sort"
@ -12,6 +13,7 @@ import (
"sync"
"github.com/hashicorp/terraform/backend"
"github.com/hashicorp/terraform/command/clistate"
"github.com/hashicorp/terraform/helper/schema"
"github.com/hashicorp/terraform/state"
"github.com/hashicorp/terraform/terraform"
@ -224,7 +226,7 @@ func (b *Local) State(name string) (state.State, error) {
// name conflicts, assume that the field is overwritten if set.
func (b *Local) Operation(ctx context.Context, op *backend.Operation) (*backend.RunningOperation, error) {
// Determine the function to call for our operation
var f func(context.Context, *backend.Operation, *backend.RunningOperation)
var f func(context.Context, context.Context, *backend.Operation, *backend.RunningOperation)
switch op.Type {
case backend.OperationTypeRefresh:
f = b.opRefresh
@ -244,20 +246,94 @@ func (b *Local) Operation(ctx context.Context, op *backend.Operation) (*backend.
b.opLock.Lock()
// Build our running operation
runningCtx, runningCtxCancel := context.WithCancel(context.Background())
runningOp := &backend.RunningOperation{Context: runningCtx}
// the runninCtx is only used to block until the operation returns.
runningCtx, done := context.WithCancel(context.Background())
runningOp := &backend.RunningOperation{
Context: runningCtx,
}
// stopCtx wraps the context passed in, and is used to signal a graceful Stop.
stopCtx, stop := context.WithCancel(ctx)
runningOp.Stop = stop
// cancelCtx is used to cancel the operation immediately, usually
// indicating that the process is exiting.
cancelCtx, cancel := context.WithCancel(context.Background())
runningOp.Cancel = cancel
if op.LockState {
op.StateLocker = clistate.NewLocker(stopCtx, op.StateLockTimeout, b.CLI, b.Colorize())
} else {
op.StateLocker = clistate.NewNoopLocker()
}
// Do it
go func() {
defer done()
defer stop()
defer cancel()
// the state was locked during context creation, unlock the state when
// the operation completes
defer func() {
runningOp.Err = op.StateLocker.Unlock(runningOp.Err)
}()
defer b.opLock.Unlock()
defer runningCtxCancel()
f(ctx, op, runningOp)
f(stopCtx, cancelCtx, op, runningOp)
}()
// Return
return runningOp, nil
}
// opWait wats for the operation to complete, and a stop signal or a
// cancelation signal.
func (b *Local) opWait(
doneCh <-chan struct{},
stopCtx context.Context,
cancelCtx context.Context,
tfCtx *terraform.Context,
opState state.State) (canceled bool) {
// Wait for the operation to finish or for us to be interrupted so
// we can handle it properly.
select {
case <-stopCtx.Done():
if b.CLI != nil {
b.CLI.Output("stopping operation...")
}
// try to force a PersistState just in case the process is terminated
// before we can complete.
if err := opState.PersistState(); err != nil {
// We can't error out from here, but warn the user if there was an error.
// If this isn't transient, we will catch it again below, and
// attempt to save the state another way.
if b.CLI != nil {
b.CLI.Error(fmt.Sprintf(earlyStateWriteErrorFmt, err))
}
}
// Stop execution
go tfCtx.Stop()
select {
case <-cancelCtx.Done():
log.Println("[WARN] running operation canceled")
// if the operation was canceled, we need to return immediately
canceled = true
case <-doneCh:
}
case <-cancelCtx.Done():
// this should not be called without first attempting to stop the
// operation
log.Println("[ERROR] running operation canceled without Stop")
canceled = true
case <-doneCh:
}
return
}
// Colorize returns the Colorize structure that can be used for colorizing
// output. This is gauranteed to always return a non-nil value and so is useful
// as a helper to wrap any potentially colored strings.

53
backend/local/backend_apply.go
View File

@ -11,7 +11,6 @@ import (
"github.com/hashicorp/errwrap"
"github.com/hashicorp/go-multierror"
"github.com/hashicorp/terraform/backend"
"github.com/hashicorp/terraform/command/clistate"
"github.com/hashicorp/terraform/command/format"
"github.com/hashicorp/terraform/config/module"
"github.com/hashicorp/terraform/state"
@ -19,7 +18,8 @@ import (
)
func (b *Local) opApply(
ctx context.Context,
stopCtx context.Context,
cancelCtx context.Context,
op *backend.Operation,
runningOp *backend.RunningOperation) {
log.Printf("[INFO] backend/local: starting Apply operation")
@ -54,25 +54,6 @@ func (b *Local) opApply(
return
}
if op.LockState {
lockCtx, cancel := context.WithTimeout(ctx, op.StateLockTimeout)
defer cancel()
lockInfo := state.NewLockInfo()
lockInfo.Operation = op.Type.String()
lockID, err := clistate.Lock(lockCtx, opState, lockInfo, b.CLI, b.Colorize())
if err != nil {
runningOp.Err = errwrap.Wrapf("Error locking state: {{err}}", err)
return
}
defer func() {
if err := clistate.Unlock(opState, lockID, b.CLI, b.Colorize()); err != nil {
runningOp.Err = multierror.Append(runningOp.Err, err)
}
}()
}
// Setup the state
runningOp.State = tfCtx.State()
@ -99,7 +80,7 @@ func (b *Local) opApply(
dispPlan := format.NewPlan(plan)
trivialPlan := dispPlan.Empty()
hasUI := op.UIOut != nil && op.UIIn != nil
mustConfirm := hasUI && ((op.Destroy && !op.DestroyForce) || (!op.Destroy && !op.AutoApprove && !trivialPlan))
mustConfirm := hasUI && ((op.Destroy && (!op.DestroyForce && !op.AutoApprove)) || (!op.Destroy && !op.AutoApprove && !trivialPlan))
if mustConfirm {
var desc, query string
if op.Destroy {
@ -153,32 +134,8 @@ func (b *Local) opApply(
applyState = tfCtx.State()
}()
// Wait for the apply to finish or for us to be interrupted so
// we can handle it properly.
err = nil
select {
case <-ctx.Done():
if b.CLI != nil {
b.CLI.Output("stopping apply operation...")
}
// try to force a PersistState just in case the process is terminated
// before we can complete.
if err := opState.PersistState(); err != nil {
// We can't error out from here, but warn the user if there was an error.
// If this isn't transient, we will catch it again below, and
// attempt to save the state another way.
if b.CLI != nil {
b.CLI.Error(fmt.Sprintf(earlyStateWriteErrorFmt, err))
}
}
// Stop execution
go tfCtx.Stop()
// Wait for completion still
<-doneCh
case <-doneCh:
if b.opWait(doneCh, stopCtx, cancelCtx, tfCtx, opState) {
return
}
// Store the final state

12
backend/local/backend_local.go
View File

@ -1,9 +1,11 @@
package local
import (
"context"
"errors"
"log"
"github.com/hashicorp/terraform/command/clistate"
"github.com/hashicorp/terraform/command/format"
"github.com/hashicorp/terraform/tfdiags"
@ -20,6 +22,12 @@ func (b *Local) Context(op *backend.Operation) (*terraform.Context, state.State,
// to ask for input/validate.
op.Type = backend.OperationTypeInvalid
if op.LockState {
op.StateLocker = clistate.NewLocker(context.Background(), op.StateLockTimeout, b.CLI, b.Colorize())
} else {
op.StateLocker = clistate.NewNoopLocker()
}
return b.context(op)
}
@ -30,6 +38,10 @@ func (b *Local) context(op *backend.Operation) (*terraform.Context, state.State,
return nil, nil, errwrap.Wrapf("Error loading state: {{err}}", err)
}
if err := op.StateLocker.Lock(s, op.Type.String()); err != nil {
return nil, nil, errwrap.Wrapf("Error locking state: {{err}}", err)
}
if err := s.RefreshState(); err != nil {
return nil, nil, errwrap.Wrapf("Error loading state: {{err}}", err)
}

39
backend/local/backend_plan.go
View File

@ -9,17 +9,15 @@ import (
"strings"
"github.com/hashicorp/errwrap"
"github.com/hashicorp/go-multierror"
"github.com/hashicorp/terraform/backend"
"github.com/hashicorp/terraform/command/clistate"
"github.com/hashicorp/terraform/command/format"
"github.com/hashicorp/terraform/config/module"
"github.com/hashicorp/terraform/state"
"github.com/hashicorp/terraform/terraform"
)
func (b *Local) opPlan(
ctx context.Context,
stopCtx context.Context,
cancelCtx context.Context,
op *backend.Operation,
runningOp *backend.RunningOperation) {
log.Printf("[INFO] backend/local: starting Plan operation")
@ -61,25 +59,6 @@ func (b *Local) opPlan(
return
}
if op.LockState {
lockCtx, cancel := context.WithTimeout(ctx, op.StateLockTimeout)
defer cancel()
lockInfo := state.NewLockInfo()
lockInfo.Operation = op.Type.String()
lockID, err := clistate.Lock(lockCtx, opState, lockInfo, b.CLI, b.Colorize())
if err != nil {
runningOp.Err = errwrap.Wrapf("Error locking state: {{err}}", err)
return
}
defer func() {
if err := clistate.Unlock(opState, lockID, b.CLI, b.Colorize()); err != nil {
runningOp.Err = multierror.Append(runningOp.Err, err)
}
}()
}
// Setup the state
runningOp.State = tfCtx.State()
@ -111,18 +90,8 @@ func (b *Local) opPlan(
plan, planErr = tfCtx.Plan()
}()
select {
case <-ctx.Done():
if b.CLI != nil {
b.CLI.Output("stopping plan operation...")
}
// Stop execution
go tfCtx.Stop()
// Wait for completion still
<-doneCh
case <-doneCh:
if b.opWait(doneCh, stopCtx, cancelCtx, tfCtx, opState) {
return
}
if planErr != nil {

39
backend/local/backend_refresh.go
View File

@ -8,16 +8,14 @@ import (
"strings"
"github.com/hashicorp/errwrap"
"github.com/hashicorp/go-multierror"
"github.com/hashicorp/terraform/backend"
"github.com/hashicorp/terraform/command/clistate"
"github.com/hashicorp/terraform/config/module"
"github.com/hashicorp/terraform/state"
"github.com/hashicorp/terraform/terraform"
)
func (b *Local) opRefresh(
ctx context.Context,
stopCtx context.Context,
cancelCtx context.Context,
op *backend.Operation,
runningOp *backend.RunningOperation) {
// Check if our state exists if we're performing a refresh operation. We
@ -52,25 +50,6 @@ func (b *Local) opRefresh(
return
}
if op.LockState {
lockCtx, cancel := context.WithTimeout(ctx, op.StateLockTimeout)
defer cancel()
lockInfo := state.NewLockInfo()
lockInfo.Operation = op.Type.String()
lockID, err := clistate.Lock(lockCtx, opState, lockInfo, b.CLI, b.Colorize())
if err != nil {
runningOp.Err = errwrap.Wrapf("Error locking state: {{err}}", err)
return
}
defer func() {
if err := clistate.Unlock(opState, lockID, b.CLI, b.Colorize()); err != nil {
runningOp.Err = multierror.Append(runningOp.Err, err)
}
}()
}
// Set our state
runningOp.State = opState.State()
if runningOp.State.Empty() || !runningOp.State.HasResources() {
@ -90,18 +69,8 @@ func (b *Local) opRefresh(
log.Printf("[INFO] backend/local: refresh calling Refresh")
}()
select {
case <-ctx.Done():
if b.CLI != nil {
b.CLI.Output("stopping refresh operation...")
}
// Stop execution
go tfCtx.Stop()
// Wait for completion still
<-doneCh
case <-doneCh:
if b.opWait(doneCh, stopCtx, cancelCtx, tfCtx, opState) {
return
}
// write the resulting state to the running op

3
backend/local/backend_test.go
View File

@ -23,7 +23,8 @@ func TestLocal_impl(t *testing.T) {
func TestLocal_backend(t *testing.T) {
defer testTmpDir(t)()
b := &Local{}
backend.TestBackend(t, b, b)
backend.TestBackendStates(t, b)
backend.TestBackendStateLocks(t, b, b)
}
func checkState(t *testing.T, path, expected string) {

5
backend/remote-state/azure/backend_test.go
View File

@ -64,7 +64,7 @@ func TestBackend(t *testing.T) {
"access_key": res.accessKey,
}).(*Backend)
backend.TestBackend(t, b, nil)
backend.TestBackendStates(t, b)
}
func TestBackendLocked(t *testing.T) {
@ -88,7 +88,8 @@ func TestBackendLocked(t *testing.T) {
"access_key": res.accessKey,
}).(*Backend)
backend.TestBackend(t, b1, b2)
backend.TestBackendStateLocks(t, b1, b2)
backend.TestBackendStateForceUnlock(t, b1, b2)
}
type testResources struct {

8
backend/remote-state/consul/backend_test.go
View File

@ -63,7 +63,8 @@ func TestBackend(t *testing.T) {
})
// Test
backend.TestBackend(t, b1, b2)
backend.TestBackendStates(t, b1)
backend.TestBackendStateLocks(t, b1, b2)
}
func TestBackend_lockDisabled(t *testing.T) {
@ -83,7 +84,8 @@ func TestBackend_lockDisabled(t *testing.T) {
})
// Test
backend.TestBackend(t, b1, b2)
backend.TestBackendStates(t, b1)
backend.TestBackendStateLocks(t, b1, b2)
}
func TestBackend_gzip(t *testing.T) {
@ -95,5 +97,5 @@ func TestBackend_gzip(t *testing.T) {
})
// Test
backend.TestBackend(t, b, nil)
backend.TestBackendStates(t, b)
}

6
backend/remote-state/etcdv3/backend_test.go
View File

@ -66,7 +66,9 @@ func TestBackend(t *testing.T) {
})
// Test
backend.TestBackend(t, b1, b2)
backend.TestBackendStates(t, b1)
backend.TestBackendStateLocks(t, b1, b2)
backend.TestBackendStateForceUnlock(t, b1, b2)
}
func TestBackend_lockDisabled(t *testing.T) {
@ -89,5 +91,5 @@ func TestBackend_lockDisabled(t *testing.T) {
})
// Test
backend.TestBackend(t, b1, b2)
backend.TestBackendStateLocks(t, b1, b2)
}

4
backend/remote-state/gcs/backend.go
View File

@ -13,7 +13,7 @@ import (
"github.com/hashicorp/terraform/backend"
"github.com/hashicorp/terraform/helper/pathorcontents"
"github.com/hashicorp/terraform/helper/schema"
"github.com/hashicorp/terraform/terraform"
"github.com/hashicorp/terraform/httpclient"
"golang.org/x/oauth2/jwt"
"google.golang.org/api/option"
)
@ -156,7 +156,7 @@ func (b *gcsBackend) configure(ctx context.Context) error {
opts = append(opts, option.WithScopes(storage.ScopeReadWrite))
}
opts = append(opts, option.WithUserAgent(terraform.UserAgentString()))
opts = append(opts, option.WithUserAgent(httpclient.UserAgentString()))
client, err := storage.NewClient(b.storageContext, opts...)
if err != nil {
return fmt.Errorf("storage.NewClient() failed: %v", err)

11
backend/remote-state/gcs/backend_test.go
View File

@ -136,8 +136,11 @@ func TestBackend(t *testing.T) {
be1 := setupBackend(t, bucket, noPrefix, noEncryptionKey)
backend.TestBackend(t, be0, be1)
backend.TestBackendStates(t, be0)
backend.TestBackendStateLocks(t, be0, be1)
backend.TestBackendStateForceUnlock(t, be0, be1)
}
func TestBackendWithPrefix(t *testing.T) {
t.Parallel()
@ -149,7 +152,8 @@ func TestBackendWithPrefix(t *testing.T) {
be1 := setupBackend(t, bucket, prefix+"/", noEncryptionKey)
backend.TestBackend(t, be0, be1)
backend.TestBackendStates(t, be0)
backend.TestBackendStateLocks(t, be0, be1)
}
func TestBackendWithEncryption(t *testing.T) {
t.Parallel()
@ -161,7 +165,8 @@ func TestBackendWithEncryption(t *testing.T) {
be1 := setupBackend(t, bucket, noPrefix, encryptionKey)
backend.TestBackend(t, be0, be1)
backend.TestBackendStates(t, be0)
backend.TestBackendStateLocks(t, be0, be1)
}
// setupBackend returns a new GCS backend.

15
backend/remote-state/gcs/client.go
View File

@ -80,6 +80,10 @@ func (c *remoteClient) Delete() error {
// Lock writes to a lock file, ensuring file creation. Returns the generation
// number, which must be passed to Unlock().
func (c *remoteClient) Lock(info *state.LockInfo) (string, error) {
// update the path we're using
// we can't set the ID until the info is written
info.Path = c.lockFileURL()
infoJson, err := json.Marshal(info)
if err != nil {
return "", err
@ -93,12 +97,12 @@ func (c *remoteClient) Lock(info *state.LockInfo) (string, error) {
}
return w.Close()
}()
if err != nil {
return "", c.lockError(fmt.Errorf("writing %q failed: %v", c.lockFileURL(), err))
}
info.ID = strconv.FormatInt(w.Attrs().Generation, 10)
info.Path = c.lockFileURL()
return info.ID, nil
}
@ -149,6 +153,15 @@ func (c *remoteClient) lockInfo() (*state.LockInfo, error) {
return nil, err
}
// We use the Generation as the ID, so overwrite the ID in the json.
// This can't be written into the Info, since the generation isn't known
// until it's written.
attrs, err := c.lockFile().Attrs(c.storageContext)
if err != nil {
return nil, err
}
info.ID = strconv.FormatInt(attrs.Generation, 10)
return info, nil
}

4
backend/remote-state/inmem/backend_test.go
View File

@ -40,7 +40,7 @@ func TestBackendConfig(t *testing.T) {
func TestBackend(t *testing.T) {
defer Reset()
b := backend.TestBackendConfig(t, New(), nil).(*Backend)
backend.TestBackend(t, b, nil)
backend.TestBackendStates(t, b)
}
func TestBackendLocked(t *testing.T) {
@ -48,7 +48,7 @@ func TestBackendLocked(t *testing.T) {
b1 := backend.TestBackendConfig(t, New(), nil).(*Backend)
b2 := backend.TestBackendConfig(t, New(), nil).(*Backend)
backend.TestBackend(t, b1, b2)
backend.TestBackendStateLocks(t, b1, b2)
}
// use the this backen to test the remote.State implementation

5
backend/remote-state/manta/backend_test.go
View File

@ -38,7 +38,7 @@ func TestBackend(t *testing.T) {
createMantaFolder(t, b.storageClient, directory)
defer deleteMantaFolder(t, b.storageClient, directory)
backend.TestBackend(t, b, nil)
backend.TestBackendStates(t, b)
}
func TestBackendLocked(t *testing.T) {
@ -60,7 +60,8 @@ func TestBackendLocked(t *testing.T) {
createMantaFolder(t, b1.storageClient, directory)
defer deleteMantaFolder(t, b1.storageClient, directory)
backend.TestBackend(t, b1, b2)
backend.TestBackendStateLocks(t, b1, b2)
backend.TestBackendStateForceUnlock(t, b1, b2)
}
func createMantaFolder(t *testing.T, mantaClient *storage.StorageClient, directoryName string) {

2
backend/remote-state/manta/client.go
View File

@ -106,7 +106,7 @@ func (c *RemoteClient) Lock(info *state.LockInfo) (string, error) {
lockErr := &state.LockError{}
lockInfo, err := c.getLockInfo()
if err != nil {
if tritonErrors.IsResourceNotFound(err) {
if !tritonErrors.IsResourceNotFound(err) {
lockErr.Err = fmt.Errorf("failed to retrieve lock info: %s", err)
return "", lockErr
}

11
backend/remote-state/s3/backend_test.go
View File

@ -103,7 +103,7 @@ func TestBackend(t *testing.T) {
createS3Bucket(t, b.s3Client, bucketName)
defer deleteS3Bucket(t, b.s3Client, bucketName)
backend.TestBackend(t, b, nil)
backend.TestBackendStates(t, b)
}
func TestBackendLocked(t *testing.T) {
@ -131,7 +131,8 @@ func TestBackendLocked(t *testing.T) {
createDynamoDBTable(t, b1.dynClient, bucketName)
defer deleteDynamoDBTable(t, b1.dynClient, bucketName)
backend.TestBackend(t, b1, b2)
backend.TestBackendStateLocks(t, b1, b2)
backend.TestBackendStateForceUnlock(t, b1, b2)
}
// add some extra junk in S3 to try and confuse the env listing.
@ -334,9 +335,9 @@ func TestKeyEnv(t *testing.T) {
t.Fatal(err)
}
backend.TestBackend(t, b0, nil)
backend.TestBackend(t, b1, nil)
backend.TestBackend(t, b2, nil)
backend.TestBackendStates(t, b0)
backend.TestBackendStates(t, b1)
backend.TestBackendStates(t, b2)
}
func testGetWorkspaceForKey(b *Backend, key string, expected string) error {

2
backend/remote-state/swift/backend_test.go
View File

@ -67,7 +67,7 @@ func TestBackend(t *testing.T) {
defer deleteSwiftContainer(t, b.client, container)
backend.TestBackend(t, b, nil)
backend.TestBackendStates(t, b)
}
func TestBackendPath(t *testing.T) {

78
backend/testing.go
View File

@ -5,6 +5,7 @@ import (
"sort"
"testing"
uuid "github.com/hashicorp/go-uuid"
"github.com/hashicorp/terraform/config"
"github.com/hashicorp/terraform/state"
"github.com/hashicorp/terraform/terraform"
@ -43,20 +44,7 @@ func TestBackendConfig(t *testing.T, b Backend, c map[string]interface{}) Backen
// assumed to already be configured. This will test state functionality.
// If the backend reports it doesn't support multi-state by returning the
// error ErrNamedStatesNotSupported, then it will not test that.
//
// If you want to test locking, two backends must be given. If b2 is nil,
// then state locking won't be tested.
func TestBackend(t *testing.T, b1, b2 Backend) {
t.Helper()
testBackendStates(t, b1)
if b2 != nil {
testBackendStateLock(t, b1, b2)
}
}
func testBackendStates(t *testing.T, b Backend) {
func TestBackendStates(t *testing.T, b Backend) {
t.Helper()
states, err := b.States()
@ -236,7 +224,23 @@ func testBackendStates(t *testing.T, b Backend) {
}
}
func testBackendStateLock(t *testing.T, b1, b2 Backend) {
// TestBackendStateLocks will test the locking functionality of the remote
// state backend.
func TestBackendStateLocks(t *testing.T, b1, b2 Backend) {
t.Helper()
testLocks(t, b1, b2, false)
}
// TestBackendStateForceUnlock verifies that the lock error is the expected
// type, and the lock can be unlocked using the ID reported in the error.
// Remote state backends that support -force-unlock should call this in at
// least one of the acceptance tests.
func TestBackendStateForceUnlock(t *testing.T, b1, b2 Backend) {
t.Helper()
testLocks(t, b1, b2, true)
}
func testLocks(t *testing.T, b1, b2 Backend, testForceUnlock bool) {
t.Helper()
// Get the default state for each
@ -286,7 +290,7 @@ func testBackendStateLock(t *testing.T, b1, b2 Backend) {
// backend, and as a remote state.
_, err = b2.State(DefaultStateName)
if err != nil {
t.Fatalf("failed to read locked state from another backend instance: %s", err)
t.Errorf("failed to read locked state from another backend instance: %s", err)
}
// If the lock ID is blank, assume locking is disabled
@ -311,11 +315,51 @@ func testBackendStateLock(t *testing.T, b1, b2 Backend) {
}
if lockIDB == lockIDA {
t.Fatalf("duplicate lock IDs: %q", lockIDB)
t.Errorf("duplicate lock IDs: %q", lockIDB)
}
if err = lockerB.Unlock(lockIDB); err != nil {
t.Fatal("error unlocking client B:", err)
}
// test the equivalent of -force-unlock, by using the id from the error
// output.
if !testForceUnlock {
return
}
// get a new ID
infoA.ID, err = uuid.GenerateUUID()
if err != nil {
panic(err)
}
lockIDA, err = lockerA.Lock(infoA)
if err != nil {
t.Fatal("unable to get re lock A:", err)
}
unlock := func() {
err := lockerA.Unlock(lockIDA)
if err != nil {
t.Fatal(err)
}
}
_, err = lockerB.Lock(infoB)
if err == nil {
unlock()
t.Fatal("client B obtained lock while held by client A")
}
infoErr, ok := err.(*state.LockError)
if !ok {
unlock()
t.Fatalf("expected type *state.LockError, got : %#v", err)
}
// try to unlock with the second unlocker, using the ID from the error
if err := lockerB.Unlock(infoErr.Info.ID); err != nil {
unlock()
t.Fatalf("could not unlock with the reported ID %q: %s", infoErr.Info.ID, err)
}
}

19
builtin/providers/terraform/data_source_state.go
View File

@ -64,7 +64,7 @@ func dataSourceRemoteState() *schema.Resource {
}
func dataSourceRemoteStateRead(d *schema.ResourceData, meta interface{}) error {
backend := d.Get("backend").(string)
backendType := d.Get("backend").(string)
// Get the configuration in a type we want.
rawConfig, err := config.NewRawConfig(d.Get("config").(map[string]interface{}))
@ -73,16 +73,16 @@ func dataSourceRemoteStateRead(d *schema.ResourceData, meta interface{}) error {
}
// Don't break people using the old _local syntax - but note warning above
if backend == "_local" {
if backendType == "_local" {
log.Println(`[INFO] Switching old (unsupported) backend "_local" to "local"`)
backend = "local"
backendType = "local"
}
// Create the client to access our remote state
log.Printf("[DEBUG] Initializing remote state backend: %s", backend)
f := backendinit.Backend(backend)
log.Printf("[DEBUG] Initializing remote state backend: %s", backendType)
f := backendinit.Backend(backendType)
if f == nil {
return fmt.Errorf("Unknown backend type: %s", backend)
return fmt.Errorf("Unknown backend type: %s", backendType)
}
b := f()
@ -94,9 +94,10 @@ func dataSourceRemoteStateRead(d *schema.ResourceData, meta interface{}) error {
// environment is deprecated in favour of workspace.
// If both keys are set workspace should win.
name := d.Get("environment").(string)
if ws, ok := d.GetOk("workspace"); ok {
if ws, ok := d.GetOk("workspace"); ok && ws != backend.DefaultStateName {
name = ws.(string)
}
state, err := b.State(name)
if err != nil {
return fmt.Errorf("error loading the remote state: %s", err)
@ -118,7 +119,9 @@ func dataSourceRemoteStateRead(d *schema.ResourceData, meta interface{}) error {
log.Println("[DEBUG] empty remote state")
} else {
for key, val := range remoteState.RootModule().Outputs {
outputMap[key] = val.Value
if val.Value != nil {
outputMap[key] = val.Value
}
}
}

54
builtin/providers/terraform/data_source_state_test.go
View File

@ -63,6 +63,20 @@ func TestState_complexOutputs(t *testing.T) {
})
}
// outputs should never have a null value, but don't crash if we ever encounter
// them.
func TestState_nullOutputs(t *testing.T) {
resource.UnitTest(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
Steps: []resource.TestStep{
{
Config: testAccState_nullOutputs,
},
},
})
}
func TestEmptyState_defaults(t *testing.T) {
resource.UnitTest(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
@ -115,6 +129,27 @@ func testAccCheckStateValue(id, name, value string) resource.TestCheckFunc {
}
}
// make sure that the deprecated environment field isn't overridden by the
// default value for workspace.
func TestState_deprecatedEnvironment(t *testing.T) {
resource.UnitTest(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
Steps: []resource.TestStep{
{
Config: testAccState_deprecatedEnvironment,
Check: resource.ComposeTestCheckFunc(
testAccCheckStateValue(
// if the workspace default value overrides the
// environment, this will get the foo value from the
// default state.
"data.terraform_remote_state.foo", "foo", ""),
),
},
},
})
}
const testAccState_basic = `
data "terraform_remote_state" "foo" {
backend = "local"
@ -142,6 +177,15 @@ resource "terraform_remote_state" "foo" {
}
}`
const testAccState_nullOutputs = `
resource "terraform_remote_state" "foo" {
backend = "local"
config {
path = "./test-fixtures/null_outputs.tfstate"
}
}`
const testAccEmptyState_defaults = `
data "terraform_remote_state" "foo" {
backend = "local"
@ -167,3 +211,13 @@ data "terraform_remote_state" "foo" {
foo = "not bar"
}
}`
const testAccState_deprecatedEnvironment = `
data "terraform_remote_state" "foo" {
backend = "local"
environment = "deprecated"
config {
path = "./test-fixtures/basic.tfstate"
}
}`

24
builtin/providers/terraform/test-fixtures/null_outputs.tfstate Normal file
View File

@ -0,0 +1,24 @@
{
"version": 3,
"terraform_version": "0.7.0",
"serial": 3,
"modules": [
{
"path": [
"root"
],
"outputs": {
"map": {
"sensitive": false,
"type": "map",
"value": null
},
"list": {
"sensitive": false,
"type": "list",
"value": null
}
}
}
]
}

46
builtin/provisioners/chef/resource_provisioner.go
View File

@ -15,7 +15,6 @@ import (
"strings"
"sync"
"text/template"
"time"
"github.com/hashicorp/terraform/communicator"
"github.com/hashicorp/terraform/communicator/remote"
@ -307,8 +306,11 @@ func applyFn(ctx context.Context) error {
return err
}
ctx, cancel := context.WithTimeout(ctx, comm.Timeout())
defer cancel()
// Wait and retry until we establish the connection
err = retryFunc(comm.Timeout(), func() error {
err = communicator.Retry(ctx, func() error {
return comm.Connect(o)
})
if err != nil {
@ -682,6 +684,13 @@ func (p *provisioner) runCommand(o terraform.UIOutput, comm communicator.Communi
errDoneCh := make(chan struct{})
go p.copyOutput(o, outR, outDoneCh)
go p.copyOutput(o, errR, errDoneCh)
go func() {
// Wait for output to clean up
outW.Close()
errW.Close()
<-outDoneCh
<-errDoneCh
}()
cmd := &remote.Cmd{
Command: command,
@ -695,18 +704,15 @@ func (p *provisioner) runCommand(o terraform.UIOutput, comm communicator.Communi
}
cmd.Wait()
if cmd.ExitStatus != 0 {
err = fmt.Errorf(
"Command %q exited with non-zero exit status: %d", cmd.Command, cmd.ExitStatus)
if cmd.Err() != nil {
return cmd.Err()
}
// Wait for output to clean up
outW.Close()
errW.Close()
<-outDoneCh
<-errDoneCh
if cmd.ExitStatus() != 0 {
return fmt.Errorf("Command %q exited with non-zero exit status: %d", cmd.Command, cmd.ExitStatus())
}
return err
return nil
}
func (p *provisioner) copyOutput(o terraform.UIOutput, r io.Reader, doneCh chan<- struct{}) {
@ -717,24 +723,6 @@ func (p *provisioner) copyOutput(o terraform.UIOutput, r io.Reader, doneCh chan<
}
}
// retryFunc is used to retry a function for a given duration
func retryFunc(timeout time.Duration, f func() error) error {
finish := time.After(timeout)
for {
err := f()
if err == nil {
return nil
}
log.Printf("Retryable error: %v", err)
select {
case <-finish:
return err
case <-time.After(3 * time.Second):
}
}
}
func decodeConfig(d *schema.ResourceData) (*provisioner, error) {
p := &provisioner{
ClientOptions: getStringList(d.Get("client_options")),

52
builtin/provisioners/file/resource_provisioner.go
View File

@ -4,9 +4,7 @@ import (
"context"
"fmt"
"io/ioutil"
"log"
"os"
"time"
"github.com/hashicorp/terraform/communicator"
"github.com/hashicorp/terraform/helper/schema"
@ -50,6 +48,9 @@ func applyFn(ctx context.Context) error {
return err
}
ctx, cancel := context.WithTimeout(ctx, comm.Timeout())
defer cancel()
// Get the source
src, deleteSource, err := getSrc(data)
if err != nil {
@ -61,21 +62,11 @@ func applyFn(ctx context.Context) error {
// Begin the file copy
dst := data.Get("destination").(string)
resultCh := make(chan error, 1)
go func() {
resultCh <- copyFiles(comm, src, dst)
}()
// Allow the file copy to complete unless there is an interrupt.
// If there is an interrupt we make no attempt to cleanly close
// the connection currently. We just abruptly exit. Because Terraform
// taints the resource, this is fine.
select {
case err := <-resultCh:
if err := copyFiles(ctx, comm, src, dst); err != nil {
return err
case <-ctx.Done():
return fmt.Errorf("file transfer interrupted")
}
return nil
}
func validateFn(c *terraform.ResourceConfig) (ws []string, es []error) {
@ -107,16 +98,21 @@ func getSrc(data *schema.ResourceData) (string, bool, error) {
}
// copyFiles is used to copy the files from a source to a destination
func copyFiles(comm communicator.Communicator, src, dst string) error {
func copyFiles(ctx context.Context, comm communicator.Communicator, src, dst string) error {
// Wait and retry until we establish the connection
err := retryFunc(comm.Timeout(), func() error {
err := comm.Connect(nil)
return err
err := communicator.Retry(ctx, func() error {
return comm.Connect(nil)
})
if err != nil {
return err
}
defer comm.Disconnect()
// disconnect when the context is canceled, which will close this after
// Apply as well.
go func() {
<-ctx.Done()
comm.Disconnect()
}()
info, err := os.Stat(src)
if err != nil {
@ -144,21 +140,3 @@ func copyFiles(comm communicator.Communicator, src, dst string) error {
}
return err
}
// retryFunc is used to retry a function for a given duration
func retryFunc(timeout time.Duration, f func() error) error {
finish := time.After(timeout)
for {
err := f()
if err == nil {
return nil
}
log.Printf("Retryable error: %v", err)
select {
case <-finish:
return err
case <-time.After(3 * time.Second):
}
}
}

88
builtin/provisioners/habitat/resource_provisioner.go
View File

@ -6,12 +6,10 @@ import (
"errors"
"fmt"
"io"
"log"
"net/url"
"path"
"strings"
"text/template"
"time"
"github.com/hashicorp/terraform/communicator"
"github.com/hashicorp/terraform/communicator/remote"
@ -54,6 +52,7 @@ type provisioner struct {
SkipInstall bool
UseSudo bool
ServiceType string
ServiceName string
URL string
Channel string
Events string
@ -79,6 +78,11 @@ func Provisioner() terraform.ResourceProvisioner {
Optional: true,
Default: "systemd",
},
"service_name": &schema.Schema{
Type: schema.TypeString,
Optional: true,
Default: "hab-supervisor",
},
"use_sudo": &schema.Schema{
Type: schema.TypeBool,
Optional: true,
@ -227,10 +231,13 @@ func applyFn(ctx context.Context) error {
return err
}
err = retryFunc(comm.Timeout(), func() error {
err = comm.Connect(o)
return err
ctx, cancel := context.WithTimeout(ctx, comm.Timeout())
defer cancel()
err = communicator.Retry(ctx, func() error {
return comm.Connect(o)
})
if err != nil {
return err
}
@ -344,6 +351,7 @@ func decodeConfig(d *schema.ResourceData) (*provisioner, error) {
Services: getServices(d.Get("service").(*schema.Set).List()),
UseSudo: d.Get("use_sudo").(bool),
ServiceType: d.Get("service_type").(string),
ServiceName: d.Get("service_name").(string),
RingKey: d.Get("ring_key").(string),
RingKeyContent: d.Get("ring_key_content").(string),
PermanentPeer: d.Get("permanent_peer").(bool),
@ -569,9 +577,9 @@ func (p *provisioner) startHabSystemd(o terraform.UIOutput, comm communicator.Co
var command string
if p.UseSudo {
command = fmt.Sprintf("sudo echo '%s' | sudo tee /etc/systemd/system/hab-supervisor.service > /dev/null", &buf)
command = fmt.Sprintf("sudo echo '%s' | sudo tee /etc/systemd/system/%s.service > /dev/null", &buf, p.ServiceName)
} else {
command = fmt.Sprintf("echo '%s' | tee /etc/systemd/system/hab-supervisor.service > /dev/null", &buf)
command = fmt.Sprintf("echo '%s' | tee /etc/systemd/system/%s.service > /dev/null", &buf, p.ServiceName)
}
if err := p.runCommand(o, comm, command); err != nil {
@ -579,15 +587,16 @@ func (p *provisioner) startHabSystemd(o terraform.UIOutput, comm communicator.Co
}
if p.UseSudo {
command = fmt.Sprintf("sudo systemctl start hab-supervisor")
command = fmt.Sprintf("sudo systemctl start %s", p.ServiceName)
} else {
command = fmt.Sprintf("systemctl start hab-supervisor")
command = fmt.Sprintf("systemctl start %s", p.ServiceName)
}
return p.runCommand(o, comm, command)
}
func (p *provisioner) createHabUser(o terraform.UIOutput, comm communicator.Communicator) error {
// Create the hab user
addUser := false
// Install busybox to get us the user tools we need
command := fmt.Sprintf("env HAB_NONINTERACTIVE=true hab install core/busybox")
if p.UseSudo {
command = fmt.Sprintf("sudo %s", command)
@ -596,11 +605,25 @@ func (p *provisioner) createHabUser(o terraform.UIOutput, comm communicator.Comm
return err
}
command = fmt.Sprintf("hab pkg exec core/busybox adduser -D -g \"\" hab")
// Check for existing hab user
command = fmt.Sprintf("hab pkg exec core/busybox id hab")
if p.UseSudo {
command = fmt.Sprintf("sudo %s", command)
}
return p.runCommand(o, comm, command)
if err := p.runCommand(o, comm, command); err != nil {
o.Output("No existing hab user detected, creating...")
addUser = true
}
if addUser {
command = fmt.Sprintf("hab pkg exec core/busybox adduser -D -g \"\" hab")
if p.UseSudo {
command = fmt.Sprintf("sudo %s", command)
}
return p.runCommand(o, comm, command)
}
return nil
}
func (p *provisioner) startHabService(o terraform.UIOutput, comm communicator.Communicator, service Service) error {
@ -706,24 +729,6 @@ func (p *provisioner) uploadUserTOML(o terraform.UIOutput, comm communicator.Com
}
func retryFunc(timeout time.Duration, f func() error) error {
finish := time.After(timeout)
for {
err := f()
if err == nil {
return nil
}
log.Printf("Retryable error: %v", err)
select {
case <-finish:
return err
case <-time.After(3 * time.Second):
}
}
}
func (p *provisioner) copyOutput(o terraform.UIOutput, r io.Reader, doneCh chan<- struct{}) {
defer close(doneCh)
lr := linereader.New(r)
@ -735,12 +740,17 @@ func (p *provisioner) copyOutput(o terraform.UIOutput, r io.Reader, doneCh chan<
func (p *provisioner) runCommand(o terraform.UIOutput, comm communicator.Communicator, command string) error {
outR, outW := io.Pipe()
errR, errW := io.Pipe()
var err error
outDoneCh := make(chan struct{})
errDoneCh := make(chan struct{})
go p.copyOutput(o, outR, outDoneCh)
go p.copyOutput(o, errR, errDoneCh)
defer func() {
outW.Close()
errW.Close()
<-outDoneCh
<-errDoneCh
}()
cmd := &remote.Cmd{
Command: command,
@ -748,22 +758,20 @@ func (p *provisioner) runCommand(o terraform.UIOutput, comm communicator.Communi
Stderr: errW,
}
if err = comm.Start(cmd); err != nil {
if err := comm.Start(cmd); err != nil {
return fmt.Errorf("Error executing command %q: %v", cmd.Command, err)
}
cmd.Wait()
if cmd.ExitStatus != 0 {
err = fmt.Errorf(
"Command %q exited with non-zero exit status: %d", cmd.Command, cmd.ExitStatus)
if cmd.Err() != nil {
return cmd.Err()
}
outW.Close()
errW.Close()
<-outDoneCh
<-errDoneCh
if cmd.ExitStatus() != 0 {
return fmt.Errorf("Command %q exited with non-zero exit status: %d", cmd.Command, cmd.ExitStatus())
}
return err
return nil
}
func getBindFromString(bind string) (Bind, error) {

33
builtin/provisioners/local-exec/resource_provisioner.go
View File

@ -28,12 +28,19 @@ func Provisioner() terraform.ResourceProvisioner {
Type: schema.TypeString,
Required: true,
},
"interpreter": &schema.Schema{
Type: schema.TypeList,
Elem: &schema.Schema{Type: schema.TypeString},
Optional: true,
},
"working_dir": &schema.Schema{
Type: schema.TypeString,
Optional: true,
},
"environment": &schema.Schema{
Type: schema.TypeMap,
Optional: true,
},
},
ApplyFunc: applyFn,
@ -45,11 +52,20 @@ func applyFn(ctx context.Context) error {
o := ctx.Value(schema.ProvOutputKey).(terraform.UIOutput)
command := data.Get("command").(string)
if command == "" {
return fmt.Errorf("local-exec provisioner command must be a non-empty string")
}
// Execute the command with env
environment := data.Get("environment").(map[string]interface{})
var env []string
env = make([]string, len(environment))
for k := range environment {
entry := fmt.Sprintf("%s=%s", k, environment[k].(string))
env = append(env, entry)
}
// Execute the command using a shell
interpreter := data.Get("interpreter").([]interface{})
@ -69,6 +85,8 @@ func applyFn(ctx context.Context) error {
}
cmdargs = append(cmdargs, command)
workingdir := data.Get("working_dir").(string)
// Setup the reader that will read the output from the command.
// We use an os.Pipe so that the *os.File can be passed directly to the
// process, and not rely on goroutines copying the data which may block.
@ -78,10 +96,21 @@ func applyFn(ctx context.Context) error {
return fmt.Errorf("failed to initialize pipe for output: %s", err)
}
var cmdEnv []string
cmdEnv = os.Environ()
cmdEnv = append(cmdEnv, env...)
// Setup the command
cmd := exec.CommandContext(ctx, cmdargs[0], cmdargs[1:]...)
cmd.Stderr = pw
cmd.Stdout = pw
// Dir specifies the working directory of the command.
// If Dir is the empty string (this is default), runs the command
// in the calling process's current directory.
cmd.Dir = workingdir
// Env specifies the environment of the command.
// By default will use the calling process's environment
cmd.Env = cmdEnv
output, _ := circbuf.NewBuffer(maxBufSize)

53
builtin/provisioners/local-exec/resource_provisioner_test.go
View File

@ -145,3 +145,56 @@ func TestResourceProvider_ApplyCustomInterpreter(t *testing.T) {
t.Errorf("wrong output\ngot: %s\nwant: %s", got, want)
}
}
func TestResourceProvider_ApplyCustomWorkingDirectory(t *testing.T) {
testdir := "working_dir_test"
os.Mkdir(testdir, 0755)
defer os.Remove(testdir)
c := testConfig(t, map[string]interface{}{
"working_dir": testdir,
"command": "echo `pwd`",
})
output := new(terraform.MockUIOutput)
p := Provisioner()
if err := p.Apply(output, nil, c); err != nil {
t.Fatalf("err: %v", err)
}
dir, err := os.Getwd()
if err != nil {
t.Fatalf("err: %v", err)
}
got := strings.TrimSpace(output.OutputMessage)
want := dir + "/" + testdir
if got != want {
t.Errorf("wrong output\ngot: %s\nwant: %s", got, want)
}
}
func TestResourceProvider_ApplyCustomEnv(t *testing.T) {
c := testConfig(t, map[string]interface{}{
"command": "echo $FOO $BAR $BAZ",
"environment": map[string]interface{}{
"FOO": "BAR",
"BAR": 1,
"BAZ": "true",
},
})
output := new(terraform.MockUIOutput)
p := Provisioner()
if err := p.Apply(output, nil, c); err != nil {
t.Fatalf("err: %v", err)
}
got := strings.TrimSpace(output.OutputMessage)
want := "BAR 1 true"
if got != want {
t.Errorf("wrong output\ngot: %s\nwant: %s", got, want)
}
}

147
builtin/provisioners/remote-exec/resource_provisioner.go
View File

@ -9,7 +9,6 @@ import (
"log"
"os"
"strings"
"sync/atomic"
"time"
"github.com/hashicorp/terraform/communicator"
@ -157,10 +156,6 @@ func runScripts(
o terraform.UIOutput,
comm communicator.Communicator,
scripts []io.ReadCloser) error {
// Wrap out context in a cancelation function that we use to
// kill the connection.
ctx, cancelFunc := context.WithCancel(ctx)
defer cancelFunc()
// Wait for the context to end and then disconnect
go func() {
@ -169,9 +164,8 @@ func runScripts(
}()
// Wait and retry until we establish the connection
err := retryFunc(ctx, comm.Timeout(), func() error {
err := comm.Connect(o)
return err
err := communicator.Retry(ctx, func() error {
return comm.Connect(o)
})
if err != nil {
return err
@ -179,49 +173,38 @@ func runScripts(
for _, script := range scripts {
var cmd *remote.Cmd
outR, outW := io.Pipe()
errR, errW := io.Pipe()
outDoneCh := make(chan struct{})
errDoneCh := make(chan struct{})
go copyOutput(o, outR, outDoneCh)
go copyOutput(o, errR, errDoneCh)
defer outW.Close()
defer errW.Close()
go copyOutput(o, outR)
go copyOutput(o, errR)
remotePath := comm.ScriptPath()
err = retryFunc(ctx, comm.Timeout(), func() error {
if err := comm.UploadScript(remotePath, script); err != nil {
return fmt.Errorf("Failed to upload script: %v", err)
}
cmd = &remote.Cmd{
Command: remotePath,
Stdout: outW,
Stderr: errW,
}
if err := comm.Start(cmd); err != nil {
return fmt.Errorf("Error starting script: %v", err)
}
return nil
})
if err == nil {
cmd.Wait()
if cmd.ExitStatus != 0 {
err = fmt.Errorf("Script exited with non-zero exit status: %d", cmd.ExitStatus)
}
if err := comm.UploadScript(remotePath, script); err != nil {
return fmt.Errorf("Failed to upload script: %v", err)
}
// If we have an error, end our context so the disconnect happens.
// This has to happen before the output cleanup below since during
// an interrupt this will cause the outputs to end.
if err != nil {
cancelFunc()
cmd = &remote.Cmd{
Command: remotePath,
Stdout: outW,
Stderr: errW,
}
if err := comm.Start(cmd); err != nil {
return fmt.Errorf("Error starting script: %v", err)
}
cmd.Wait()
if err := cmd.Err(); err != nil {
return fmt.Errorf("Remote command exited with error: %s", err)
}
// Wait for output to clean up
outW.Close()
errW.Close()
<-outDoneCh
<-errDoneCh
if cmd.ExitStatus() != 0 {
err = fmt.Errorf("Script exited with non-zero exit status: %d", cmd.ExitStatus())
}
// Upload a blank follow up file in the same path to prevent residual
// script contents from remaining on remote machine
@ -230,93 +213,15 @@ func runScripts(
// This feature is best-effort.
log.Printf("[WARN] Failed to upload empty follow up script: %v", err)
}
// If we have an error, return it out now that we've cleaned up
if err != nil {
return err
}
}
return nil
}
func copyOutput(
o terraform.UIOutput, r io.Reader, doneCh chan<- struct{}) {
defer close(doneCh)
o terraform.UIOutput, r io.Reader) {
lr := linereader.New(r)
for line := range lr.Ch {
o.Output(line)
}
}
// retryFunc is used to retry a function for a given duration
func retryFunc(ctx context.Context, timeout time.Duration, f func() error) error {
// Build a new context with the timeout
ctx, done := context.WithTimeout(ctx, timeout)
defer done()
// container for atomic error value
type errWrap struct {
E error
}
// Try the function in a goroutine
var errVal atomic.Value
doneCh := make(chan struct{})
go func() {
defer close(doneCh)
delay := time.Duration(0)
for {
// If our context ended, we want to exit right away.
select {
case <-ctx.Done():
return
case <-time.After(delay):
}
// Try the function call
err := f()
errVal.Store(&errWrap{err})
if err == nil {
return
}
log.Printf("[WARN] retryable error: %v", err)
delay *= 2
if delay == 0 {
delay = initialBackoffDelay
}
if delay > maxBackoffDelay {
delay = maxBackoffDelay
}
log.Printf("[INFO] sleeping for %s", delay)
}
}()
// Wait for completion
select {
case <-ctx.Done():
case <-doneCh:
}
// Check if we have a context error to check if we're interrupted or timeout
switch ctx.Err() {
case context.Canceled:
return fmt.Errorf("interrupted")
case context.DeadlineExceeded:
return fmt.Errorf("timeout")
}
// Check if we got an error executing
if ev, ok := errVal.Load().(errWrap); ok {
return ev.E
}
return nil
}

62
builtin/provisioners/remote-exec/resource_provisioner_test.go
View File

@ -2,12 +2,8 @@ package remoteexec
import (
"bytes"
"context"
"errors"
"io"
"net"
"testing"
"time"
"strings"
@ -210,64 +206,6 @@ func TestResourceProvider_CollectScripts_scriptsEmpty(t *testing.T) {
}
}
func TestRetryFunc(t *testing.T) {
origMax := maxBackoffDelay
maxBackoffDelay = time.Second
origStart := initialBackoffDelay
initialBackoffDelay = 10 * time.Millisecond
defer func() {
maxBackoffDelay = origMax
initialBackoffDelay = origStart
}()
// succeed on the third try
errs := []error{io.EOF, &net.OpError{Err: errors.New("ERROR")}, nil}
count := 0
err := retryFunc(context.Background(), time.Second, func() error {
if count >= len(errs) {
return errors.New("failed to stop after nil error")
}
err := errs[count]
count++
return err
})
if count != 3 {
t.Fatal("retry func should have been called 3 times")
}
if err != nil {
t.Fatal(err)
}
}
func TestRetryFuncBackoff(t *testing.T) {
origMax := maxBackoffDelay
maxBackoffDelay = time.Second
origStart := initialBackoffDelay
initialBackoffDelay = 100 * time.Millisecond
defer func() {
maxBackoffDelay = origMax
initialBackoffDelay = origStart
}()
count := 0
retryFunc(context.Background(), time.Second, func() error {
count++
return io.EOF
})
if count > 4 {
t.Fatalf("retry func failed to backoff. called %d times", count)
}
}
func testConfig(t *testing.T, c map[string]interface{}) *terraform.ResourceConfig {
r, err := config.NewRawConfig(c)
if err != nil {

67
builtin/provisioners/salt-masterless/resource_provisioner.go
View File

@ -131,6 +131,24 @@ func applyFn(ctx context.Context) error {
return err
}
ctx, cancelFunc := context.WithTimeout(ctx, comm.Timeout())
defer cancelFunc()
// Wait for the context to end and then disconnect
go func() {
<-ctx.Done()
comm.Disconnect()
}()
// Wait and retry until we establish the connection
err = communicator.Retry(ctx, func() error {
return comm.Connect(o)
})
if err != nil {
return err
}
var src, dst string
o.Output("Provisioning with Salt...")
@ -146,8 +164,10 @@ func applyFn(ctx context.Context) error {
if err == nil {
cmd.Wait()
if cmd.ExitStatus != 0 {
err = fmt.Errorf("Script exited with non-zero exit status: %d", cmd.ExitStatus)
if cmd.Err() != nil {
err = cmd.Err()
} else if cmd.ExitStatus() != 0 {
err = fmt.Errorf("Curl exited with non-zero exit status: %d", cmd.ExitStatus())
}
}
@ -170,8 +190,10 @@ func applyFn(ctx context.Context) error {
if err == nil {
cmd.Wait()
if cmd.ExitStatus != 0 {
err = fmt.Errorf("Script exited with non-zero exit status: %d", cmd.ExitStatus)
if cmd.Err() != nil {
err = cmd.Err()
} else if cmd.ExitStatus() != 0 {
err = fmt.Errorf("install_salt.sh exited with non-zero exit status: %d", cmd.ExitStatus())
}
}
// Wait for output to clean up
@ -259,17 +281,16 @@ func applyFn(ctx context.Context) error {
Stdout: outW,
Stderr: errW,
}
if err = comm.Start(cmd); err != nil || cmd.ExitStatus != 0 {
if err == nil {
err = fmt.Errorf("Bad exit status: %d", cmd.ExitStatus)
}
if err = comm.Start(cmd); err != nil {
err = fmt.Errorf("Error executing salt-call: %s", err)
}
if err == nil {
cmd.Wait()
if cmd.ExitStatus != 0 {
err = fmt.Errorf("Script exited with non-zero exit status: %d", cmd.ExitStatus)
if cmd.Err() != nil {
err = cmd.Err()
} else if cmd.ExitStatus() != 0 {
err = fmt.Errorf("Script exited with non-zero exit status: %d", cmd.ExitStatus())
}
}
// Wait for output to clean up
@ -336,14 +357,15 @@ func (p *provisioner) uploadFile(o terraform.UIOutput, comm communicator.Communi
func (p *provisioner) moveFile(o terraform.UIOutput, comm communicator.Communicator, dst, src string) error {
o.Output(fmt.Sprintf("Moving %s to %s", src, dst))
cmd := &remote.Cmd{Command: fmt.Sprintf(p.sudo("mv %s %s"), src, dst)}
if err := comm.Start(cmd); err != nil || cmd.ExitStatus != 0 {
if err == nil {
err = fmt.Errorf("Bad exit status: %d", cmd.ExitStatus)
}
if err := comm.Start(cmd); err != nil {
return fmt.Errorf("Unable to move %s to %s: %s", src, dst, err)
}
return nil
cmd.Wait()
if cmd.ExitStatus() != 0 {
return fmt.Errorf("Unable to move %s to %s: exit status: %d", src, dst, cmd.ExitStatus())
}
return cmd.Err()
}
func (p *provisioner) createDir(o terraform.UIOutput, comm communicator.Communicator, dir string) error {
@ -354,10 +376,12 @@ func (p *provisioner) createDir(o terraform.UIOutput, comm communicator.Communic
if err := comm.Start(cmd); err != nil {
return err
}
if cmd.ExitStatus != 0 {
cmd.Wait()
if cmd.ExitStatus() != 0 {
return fmt.Errorf("Non-zero exit status.")
}
return nil
return cmd.Err()
}
func (p *provisioner) removeDir(o terraform.UIOutput, comm communicator.Communicator, dir string) error {
@ -368,10 +392,11 @@ func (p *provisioner) removeDir(o terraform.UIOutput, comm communicator.Communic
if err := comm.Start(cmd); err != nil {
return err
}
if cmd.ExitStatus != 0 {
cmd.Wait()
if cmd.ExitStatus() != 0 {
return fmt.Errorf("Non-zero exit status.")
}
return nil
return cmd.Err()
}
func (p *provisioner) uploadDir(o terraform.UIOutput, comm communicator.Communicator, dst, src string, ignore []string) error {

46
command/apply.go
View File

@ -2,7 +2,6 @@ package command
import (
"bytes"
"context"
"fmt"
"os"
"sort"
@ -40,13 +39,11 @@ func (c *ApplyCommand) Run(args []string) int {
}
cmdFlags := c.Meta.flagSet(cmdName)
cmdFlags.BoolVar(&autoApprove, "auto-approve", false, "skip interactive approval of plan before applying")
if c.Destroy {
cmdFlags.BoolVar(&destroyForce, "force", false, "force")
cmdFlags.BoolVar(&destroyForce, "force", false, "deprecated: same as auto-approve")
}
cmdFlags.BoolVar(&refresh, "refresh", true, "refresh")
if !c.Destroy {
cmdFlags.BoolVar(&autoApprove, "auto-approve", false, "skip interactive approval of plan before applying")
}
cmdFlags.IntVar(
&c.Meta.parallelism, "parallelism", DefaultParallelism, "parallelism")
cmdFlags.StringVar(&c.Meta.statePath, "state", "", "path")
@ -158,38 +155,9 @@ func (c *ApplyCommand) Run(args []string) int {
opReq.AutoApprove = autoApprove
opReq.DestroyForce = destroyForce
// Perform the operation
ctx, ctxCancel := context.WithCancel(context.Background())
defer ctxCancel()
op, err := b.Operation(ctx, opReq)
op, err := c.RunOperation(b, opReq)
if err != nil {
c.Ui.Error(fmt.Sprintf("Error starting operation: %s", err))
return 1
}
// Wait for the operation to complete or an interrupt to occur
select {
case <-c.ShutdownCh:
// Cancel our context so we can start gracefully exiting
ctxCancel()
// Notify the user
c.Ui.Output(outputInterrupt)
// Still get the result, since there is still one
select {
case <-c.ShutdownCh:
c.Ui.Error(
"Two interrupts received. Exiting immediately. Note that data\n" +
"loss may have occurred.")
return 1
case <-op.Done():
}
case <-op.Done():
if err := op.Err; err != nil {
diags = diags.Append(err)
}
diags = diags.Append(err)
}
c.showDiagnostics(diags)
@ -250,8 +218,6 @@ Options:
-lock-timeout=0s Duration to retry a state lock.
-auto-approve Skip interactive approval of plan before applying.
-input=true Ask for input for variables if not directly set.
-no-color If specified, output won't contain any color.
@ -297,7 +263,9 @@ Options:
modifying. Defaults to the "-state-out" path with
".backup" extension. Set to "-" to disable backup.
-force Don't ask for input for destroy confirmation.
-auto-approve Skip interactive approval before destroying.
-force Deprecated: same as auto-approve.
-lock=true Lock the state file when locking is supported.

6
command/apply_destroy_test.go
View File

@ -40,7 +40,7 @@ func TestApply_destroy(t *testing.T) {
// Run the apply command pointing to our existing state
args := []string{
"-force",
"-auto-approve",
"-state", statePath,
testFixturePath("apply"),
}
@ -130,7 +130,7 @@ func TestApply_destroyLockedState(t *testing.T) {
// Run the apply command pointing to our existing state
args := []string{
"-force",
"-auto-approve",
"-state", statePath,
testFixturePath("apply"),
}
@ -206,7 +206,7 @@ func TestApply_destroyTargeted(t *testing.T) {
// Run the apply command pointing to our existing state
args := []string{
"-force",
"-auto-approve",
"-target", "test_instance.foo",
"-state", statePath,
testFixturePath("apply-destroy-targeted"),

29
command/apply_test.go
View File

@ -824,12 +824,11 @@ func TestApply_refresh(t *testing.T) {
}
func TestApply_shutdown(t *testing.T) {
cancelled := false
stopped := make(chan struct{})
cancelled := make(chan struct{})
shutdownCh := make(chan struct{})
statePath := testTempFile(t)
p := testProvider()
shutdownCh := make(chan struct{})
ui := new(cli.MockUi)
c := &ApplyCommand{
@ -841,8 +840,7 @@ func TestApply_shutdown(t *testing.T) {
}
p.StopFn = func() error {
close(stopped)
cancelled = true
close(cancelled)
return nil
}
@ -858,15 +856,26 @@ func TestApply_shutdown(t *testing.T) {
},
}, nil
}
var once sync.Once
p.ApplyFn = func(
*terraform.InstanceInfo,
*terraform.InstanceState,
*terraform.InstanceDiff) (*terraform.InstanceState, error) {
// only cancel once
if !cancelled {
once.Do(func() {
shutdownCh <- struct{}{}
<-stopped
}
})
// Because of the internal lock in the MockProvider, we can't
// coordiante directly with the calling of Stop, and making the
// MockProvider concurrent is disruptive to a lot of existing tests.
// Wait here a moment to help make sure the main goroutine gets to the
// Stop call before we exit, or the plan may finish before it can be
// canceled.
time.Sleep(200 * time.Millisecond)
return &terraform.InstanceState{
ID: "foo",
Attributes: map[string]string{
@ -888,7 +897,9 @@ func TestApply_shutdown(t *testing.T) {
t.Fatalf("err: %s", err)
}
if !cancelled {
select {
case <-cancelled:
default:
t.Fatal("command not cancelled")
}

120
command/clistate/state.go
View File

@ -8,9 +8,11 @@ import (
"context"
"fmt"
"strings"
"sync"
"time"
"github.com/hashicorp/errwrap"
multierror "github.com/hashicorp/go-multierror"
"github.com/hashicorp/terraform/helper/slowmessage"
"github.com/hashicorp/terraform/state"
"github.com/mitchellh/cli"
@ -48,47 +50,125 @@ that no one else is holding a lock.
`
)
// Lock locks the given state and outputs to the user if locking
// is taking longer than the threshold. The lock is retried until the context
// is cancelled.
func Lock(ctx context.Context, s state.State, info *state.LockInfo, ui cli.Ui, color *colorstring.Colorize) (string, error) {
var lockID string
// Locker allows for more convenient usage of the lower-level state.Locker
// implementations.
// The state.Locker API requires passing in a state.LockInfo struct. Locker
// implementations are expected to create the required LockInfo struct when
// Lock is called, populate the Operation field with the "reason" string
// provided, and pass that on to the underlying state.Locker.
// Locker implementations are also expected to store any state required to call
// Unlock, which is at a minimum the LockID string returned by the
// state.Locker.
type Locker interface {
// Lock the provided state, storing the reason string in the LockInfo.
Lock(s state.State, reason string) error
// Unlock the previously locked state.
// An optional error can be passed in, and will be combined with any error
// from the Unlock operation.
Unlock(error) error
}
type locker struct {
mu sync.Mutex
ctx context.Context
timeout time.Duration
state state.State
ui cli.Ui
color *colorstring.Colorize
lockID string
}
// Create a new Locker.
// This Locker uses state.LockWithContext to retry the lock until the provided
// timeout is reached, or the context is canceled. Lock progress will be be
// reported to the user through the provided UI.
func NewLocker(
ctx context.Context,
timeout time.Duration,
ui cli.Ui,
color *colorstring.Colorize) Locker {
l := &locker{
ctx: ctx,
timeout: timeout,
ui: ui,
color: color,
}
return l
}
// Locker locks the given state and outputs to the user if locking is taking
// longer than the threshold. The lock is retried until the context is
// cancelled.
func (l *locker) Lock(s state.State, reason string) error {
l.mu.Lock()
defer l.mu.Unlock()
l.state = s
ctx, cancel := context.WithTimeout(l.ctx, l.timeout)
defer cancel()
lockInfo := state.NewLockInfo()
lockInfo.Operation = reason
err := slowmessage.Do(LockThreshold, func() error {
id, err := state.LockWithContext(ctx, s, info)
lockID = id
id, err := state.LockWithContext(ctx, s, lockInfo)
l.lockID = id
return err
}, func() {
if ui != nil {
ui.Output(color.Color(LockMessage))
if l.ui != nil {
l.ui.Output(l.color.Color(LockMessage))
}
})
if err != nil {
err = errwrap.Wrapf(strings.TrimSpace(LockErrorMessage), err)
return errwrap.Wrapf(strings.TrimSpace(LockErrorMessage), err)
}
return lockID, err
return nil
}
// Unlock unlocks the given state and outputs to the user if the
// unlock fails what can be done.
func Unlock(s state.State, id string, ui cli.Ui, color *colorstring.Colorize) error {
func (l *locker) Unlock(parentErr error) error {
l.mu.Lock()
defer l.mu.Unlock()
if l.lockID == "" {
return parentErr
}
err := slowmessage.Do(LockThreshold, func() error {
return s.Unlock(id)
return l.state.Unlock(l.lockID)
}, func() {
if ui != nil {
ui.Output(color.Color(UnlockMessage))
if l.ui != nil {
l.ui.Output(l.color.Color(UnlockMessage))
}
})
if err != nil {
ui.Output(color.Color(fmt.Sprintf(
l.ui.Output(l.color.Color(fmt.Sprintf(
"\n"+strings.TrimSpace(UnlockErrorMessage)+"\n", err)))
err = fmt.Errorf(
"Error releasing the state lock. Please see the longer error message above.")
if parentErr != nil {
parentErr = multierror.Append(parentErr, err)
}
}
return parentErr
}
type noopLocker struct{}
// NewNoopLocker returns a valid Locker that does nothing.
func NewNoopLocker() Locker {
return noopLocker{}
}
func (l noopLocker) Lock(state.State, string) error {
return nil
}
func (l noopLocker) Unlock(err error) error {
return err
}

2
command/e2etest/primary_test.go
View File

@ -111,7 +111,7 @@ func TestPrimarySeparatePlan(t *testing.T) {
}
//// DESTROY
stdout, stderr, err = tf.Run("destroy", "-force")
stdout, stderr, err = tf.Run("destroy", "-auto-approve")
if err != nil {
t.Fatalf("unexpected destroy error: %s\nstderr:\n%s", err, stderr)
}

2
command/format/plan_test.go
View File

@ -408,7 +408,7 @@ func TestPlanStats(t *testing.T) {
},
},
PlanStats{
// data resource refreshes are not counted in our stats
// data resource refreshes are not counted in our stats
},
},
"replace": {

49
command/meta.go
View File

@ -3,6 +3,7 @@ package command
import (
"bufio"
"bytes"
"context"
"errors"
"flag"
"fmt"
@ -259,6 +260,54 @@ func (m *Meta) StdinPiped() bool {
return fi.Mode()&os.ModeNamedPipe != 0
}
func (m *Meta) RunOperation(b backend.Enhanced, opReq *backend.Operation) (*backend.RunningOperation, error) {
op, err := b.Operation(context.Background(), opReq)
if err != nil {
return nil, fmt.Errorf("error starting operation: %s", err)
}
// Wait for the operation to complete or an interrupt to occur
select {
case <-m.ShutdownCh:
// gracefully stop the operation
op.Stop()
// Notify the user
m.Ui.Output(outputInterrupt)
// Still get the result, since there is still one
select {
case <-m.ShutdownCh:
m.Ui.Error(
"Two interrupts received. Exiting immediately. Note that data\n" +
"loss may have occurred.")
// cancel the operation completely
op.Cancel()
// the operation should return asap
// but timeout just in case
select {
case <-op.Done():
case <-time.After(5 * time.Second):
}
return nil, errors.New("operation canceled")
case <-op.Done():
// operation completed after Stop
}
case <-op.Done():
// operation completed normally
}
if op.Err != nil {
return op, op.Err
}
return op, nil
}
const (
ProviderSkipVerifyEnvVar = "TF_SKIP_PROVIDER_VERIFY"
)

52
command/meta_backend.go
View File

@ -583,18 +583,11 @@ func (m *Meta) backendFromPlan(opts *BackendOpts) (backend.Backend, error) {
}
if m.stateLock {
lockCtx, cancel := context.WithTimeout(context.Background(), m.stateLockTimeout)
defer cancel()
// Lock the state if we can
lockInfo := state.NewLockInfo()
lockInfo.Operation = "backend from plan"
lockID, err := clistate.Lock(lockCtx, realMgr, lockInfo, m.Ui, m.Colorize())
if err != nil {
stateLocker := clistate.NewLocker(context.Background(), m.stateLockTimeout, m.Ui, m.Colorize())
if err := stateLocker.Lock(realMgr, "backend from plan"); err != nil {
return nil, fmt.Errorf("Error locking state: %s", err)
}
defer clistate.Unlock(realMgr, lockID, m.Ui, m.Colorize())
defer stateLocker.Unlock(nil)
}
if err := realMgr.RefreshState(); err != nil {
@ -964,18 +957,11 @@ func (m *Meta) backend_C_r_s(
}
if m.stateLock {
lockCtx, cancel := context.WithTimeout(context.Background(), m.stateLockTimeout)
defer cancel()
// Lock the state if we can
lockInfo := state.NewLockInfo()
lockInfo.Operation = "backend from config"
lockID, err := clistate.Lock(lockCtx, sMgr, lockInfo, m.Ui, m.Colorize())
if err != nil {
stateLocker := clistate.NewLocker(context.Background(), m.stateLockTimeout, m.Ui, m.Colorize())
if err := stateLocker.Lock(sMgr, "backend from plan"); err != nil {
return nil, fmt.Errorf("Error locking state: %s", err)
}
defer clistate.Unlock(sMgr, lockID, m.Ui, m.Colorize())
defer stateLocker.Unlock(nil)
}
// Store the metadata in our saved state location
@ -1047,18 +1033,11 @@ func (m *Meta) backend_C_r_S_changed(
}
if m.stateLock {
lockCtx, cancel := context.WithTimeout(context.Background(), m.stateLockTimeout)
defer cancel()
// Lock the state if we can
lockInfo := state.NewLockInfo()
lockInfo.Operation = "backend from config"
lockID, err := clistate.Lock(lockCtx, sMgr, lockInfo, m.Ui, m.Colorize())
if err != nil {
stateLocker := clistate.NewLocker(context.Background(), m.stateLockTimeout, m.Ui, m.Colorize())
if err := stateLocker.Lock(sMgr, "backend from plan"); err != nil {
return nil, fmt.Errorf("Error locking state: %s", err)
}
defer clistate.Unlock(sMgr, lockID, m.Ui, m.Colorize())
defer stateLocker.Unlock(nil)
}
// Update the backend state
@ -1190,18 +1169,11 @@ func (m *Meta) backend_C_R_S_unchanged(
}
if m.stateLock {
lockCtx, cancel := context.WithTimeout(context.Background(), m.stateLockTimeout)
defer cancel()
// Lock the state if we can
lockInfo := state.NewLockInfo()
lockInfo.Operation = "backend from config"
lockID, err := clistate.Lock(lockCtx, sMgr, lockInfo, m.Ui, m.Colorize())
if err != nil {
stateLocker := clistate.NewLocker(context.Background(), m.stateLockTimeout, m.Ui, m.Colorize())
if err := stateLocker.Lock(sMgr, "backend from plan"); err != nil {
return nil, fmt.Errorf("Error locking state: %s", err)
}
defer clistate.Unlock(sMgr, lockID, m.Ui, m.Colorize())
defer stateLocker.Unlock(nil)
}
// Unset the remote state

23
command/meta_backend_migrate.go
View File

@ -233,28 +233,19 @@ func (m *Meta) backendMigrateState_s_s(opts *backendMigrateOpts) error {
}
if m.stateLock {
lockCtx, cancel := context.WithTimeout(context.Background(), m.stateLockTimeout)
defer cancel()
lockCtx := context.Background()
lockInfoOne := state.NewLockInfo()
lockInfoOne.Operation = "migration"
lockInfoOne.Info = "source state"
lockIDOne, err := clistate.Lock(lockCtx, stateOne, lockInfoOne, m.Ui, m.Colorize())
if err != nil {
lockerOne := clistate.NewLocker(lockCtx, m.stateLockTimeout, m.Ui, m.Colorize())
if err := lockerOne.Lock(stateOne, "migration source state"); err != nil {
return fmt.Errorf("Error locking source state: %s", err)
}
defer clistate.Unlock(stateOne, lockIDOne, m.Ui, m.Colorize())
defer lockerOne.Unlock(nil)
lockInfoTwo := state.NewLockInfo()
lockInfoTwo.Operation = "migration"
lockInfoTwo.Info = "destination state"
lockIDTwo, err := clistate.Lock(lockCtx, stateTwo, lockInfoTwo, m.Ui, m.Colorize())
if err != nil {
lockerTwo := clistate.NewLocker(lockCtx, m.stateLockTimeout, m.Ui, m.Colorize())
if err := lockerTwo.Lock(stateTwo, "migration destination state"); err != nil {
return fmt.Errorf("Error locking destination state: %s", err)
}
defer clistate.Unlock(stateTwo, lockIDTwo, m.Ui, m.Colorize())
defer lockerTwo.Unlock(nil)
// We now own a lock, so double check that we have the version
// corresponding to the lock.

31
command/plan.go
View File

@ -1,7 +1,6 @@
package command
import (
"context"
"fmt"
"strings"
@ -107,35 +106,9 @@ func (c *PlanCommand) Run(args []string) int {
opReq.Type = backend.OperationTypePlan
// Perform the operation
ctx, ctxCancel := context.WithCancel(context.Background())
defer ctxCancel()
op, err := b.Operation(ctx, opReq)
op, err := c.RunOperation(b, opReq)
if err != nil {
c.Ui.Error(fmt.Sprintf("Error starting operation: %s", err))
return 1
}
select {
case <-c.ShutdownCh:
// Cancel our context so we can start gracefully exiting
ctxCancel()
// Notify the user
c.Ui.Output(outputInterrupt)
// Still get the result, since there is still one
select {
case <-c.ShutdownCh:
c.Ui.Error(
"Two interrupts received. Exiting immediately")
return 1
case <-op.Done():
}
case <-op.Done():
if err := op.Err; err != nil {
diags = diags.Append(err)
}
diags = diags.Append(err)
}
c.showDiagnostics(diags)

18
command/plan_test.go
View File

@ -835,8 +835,8 @@ func TestPlan_detailedExitcode_emptyDiff(t *testing.T) {
func TestPlan_shutdown(t *testing.T) {
cancelled := make(chan struct{})
shutdownCh := make(chan struct{})
p := testProvider()
ui := new(cli.MockUi)
c := &PlanCommand{
@ -863,6 +863,14 @@ func TestPlan_shutdown(t *testing.T) {
shutdownCh <- struct{}{}
})
// Because of the internal lock in the MockProvider, we can't
// coordiante directly with the calling of Stop, and making the
// MockProvider concurrent is disruptive to a lot of existing tests.
// Wait here a moment to help make sure the main goroutine gets to the
// Stop call before we exit, or the plan may finish before it can be
// canceled.
time.Sleep(200 * time.Millisecond)
return &terraform.InstanceDiff{
Attributes: map[string]*terraform.ResourceAttrDiff{
"ami": &terraform.ResourceAttrDiff{
@ -872,13 +880,15 @@ func TestPlan_shutdown(t *testing.T) {
}, nil
}
if code := c.Run([]string{testFixturePath("apply-shutdown")}); code != 0 {
t.Fatalf("bad: %d\n\n%s", code, ui.ErrorWriter.String())
if code := c.Run([]string{testFixturePath("apply-shutdown")}); code != 1 {
// FIXME: we should be able to avoid the error during evaluation
// the early exit isn't caught before the interpolation is evaluated
t.Fatal(ui.OutputWriter.String())
}
select {
case <-cancelled:
case <-time.After(5 * time.Second):
default:
t.Fatal("command not cancelled")
}
}

11
command/refresh.go
View File

@ -1,7 +1,6 @@
package command
import (
"context"
"fmt"
"strings"
@ -75,16 +74,8 @@ func (c *RefreshCommand) Run(args []string) int {
opReq.Type = backend.OperationTypeRefresh
opReq.Module = mod
// Perform the operation
op, err := b.Operation(context.Background(), opReq)
op, err := c.RunOperation(b, opReq)
if err != nil {
c.Ui.Error(fmt.Sprintf("Error starting operation: %s", err))
return 1
}
// Wait for the operation to complete
<-op.Done()
if err := op.Err; err != nil {
diags = diags.Append(err)
}

9
command/state_list.go
View File

@ -23,6 +23,7 @@ func (c *StateListCommand) Run(args []string) int {
cmdFlags := c.Meta.flagSet("state list")
cmdFlags.StringVar(&c.Meta.statePath, "state", DefaultStateFilename, "path")
lookupId := cmdFlags.String("id", "", "Restrict output to paths with a resource having the specified ID.")
if err := cmdFlags.Parse(args); err != nil {
return cli.RunResultHelp
}
@ -62,8 +63,10 @@ func (c *StateListCommand) Run(args []string) int {
}
for _, result := range results {
if _, ok := result.Value.(*terraform.InstanceState); ok {
c.Ui.Output(result.Address)
if i, ok := result.Value.(*terraform.InstanceState); ok {
if *lookupId == "" || i.ID == *lookupId {
c.Ui.Output(result.Address)
}
}
}
@ -94,6 +97,8 @@ Options:
up Terraform-managed resources. By default it will
use the state "terraform.tfstate" if it exists.
-id=ID Restricts the output to objects whose id is ID.
`
return strings.TrimSpace(helpText)
}

59
command/state_list_test.go
View File

@ -37,6 +37,65 @@ func TestStateList(t *testing.T) {
}
}
func TestStateListWithID(t *testing.T) {
state := testState()
statePath := testStateFile(t, state)
p := testProvider()
ui := new(cli.MockUi)
c := &StateListCommand{
Meta: Meta{
testingOverrides: metaOverridesForProvider(p),
Ui: ui,
},
}
args := []string{
"-state", statePath,
"-id", "bar",
}
if code := c.Run(args); code != 0 {
t.Fatalf("bad: %d\n\n%s", code, ui.ErrorWriter.String())
}
// Test that outputs were displayed
expected := strings.TrimSpace(testStateListOutput) + "\n"
actual := ui.OutputWriter.String()
if actual != expected {
t.Fatalf("Expected:\n%q\n\nTo equal: %q", actual, expected)
}
}
func TestStateListWithNonExistentID(t *testing.T) {
state := testState()
statePath := testStateFile(t, state)
p := testProvider()
ui := new(cli.MockUi)
c := &StateListCommand{
Meta: Meta{
testingOverrides: metaOverridesForProvider(p),
Ui: ui,
},
}
args := []string{
"-state", statePath,
"-id", "baz",
}
if code := c.Run(args); code != 0 {
t.Fatalf("bad: %d\n\n%s", code, ui.ErrorWriter.String())
}
// Test that output is empty
if ui.OutputWriter != nil {
actual := ui.OutputWriter.String()
if actual != "" {
t.Fatalf("Expected an empty output but got: %q", actual)
}
}
}
func TestStateList_backendState(t *testing.T) {
// Create a temporary working directory that is empty
td := tempDir(t)

13
command/taint.go
View File

@ -7,7 +7,6 @@ import (
"strings"
"github.com/hashicorp/terraform/command/clistate"
"github.com/hashicorp/terraform/state"
"github.com/hashicorp/terraform/terraform"
)
@ -84,18 +83,12 @@ func (c *TaintCommand) Run(args []string) int {
}
if c.stateLock {
lockCtx, cancel := context.WithTimeout(context.Background(), c.stateLockTimeout)
defer cancel()
lockInfo := state.NewLockInfo()
lockInfo.Operation = "taint"
lockID, err := clistate.Lock(lockCtx, st, lockInfo, c.Ui, c.Colorize())
if err != nil {
stateLocker := clistate.NewLocker(context.Background(), c.stateLockTimeout, c.Ui, c.Colorize())
if err := stateLocker.Lock(st, "taint"); err != nil {
c.Ui.Error(fmt.Sprintf("Error locking state: %s", err))
return 1
}
defer clistate.Unlock(st, lockID, c.Ui, c.Colorize())
defer stateLocker.Unlock(nil)
}
// Get the actual state structure

13
command/untaint.go
View File

@ -7,7 +7,6 @@ import (
"strings"
"github.com/hashicorp/terraform/command/clistate"
"github.com/hashicorp/terraform/state"
)
// UntaintCommand is a cli.Command implementation that manually untaints
@ -72,18 +71,12 @@ func (c *UntaintCommand) Run(args []string) int {
}
if c.stateLock {
lockCtx, cancel := context.WithTimeout(context.Background(), c.stateLockTimeout)
defer cancel()
lockInfo := state.NewLockInfo()
lockInfo.Operation = "untaint"
lockID, err := clistate.Lock(lockCtx, st, lockInfo, c.Ui, c.Colorize())
if err != nil {
stateLocker := clistate.NewLocker(context.Background(), c.stateLockTimeout, c.Ui, c.Colorize())
if err := stateLocker.Lock(st, "untaint"); err != nil {
c.Ui.Error(fmt.Sprintf("Error locking state: %s", err))
return 1
}
defer clistate.Unlock(st, lockID, c.Ui, c.Colorize())
defer stateLocker.Unlock(nil)
}
// Get the actual state structure

47
command/workspace_delete.go
View File

@ -6,7 +6,6 @@ import (
"strings"
"github.com/hashicorp/terraform/command/clistate"
"github.com/hashicorp/terraform/state"
"github.com/mitchellh/cli"
"github.com/posener/complete"
)
@ -97,6 +96,17 @@ func (c *WorkspaceDeleteCommand) Run(args []string) int {
return 1
}
var stateLocker clistate.Locker
if c.stateLock {
stateLocker = clistate.NewLocker(context.Background(), c.stateLockTimeout, c.Ui, c.Colorize())
if err := stateLocker.Lock(sMgr, "workspace_delete"); err != nil {
c.Ui.Error(fmt.Sprintf("Error locking state: %s", err))
return 1
}
} else {
stateLocker = clistate.NewNoopLocker()
}
if err := sMgr.RefreshState(); err != nil {
c.Ui.Error(err.Error())
return 1
@ -109,31 +119,16 @@ func (c *WorkspaceDeleteCommand) Run(args []string) int {
return 1
}
// Honor the lock request, for consistency and one final safety check.
if c.stateLock {
lockCtx, cancel := context.WithTimeout(context.Background(), c.stateLockTimeout)
defer cancel()
// Lock the state if we can
lockInfo := state.NewLockInfo()
lockInfo.Operation = "workspace delete"
lockID, err := clistate.Lock(lockCtx, sMgr, lockInfo, c.Ui, c.Colorize())
if err != nil {
c.Ui.Error(fmt.Sprintf("Error locking state: %s", err))
return 1
}
// We need to release the lock just before deleting the state, in case
// the backend can't remove the resource while holding the lock. This
// is currently true for Windows local files.
//
// TODO: While there is little safety in locking while deleting the
// state, it might be nice to be able to coordinate processes around
// state deletion, i.e. in a CI environment. Adding Delete() as a
// required method of States would allow the removal of the resource to
// be delegated from the Backend to the State itself.
clistate.Unlock(sMgr, lockID, c.Ui, c.Colorize())
}
// We need to release the lock just before deleting the state, in case
// the backend can't remove the resource while holding the lock. This
// is currently true for Windows local files.
//
// TODO: While there is little safety in locking while deleting the
// state, it might be nice to be able to coordinate processes around
// state deletion, i.e. in a CI environment. Adding Delete() as a
// required method of States would allow the removal of the resource to
// be delegated from the Backend to the State itself.
stateLocker.Unlock(nil)
err = b.DeleteState(delEnv)
if err != nil {

13
command/workspace_new.go
View File

@ -7,7 +7,6 @@ import (
"strings"
"github.com/hashicorp/terraform/command/clistate"
"github.com/hashicorp/terraform/state"
"github.com/hashicorp/terraform/terraform"
"github.com/mitchellh/cli"
"github.com/posener/complete"
@ -115,18 +114,12 @@ func (c *WorkspaceNewCommand) Run(args []string) int {
}
if c.stateLock {
lockCtx, cancel := context.WithTimeout(context.Background(), c.stateLockTimeout)
defer cancel()
// Lock the state if we can
lockInfo := state.NewLockInfo()
lockInfo.Operation = "workspace new"
lockID, err := clistate.Lock(lockCtx, sMgr, lockInfo, c.Ui, c.Colorize())
if err != nil {
stateLocker := clistate.NewLocker(context.Background(), c.stateLockTimeout, c.Ui, c.Colorize())
if err := stateLocker.Lock(sMgr, "workspace_delete"); err != nil {
c.Ui.Error(fmt.Sprintf("Error locking state: %s", err))
return 1
}
defer clistate.Unlock(sMgr, lockID, c.Ui, c.Colorize())
defer stateLocker.Unlock(nil)
}
// read the existing state file

96
communicator/communicator.go
View File

@ -1,8 +1,11 @@
package communicator
import (
"context"
"fmt"
"io"
"log"
"sync/atomic"
"time"
"github.com/hashicorp/terraform/communicator/remote"
@ -51,3 +54,96 @@ func New(s *terraform.InstanceState) (Communicator, error) {
return nil, fmt.Errorf("connection type '%s' not supported", connType)
}
}
// maxBackoffDelay is the maximum delay between retry attempts
var maxBackoffDelay = 20 * time.Second
var initialBackoffDelay = time.Second
// Fatal is an interface that error values can return to halt Retry
type Fatal interface {
FatalError() error
}
// Retry retries the function f until it returns a nil error, a Fatal error, or
// the context expires.
func Retry(ctx context.Context, f func() error) error {
// container for atomic error value
type errWrap struct {
E error
}
// Try the function in a goroutine
var errVal atomic.Value
doneCh := make(chan struct{})
go func() {
defer close(doneCh)
delay := time.Duration(0)
for {
// If our context ended, we want to exit right away.
select {
case <-ctx.Done():
return
case <-time.After(delay):
}
// Try the function call
err := f()
// return if we have no error, or a FatalError
done := false
switch e := err.(type) {
case nil:
done = true
case Fatal:
err = e.FatalError()
done = true
}
errVal.Store(errWrap{err})
if done {
return
}
log.Printf("[WARN] retryable error: %v", err)
delay *= 2
if delay == 0 {
delay = initialBackoffDelay
}
if delay > maxBackoffDelay {
delay = maxBackoffDelay
}
log.Printf("[INFO] sleeping for %s", delay)
}
}()
// Wait for completion
select {
case <-ctx.Done():
case <-doneCh:
}
var lastErr error
// Check if we got an error executing
if ev, ok := errVal.Load().(errWrap); ok {
lastErr = ev.E
}
// Check if we have a context error to check if we're interrupted or timeout
switch ctx.Err() {
case context.Canceled:
return fmt.Errorf("interrupted - last error: %v", lastErr)
case context.DeadlineExceeded:
return fmt.Errorf("timeout - last error: %v", lastErr)
}
if lastErr != nil {
return lastErr
}
return nil
}

4
communicator/communicator_mock.go
View File

@ -42,11 +42,13 @@ func (c *MockCommunicator) ScriptPath() string {
// Start implementation of communicator.Communicator interface
func (c *MockCommunicator) Start(r *remote.Cmd) error {
r.Init()
if !c.Commands[r.Command] {
return fmt.Errorf("Command not found!")
}
r.SetExited(0)
r.SetExitStatus(0, nil)
return nil
}

68
communicator/communicator_test.go
View File

@ -1,7 +1,12 @@
package communicator
import (
"context"
"errors"
"io"
"net"
"testing"
"time"
"github.com/hashicorp/terraform/terraform"
)
@ -28,3 +33,66 @@ func TestCommunicator_new(t *testing.T) {
t.Fatalf("err: %v", err)
}
}
func TestRetryFunc(t *testing.T) {
origMax := maxBackoffDelay
maxBackoffDelay = time.Second
origStart := initialBackoffDelay
initialBackoffDelay = 10 * time.Millisecond
defer func() {
maxBackoffDelay = origMax
initialBackoffDelay = origStart
}()
// succeed on the third try
errs := []error{io.EOF, &net.OpError{Err: errors.New("ERROR")}, nil}
count := 0
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
err := Retry(ctx, func() error {
if count >= len(errs) {
return errors.New("failed to stop after nil error")
}
err := errs[count]
count++
return err
})
if count != 3 {
t.Fatal("retry func should have been called 3 times")
}
if err != nil {
t.Fatal(err)
}
}
func TestRetryFuncBackoff(t *testing.T) {
origMax := maxBackoffDelay
maxBackoffDelay = time.Second
origStart := initialBackoffDelay
initialBackoffDelay = 100 * time.Millisecond
defer func() {
maxBackoffDelay = origMax
initialBackoffDelay = origStart
}()
count := 0
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
Retry(ctx, func() error {
count++
return io.EOF
})
if count > 4 {
t.Fatalf("retry func failed to backoff. called %d times", count)
}
}

70
communicator/remote/command.go
View File

@ -23,45 +23,59 @@ type Cmd struct {
Stdout io.Writer
Stderr io.Writer
// This will be set to true when the remote command has exited. It
// shouldn't be set manually by the user, but there is no harm in
// doing so.
Exited bool
// Once Exited is true, this will contain the exit code of the process.
ExitStatus int
// Once Wait returns, his will contain the exit code of the process.
exitStatus int
// Internal fields
exitCh chan struct{}
// err is used to store any error reported by the Communicator during
// execution.
err error
// This thing is a mutex, lock when making modifications concurrently
sync.Mutex
}
// SetExited is a helper for setting that this process is exited. This
// should be called by communicators who are running a remote command in
// order to set that the command is done.
func (r *Cmd) SetExited(status int) {
r.Lock()
defer r.Unlock()
// Init must be called by the Communicator before executing the command.
func (c *Cmd) Init() {
c.Lock()
defer c.Unlock()
if r.exitCh == nil {
r.exitCh = make(chan struct{})
}
c.exitCh = make(chan struct{})
}
r.Exited = true
r.ExitStatus = status
close(r.exitCh)
// SetExitStatus stores the exit status of the remote command as well as any
// communicator related error. SetExitStatus then unblocks any pending calls
// to Wait.
// This should only be called by communicators executing the remote.Cmd.
func (c *Cmd) SetExitStatus(status int, err error) {
c.Lock()
defer c.Unlock()
c.exitStatus = status
c.err = err
close(c.exitCh)
}
// Err returns any communicator related error.
func (c *Cmd) Err() error {
c.Lock()
defer c.Unlock()
return c.err
}
// ExitStatus returns the exit status of the remote command
func (c *Cmd) ExitStatus() int {
c.Lock()
defer c.Unlock()
return c.exitStatus
}
// Wait waits for the remote command to complete.
func (r *Cmd) Wait() {
// Make sure our condition variable is initialized.
r.Lock()
if r.exitCh == nil {
r.exitCh = make(chan struct{})
}
r.Unlock()
<-r.exitCh
func (c *Cmd) Wait() {
<-c.exitCh
}

32
communicator/ssh/communicator.go
View File

@ -63,6 +63,14 @@ type sshConfig struct {
sshAgent *sshAgent
}
type fatalError struct {
error
}
func (e fatalError) FatalError() error {
return e.error
}
// New creates a new communicator implementation over SSH.
func New(s *terraform.InstanceState) (*Communicator, error) {
connInfo, err := parseConnectionInfo(s)
@ -117,11 +125,13 @@ func (c *Communicator) Connect(o terraform.UIOutput) (err error) {
" User: %s\n"+
" Password: %t\n"+
" Private key: %t\n"+
" SSH Agent: %t",
" SSH Agent: %t\n"+
" Checking Host Key: %t",
c.connInfo.Host, c.connInfo.User,
c.connInfo.Password != "",
c.connInfo.PrivateKey != "",
c.connInfo.Agent,
c.connInfo.HostKey != "",
))
if c.connInfo.BastionHost != "" {
@ -131,11 +141,13 @@ func (c *Communicator) Connect(o terraform.UIOutput) (err error) {
" User: %s\n"+
" Password: %t\n"+
" Private key: %t\n"+
" SSH Agent: %t",
" SSH Agent: %t\n"+
" Checking Host Key: %t",
c.connInfo.BastionHost, c.connInfo.BastionUser,
c.connInfo.BastionPassword != "",
c.connInfo.BastionPrivateKey != "",
c.connInfo.Agent,
c.connInfo.BastionHostKey != "",
))
}
}
@ -159,8 +171,8 @@ func (c *Communicator) Connect(o terraform.UIOutput) (err error) {
host := fmt.Sprintf("%s:%d", c.connInfo.Host, c.connInfo.Port)
sshConn, sshChan, req, err := ssh.NewClientConn(c.conn, host, c.config.config)
if err != nil {
log.Printf("handshake error: %s", err)
return err
log.Printf("fatal handshake error: %s", err)
return fatalError{err}
}
c.client = ssh.NewClient(sshConn, sshChan, req)
@ -168,7 +180,7 @@ func (c *Communicator) Connect(o terraform.UIOutput) (err error) {
if c.config.sshAgent != nil {
log.Printf("[DEBUG] Telling SSH config to forward to agent")
if err := c.config.sshAgent.ForwardToAgent(c.client); err != nil {
return err
return fatalError{err}
}
log.Printf("[DEBUG] Setting up a session to request agent forwarding")
@ -231,6 +243,8 @@ func (c *Communicator) ScriptPath() string {
// Start implementation of communicator.Communicator interface
func (c *Communicator) Start(cmd *remote.Cmd) error {
cmd.Init()
session, err := c.newSession()
if err != nil {
return err
@ -255,7 +269,7 @@ func (c *Communicator) Start(cmd *remote.Cmd) error {
}
log.Printf("starting remote command: %s", cmd.Command)
err = session.Start(cmd.Command + "\n")
err = session.Start(strings.TrimSpace(cmd.Command) + "\n")
if err != nil {
return err
}
@ -274,8 +288,8 @@ func (c *Communicator) Start(cmd *remote.Cmd) error {
}
}
cmd.SetExitStatus(exitStatus, err)
log.Printf("remote command exited with '%d': %s", exitStatus, cmd.Command)
cmd.SetExited(exitStatus)
}()
return nil
@ -346,10 +360,10 @@ func (c *Communicator) UploadScript(path string, input io.Reader) error {
"machine: %s", err)
}
cmd.Wait()
if cmd.ExitStatus != 0 {
if cmd.ExitStatus() != 0 {
return fmt.Errorf(
"Error chmodding script file to 0777 in remote "+
"machine %d: %s %s", cmd.ExitStatus, stdout.String(), stderr.String())
"machine %d: %s %s", cmd.ExitStatus(), stdout.String(), stderr.String())
}
return nil

225
communicator/ssh/communicator_test.go
View File

@ -5,6 +5,7 @@ package ssh
import (
"bufio"
"bytes"
"encoding/base64"
"fmt"
"io"
"io/ioutil"
@ -13,8 +14,10 @@ import (
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"testing"
"time"
"github.com/hashicorp/terraform/communicator/remote"
"github.com/hashicorp/terraform/terraform"
@ -50,21 +53,26 @@ gqnBycHj6AhEycjda75cs+0zybZvN4x65KZHOGW/O/7OAWEcZP5TPb3zf9ned3Hl
NsZoFj52ponUM6+99A2CmezFCN16c4mbA//luWF+k3VVqR6BpkrhKw==
-----END RSA PRIVATE KEY-----`
var serverConfig = &ssh.ServerConfig{
PasswordCallback: acceptUserPass("user", "pass"),
PublicKeyCallback: acceptPublicKey(testClientPublicKey),
}
// this cert was signed by the key from testCAPublicKey
const testServerHostCert = `ssh-rsa-cert-v01@openssh.com AAAAHHNzaC1yc2EtY2VydC12MDFAb3BlbnNzaC5jb20AAAAgvQ3Bs1ex7277b9q6I0fNaWsVEC16f+LcT8RLPSVMEVMAAAADAQABAAABAQDX2UZWxOohPmKI1hGCehjULCRsRNblyr5HOTm/+ROV/fVelJTvQdVaRtMREQKNph1czaAZxtv6zGmroa1d/UzeRWibJyqHHCE+/gKvpenhZP+OQXH3P4UXOl6h0YlaM4fovYfm5fUK+v0QN1Cn2338nfb+oEWe1jwbChQj/L/UxJOYyIW26l0w4M3Tri93eDIwpPCuVDy1kzppi7I4+y60uVRjsznHkXAwNi+c8NJ7JP8jDTOzcH40LKp54x3ZPtjNAWdEBOPQzuszkuhKzsNWpWuI4QAGywXIuPfU9uhqguE4qByqgz2SGQ3OvsUdW+L4OFgzaMPQPC+pks3o2acvAAAAAAAAAAAAAAACAAAAB2NhLXRlc3QAAAANAAAACTEyNy4wLjAuMQAAAABag0jkAAAAAHDcHtAAAAAAAAAAAAAAAAAAAAEXAAAAB3NzaC1yc2EAAAADAQABAAABAQCrozyZIhdEvalCn+eSzHH94cO9ykiywA13ntWI7mJcHBwYTeCYWG8E9zGXyp2iDOjCGudM0Tdt8o0OofKChk9Z/qiUN0G8y1kmaXBlBM3qA5R9NPpvMYMNkYLfX6ivtZCnqrsbzaoqN2Oc/7H2StHzJWh/XCGu9otQZA6vdv1oSmAsZOjw/xIGaGQqDUaLq21J280PP1qSbdJHf76iSHE+TWe3YpqV946JWM5tCh0DykZ10VznvxYpUjzhr07IN3tVKxOXbPnnU7lX6IaLIWgfzLqwSyheeux05c3JLF9iF4sFu8ou4hwQz1iuUTU1jxgwZP0w/bkXgFFs0949lW81AAABDwAAAAdzc2gtcnNhAAABAEyoiVkZ5z79nh3WSU5mU2U7e2BItnnEqsJIm9EN+35uG0yORSXmQoaa9mtli7G3r79tyqEJd/C95EdNvU/9TjaoDcbH8OHP+Ue9XSfUzBuQ6bGSXe6mlZlO7QJ1cIyWphFP3MkrweDSiJ+SpeXzLzZkiJ7zKv5czhBEyG/MujFgvikotL+eUNG42y2cgsesXSjENSBS3l11q55a+RM2QKt3W32im8CsSxrH6Mz6p4JXQNgsVvZRknLxNlWXULFB2HLTunPKzJNMTf6xZf66oivSBAXVIdNKhlVpAQ3dT/dW5K6J4aQF/hjWByyLprFwZ16cPDqvtalnTCpbRYelNbw=`
func init() {
// Parse and set the private key of the server, required to accept connections
signer, err := ssh.ParsePrivateKey([]byte(testServerPrivateKey))
if err != nil {
panic("unable to parse private key: " + err.Error())
const testCAPublicKey = `ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCrozyZIhdEvalCn+eSzHH94cO9ykiywA13ntWI7mJcHBwYTeCYWG8E9zGXyp2iDOjCGudM0Tdt8o0OofKChk9Z/qiUN0G8y1kmaXBlBM3qA5R9NPpvMYMNkYLfX6ivtZCnqrsbzaoqN2Oc/7H2StHzJWh/XCGu9otQZA6vdv1oSmAsZOjw/xIGaGQqDUaLq21J280PP1qSbdJHf76iSHE+TWe3YpqV946JWM5tCh0DykZ10VznvxYpUjzhr07IN3tVKxOXbPnnU7lX6IaLIWgfzLqwSyheeux05c3JLF9iF4sFu8ou4hwQz1iuUTU1jxgwZP0w/bkXgFFs0949lW81`
func newMockLineServer(t *testing.T, signer ssh.Signer) string {
serverConfig := &ssh.ServerConfig{
PasswordCallback: acceptUserPass("user", "pass"),
PublicKeyCallback: acceptPublicKey(testClientPublicKey),
}
var err error
if signer == nil {
signer, err = ssh.ParsePrivateKey([]byte(testServerPrivateKey))
if err != nil {
t.Fatalf("unable to parse private key: %s", err)
}
}
serverConfig.AddHostKey(signer)
}
func newMockLineServer(t *testing.T) string {
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("Unable to listen for connection: %s", err)
@ -111,7 +119,7 @@ func newMockLineServer(t *testing.T) string {
}
func TestNew_Invalid(t *testing.T) {
address := newMockLineServer(t)
address := newMockLineServer(t, nil)
parts := strings.Split(address, ":")
r := &terraform.InstanceState{
@ -139,7 +147,7 @@ func TestNew_Invalid(t *testing.T) {
}
func TestStart(t *testing.T) {
address := newMockLineServer(t)
address := newMockLineServer(t, nil)
parts := strings.Split(address, ":")
r := &terraform.InstanceState{
@ -171,6 +179,197 @@ func TestStart(t *testing.T) {
}
}
func TestLostConnection(t *testing.T) {
address := newMockLineServer(t, nil)
parts := strings.Split(address, ":")
r := &terraform.InstanceState{
Ephemeral: terraform.EphemeralState{
ConnInfo: map[string]string{
"type": "ssh",
"user": "user",
"password": "pass",
"host": parts[0],
"port": parts[1],
"timeout": "30s",
},
},
}
c, err := New(r)
if err != nil {
t.Fatalf("error creating communicator: %s", err)
}
var cmd remote.Cmd
stdout := new(bytes.Buffer)
cmd.Command = "echo foo"
cmd.Stdout = stdout
err = c.Start(&cmd)
if err != nil {
t.Fatalf("error executing remote command: %s", err)
}
// The test server can't execute anything, so Wait will block, unless
// there's an error. Disconnect the communicator transport, to cause the
// command to fail.
go func() {
time.Sleep(100 * time.Millisecond)
c.Disconnect()
}()
cmd.Wait()
if cmd.Err() == nil {
t.Fatal("expected communicator error")
}
if cmd.ExitStatus() != 0 {
t.Fatal("command should not have returned an exit status")
}
}
func TestHostKey(t *testing.T) {
// get the server's public key
signer, err := ssh.ParsePrivateKey([]byte(testServerPrivateKey))
if err != nil {
panic("unable to parse private key: " + err.Error())
}
pubKey := fmt.Sprintf("ssh-rsa %s", base64.StdEncoding.EncodeToString(signer.PublicKey().Marshal()))
address := newMockLineServer(t, nil)
host, p, _ := net.SplitHostPort(address)
port, _ := strconv.Atoi(p)
connInfo := &connectionInfo{
User: "user",
Password: "pass",
Host: host,
HostKey: pubKey,
Port: port,
Timeout: "30s",
}
cfg, err := prepareSSHConfig(connInfo)
if err != nil {
t.Fatal(err)
}
c := &Communicator{
connInfo: connInfo,
config: cfg,
}
var cmd remote.Cmd
stdout := new(bytes.Buffer)
cmd.Command = "echo foo"
cmd.Stdout = stdout
if err := c.Start(&cmd); err != nil {
t.Fatal(err)
}
if err := c.Disconnect(); err != nil {
t.Fatal(err)
}
// now check with the wrong HostKey
address = newMockLineServer(t, nil)
_, p, _ = net.SplitHostPort(address)
port, _ = strconv.Atoi(p)
connInfo.HostKey = testClientPublicKey
connInfo.Port = port
cfg, err = prepareSSHConfig(connInfo)
if err != nil {
t.Fatal(err)
}
c = &Communicator{
connInfo: connInfo,
config: cfg,
}
err = c.Start(&cmd)
if err == nil || !strings.Contains(err.Error(), "mismatch") {
t.Fatalf("expected host key mismatch, got error:%v", err)
}
}
func TestHostCert(t *testing.T) {
pk, _, _, _, err := ssh.ParseAuthorizedKey([]byte(testServerHostCert))
if err != nil {
t.Fatal(err)
}
signer, err := ssh.ParsePrivateKey([]byte(testServerPrivateKey))
if err != nil {
t.Fatal(err)
}
signer, err = ssh.NewCertSigner(pk.(*ssh.Certificate), signer)
if err != nil {
t.Fatal(err)
}
address := newMockLineServer(t, signer)
host, p, _ := net.SplitHostPort(address)
port, _ := strconv.Atoi(p)
connInfo := &connectionInfo{
User: "user",
Password: "pass",
Host: host,
HostKey: testCAPublicKey,
Port: port,
Timeout: "30s",
}
cfg, err := prepareSSHConfig(connInfo)
if err != nil {
t.Fatal(err)
}
c := &Communicator{
connInfo: connInfo,
config: cfg,
}
var cmd remote.Cmd
stdout := new(bytes.Buffer)
cmd.Command = "echo foo"
cmd.Stdout = stdout
if err := c.Start(&cmd); err != nil {
t.Fatal(err)
}
if err := c.Disconnect(); err != nil {
t.Fatal(err)
}
// now check with the wrong HostKey
address = newMockLineServer(t, signer)
_, p, _ = net.SplitHostPort(address)
port, _ = strconv.Atoi(p)
connInfo.HostKey = testClientPublicKey
connInfo.Port = port
cfg, err = prepareSSHConfig(connInfo)
if err != nil {
t.Fatal(err)
}
c = &Communicator{
connInfo: connInfo,
config: cfg,
}
err = c.Start(&cmd)
if err == nil || !strings.Contains(err.Error(), "authorities") {
t.Fatalf("expected host key mismatch, got error:%v", err)
}
}
func TestAccUploadFile(t *testing.T) {
// use the local ssh server and scp binary to check uploads
if ok := os.Getenv("SSH_UPLOAD_TEST"); ok == "" {

51
communicator/ssh/provisioner.go
View File

@ -18,6 +18,7 @@ import (
"github.com/xanzy/ssh-agent"
"golang.org/x/crypto/ssh"
"golang.org/x/crypto/ssh/agent"
"golang.org/x/crypto/ssh/knownhosts"
)
const (
@ -43,6 +44,7 @@ type connectionInfo struct {
Password string
PrivateKey string `mapstructure:"private_key"`
Host string
HostKey string `mapstructure:"host_key"`
Port int
Agent bool
Timeout string
@ -53,6 +55,7 @@ type connectionInfo struct {
BastionPassword string `mapstructure:"bastion_password"`
BastionPrivateKey string `mapstructure:"bastion_private_key"`
BastionHost string `mapstructure:"bastion_host"`
BastionHostKey string `mapstructure:"bastion_host_key"`
BastionPort int `mapstructure:"bastion_port"`
AgentIdentity string `mapstructure:"agent_identity"`
@ -144,34 +147,38 @@ func prepareSSHConfig(connInfo *connectionInfo) (*sshConfig, error) {
return nil, err
}
host := fmt.Sprintf("%s:%d", connInfo.Host, connInfo.Port)
sshConf, err := buildSSHClientConfig(sshClientConfigOpts{
user: connInfo.User,
host: host,
privateKey: connInfo.PrivateKey,
password: connInfo.Password,
hostKey: connInfo.HostKey,
sshAgent: sshAgent,
})
if err != nil {
return nil, err
}
connectFunc := ConnectFunc("tcp", host)
var bastionConf *ssh.ClientConfig
if connInfo.BastionHost != "" {
bastionHost := fmt.Sprintf("%s:%d", connInfo.BastionHost, connInfo.BastionPort)
bastionConf, err = buildSSHClientConfig(sshClientConfigOpts{
user: connInfo.BastionUser,
host: bastionHost,
privateKey: connInfo.BastionPrivateKey,
password: connInfo.BastionPassword,
hostKey: connInfo.HostKey,
sshAgent: sshAgent,
})
if err != nil {
return nil, err
}
}
host := fmt.Sprintf("%s:%d", connInfo.Host, connInfo.Port)
connectFunc := ConnectFunc("tcp", host)
if bastionConf != nil {
bastionHost := fmt.Sprintf("%s:%d", connInfo.BastionHost, connInfo.BastionPort)
connectFunc = BastionConnectFunc("tcp", bastionHost, bastionConf, "tcp", host)
}
@ -188,11 +195,41 @@ type sshClientConfigOpts struct {
password string
sshAgent *sshAgent
user string
host string
hostKey string
}
func buildSSHClientConfig(opts sshClientConfigOpts) (*ssh.ClientConfig, error) {
hkCallback := ssh.InsecureIgnoreHostKey()
if opts.hostKey != "" {
// The knownhosts package only takes paths to files, but terraform
// generally wants to handle config data in-memory. Rather than making
// the known_hosts file an exception, write out the data to a temporary
// file to create the HostKeyCallback.
tf, err := ioutil.TempFile("", "tf-known_hosts")
if err != nil {
return nil, fmt.Errorf("failed to create temp known_hosts file: %s", err)
}
defer tf.Close()
defer os.RemoveAll(tf.Name())
// we mark this as a CA as well, but the host key fallback will still
// use it as a direct match if the remote host doesn't return a
// certificate.
if _, err := tf.WriteString(fmt.Sprintf("@cert-authority %s %s\n", opts.host, opts.hostKey)); err != nil {
return nil, fmt.Errorf("failed to write temp known_hosts file: %s", err)
}
tf.Sync()
hkCallback, err = knownhosts.New(tf.Name())
if err != nil {
return nil, err
}
}
conf := &ssh.ClientConfig{
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
HostKeyCallback: hkCallback,
User: opts.user,
}

5
communicator/winrm/communicator.go
View File

@ -131,6 +131,8 @@ func (c *Communicator) ScriptPath() string {
// Start implementation of communicator.Communicator interface
func (c *Communicator) Start(rc *remote.Cmd) error {
rc.Init()
err := c.Connect(nil)
if err != nil {
return err
@ -168,7 +170,8 @@ func runCommand(shell *winrm.Shell, cmd *winrm.Command, rc *remote.Cmd) {
cmd.Wait()
wg.Wait()
rc.SetExited(cmd.ExitCode())
rc.SetExitStatus(cmd.ExitCode(), nil)
}
// Upload implementation of communicator.Communicator interface

24
configs/backend.go Normal file
View File

@ -0,0 +1,24 @@
package configs
import (
"github.com/hashicorp/hcl2/hcl"
)
// Backend represents a "backend" block inside a "terraform" block in a module
// or file.
type Backend struct {
Type string
Config hcl.Body
TypeRange hcl.Range
DeclRange hcl.Range
}
func decodeBackendBlock(block *hcl.Block) (*Backend, hcl.Diagnostics) {
return &Backend{
Type: block.Labels[0],
TypeRange: block.LabelRanges[0],
Config: block.Body,
DeclRange: block.DefRange,
}, nil
}

106
configs/compat_shim.go Normal file
View File

@ -0,0 +1,106 @@
package configs
import (
"github.com/hashicorp/hcl2/hcl"
"github.com/hashicorp/hcl2/hcl/hclsyntax"
"github.com/zclconf/go-cty/cty"
)
// -------------------------------------------------------------------------
// Functions in this file are compatibility shims intended to ease conversion
// from the old configuration loader. Any use of these functions that makes
// a change should generate a deprecation warning explaining to the user how
// to update their code for new patterns.
//
// Shims are particularly important for any patterns that have been widely
// documented in books, tutorials, etc. Users will still be starting from
// these examples and we want to help them adopt the latest patterns rather
// than leave them stranded.
// -------------------------------------------------------------------------
// shimTraversalInString takes any arbitrary expression and checks if it is
// a quoted string in the native syntax. If it _is_, then it is parsed as a
// traversal and re-wrapped into a synthetic traversal expression and a
// warning is generated. Otherwise, the given expression is just returned
// verbatim.
//
// This function has no effect on expressions from the JSON syntax, since
// traversals in strings are the required pattern in that syntax.
//
// If wantKeyword is set, the generated warning diagnostic will talk about
// keywords rather than references. The behavior is otherwise unchanged, and
// the caller remains responsible for checking that the result is indeed
// a keyword, e.g. using hcl.ExprAsKeyword.
func shimTraversalInString(expr hcl.Expression, wantKeyword bool) (hcl.Expression, hcl.Diagnostics) {
if !exprIsNativeQuotedString(expr) {
return expr, nil
}
strVal, diags := expr.Value(nil)
if diags.HasErrors() || strVal.IsNull() || !strVal.IsKnown() {
// Since we're not even able to attempt a shim here, we'll discard
// the diagnostics we saw so far and let the caller's own error
// handling take care of reporting the invalid expression.
return expr, nil
}
// The position handling here isn't _quite_ right because it won't
// take into account any escape sequences in the literal string, but
// it should be close enough for any error reporting to make sense.
srcRange := expr.Range()
startPos := srcRange.Start // copy
startPos.Column++ // skip initial quote
startPos.Byte++ // skip initial quote
traversal, tDiags := hclsyntax.ParseTraversalAbs(
[]byte(strVal.AsString()),
srcRange.Filename,
startPos,
)
diags = append(diags, tDiags...)
// For initial release our deprecation warnings are disabled to allow
// a period where modules can be compatible with both old and new
// conventions.
// FIXME: Re-enable these deprecation warnings in a release prior to
// Terraform 0.13 and then remove the shims altogether for 0.13.
/*
if wantKeyword {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagWarning,
Summary: "Quoted keywords are deprecated",
Detail: "In this context, keywords are expected literally rather than in quotes. Previous versions of Terraform required quotes, but that usage is now deprecated. Remove the quotes surrounding this keyword to silence this warning.",
Subject: &srcRange,
})
} else {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagWarning,
Summary: "Quoted references are deprecated",
Detail: "In this context, references are expected literally rather than in quotes. Previous versions of Terraform required quotes, but that usage is now deprecated. Remove the quotes surrounding this reference to silence this warning.",
Subject: &srcRange,
})
}
*/
return &hclsyntax.ScopeTraversalExpr{
Traversal: traversal,
SrcRange: srcRange,
}, diags
}
// shimIsIgnoreChangesStar returns true if the given expression seems to be
// a string literal whose value is "*". This is used to support a legacy
// form of ignore_changes = all .
//
// This function does not itself emit any diagnostics, so it's the caller's
// responsibility to emit a warning diagnostic when this function returns true.
func shimIsIgnoreChangesStar(expr hcl.Expression) bool {
val, valDiags := expr.Value(nil)
if valDiags.HasErrors() {
return false
}
if val.Type() != cty.String || val.IsNull() || !val.IsKnown() {
return false
}
return val.AsString() == "*"
}

115
configs/config.go Normal file
View File

@ -0,0 +1,115 @@
package configs
import (
version "github.com/hashicorp/go-version"
"github.com/hashicorp/hcl2/hcl"
)
// A Config is a node in the tree of modules within a configuration.
//
// The module tree is constructed by following ModuleCall instances recursively
// through the root module transitively into descendent modules.
//
// A module tree described in *this* package represents the static tree
// represented by configuration. During evaluation a static ModuleNode may
// expand into zero or more module instances depending on the use of count and
// for_each configuration attributes within each call.
type Config struct {
// RootModule points to the Config for the root module within the same
// module tree as this module. If this module _is_ the root module then
// this is self-referential.
Root *Config
// ParentModule points to the Config for the module that directly calls
// this module. If this is the root module then this field is nil.
Parent *Config
// Path is a sequence of module logical names that traverse from the root
// module to this config. Path is empty for the root module.
//
// This should not be used to display a path to the end-user, since
// our UI conventions call for us to return a module address string in that
// case, and a module address string ought to be built from the dynamic
// module tree (resulting from evaluating "count" and "for_each" arguments
// on our calls to produce potentially multiple child instances per call)
// rather than from our static module tree.
Path []string
// ChildModules points to the Config for each of the direct child modules
// called from this module. The keys in this map match the keys in
// Module.ModuleCalls.
Children map[string]*Config
// Module points to the object describing the configuration for the
// various elements (variables, resources, etc) defined by this module.
Module *Module
// CallRange is the source range for the header of the module block that
// requested this module.
//
// This field is meaningless for the root module, where its contents are undefined.
CallRange hcl.Range
// SourceAddr is the source address that the referenced module was requested
// from, as specified in configuration.
//
// This field is meaningless for the root module, where its contents are undefined.
SourceAddr string
// SourceAddrRange is the location in the configuration source where the
// SourceAddr value was set, for use in diagnostic messages.
//
// This field is meaningless for the root module, where its contents are undefined.
SourceAddrRange hcl.Range
// Version is the specific version that was selected for this module,
// based on version constraints given in configuration.
//
// This field is nil if the module was loaded from a non-registry source,
// since versions are not supported for other sources.
//
// This field is meaningless for the root module, where it will always
// be nil.
Version *version.Version
}
// Depth returns the number of "hops" the receiver is from the root of its
// module tree, with the root module having a depth of zero.
func (c *Config) Depth() int {
ret := 0
this := c
for this.Parent != nil {
ret++
this = this.Parent
}
return ret
}
// DeepEach calls the given function once for each module in the tree, starting
// with the receiver.
//
// A parent is always called before its children and children of a particular
// node are visited in lexicographic order by their names.
func (c *Config) DeepEach(cb func(c *Config)) {
cb(c)
names := make([]string, 0, len(c.Children))
for name := range c.Children {
names = append(names, name)
}
for _, name := range names {
c.Children[name].DeepEach(cb)
}
}
// AllModules returns a slice of all the receiver and all of its descendent
// nodes in the module tree, in the same order they would be visited by
// DeepEach.
func (c *Config) AllModules() []*Config {
var ret []*Config
c.DeepEach(func(c *Config) {
ret = append(ret, c)
})
return ret
}

158
configs/config_build.go Normal file
View File

@ -0,0 +1,158 @@
package configs
import (
"sort"
version "github.com/hashicorp/go-version"
"github.com/hashicorp/hcl2/hcl"
)
// BuildConfig constructs a Config from a root module by loading all of its
// descendent modules via the given ModuleWalker.
//
// The result is a module tree that has so far only had basic module- and
// file-level invariants validated. If the returned diagnostics contains errors,
// the returned module tree may be incomplete but can still be used carefully
// for static analysis.
func BuildConfig(root *Module, walker ModuleWalker) (*Config, hcl.Diagnostics) {
var diags hcl.Diagnostics
cfg := &Config{
Module: root,
}
cfg.Root = cfg // Root module is self-referential.
cfg.Children, diags = buildChildModules(cfg, walker)
return cfg, diags
}
func buildChildModules(parent *Config, walker ModuleWalker) (map[string]*Config, hcl.Diagnostics) {
var diags hcl.Diagnostics
ret := map[string]*Config{}
calls := parent.Module.ModuleCalls
// We'll sort the calls by their local names so that they'll appear in a
// predictable order in any logging that's produced during the walk.
callNames := make([]string, 0, len(calls))
for k := range calls {
callNames = append(callNames, k)
}
sort.Strings(callNames)
for _, callName := range callNames {
call := calls[callName]
path := make([]string, len(parent.Path)+1)
copy(path, parent.Path)
path[len(path)-1] = call.Name
req := ModuleRequest{
Name: call.Name,
Path: path,
SourceAddr: call.SourceAddr,
SourceAddrRange: call.SourceAddrRange,
VersionConstraint: call.Version,
Parent: parent,
CallRange: call.DeclRange,
}
mod, ver, modDiags := walker.LoadModule(&req)
diags = append(diags, modDiags...)
if mod == nil {
// nil can be returned if the source address was invalid and so
// nothing could be loaded whatsoever. LoadModule should've
// returned at least one error diagnostic in that case.
continue
}
child := &Config{
Parent: parent,
Root: parent.Root,
Path: path,
Module: mod,
CallRange: call.DeclRange,
SourceAddr: call.SourceAddr,
SourceAddrRange: call.SourceAddrRange,
Version: ver,
}
child.Children, modDiags = buildChildModules(child, walker)
ret[call.Name] = child
}
return ret, diags
}
// A ModuleWalker knows how to find and load a child module given details about
// the module to be loaded and a reference to its partially-loaded parent
// Config.
type ModuleWalker interface {
// LoadModule finds and loads a requested child module.
//
// If errors are detected during loading, implementations should return them
// in the diagnostics object. If the diagnostics object contains any errors
// then the caller will tolerate the returned module being nil or incomplete.
// If no errors are returned, it should be non-nil and complete.
//
// Full validation need not have been performed but an implementation should
// ensure that the basic file- and module-validations performed by the
// LoadConfigDir function (valid syntax, no namespace collisions, etc) have
// been performed before returning a module.
LoadModule(req *ModuleRequest) (*Module, *version.Version, hcl.Diagnostics)
}
// ModuleWalkerFunc is an implementation of ModuleWalker that directly wraps
// a callback function, for more convenient use of that interface.
type ModuleWalkerFunc func(req *ModuleRequest) (*Module, *version.Version, hcl.Diagnostics)
// LoadModule implements ModuleWalker.
func (f ModuleWalkerFunc) LoadModule(req *ModuleRequest) (*Module, *version.Version, hcl.Diagnostics) {
return f(req)
}
// ModuleRequest is used with the ModuleWalker interface to describe a child
// module that must be loaded.
type ModuleRequest struct {
// Name is the "logical name" of the module call within configuration.
// This is provided in case the name is used as part of a storage key
// for the module, but implementations must otherwise treat it as an
// opaque string. It is guaranteed to have already been validated as an
// HCL identifier and UTF-8 encoded.
Name string
// Path is a list of logical names that traverse from the root module to
// this module. This can be used, for example, to form a lookup key for
// each distinct module call in a configuration, allowing for multiple
// calls with the same name at different points in the tree.
Path []string
// SourceAddr is the source address string provided by the user in
// configuration.
SourceAddr string
// SourceAddrRange is the source range for the SourceAddr value as it
// was provided in configuration. This can and should be used to generate
// diagnostics about the source address having invalid syntax, referring
// to a non-existent object, etc.
SourceAddrRange hcl.Range
// VersionConstraint is the version constraint applied to the module in
// configuration. This data structure includes the source range for
// the constraint, which can and should be used to generate diagnostics
// about constraint-related issues, such as constraints that eliminate all
// available versions of a module whose source is otherwise valid.
VersionConstraint VersionConstraint
// Parent is the partially-constructed module tree node that the loaded
// module will be added to. Callers may refer to any field of this
// structure except Children, which is still under construction when
// ModuleRequest objects are created and thus has undefined content.
// The main reason this is provided is so that full module paths can
// be constructed for uniqueness.
Parent *Config
// CallRange is the source range for the header of the "module" block
// in configuration that prompted this request. This can be used as the
// subject of an error diagnostic that relates to the module call itself,
// rather than to either its source address or its version number.
CallRange hcl.Range
}

71
configs/config_build_test.go Normal file
View File

@ -0,0 +1,71 @@
package configs
import (
"fmt"
"path/filepath"
"reflect"
"sort"
"strings"
"testing"
"github.com/davecgh/go-spew/spew"
version "github.com/hashicorp/go-version"
"github.com/hashicorp/hcl2/hcl"
)
func TestBuildConfig(t *testing.T) {
parser := NewParser(nil)
mod, diags := parser.LoadConfigDir("test-fixtures/config-build")
assertNoDiagnostics(t, diags)
if mod == nil {
t.Fatal("got nil root module; want non-nil")
}
versionI := 0
cfg, diags := BuildConfig(mod, ModuleWalkerFunc(
func(req *ModuleRequest) (*Module, *version.Version, hcl.Diagnostics) {
// For the sake of this test we're going to just treat our
// SourceAddr as a path relative to our fixture directory.
// A "real" implementation of ModuleWalker should accept the
// various different source address syntaxes Terraform supports.
sourcePath := filepath.Join("test-fixtures/config-build", req.SourceAddr)
mod, diags := parser.LoadConfigDir(sourcePath)
version, _ := version.NewVersion(fmt.Sprintf("1.0.%d", versionI))
versionI++
return mod, version, diags
},
))
assertNoDiagnostics(t, diags)
if cfg == nil {
t.Fatal("got nil config; want non-nil")
}
var got []string
cfg.DeepEach(func(c *Config) {
got = append(got, fmt.Sprintf("%s %s", strings.Join(c.Path, "."), c.Version))
})
sort.Strings(got)
want := []string{
" <nil>",
"child_a 1.0.0",
"child_a.child_c 1.0.1",
"child_b 1.0.2",
"child_b.child_c 1.0.3",
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("wrong result\ngot: %swant: %s", spew.Sdump(got), spew.Sdump(want))
}
if _, exists := cfg.Children["child_a"].Children["child_c"].Module.Outputs["hello"]; !exists {
t.Fatalf("missing output 'hello' in child_a.child_c")
}
if _, exists := cfg.Children["child_b"].Children["child_c"].Module.Outputs["hello"]; !exists {
t.Fatalf("missing output 'hello' in child_b.child_c")
}
if cfg.Children["child_a"].Children["child_c"].Module == cfg.Children["child_b"].Children["child_c"].Module {
t.Fatalf("child_a.child_c is same object as child_b.child_c; should not be")
}
}

114
configs/configload/copy_dir.go Normal file
View File

@ -0,0 +1,114 @@
package configload
import (
"io"
"os"
"path/filepath"
"strings"
)
// copyDir copies the src directory contents into dst. Both directories
// should already exist.
func copyDir(dst, src string) error {
src, err := filepath.EvalSymlinks(src)
if err != nil {
return err
}
walkFn := func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if path == src {
return nil
}
if strings.HasPrefix(filepath.Base(path), ".") {
// Skip any dot files
if info.IsDir() {
return filepath.SkipDir
} else {
return nil
}
}
// The "path" has the src prefixed to it. We need to join our
// destination with the path without the src on it.
dstPath := filepath.Join(dst, path[len(src):])
// we don't want to try and copy the same file over itself.
if eq, err := sameFile(path, dstPath); eq {
return nil
} else if err != nil {
return err
}
// If we have a directory, make that subdirectory, then continue
// the walk.
if info.IsDir() {
if path == filepath.Join(src, dst) {
// dst is in src; don't walk it.
return nil
}
if err := os.MkdirAll(dstPath, 0755); err != nil {
return err
}
return nil
}
// If we have a file, copy the contents.
srcF, err := os.Open(path)
if err != nil {
return err
}
defer srcF.Close()
dstF, err := os.Create(dstPath)
if err != nil {
return err
}
defer dstF.Close()
if _, err := io.Copy(dstF, srcF); err != nil {
return err
}
// Chmod it
return os.Chmod(dstPath, info.Mode())
}
return filepath.Walk(src, walkFn)
}
// sameFile tried to determine if to paths are the same file.
// If the paths don't match, we lookup the inode on supported systems.
func sameFile(a, b string) (bool, error) {
if a == b {
return true, nil
}
aIno, err := inode(a)
if err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, err
}
bIno, err := inode(b)
if err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, err
}
if aIno > 0 && aIno == bIno {
return true, nil
}
return false, nil
}

4
configs/configload/doc.go Normal file
View File

@ -0,0 +1,4 @@
// Package configload knows how to install modules into the .terraform/modules
// directory and to load modules from those installed locations. It is used
// in conjunction with the LoadConfig function in the parent package.
package configload

122
configs/configload/getter.go Normal file
View File

@ -0,0 +1,122 @@
package configload
import (
"log"
"path/filepath"
cleanhttp "github.com/hashicorp/go-cleanhttp"
getter "github.com/hashicorp/go-getter"
)
// We configure our own go-getter detector and getter sets here, because
// the set of sources we support is part of Terraform's documentation and
// so we don't want any new sources introduced in go-getter to sneak in here
// and work even though they aren't documented. This also insulates us from
// any meddling that might be done by other go-getter callers linked into our
// executable.
var goGetterDetectors = []getter.Detector{
new(getter.GitHubDetector),
new(getter.BitBucketDetector),
new(getter.S3Detector),
new(getter.FileDetector),
}
var goGetterNoDetectors = []getter.Detector{}
var goGetterDecompressors = map[string]getter.Decompressor{
"bz2": new(getter.Bzip2Decompressor),
"gz": new(getter.GzipDecompressor),
"xz": new(getter.XzDecompressor),
"zip": new(getter.ZipDecompressor),
"tar.bz2": new(getter.TarBzip2Decompressor),
"tar.tbz2": new(getter.TarBzip2Decompressor),
"tar.gz": new(getter.TarGzipDecompressor),
"tgz": new(getter.TarGzipDecompressor),
"tar.xz": new(getter.TarXzDecompressor),
"txz": new(getter.TarXzDecompressor),
}
var goGetterGetters = map[string]getter.Getter{
"file": new(getter.FileGetter),
"git": new(getter.GitGetter),
"hg": new(getter.HgGetter),
"s3": new(getter.S3Getter),
"http": getterHTTPGetter,
"https": getterHTTPGetter,
}
var getterHTTPClient = cleanhttp.DefaultClient()
var getterHTTPGetter = &getter.HttpGetter{
Client: getterHTTPClient,
Netrc: true,
}
// getWithGoGetter retrieves the package referenced in the given address
// into the installation path and then returns the full path to any subdir
// indicated in the address.
//
// The errors returned by this function are those surfaced by the underlying
// go-getter library, which have very inconsistent quality as
// end-user-actionable error messages. At this time we do not have any
// reasonable way to improve these error messages at this layer because
// the underlying errors are not separatelyr recognizable.
func getWithGoGetter(instPath, addr string) (string, error) {
packageAddr, subDir := splitAddrSubdir(addr)
log.Printf("[DEBUG] will download %q to %s", packageAddr, instPath)
realAddr, err := getter.Detect(packageAddr, instPath, getter.Detectors)
if err != nil {
return "", err
}
var realSubDir string
realAddr, realSubDir = splitAddrSubdir(realAddr)
if realSubDir != "" {
subDir = filepath.Join(realSubDir, subDir)
}
if realAddr != packageAddr {
log.Printf("[TRACE] go-getter detectors rewrote %q to %q", packageAddr, realAddr)
}
client := getter.Client{
Src: realAddr,
Dst: instPath,
Pwd: instPath,
Mode: getter.ClientModeDir,
Detectors: goGetterNoDetectors, // we already did detection above
Decompressors: goGetterDecompressors,
Getters: goGetterGetters,
}
err = client.Get()
if err != nil {
return "", err
}
// Our subDir string can contain wildcards until this point, so that
// e.g. a subDir of * can expand to one top-level directory in a .tar.gz
// archive. Now that we've expanded the archive successfully we must
// resolve that into a concrete path.
var finalDir string
if subDir != "" {
finalDir, err = getter.SubdirGlob(instPath, subDir)
log.Printf("[TRACE] expanded %q to %q", subDir, finalDir)
if err != nil {
return "", err
}
} else {
finalDir = instPath
}
// If we got this far then we have apparently succeeded in downloading
// the requested object!
return filepath.Clean(finalDir), nil
}

21
configs/configload/inode.go Normal file
View File

@ -0,0 +1,21 @@
// +build linux darwin openbsd netbsd solaris dragonfly
package configload
import (
"fmt"
"os"
"syscall"
)
// lookup the inode of a file on posix systems
func inode(path string) (uint64, error) {
stat, err := os.Stat(path)
if err != nil {
return 0, err
}
if st, ok := stat.Sys().(*syscall.Stat_t); ok {
return st.Ino, nil
}
return 0, fmt.Errorf("could not determine file inode")
}

21
configs/configload/inode_freebsd.go Normal file
View File

@ -0,0 +1,21 @@
// +build freebsd
package configload
import (
"fmt"
"os"
"syscall"
)
// lookup the inode of a file on posix systems
func inode(path string) (uint64, error) {
stat, err := os.Stat(path)
if err != nil {
return 0, err
}
if st, ok := stat.Sys().(*syscall.Stat_t); ok {
return uint64(st.Ino), nil
}
return 0, fmt.Errorf("could not determine file inode")
}

8
configs/configload/inode_windows.go Normal file
View File

@ -0,0 +1,8 @@
// +build windows
package configload
// no syscall.Stat_t on windows, return 0 for inodes
func inode(path string) (uint64, error) {
return 0, nil
}

92
configs/configload/loader.go Normal file
View File

@ -0,0 +1,92 @@
package configload
import (
"fmt"
"github.com/hashicorp/terraform/configs"
"github.com/hashicorp/terraform/registry"
"github.com/hashicorp/terraform/svchost/auth"
"github.com/hashicorp/terraform/svchost/disco"
"github.com/spf13/afero"
)
// A Loader instance is the main entry-point for loading configurations via
// this package.
//
// It extends the general config-loading functionality in the parent package
// "configs" to support installation of modules from remote sources and
// loading full configurations using modules that were previously installed.
type Loader struct {
// parser is used to read configuration
parser *configs.Parser
// modules is used to install and locate descendent modules that are
// referenced (directly or indirectly) from the root module.
modules moduleMgr
}
// Config is used with NewLoader to specify configuration arguments for the
// loader.
type Config struct {
// ModulesDir is a path to a directory where descendent modules are
// (or should be) installed. (This is usually the
// .terraform/modules directory, in the common case where this package
// is being loaded from the main Terraform CLI package.)
ModulesDir string
// Services is the service discovery client to use when locating remote
// module registry endpoints. If this is nil then registry sources are
// not supported, which should be true only in specialized circumstances
// such as in tests.
Services *disco.Disco
// Creds is a credentials store for communicating with remote module
// registry endpoints. If this is nil then no credentials will be used.
Creds auth.CredentialsSource
}
// NewLoader creates and returns a loader that reads configuration from the
// real OS filesystem.
//
// The loader has some internal state about the modules that are currently
// installed, which is read from disk as part of this function. If that
// manifest cannot be read then an error will be returned.
func NewLoader(config *Config) (*Loader, error) {
fs := afero.NewOsFs()
parser := configs.NewParser(fs)
reg := registry.NewClient(config.Services, config.Creds, nil)
ret := &Loader{
parser: parser,
modules: moduleMgr{
FS: afero.Afero{Fs: fs},
CanInstall: true,
Dir: config.ModulesDir,
Services: config.Services,
Creds: config.Creds,
Registry: reg,
},
}
err := ret.modules.readModuleManifestSnapshot()
if err != nil {
return nil, fmt.Errorf("failed to read module manifest: %s", err)
}
return ret, nil
}
// Parser returns the underlying parser for this loader.
//
// This is useful for loading other sorts of files than the module directories
// that a loader deals with, since then they will share the source code cache
// for this loader and can thus be shown as snippets in diagnostic messages.
func (l *Loader) Parser() *configs.Parser {
return l.parser
}
// Sources returns the source code cache for the underlying parser of this
// loader. This is a shorthand for l.Parser().Sources().
func (l *Loader) Sources() map[string][]byte {
return l.parser.Sources()
}

372
configs/configload/loader_init_from_module.go Normal file
View File

@ -0,0 +1,372 @@
package configload
import (
"fmt"
"log"
"os"
"path/filepath"
"sort"
"strings"
version "github.com/hashicorp/go-version"
"github.com/hashicorp/hcl2/hcl"
"github.com/hashicorp/terraform/configs"
)
const initFromModuleRootCallName = "root"
const initFromModuleRootKeyPrefix = initFromModuleRootCallName + "."
// InitDirFromModule populates the given directory (which must exist and be
// empty) with the contents of the module at the given source address.
//
// It does this by installing the given module and all of its descendent
// modules in a temporary root directory and then copying the installed
// files into suitable locations. As a consequence, any diagnostics it
// generates will reveal the location of this temporary directory to the
// user.
//
// This rather roundabout installation approach is taken to ensure that
// installation proceeds in a manner identical to normal module installation.
//
// If the given source address specifies a sub-directory of the given
// package then only the sub-directory and its descendents will be copied
// into the given root directory, which will cause any relative module
// references using ../ from that module to be unresolvable. Error diagnostics
// are produced in that case, to prompt the user to rewrite the source strings
// to be absolute references to the original remote module.
//
// This can be installed only on a loder that can install modules, and will
// panic otherwise. Use CanInstallModules to determine if this method can be
// used, or refer to the documentation of that method for situations where
// install ability is guaranteed.
func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHooks) hcl.Diagnostics {
var diags hcl.Diagnostics
// The way this function works is pretty ugly, but we accept it because
// -from-module is a less important case than normal module installation
// and so it's better to keep this ugly complexity out here rather than
// adding even more complexity to the normal module installer.
// The target directory must exist but be empty.
{
entries, err := l.modules.FS.ReadDir(rootDir)
if err != nil {
if os.IsNotExist(err) {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Target directory does not exist",
Detail: fmt.Sprintf("Cannot initialize non-existent directory %s.", rootDir),
})
} else {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Failed to read target directory",
Detail: fmt.Sprintf("Error reading %s to ensure it is empty: %s.", rootDir, err),
})
}
return diags
}
haveEntries := false
for _, entry := range entries {
if entry.Name() == "." || entry.Name() == ".." || entry.Name() == ".terraform" {
continue
}
haveEntries = true
}
if haveEntries {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Can't populate non-empty directory",
Detail: fmt.Sprintf("The target directory %s is not empty, so it cannot be initialized with the -from-module=... option.", rootDir),
})
return diags
}
}
// We use a hidden sub-loader to manage our inner installation directory,
// but it shares dependencies with the receiver to allow it to access the
// same remote resources and ensure it populates the same source code
// cache in case .
subLoader := &Loader{
parser: l.parser,
modules: l.modules, // this is a shallow copy, so we can safely mutate below
}
// Our sub-loader will have its own independent manifest and install
// directory, so we can install with it and know we won't interfere
// with the receiver.
subLoader.modules.manifest = make(moduleManifest)
subLoader.modules.Dir = filepath.Join(rootDir, ".terraform/init-from-module")
log.Printf("[DEBUG] using a child module loader in %s to initialize working directory from %q", subLoader.modules.Dir, sourceAddr)
subLoader.modules.FS.RemoveAll(subLoader.modules.Dir) // if this fails then we'll fail on MkdirAll below too
err := subLoader.modules.FS.MkdirAll(subLoader.modules.Dir, os.ModePerm)
if err != nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Failed to create temporary directory",
Detail: fmt.Sprintf("Failed to create temporary directory %s: %s.", subLoader.modules.Dir, err),
})
return diags
}
fakeFilename := fmt.Sprintf("-from-module=%q", sourceAddr)
fakeRange := hcl.Range{
Filename: fakeFilename,
Start: hcl.Pos{
Line: 1,
Column: 1,
Byte: 0,
},
End: hcl.Pos{
Line: 1,
Column: len(fakeFilename) + 1, // not accurate if the address contains unicode, but irrelevant since we have no source cache for this anyway
Byte: len(fakeFilename),
},
}
// Now we need to create an artificial root module that will seed our
// installation process.
fakeRootModule := &configs.Module{
ModuleCalls: map[string]*configs.ModuleCall{
initFromModuleRootCallName: &configs.ModuleCall{
Name: initFromModuleRootCallName,
SourceAddr: sourceAddr,
SourceAddrRange: fakeRange,
SourceSet: true,
DeclRange: fakeRange,
},
},
}
// wrapHooks filters hook notifications to only include Download calls
// and to trim off the initFromModuleRootCallName prefix. We'll produce
// our own Install notifications directly below.
wrapHooks := installHooksInitDir{
Wrapped: hooks,
}
instDiags := subLoader.installDescendentModules(fakeRootModule, rootDir, true, wrapHooks)
diags = append(diags, instDiags...)
if instDiags.HasErrors() {
return diags
}
// If all of that succeeded then we'll now migrate what was installed
// into the final directory structure.
modulesDir := l.modules.Dir
err = subLoader.modules.FS.MkdirAll(modulesDir, os.ModePerm)
if err != nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Failed to create local modules directory",
Detail: fmt.Sprintf("Failed to create modules directory %s: %s.", modulesDir, err),
})
return diags
}
manifest := subLoader.modules.manifest
recordKeys := make([]string, 0, len(manifest))
for k := range manifest {
recordKeys = append(recordKeys, k)
}
sort.Strings(recordKeys)
for _, recordKey := range recordKeys {
record := manifest[recordKey]
if record.Key == initFromModuleRootCallName {
// We've found the module the user requested, which we must
// now copy into rootDir so it can be used directly.
log.Printf("[TRACE] copying new root module from %s to %s", record.Dir, rootDir)
err := copyDir(rootDir, record.Dir)
if err != nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Failed to copy root module",
Detail: fmt.Sprintf("Error copying root module %q from %s to %s: %s.", sourceAddr, record.Dir, rootDir, err),
})
continue
}
// We'll try to load the newly-copied module here just so we can
// sniff for any module calls that ../ out of the root directory
// and must thus be rewritten to be absolute addresses again.
// For now we can't do this rewriting automatically, but we'll
// generate an error to help the user do it manually.
mod, _ := l.parser.LoadConfigDir(rootDir) // ignore diagnostics since we're just doing value-add here anyway
for _, mc := range mod.ModuleCalls {
if pathTraversesUp(sourceAddr) {
packageAddr, givenSubdir := splitAddrSubdir(sourceAddr)
newSubdir := filepath.Join(givenSubdir, mc.SourceAddr)
if pathTraversesUp(newSubdir) {
// This should never happen in any reasonable
// configuration since this suggests a path that
// traverses up out of the package root. We'll just
// ignore this, since we'll fail soon enough anyway
// trying to resolve this path when this module is
// loaded.
continue
}
var newAddr = packageAddr
if newSubdir != "" {
newAddr = fmt.Sprintf("%s//%s", newAddr, filepath.ToSlash(newSubdir))
}
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Root module references parent directory",
Detail: fmt.Sprintf("The requested module %q refers to a module via its parent directory. To use this as a new root module this source string must be rewritten as a remote source address, such as %q.", sourceAddr, newAddr),
Subject: &mc.SourceAddrRange,
})
continue
}
}
l.modules.manifest[""] = moduleRecord{
Key: "",
Dir: rootDir,
}
continue
}
if !strings.HasPrefix(record.Key, initFromModuleRootKeyPrefix) {
// Ignore the *real* root module, whose key is empty, since
// we're only interested in the module named "root" and its
// descendents.
continue
}
newKey := record.Key[len(initFromModuleRootKeyPrefix):]
instPath := filepath.Join(l.modules.Dir, newKey)
tempPath := filepath.Join(subLoader.modules.Dir, record.Key)
// tempPath won't be present for a module that was installed from
// a relative path, so in that case we just record the installation
// directory and assume it was already copied into place as part
// of its parent.
if _, err := os.Stat(tempPath); err != nil {
if !os.IsNotExist(err) {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Failed to stat temporary module install directory",
Detail: fmt.Sprintf("Error from stat %s for module %s: %s.", instPath, newKey, err),
})
continue
}
var parentKey string
if lastDot := strings.LastIndexByte(newKey, '.'); lastDot != -1 {
parentKey = newKey[:lastDot]
} else {
parentKey = "" // parent is the root module
}
parentOld := manifest[initFromModuleRootKeyPrefix+parentKey]
parentNew := l.modules.manifest[parentKey]
// We need to figure out which portion of our directory is the
// parent package path and which portion is the subdirectory
// under that.
baseDirRel, err := filepath.Rel(parentOld.Dir, record.Dir)
if err != nil {
// Should never happen, because we constructed both directories
// from the same base and so they must have a common prefix.
panic(err)
}
newDir := filepath.Join(parentNew.Dir, baseDirRel)
log.Printf("[TRACE] relative reference for %s rewritten from %s to %s", newKey, record.Dir, newDir)
newRecord := record // shallow copy
newRecord.Dir = newDir
newRecord.Key = newKey
l.modules.manifest[newKey] = newRecord
hooks.Install(newRecord.Key, newRecord.Version, newRecord.Dir)
continue
}
err = subLoader.modules.FS.MkdirAll(instPath, os.ModePerm)
if err != nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Failed to create module install directory",
Detail: fmt.Sprintf("Error creating directory %s for module %s: %s.", instPath, newKey, err),
})
continue
}
// We copy rather than "rename" here because renaming between directories
// can be tricky in edge-cases like network filesystems, etc.
log.Printf("[TRACE] copying new module %s from %s to %s", newKey, record.Dir, instPath)
err := copyDir(instPath, tempPath)
if err != nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Failed to copy descendent module",
Detail: fmt.Sprintf("Error copying module %q from %s to %s: %s.", newKey, tempPath, rootDir, err),
})
continue
}
subDir, err := filepath.Rel(tempPath, record.Dir)
if err != nil {
// Should never happen, because we constructed both directories
// from the same base and so they must have a common prefix.
panic(err)
}
newRecord := record // shallow copy
newRecord.Dir = filepath.Join(instPath, subDir)
newRecord.Key = newKey
l.modules.manifest[newKey] = newRecord
hooks.Install(newRecord.Key, newRecord.Version, newRecord.Dir)
}
err = l.modules.writeModuleManifestSnapshot()
if err != nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Failed to write module manifest",
Detail: fmt.Sprintf("Error writing module manifest: %s.", err),
})
}
if !diags.HasErrors() {
// Try to clean up our temporary directory, but don't worry if we don't
// succeed since it shouldn't hurt anything.
subLoader.modules.FS.RemoveAll(subLoader.modules.Dir)
}
return diags
}
func pathTraversesUp(path string) bool {
return strings.HasPrefix(filepath.ToSlash(path), "../")
}
// installHooksInitDir is an adapter wrapper for an InstallHooks that
// does some fakery to make downloads look like they are happening in their
// final locations, rather than in the temporary loader we use.
//
// It also suppresses "Install" calls entirely, since InitDirFromModule
// does its own installation steps after the initial installation pass
// has completed.
type installHooksInitDir struct {
Wrapped InstallHooks
InstallHooksImpl
}
func (h installHooksInitDir) Download(moduleAddr, packageAddr string, version *version.Version) {
if !strings.HasPrefix(moduleAddr, initFromModuleRootKeyPrefix) {
// We won't announce the root module, since hook implementations
// don't expect to see that and the caller will usually have produced
// its own user-facing notification about what it's doing anyway.
return
}
trimAddr := moduleAddr[len(initFromModuleRootKeyPrefix):]
h.Wrapped.Download(trimAddr, packageAddr, version)
}

92
configs/configload/loader_init_from_module_test.go Normal file
View File

@ -0,0 +1,92 @@
package configload
import (
"os"
"path/filepath"
"strings"
"testing"
version "github.com/hashicorp/go-version"
"github.com/hashicorp/terraform/configs"
)
func TestLoaderInitDirFromModule_registry(t *testing.T) {
if os.Getenv("TF_ACC") == "" {
t.Skip("this test accesses registry.terraform.io and github.com; set TF_ACC=1 to run it")
}
fixtureDir := filepath.Clean("test-fixtures/empty")
loader, done := tempChdirLoader(t, fixtureDir)
defer done()
hooks := &testInstallHooks{}
diags := loader.InitDirFromModule(".", "hashicorp/module-installer-acctest/aws//examples/main", hooks)
assertNoDiagnostics(t, diags)
v := version.Must(version.NewVersion("0.0.1"))
wantCalls := []testInstallHookCall{
// The module specified to populate the root directory is not mentioned
// here, because the hook mechanism is defined to talk about descendent
// modules only and so a caller to InitDirFromModule is expected to
// produce its own user-facing announcement about the root module being
// installed.
// Note that "root" in the following examples is, confusingly, the
// label on the module block in the example we've installed here:
// module "root" {
{
Name: "Download",
ModuleAddr: "root",
PackageAddr: "hashicorp/module-installer-acctest/aws",
Version: v,
},
{
Name: "Install",
ModuleAddr: "root",
Version: v,
LocalPath: ".terraform/modules/root/hashicorp-terraform-aws-module-installer-acctest-5e87aff",
},
{
Name: "Install",
ModuleAddr: "root.child_a",
LocalPath: ".terraform/modules/root/hashicorp-terraform-aws-module-installer-acctest-5e87aff/modules/child_a",
},
{
Name: "Install",
ModuleAddr: "root.child_a.child_b",
LocalPath: ".terraform/modules/root/hashicorp-terraform-aws-module-installer-acctest-5e87aff/modules/child_b",
},
}
if assertResultDeepEqual(t, hooks.Calls, wantCalls) {
return
}
// Make sure the configuration is loadable now.
// (This ensures that correct information is recorded in the manifest.)
config, loadDiags := loader.LoadConfig(".")
if assertNoDiagnostics(t, loadDiags) {
return
}
wantTraces := map[string]string{
"": "in example",
"root": "in root module",
"root.child_a": "in child_a module",
"root.child_a.child_b": "in child_b module",
}
gotTraces := map[string]string{}
config.DeepEach(func(c *configs.Config) {
path := strings.Join(c.Path, ".")
if c.Module.Variables["v"] == nil {
gotTraces[path] = "<missing>"
return
}
varDesc := c.Module.Variables["v"].Description
gotTraces[path] = varDesc
})
assertResultDeepEqual(t, gotTraces, wantTraces)
}

522
configs/configload/loader_install.go Normal file
View File

@ -0,0 +1,522 @@
package configload
import (
"fmt"
"log"
"os"
"path/filepath"
"strings"
version "github.com/hashicorp/go-version"
"github.com/hashicorp/hcl2/hcl"
"github.com/hashicorp/terraform/configs"
"github.com/hashicorp/terraform/registry"
"github.com/hashicorp/terraform/registry/regsrc"
)
// InstallModules analyses the root module in the given directory and installs
// all of its direct and transitive dependencies into the loader's modules
// directory, which must already exist.
//
// Since InstallModules makes possibly-time-consuming calls to remote services,
// a hook interface is supported to allow the caller to be notified when
// each module is installed and, for remote modules, when downloading begins.
// LoadConfig guarantees that two hook calls will not happen concurrently but
// it does not guarantee any particular ordering of hook calls. This mechanism
// is for UI feedback only and does not give the caller any control over the
// process.
//
// If modules are already installed in the target directory, they will be
// skipped unless their source address or version have changed or unless
// the upgrade flag is set.
//
// InstallModules never deletes any directory, except in the case where it
// needs to replace a directory that is already present with a newly-extracted
// package.
//
// If the returned diagnostics contains errors then the module installation
// may have wholly or partially completed. Modules must be loaded in order
// to find their dependencies, so this function does many of the same checks
// as LoadConfig as a side-effect.
//
// This function will panic if called on a loader that cannot install modules.
// Use CanInstallModules to determine if a loader can install modules, or
// refer to the documentation for that method for situations where module
// installation capability is guaranteed.
func (l *Loader) InstallModules(rootDir string, upgrade bool, hooks InstallHooks) hcl.Diagnostics {
if !l.CanInstallModules() {
panic(fmt.Errorf("InstallModules called on loader that cannot install modules"))
}
rootMod, diags := l.parser.LoadConfigDir(rootDir)
if rootMod == nil {
return diags
}
instDiags := l.installDescendentModules(rootMod, rootDir, upgrade, hooks)
diags = append(diags, instDiags...)
return diags
}
func (l *Loader) installDescendentModules(rootMod *configs.Module, rootDir string, upgrade bool, hooks InstallHooks) hcl.Diagnostics {
var diags hcl.Diagnostics
if hooks == nil {
// Use our no-op implementation as a placeholder
hooks = InstallHooksImpl{}
}
// Create a manifest record for the root module. This will be used if
// there are any relative-pathed modules in the root.
l.modules.manifest[""] = moduleRecord{
Key: "",
Dir: rootDir,
}
_, cDiags := configs.BuildConfig(rootMod, configs.ModuleWalkerFunc(
func(req *configs.ModuleRequest) (*configs.Module, *version.Version, hcl.Diagnostics) {
key := manifestKey(req.Path)
instPath := l.packageInstallPath(req.Path)
log.Printf("[DEBUG] Module installer: begin %s", key)
// First we'll check if we need to upgrade/replace an existing
// installed module, and delete it out of the way if so.
replace := upgrade
if !replace {
record, recorded := l.modules.manifest[key]
switch {
case !recorded:
log.Printf("[TRACE] %s is not yet installed", key)
replace = true
case record.SourceAddr != req.SourceAddr:
log.Printf("[TRACE] %s source address has changed from %q to %q", key, record.SourceAddr, req.SourceAddr)
replace = true
case record.Version != nil && !req.VersionConstraint.Required.Check(record.Version):
log.Printf("[TRACE] %s version %s no longer compatible with constraints %s", key, record.Version, req.VersionConstraint.Required)
replace = true
}
}
// If we _are_ planning to replace this module, then we'll remove
// it now so our installation code below won't conflict with any
// existing remnants.
if replace {
if _, recorded := l.modules.manifest[key]; recorded {
log.Printf("[TRACE] discarding previous record of %s prior to reinstall", key)
}
delete(l.modules.manifest, key)
// Deleting a module invalidates all of its descendent modules too.
keyPrefix := key + "."
for subKey := range l.modules.manifest {
if strings.HasPrefix(subKey, keyPrefix) {
if _, recorded := l.modules.manifest[subKey]; recorded {
log.Printf("[TRACE] also discarding downstream %s", subKey)
}
delete(l.modules.manifest, subKey)
}
}
}
record, recorded := l.modules.manifest[key]
if !recorded {
// Clean up any stale cache directory that might be present.
// If this is a local (relative) source then the dir will
// not exist, but we'll ignore that.
log.Printf("[TRACE] cleaning directory %s prior to install of %s", instPath, key)
err := l.modules.FS.RemoveAll(instPath)
if err != nil && !os.IsNotExist(err) {
log.Printf("[TRACE] failed to remove %s: %s", key, err)
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Failed to remove local module cache",
Detail: fmt.Sprintf(
"Terraform tried to remove %s in order to reinstall this module, but encountered an error: %s",
instPath, err,
),
Subject: &req.CallRange,
})
return nil, nil, diags
}
} else {
// If this module is already recorded and its root directory
// exists then we will just load what's already there and
// keep our existing record.
info, err := l.modules.FS.Stat(record.Dir)
if err == nil && info.IsDir() {
mod, mDiags := l.parser.LoadConfigDir(record.Dir)
diags = append(diags, mDiags...)
log.Printf("[TRACE] Module installer: %s %s already installed in %s", key, record.Version, record.Dir)
return mod, record.Version, diags
}
}
// If we get down here then it's finally time to actually install
// the module. There are some variants to this process depending
// on what type of module source address we have.
switch {
case isLocalSourceAddr(req.SourceAddr):
log.Printf("[TRACE] %s has local path %q", key, req.SourceAddr)
mod, mDiags := l.installLocalModule(req, key, hooks)
diags = append(diags, mDiags...)
return mod, nil, diags
case isRegistrySourceAddr(req.SourceAddr):
addr, err := regsrc.ParseModuleSource(req.SourceAddr)
if err != nil {
// Should never happen because isRegistrySourceAddr already validated
panic(err)
}
log.Printf("[TRACE] %s is a registry module at %s", key, addr)
mod, v, mDiags := l.installRegistryModule(req, key, instPath, addr, hooks)
diags = append(diags, mDiags...)
return mod, v, diags
default:
log.Printf("[TRACE] %s address %q will be handled by go-getter", key, req.SourceAddr)
mod, mDiags := l.installGoGetterModule(req, key, instPath, hooks)
diags = append(diags, mDiags...)
return mod, nil, diags
}
},
))
diags = append(diags, cDiags...)
err := l.modules.writeModuleManifestSnapshot()
if err != nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Failed to update module manifest",
Detail: fmt.Sprintf("Unable to write the module manifest file: %s", err),
})
}
return diags
}
// CanInstallModules returns true if InstallModules can be used with this
// loader.
//
// Loaders created with NewLoader can always install modules. Loaders created
// from plan files (where the configuration is embedded in the plan file itself)
// cannot install modules, because the plan file is read-only.
func (l *Loader) CanInstallModules() bool {
return l.modules.CanInstall
}
func (l *Loader) installLocalModule(req *configs.ModuleRequest, key string, hooks InstallHooks) (*configs.Module, hcl.Diagnostics) {
var diags hcl.Diagnostics
parentKey := manifestKey(req.Parent.Path)
parentRecord, recorded := l.modules.manifest[parentKey]
if !recorded {
// This is indicative of a bug rather than a user-actionable error
panic(fmt.Errorf("missing manifest record for parent module %s", parentKey))
}
if len(req.VersionConstraint.Required) != 0 {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid version constraint",
Detail: "A version constraint cannot be applied to a module at a relative local path.",
Subject: &req.VersionConstraint.DeclRange,
})
}
// For local sources we don't actually need to modify the
// filesystem at all because the parent already wrote
// the files we need, and so we just load up what's already here.
newDir := filepath.Join(parentRecord.Dir, req.SourceAddr)
log.Printf("[TRACE] %s uses directory from parent: %s", key, newDir)
mod, mDiags := l.parser.LoadConfigDir(newDir)
if mod == nil {
// nil indicates missing or unreadable directory, so we'll
// discard the returned diags and return a more specific
// error message here.
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Unreadable module directory",
Detail: fmt.Sprintf("The directory %s could not be read.", newDir),
Subject: &req.SourceAddrRange,
})
} else {
diags = append(diags, mDiags...)
}
// Note the local location in our manifest.
l.modules.manifest[key] = moduleRecord{
Key: key,
Dir: newDir,
SourceAddr: req.SourceAddr,
}
log.Printf("[DEBUG] Module installer: %s installed at %s", key, newDir)
hooks.Install(key, nil, newDir)
return mod, diags
}
func (l *Loader) installRegistryModule(req *configs.ModuleRequest, key string, instPath string, addr *regsrc.Module, hooks InstallHooks) (*configs.Module, *version.Version, hcl.Diagnostics) {
var diags hcl.Diagnostics
hostname, err := addr.SvcHost()
if err != nil {
// If it looks like the user was trying to use punycode then we'll generate
// a specialized error for that case. We require the unicode form of
// hostname so that hostnames are always human-readable in configuration
// and punycode can't be used to hide a malicious module hostname.
if strings.HasPrefix(addr.RawHost.Raw, "xn--") {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid module registry hostname",
Detail: "The hostname portion of this source address is not an acceptable hostname. Internationalized domain names must be given in unicode form rather than ASCII (\"punycode\") form.",
Subject: &req.SourceAddrRange,
})
} else {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid module registry hostname",
Detail: "The hostname portion of this source address is not a valid hostname.",
Subject: &req.SourceAddrRange,
})
}
return nil, nil, diags
}
reg := l.modules.Registry
log.Printf("[DEBUG] %s listing available versions of %s at %s", key, addr, hostname)
resp, err := reg.Versions(addr)
if err != nil {
if registry.IsModuleNotFound(err) {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Module not found",
Detail: fmt.Sprintf("The specified module could not be found in the module registry at %s.", hostname),
Subject: &req.SourceAddrRange,
})
} else {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Error accessing remote module registry",
Detail: fmt.Sprintf("Failed to retrieve available versions for this module from %s: %s.", hostname, err),
Subject: &req.SourceAddrRange,
})
}
return nil, nil, diags
}
// The response might contain information about dependencies to allow us
// to potentially optimize future requests, but we don't currently do that
// and so for now we'll just take the first item which is guaranteed to
// be the address we requested.
if len(resp.Modules) < 1 {
// Should never happen, but since this is a remote service that may
// be implemented by third-parties we will handle it gracefully.
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid response from remote module registry",
Detail: fmt.Sprintf("The registry at %s returned an invalid response when Terraform requested available versions for this module.", hostname),
Subject: &req.SourceAddrRange,
})
return nil, nil, diags
}
modMeta := resp.Modules[0]
var latestMatch *version.Version
var latestVersion *version.Version
for _, mv := range modMeta.Versions {
v, err := version.NewVersion(mv.Version)
if err != nil {
// Should never happen if the registry server is compliant with
// the protocol, but we'll warn if not to assist someone who
// might be developing a module registry server.
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagWarning,
Summary: "Invalid response from remote module registry",
Detail: fmt.Sprintf("The registry at %s returned an invalid version string %q for this module, which Terraform ignored.", hostname, mv.Version),
Subject: &req.SourceAddrRange,
})
continue
}
// If we've found a pre-release version then we'll ignore it unless
// it was exactly requested.
if v.Prerelease() != "" && req.VersionConstraint.Required.String() != v.String() {
log.Printf("[TRACE] %s ignoring %s because it is a pre-release and was not requested exactly", key, v)
continue
}
if latestVersion == nil || v.GreaterThan(latestVersion) {
latestVersion = v
}
if req.VersionConstraint.Required.Check(v) {
if latestMatch == nil || v.GreaterThan(latestMatch) {
latestMatch = v
}
}
}
if latestVersion == nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Module has no versions",
Detail: fmt.Sprintf("The specified module does not have any available versions."),
Subject: &req.SourceAddrRange,
})
return nil, nil, diags
}
if latestMatch == nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Unresolvable module version constraint",
Detail: fmt.Sprintf("There is no available version of %q that matches the given version constraint. The newest available version is %s.", addr, latestVersion),
Subject: &req.VersionConstraint.DeclRange,
})
return nil, nil, diags
}
// Report up to the caller that we're about to start downloading.
packageAddr, _ := splitAddrSubdir(req.SourceAddr)
hooks.Download(key, packageAddr, latestMatch)
// If we manage to get down here then we've found a suitable version to
// install, so we need to ask the registry where we should download it from.
// The response to this is a go-getter-style address string.
dlAddr, err := reg.Location(addr, latestMatch.String())
if err != nil {
log.Printf("[ERROR] %s from %s %s: %s", key, addr, latestMatch, err)
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid response from remote module registry",
Detail: fmt.Sprintf("The remote registry at %s failed to return a download URL for %s %s.", hostname, addr, latestMatch),
Subject: &req.VersionConstraint.DeclRange,
})
return nil, nil, diags
}
log.Printf("[TRACE] %s %s %s is available at %q", key, addr, latestMatch, dlAddr)
modDir, err := getWithGoGetter(instPath, dlAddr)
if err != nil {
// Errors returned by go-getter have very inconsistent quality as
// end-user error messages, but for now we're accepting that because
// we have no way to recognize any specific errors to improve them
// and masking the error entirely would hide valuable diagnostic
// information from the user.
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Failed to download module",
Detail: fmt.Sprintf("Error attempting to download module source code from %q: %s", dlAddr, err),
Subject: &req.CallRange,
})
return nil, nil, diags
}
log.Printf("[TRACE] %s %q was downloaded to %s", key, dlAddr, modDir)
if addr.RawSubmodule != "" {
// Append the user's requested subdirectory to any subdirectory that
// was implied by any of the nested layers we expanded within go-getter.
modDir = filepath.Join(modDir, addr.RawSubmodule)
}
log.Printf("[TRACE] %s should now be at %s", key, modDir)
// Finally we are ready to try actually loading the module.
mod, mDiags := l.parser.LoadConfigDir(modDir)
if mod == nil {
// nil indicates missing or unreadable directory, so we'll
// discard the returned diags and return a more specific
// error message here. For registry modules this actually
// indicates a bug in the code above, since it's not the
// user's responsibility to create the directory in this case.
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Unreadable module directory",
Detail: fmt.Sprintf("The directory %s could not be read. This is a bug in Terraform and should be reported.", modDir),
Subject: &req.CallRange,
})
} else {
diags = append(diags, mDiags...)
}
// Note the local location in our manifest.
l.modules.manifest[key] = moduleRecord{
Key: key,
Version: latestMatch,
Dir: modDir,
SourceAddr: req.SourceAddr,
}
log.Printf("[DEBUG] Module installer: %s installed at %s", key, modDir)
hooks.Install(key, latestMatch, modDir)
return mod, latestMatch, diags
}
func (l *Loader) installGoGetterModule(req *configs.ModuleRequest, key string, instPath string, hooks InstallHooks) (*configs.Module, hcl.Diagnostics) {
var diags hcl.Diagnostics
// Report up to the caller that we're about to start downloading.
packageAddr, _ := splitAddrSubdir(req.SourceAddr)
hooks.Download(key, packageAddr, nil)
modDir, err := getWithGoGetter(instPath, req.SourceAddr)
if err != nil {
// Errors returned by go-getter have very inconsistent quality as
// end-user error messages, but for now we're accepting that because
// we have no way to recognize any specific errors to improve them
// and masking the error entirely would hide valuable diagnostic
// information from the user.
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Failed to download module",
Detail: fmt.Sprintf("Error attempting to download module source code from %q: %s", packageAddr, err),
Subject: &req.SourceAddrRange,
})
return nil, diags
}
log.Printf("[TRACE] %s %q was downloaded to %s", key, req.SourceAddr, modDir)
mod, mDiags := l.parser.LoadConfigDir(modDir)
if mod == nil {
// nil indicates missing or unreadable directory, so we'll
// discard the returned diags and return a more specific
// error message here. For registry modules this actually
// indicates a bug in the code above, since it's not the
// user's responsibility to create the directory in this case.
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Unreadable module directory",
Detail: fmt.Sprintf("The directory %s could not be read. This is a bug in Terraform and should be reported.", modDir),
Subject: &req.CallRange,
})
} else {
diags = append(diags, mDiags...)
}
// Note the local location in our manifest.
l.modules.manifest[key] = moduleRecord{
Key: key,
Dir: modDir,
SourceAddr: req.SourceAddr,
}
log.Printf("[DEBUG] Module installer: %s installed at %s", key, modDir)
hooks.Install(key, nil, modDir)
return mod, diags
}
func (l *Loader) packageInstallPath(modulePath []string) string {
return filepath.Join(l.modules.Dir, strings.Join(modulePath, "."))
}

34
configs/configload/loader_install_hooks.go Normal file
View File

@ -0,0 +1,34 @@
package configload
import version "github.com/hashicorp/go-version"
// InstallHooks is an interface used to provide notifications about the
// installation process being orchestrated by InstallModules.
//
// This interface may have new methods added in future, so implementers should
// embed InstallHooksImpl to get no-op implementations of any unimplemented
// methods.
type InstallHooks interface {
// Download is called for modules that are retrieved from a remote source
// before that download begins, to allow a caller to give feedback
// on progress through a possibly-long sequence of downloads.
Download(moduleAddr, packageAddr string, version *version.Version)
// Install is called for each module that is installed, even if it did
// not need to be downloaded from a remote source.
Install(moduleAddr string, version *version.Version, localPath string)
}
// InstallHooksImpl is a do-nothing implementation of InstallHooks that
// can be embedded in another implementation struct to allow only partial
// implementation of the interface.
type InstallHooksImpl struct {
}
func (h InstallHooksImpl) Download(moduleAddr, packageAddr string, version *version.Version) {
}
func (h InstallHooksImpl) Install(moduleAddr string, version *version.Version, localPath string) {
}
var _ InstallHooks = InstallHooksImpl{}

323
configs/configload/loader_install_test.go Normal file
View File

@ -0,0 +1,323 @@
package configload
import (
"os"
"path/filepath"
"strings"
"testing"
version "github.com/hashicorp/go-version"
"github.com/hashicorp/terraform/configs"
)
func TestLoaderInstallModules_local(t *testing.T) {
fixtureDir := filepath.Clean("test-fixtures/local-modules")
loader, done := tempChdirLoader(t, fixtureDir)
defer done()
hooks := &testInstallHooks{}
diags := loader.InstallModules(".", false, hooks)
assertNoDiagnostics(t, diags)
wantCalls := []testInstallHookCall{
{
Name: "Install",
ModuleAddr: "child_a",
PackageAddr: "",
LocalPath: "child_a",
},
{
Name: "Install",
ModuleAddr: "child_a.child_b",
PackageAddr: "",
LocalPath: "child_a/child_b",
},
}
if assertResultDeepEqual(t, hooks.Calls, wantCalls) {
return
}
// Make sure the configuration is loadable now.
// (This ensures that correct information is recorded in the manifest.)
config, loadDiags := loader.LoadConfig(".")
assertNoDiagnostics(t, loadDiags)
wantTraces := map[string]string{
"": "in root module",
"child_a": "in child_a module",
"child_a.child_b": "in child_b module",
}
gotTraces := map[string]string{}
config.DeepEach(func(c *configs.Config) {
path := strings.Join(c.Path, ".")
if c.Module.Variables["v"] == nil {
gotTraces[path] = "<missing>"
return
}
varDesc := c.Module.Variables["v"].Description
gotTraces[path] = varDesc
})
assertResultDeepEqual(t, gotTraces, wantTraces)
}
func TestLoaderInstallModules_registry(t *testing.T) {
if os.Getenv("TF_ACC") == "" {
t.Skip("this test accesses registry.terraform.io and github.com; set TF_ACC=1 to run it")
}
fixtureDir := filepath.Clean("test-fixtures/registry-modules")
loader, done := tempChdirLoader(t, fixtureDir)
defer done()
hooks := &testInstallHooks{}
diags := loader.InstallModules(".", false, hooks)
assertNoDiagnostics(t, diags)
v := version.Must(version.NewVersion("0.0.1"))
wantCalls := []testInstallHookCall{
// the configuration builder visits each level of calls in lexicographical
// order by name, so the following list is kept in the same order.
// acctest_child_a accesses //modules/child_a directly
{
Name: "Download",
ModuleAddr: "acctest_child_a",
PackageAddr: "hashicorp/module-installer-acctest/aws", // intentionally excludes the subdir because we're downloading the whole package here
Version: v,
},
{
Name: "Install",
ModuleAddr: "acctest_child_a",
Version: v,
LocalPath: ".terraform/modules/acctest_child_a/hashicorp-terraform-aws-module-installer-acctest-853d038/modules/child_a",
},
// acctest_child_a.child_b
// (no download because it's a relative path inside acctest_child_a)
{
Name: "Install",
ModuleAddr: "acctest_child_a.child_b",
LocalPath: ".terraform/modules/acctest_child_a/hashicorp-terraform-aws-module-installer-acctest-853d038/modules/child_b",
},
// acctest_child_b accesses //modules/child_b directly
{
Name: "Download",
ModuleAddr: "acctest_child_b",
PackageAddr: "hashicorp/module-installer-acctest/aws", // intentionally excludes the subdir because we're downloading the whole package here
Version: v,
},
{
Name: "Install",
ModuleAddr: "acctest_child_b",
Version: v,
LocalPath: ".terraform/modules/acctest_child_b/hashicorp-terraform-aws-module-installer-acctest-853d038/modules/child_b",
},
// acctest_root
{
Name: "Download",
ModuleAddr: "acctest_root",
PackageAddr: "hashicorp/module-installer-acctest/aws",
Version: v,
},
{
Name: "Install",
ModuleAddr: "acctest_root",
Version: v,
LocalPath: ".terraform/modules/acctest_root/hashicorp-terraform-aws-module-installer-acctest-853d038",
},
// acctest_root.child_a
// (no download because it's a relative path inside acctest_root)
{
Name: "Install",
ModuleAddr: "acctest_root.child_a",
LocalPath: ".terraform/modules/acctest_root/hashicorp-terraform-aws-module-installer-acctest-853d038/modules/child_a",
},
// acctest_root.child_a.child_b
// (no download because it's a relative path inside acctest_root, via acctest_root.child_a)
{
Name: "Install",
ModuleAddr: "acctest_root.child_a.child_b",
LocalPath: ".terraform/modules/acctest_root/hashicorp-terraform-aws-module-installer-acctest-853d038/modules/child_b",
},
}
if assertResultDeepEqual(t, hooks.Calls, wantCalls) {
return
}
// Make sure the configuration is loadable now.
// (This ensures that correct information is recorded in the manifest.)
config, loadDiags := loader.LoadConfig(".")
assertNoDiagnostics(t, loadDiags)
wantTraces := map[string]string{
"": "in local caller for registry-modules",
"acctest_root": "in root module",
"acctest_root.child_a": "in child_a module",
"acctest_root.child_a.child_b": "in child_b module",
"acctest_child_a": "in child_a module",
"acctest_child_a.child_b": "in child_b module",
"acctest_child_b": "in child_b module",
}
gotTraces := map[string]string{}
config.DeepEach(func(c *configs.Config) {
path := strings.Join(c.Path, ".")
if c.Module.Variables["v"] == nil {
gotTraces[path] = "<missing>"
return
}
varDesc := c.Module.Variables["v"].Description
gotTraces[path] = varDesc
})
assertResultDeepEqual(t, gotTraces, wantTraces)
}
func TestLoaderInstallModules_goGetter(t *testing.T) {
if os.Getenv("TF_ACC") == "" {
t.Skip("this test accesses github.com; set TF_ACC=1 to run it")
}
fixtureDir := filepath.Clean("test-fixtures/go-getter-modules")
loader, done := tempChdirLoader(t, fixtureDir)
defer done()
hooks := &testInstallHooks{}
diags := loader.InstallModules(".", false, hooks)
assertNoDiagnostics(t, diags)
wantCalls := []testInstallHookCall{
// the configuration builder visits each level of calls in lexicographical
// order by name, so the following list is kept in the same order.
// acctest_child_a accesses //modules/child_a directly
{
Name: "Download",
ModuleAddr: "acctest_child_a",
PackageAddr: "github.com/hashicorp/terraform-aws-module-installer-acctest?ref=v0.0.1", // intentionally excludes the subdir because we're downloading the whole repo here
},
{
Name: "Install",
ModuleAddr: "acctest_child_a",
LocalPath: ".terraform/modules/acctest_child_a/modules/child_a",
},
// acctest_child_a.child_b
// (no download because it's a relative path inside acctest_child_a)
{
Name: "Install",
ModuleAddr: "acctest_child_a.child_b",
LocalPath: ".terraform/modules/acctest_child_a/modules/child_b",
},
// acctest_child_b accesses //modules/child_b directly
{
Name: "Download",
ModuleAddr: "acctest_child_b",
PackageAddr: "github.com/hashicorp/terraform-aws-module-installer-acctest?ref=v0.0.1", // intentionally excludes the subdir because we're downloading the whole package here
},
{
Name: "Install",
ModuleAddr: "acctest_child_b",
LocalPath: ".terraform/modules/acctest_child_b/modules/child_b",
},
// acctest_root
{
Name: "Download",
ModuleAddr: "acctest_root",
PackageAddr: "github.com/hashicorp/terraform-aws-module-installer-acctest?ref=v0.0.1",
},
{
Name: "Install",
ModuleAddr: "acctest_root",
LocalPath: ".terraform/modules/acctest_root",
},
// acctest_root.child_a
// (no download because it's a relative path inside acctest_root)
{
Name: "Install",
ModuleAddr: "acctest_root.child_a",
LocalPath: ".terraform/modules/acctest_root/modules/child_a",
},
// acctest_root.child_a.child_b
// (no download because it's a relative path inside acctest_root, via acctest_root.child_a)
{
Name: "Install",
ModuleAddr: "acctest_root.child_a.child_b",
LocalPath: ".terraform/modules/acctest_root/modules/child_b",
},
}
if assertResultDeepEqual(t, hooks.Calls, wantCalls) {
return
}
// Make sure the configuration is loadable now.
// (This ensures that correct information is recorded in the manifest.)
config, loadDiags := loader.LoadConfig(".")
assertNoDiagnostics(t, loadDiags)
wantTraces := map[string]string{
"": "in local caller for go-getter-modules",
"acctest_root": "in root module",
"acctest_root.child_a": "in child_a module",
"acctest_root.child_a.child_b": "in child_b module",
"acctest_child_a": "in child_a module",
"acctest_child_a.child_b": "in child_b module",
"acctest_child_b": "in child_b module",
}
gotTraces := map[string]string{}
config.DeepEach(func(c *configs.Config) {
path := strings.Join(c.Path, ".")
if c.Module.Variables["v"] == nil {
gotTraces[path] = "<missing>"
return
}
varDesc := c.Module.Variables["v"].Description
gotTraces[path] = varDesc
})
assertResultDeepEqual(t, gotTraces, wantTraces)
}
type testInstallHooks struct {
Calls []testInstallHookCall
}
type testInstallHookCall struct {
Name string
ModuleAddr string
PackageAddr string
Version *version.Version
LocalPath string
}
func (h *testInstallHooks) Download(moduleAddr, packageAddr string, version *version.Version) {
h.Calls = append(h.Calls, testInstallHookCall{
Name: "Download",
ModuleAddr: moduleAddr,
PackageAddr: packageAddr,
Version: version,
})
}
func (h *testInstallHooks) Install(moduleAddr string, version *version.Version, localPath string) {
h.Calls = append(h.Calls, testInstallHookCall{
Name: "Install",
ModuleAddr: moduleAddr,
Version: version,
LocalPath: localPath,
})
}

97
configs/configload/loader_load.go Normal file
View File

@ -0,0 +1,97 @@
package configload
import (
"fmt"
version "github.com/hashicorp/go-version"
"github.com/hashicorp/hcl2/hcl"
"github.com/hashicorp/terraform/configs"
)
// LoadConfig reads the Terraform module in the given directory and uses it as the
// root module to build the static module tree that represents a configuration,
// assuming that all required descendent modules have already been installed.
//
// If error diagnostics are returned, the returned configuration may be either
// nil or incomplete. In the latter case, cautious static analysis is possible
// in spite of the errors.
//
// LoadConfig performs the basic syntax and uniqueness validations that are
// required to process the individual modules, and also detects
func (l *Loader) LoadConfig(rootDir string) (*configs.Config, hcl.Diagnostics) {
rootMod, diags := l.parser.LoadConfigDir(rootDir)
if rootMod == nil {
return nil, diags
}
cfg, cDiags := configs.BuildConfig(rootMod, configs.ModuleWalkerFunc(l.moduleWalkerLoad))
diags = append(diags, cDiags...)
return cfg, diags
}
// moduleWalkerLoad is a configs.ModuleWalkerFunc for loading modules that
// are presumed to have already been installed. A different function
// (moduleWalkerInstall) is used for installation.
func (l *Loader) moduleWalkerLoad(req *configs.ModuleRequest) (*configs.Module, *version.Version, hcl.Diagnostics) {
// Since we're just loading here, we expect that all referenced modules
// will be already installed and described in our manifest. However, we
// do verify that the manifest and the configuration are in agreement
// so that we can prompt the user to run "terraform init" if not.
key := manifestKey(req.Path)
record, exists := l.modules.manifest[key]
if !exists {
return nil, nil, hcl.Diagnostics{
{
Severity: hcl.DiagError,
Summary: "Module not installed",
Detail: "This module is not yet installed. Run \"terraform init\" to install all modules required by this configuration.",
Subject: &req.CallRange,
},
}
}
var diags hcl.Diagnostics
// Check for inconsistencies between manifest and config
if req.SourceAddr != record.SourceAddr {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Module source has changed",
Detail: "The source address was changed since this module was installed. Run \"terraform init\" to install all modules required by this configuration.",
Subject: &req.SourceAddrRange,
})
}
if !req.VersionConstraint.Required.Check(record.Version) {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Module version requirements have changed",
Detail: fmt.Sprintf(
"The version requirements have changed since this module was installed and the installed version (%s) is no longer acceptable. Run \"terraform init\" to install all modules required by this configuration.",
record.Version,
),
Subject: &req.SourceAddrRange,
})
}
mod, mDiags := l.parser.LoadConfigDir(record.Dir)
diags = append(diags, mDiags...)
if mod == nil {
// nil specifically indicates that the directory does not exist or
// cannot be read, so in this case we'll discard any generic diagnostics
// returned from LoadConfigDir and produce our own context-sensitive
// error message.
return nil, nil, hcl.Diagnostics{
{
Severity: hcl.DiagError,
Summary: "Module not installed",
Detail: fmt.Sprintf("This module's local cache directory %s could not be read. Run \"terraform init\" to install all modules required by this configuration.", record.Dir),
Subject: &req.CallRange,
},
}
}
return mod, record.Version, diags
}

60
configs/configload/loader_load_test.go Normal file
View File

@ -0,0 +1,60 @@
package configload
import (
"path/filepath"
"reflect"
"sort"
"strings"
"testing"
"github.com/davecgh/go-spew/spew"
"github.com/zclconf/go-cty/cty"
"github.com/hashicorp/terraform/configs"
)
func TestLoaderLoadConfig_okay(t *testing.T) {
fixtureDir := filepath.Clean("test-fixtures/already-installed")
loader, err := NewLoader(&Config{
ModulesDir: filepath.Join(fixtureDir, ".terraform/modules"),
})
if err != nil {
t.Fatalf("unexpected error from NewLoader: %s", err)
}
cfg, diags := loader.LoadConfig(fixtureDir)
assertNoDiagnostics(t, diags)
if cfg == nil {
t.Fatalf("config is nil; want non-nil")
}
var gotPaths []string
cfg.DeepEach(func(c *configs.Config) {
gotPaths = append(gotPaths, strings.Join(c.Path, "."))
})
sort.Strings(gotPaths)
wantPaths := []string{
"", // root module
"child_a",
"child_a.child_c",
"child_b",
"child_b.child_d",
}
if !reflect.DeepEqual(gotPaths, wantPaths) {
t.Fatalf("wrong module paths\ngot: %swant %s", spew.Sdump(gotPaths), spew.Sdump(wantPaths))
}
t.Run("child_a.child_c output", func(t *testing.T) {
output := cfg.Children["child_a"].Children["child_c"].Module.Outputs["hello"]
got, diags := output.Expr.Value(nil)
assertNoDiagnostics(t, diags)
assertResultCtyEqual(t, got, cty.StringVal("Hello from child_c"))
})
t.Run("child_b.child_d output", func(t *testing.T) {
output := cfg.Children["child_b"].Children["child_d"].Module.Outputs["hello"]
got, diags := output.Expr.Value(nil)
assertNoDiagnostics(t, diags)
assertResultCtyEqual(t, got, cty.StringVal("Hello from child_d"))
})
}

141
configs/configload/loader_test.go Normal file
View File

@ -0,0 +1,141 @@
package configload
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/go-test/deep"
"github.com/hashicorp/hcl2/hcl"
"github.com/zclconf/go-cty/cty"
)
// tempChdir copies the contents of the given directory to a temporary
// directory and changes the test process's current working directory to
// point to that directory. Also returned is a function that should be
// called at the end of the test (e.g. via "defer") to restore the previous
// working directory.
//
// Tests using this helper cannot safely be run in parallel with other tests.
func tempChdir(t *testing.T, sourceDir string) (string, func()) {
t.Helper()
tmpDir, err := ioutil.TempDir("", "terraform-configload")
if err != nil {
t.Fatalf("failed to create temporary directory: %s", err)
return "", nil
}
if err := copyDir(tmpDir, sourceDir); err != nil {
t.Fatalf("failed to copy fixture to temporary directory: %s", err)
return "", nil
}
oldDir, err := os.Getwd()
if err != nil {
t.Fatalf("failed to determine current working directory: %s", err)
return "", nil
}
err = os.Chdir(tmpDir)
if err != nil {
t.Fatalf("failed to switch to temp dir %s: %s", tmpDir, err)
return "", nil
}
t.Logf("tempChdir switched to %s after copying from %s", tmpDir, sourceDir)
return tmpDir, func() {
err := os.Chdir(oldDir)
if err != nil {
panic(fmt.Errorf("failed to restore previous working directory %s: %s", oldDir, err))
}
if os.Getenv("TF_CONFIGLOAD_TEST_KEEP_TMP") == "" {
os.RemoveAll(tmpDir)
}
}
}
// tempChdirLoader is a wrapper around tempChdir that also returns a Loader
// whose modules directory is at the conventional location within the
// created temporary directory.
func tempChdirLoader(t *testing.T, sourceDir string) (*Loader, func()) {
t.Helper()
_, done := tempChdir(t, sourceDir)
modulesDir := filepath.Clean(".terraform/modules")
err := os.MkdirAll(modulesDir, os.ModePerm)
if err != nil {
done() // undo the chdir in tempChdir so we can safely run other tests
t.Fatalf("failed to create modules directory: %s", err)
return nil, nil
}
loader, err := NewLoader(&Config{
ModulesDir: modulesDir,
})
if err != nil {
done() // undo the chdir in tempChdir so we can safely run other tests
t.Fatalf("failed to create loader: %s", err)
return nil, nil
}
return loader, done
}
func assertNoDiagnostics(t *testing.T, diags hcl.Diagnostics) bool {
t.Helper()
return assertDiagnosticCount(t, diags, 0)
}
func assertDiagnosticCount(t *testing.T, diags hcl.Diagnostics, want int) bool {
t.Helper()
if len(diags) != 0 {
t.Errorf("wrong number of diagnostics %d; want %d", len(diags), want)
for _, diag := range diags {
t.Logf("- %s", diag)
}
return true
}
return false
}
func assertDiagnosticSummary(t *testing.T, diags hcl.Diagnostics, want string) bool {
t.Helper()
for _, diag := range diags {
if diag.Summary == want {
return false
}
}
t.Errorf("missing diagnostic summary %q", want)
for _, diag := range diags {
t.Logf("- %s", diag)
}
return true
}
func assertResultDeepEqual(t *testing.T, got, want interface{}) bool {
t.Helper()
if diff := deep.Equal(got, want); diff != nil {
for _, problem := range diff {
t.Errorf("%s", problem)
}
return true
}
return false
}
func assertResultCtyEqual(t *testing.T, got, want cty.Value) bool {
t.Helper()
if !got.RawEquals(want) {
t.Errorf("wrong result\ngot: %#v\nwant: %#v", got, want)
return true
}
return false
}

132
configs/configload/module_manifest.go Normal file
View File

@ -0,0 +1,132 @@
package configload
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
version "github.com/hashicorp/go-version"
)
// moduleRecord represents some metadata about an installed module, as part
// of a moduleManifest.
type moduleRecord struct {
// Key is a unique identifier for this particular module, based on its
// position within the static module tree.
Key string `json:"Key"`
// SourceAddr is the source address given for this module in configuration.
// This is used only to detect if the source was changed in configuration
// since the module was last installed, which means that the installer
// must re-install it.
SourceAddr string `json:"Source"`
// Version is the exact version of the module, which results from parsing
// VersionStr. nil for un-versioned modules.
Version *version.Version `json:"-"`
// VersionStr is the version specifier string. This is used only for
// serialization in snapshots and should not be accessed or updated
// by any other codepaths; use "Version" instead.
VersionStr string `json:"Version,omitempty"`
// Dir is the path to the local directory where the module is installed.
Dir string `json:"Dir"`
}
// moduleManifest is a map used to keep track of the filesystem locations
// and other metadata about installed modules.
//
// The configuration loader refers to this, while the module installer updates
// it to reflect any changes to the installed modules.
type moduleManifest map[string]moduleRecord
func manifestKey(path []string) string {
return strings.Join(path, ".")
}
// manifestSnapshotFile is an internal struct used only to assist in our JSON
// serializtion of manifest snapshots. It should not be used for any other
// purposes.
type manifestSnapshotFile struct {
Records []moduleRecord `json:"Modules"`
}
const manifestFilename = "modules.json"
func (m *moduleMgr) manifestSnapshotPath() string {
return filepath.Join(m.Dir, manifestFilename)
}
// readModuleManifestSnapshot loads a manifest snapshot from the filesystem.
func (m *moduleMgr) readModuleManifestSnapshot() error {
src, err := m.FS.ReadFile(m.manifestSnapshotPath())
if err != nil {
if os.IsNotExist(err) {
// We'll treat a missing file as an empty manifest
m.manifest = make(moduleManifest)
return nil
}
return err
}
if len(src) == 0 {
// This should never happen, but we'll tolerate it as if it were
// a valid empty JSON object.
m.manifest = make(moduleManifest)
return nil
}
var read manifestSnapshotFile
err = json.Unmarshal(src, &read)
new := make(moduleManifest)
for _, record := range read.Records {
if record.VersionStr != "" {
record.Version, err = version.NewVersion(record.VersionStr)
if err != nil {
return fmt.Errorf("invalid version %q for %s: %s", record.VersionStr, record.Key, err)
}
}
if _, exists := new[record.Key]; exists {
// This should never happen in any valid file, so we'll catch it
// and report it to avoid confusing/undefined behavior if the
// snapshot file was edited incorrectly outside of Terraform.
return fmt.Errorf("snapshot file contains two records for path %s", record.Key)
}
new[record.Key] = record
}
m.manifest = new
return nil
}
// writeModuleManifestSnapshot writes a snapshot of the current manifest
// to the filesystem.
//
// The caller must guarantee no concurrent modifications of the manifest for
// the duration of a call to this function, or the behavior is undefined.
func (m *moduleMgr) writeModuleManifestSnapshot() error {
var write manifestSnapshotFile
for _, record := range m.manifest {
// Make sure VersionStr is in sync with Version, since we encourage
// callers to manipulate Version and ignore VersionStr.
if record.Version != nil {
record.VersionStr = record.Version.String()
} else {
record.VersionStr = ""
}
write.Records = append(write.Records, record)
}
src, err := json.Marshal(write)
if err != nil {
return err
}
return m.FS.WriteFile(m.manifestSnapshotPath(), src, os.ModePerm)
}

42
configs/configload/module_mgr.go Normal file
View File

@ -0,0 +1,42 @@
package configload
import (
"github.com/hashicorp/terraform/registry"
"github.com/hashicorp/terraform/svchost/auth"
"github.com/hashicorp/terraform/svchost/disco"
"github.com/spf13/afero"
)
type moduleMgr struct {
FS afero.Afero
// CanInstall is true for a module manager that can support installation.
//
// This must be set only if FS is an afero.OsFs, because the installer
// (which uses go-getter) is not aware of the virtual filesystem
// abstraction and will always write into the "real" filesystem.
CanInstall bool
// Dir is the path where descendent modules are (or will be) installed.
Dir string
// Services is a service discovery client that will be used to find
// remote module registry endpoints. This object may be pre-loaded with
// cached discovery information.
Services *disco.Disco
// Creds provides optional credentials for communicating with service hosts.
Creds auth.CredentialsSource
// Registry is a client for the module registry protocol, which is used
// when a module is requested from a registry source.
Registry *registry.Client
// manifest tracks the currently-installed modules for this manager.
//
// The loader may read this. Only the installer may write to it, and
// after a set of updates are completed the installer must call
// writeModuleManifestSnapshot to persist a snapshot of the manifest
// to disk for use on subsequent runs.
manifest moduleManifest
}

45
configs/configload/source_addr.go Normal file
View File

@ -0,0 +1,45 @@
package configload
import (
"strings"
"github.com/hashicorp/go-getter"
"github.com/hashicorp/terraform/registry/regsrc"
)
var localSourcePrefixes = []string{
"./",
"../",
".\\",
"..\\",
}
func isLocalSourceAddr(addr string) bool {
for _, prefix := range localSourcePrefixes {
if strings.HasPrefix(addr, prefix) {
return true
}
}
return false
}
func isRegistrySourceAddr(addr string) bool {
_, err := regsrc.ParseModuleSource(addr)
return err == nil
}
// splitAddrSubdir splits the given address (which is assumed to be a
// registry address or go-getter-style address) into a package portion
// and a sub-directory portion.
//
// The package portion defines what should be downloaded and then the
// sub-directory portion, if present, specifies a sub-directory within
// the downloaded object (an archive, VCS repository, etc) that contains
// the module's configuration files.
//
// The subDir portion will be returned as empty if no subdir separator
// ("//") is present in the address.
func splitAddrSubdir(addr string) (packageAddr, subDir string) {
return getter.SourceDirSubdir(addr)
}

4
configs/configload/test-fixtures/already-installed/.terraform/modules/child_a/child_a.tf Normal file
View File

@ -0,0 +1,4 @@
module "child_c" {
source = "./child_c"
}

4
configs/configload/test-fixtures/already-installed/.terraform/modules/child_a/child_c/child_c.tf Normal file
View File

@ -0,0 +1,4 @@
output "hello" {
value = "Hello from child_c"
}

4
configs/configload/test-fixtures/already-installed/.terraform/modules/child_b.child_d/child_d.tf Normal file
View File

@ -0,0 +1,4 @@
output "hello" {
value = "Hello from child_d"
}

5
configs/configload/test-fixtures/already-installed/.terraform/modules/child_b/child_b.tf Normal file
View File

@ -0,0 +1,5 @@
module "child_d" {
source = "example.com/foo/bar_d/baz"
# Intentionally no version here
}

1
configs/configload/test-fixtures/already-installed/.terraform/modules/modules.json Normal file
View File

@ -0,0 +1 @@
{"Modules":[{"Key":"","Source":"","Dir":"test-fixtures/already-installed"},{"Key":"child_a","Source":"example.com/foo/bar_a/baz","Version":"1.0.1","Dir":"test-fixtures/already-installed/.terraform/modules/child_a"},{"Key":"child_b","Source":"example.com/foo/bar_b/baz","Version":"1.0.0","Dir":"test-fixtures/already-installed/.terraform/modules/child_b"},{"Key":"child_a.child_c","Source":"./child_c","Dir":"test-fixtures/already-installed/.terraform/modules/child_a/child_c"},{"Key":"child_b.child_d","Source":"example.com/foo/bar_d/baz","Version":"1.2.0","Dir":"test-fixtures/already-installed/.terraform/modules/child_b.child_d"}]}

10
configs/configload/test-fixtures/already-installed/root.tf Normal file
View File

@ -0,0 +1,10 @@
module "child_a" {
source = "example.com/foo/bar_a/baz"
version = ">= 1.0.0"
}
module "child_b" {
source = "example.com/foo/bar_b/baz"
version = ">= 1.0.0"
}

0
configs/configload/test-fixtures/empty/.gitignore vendored Normal file
View File

1
configs/configload/test-fixtures/go-getter-modules/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
.terraform/*

21
configs/configload/test-fixtures/go-getter-modules/root.tf Normal file
View File

@ -0,0 +1,21 @@
# This fixture depends on a github repo at:
# https://github.com/hashicorp/terraform-aws-module-installer-acctest
# ...and expects its v0.0.1 tag to be pointing at the following commit:
# d676ab2559d4e0621d59e3c3c4cbb33958ac4608
variable "v" {
description = "in local caller for go-getter-modules"
default = ""
}
module "acctest_root" {
source = "github.com/hashicorp/terraform-aws-module-installer-acctest?ref=v0.0.1"
}
module "acctest_child_a" {
source = "github.com/hashicorp/terraform-aws-module-installer-acctest//modules/child_a?ref=v0.0.1"
}
module "acctest_child_b" {
source = "github.com/hashicorp/terraform-aws-module-installer-acctest//modules/child_b?ref=v0.0.1"
}

9
configs/configload/test-fixtures/local-modules/child_a/child_a.tf Normal file
View File

@ -0,0 +1,9 @@
variable "v" {
description = "in child_a module"
default = ""
}
module "child_b" {
source = "./child_b"
}

9
configs/configload/test-fixtures/local-modules/child_a/child_b/child_b.tf Normal file
View File

@ -0,0 +1,9 @@
variable "v" {
description = "in child_b module"
default = ""
}
output "hello" {
value = "Hello from child_b!"
}

9
configs/configload/test-fixtures/local-modules/root.tf Normal file
View File

@ -0,0 +1,9 @@
variable "v" {
description = "in root module"
default = ""
}
module "child_a" {
source = "./child_a"
}

1
configs/configload/test-fixtures/registry-modules/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
.terraform/*

33
configs/configload/test-fixtures/registry-modules/root.tf Normal file
View File

@ -0,0 +1,33 @@
# This fixture indirectly depends on a github repo at:
# https://github.com/hashicorp/terraform-aws-module-installer-acctest
# ...and expects its v0.0.1 tag to be pointing at the following commit:
# d676ab2559d4e0621d59e3c3c4cbb33958ac4608
#
# This repository is accessed indirectly via:
# https://registry.terraform.io/modules/hashicorp/module-installer-acctest/aws/0.0.1
#
# Since the tag's id is included in a downloaded archive, it is expected to
# have the following id:
# 853d03855b3290a3ca491d4c3a7684572dd42237
# (this particular assumption is encoded in the tests that use this fixture)
variable "v" {
description = "in local caller for registry-modules"
default = ""
}
module "acctest_root" {
source = "hashicorp/module-installer-acctest/aws"
version = "0.0.1"
}
module "acctest_child_a" {
source = "hashicorp/module-installer-acctest/aws//modules/child_a"
version = "0.0.1"
}
module "acctest_child_b" {
source = "hashicorp/module-installer-acctest/aws//modules/child_b"
version = "0.0.1"
}

23
configs/depends_on.go Normal file
View File

@ -0,0 +1,23 @@
package configs
import (
"github.com/hashicorp/hcl2/hcl"
)
func decodeDependsOn(attr *hcl.Attribute) ([]hcl.Traversal, hcl.Diagnostics) {
var ret []hcl.Traversal
exprs, diags := hcl.ExprList(attr.Expr)
for _, expr := range exprs {
expr, shimDiags := shimTraversalInString(expr, false)
diags = append(diags, shimDiags...)
traversal, travDiags := hcl.AbsTraversalForExpr(expr)
diags = append(diags, travDiags...)
if len(traversal) != 0 {
ret = append(ret, traversal)
}
}
return ret, diags
}

19
configs/doc.go Normal file
View File

@ -0,0 +1,19 @@
// Package configs contains types that represent Terraform configurations and
// the different elements thereof.
//
// The functionality in this package can be used for some static analyses of
// Terraform configurations, but this package generally exposes representations
// of the configuration source code rather than the result of evaluating these
// objects. The sibling package "lang" deals with evaluation of structures
// and expressions in the configuration.
//
// Due to its close relationship with HCL, this package makes frequent use
// of types from the HCL API, including raw HCL diagnostic messages. Such
// diagnostics can be converted into Terraform-flavored diagnostics, if needed,
// using functions in the sibling package tfdiags.
//
// The Parser type is the main entry-point into this package. The LoadConfigDir
// method can be used to load a single module directory, and then a full
// configuration (including any descendent modules) can be produced using
// the top-level BuildConfig method.
package configs

376
configs/module.go Normal file
View File

@ -0,0 +1,376 @@
package configs
import (
"fmt"
"github.com/hashicorp/hcl2/hcl"
)
// Module is a container for a set of configuration constructs that are
// evaluated within a common namespace.
type Module struct {
CoreVersionConstraints []VersionConstraint
Backend *Backend
ProviderConfigs map[string]*Provider
ProviderRequirements map[string][]VersionConstraint
Variables map[string]*Variable
Locals map[string]*Local
Outputs map[string]*Output
ModuleCalls map[string]*ModuleCall
ManagedResources map[string]*ManagedResource
DataResources map[string]*DataResource
}
// File describes the contents of a single configuration file.
//
// Individual files are not usually used alone, but rather combined together
// with other files (conventionally, those in the same directory) to produce
// a *Module, using NewModule.
//
// At the level of an individual file we represent directly the structural
// elements present in the file, without any attempt to detect conflicting
// declarations. A File object can therefore be used for some basic static
// analysis of individual elements, but must be built into a Module to detect
// duplicate declarations.
type File struct {
CoreVersionConstraints []VersionConstraint
Backends []*Backend
ProviderConfigs []*Provider
ProviderRequirements []*ProviderRequirement
Variables []*Variable
Locals []*Local
Outputs []*Output
ModuleCalls []*ModuleCall
ManagedResources []*ManagedResource
DataResources []*DataResource
}
// NewModule takes a list of primary files and a list of override files and
// produces a *Module by combining the files together.
//
// If there are any conflicting declarations in the given files -- for example,
// if the same variable name is defined twice -- then the resulting module
// will be incomplete and error diagnostics will be returned. Careful static
// analysis of the returned Module is still possible in this case, but the
// module will probably not be semantically valid.
func NewModule(primaryFiles, overrideFiles []*File) (*Module, hcl.Diagnostics) {
var diags hcl.Diagnostics
mod := &Module{
ProviderConfigs: map[string]*Provider{},
ProviderRequirements: map[string][]VersionConstraint{},
Variables: map[string]*Variable{},
Locals: map[string]*Local{},
Outputs: map[string]*Output{},
ModuleCalls: map[string]*ModuleCall{},
ManagedResources: map[string]*ManagedResource{},
DataResources: map[string]*DataResource{},
}
for _, file := range primaryFiles {
fileDiags := mod.appendFile(file)
diags = append(diags, fileDiags...)
}
for _, file := range overrideFiles {
fileDiags := mod.mergeFile(file)
diags = append(diags, fileDiags...)
}
return mod, diags
}
func (m *Module) appendFile(file *File) hcl.Diagnostics {
var diags hcl.Diagnostics
for _, constraint := range file.CoreVersionConstraints {
// If there are any conflicting requirements then we'll catch them
// when we actually check these constraints.
m.CoreVersionConstraints = append(m.CoreVersionConstraints, constraint)
}
for _, b := range file.Backends {
if m.Backend != nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Duplicate backend configuration",
Detail: fmt.Sprintf("A module may have only one backend configuration. The backend was previously configured at %s.", m.Backend.DeclRange),
Subject: &b.DeclRange,
})
continue
}
m.Backend = b
}
for _, pc := range file.ProviderConfigs {
key := pc.moduleUniqueKey()
if existing, exists := m.ProviderConfigs[key]; exists {
if existing.Alias == "" {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Duplicate provider configuration",
Detail: fmt.Sprintf("A default (non-aliased) provider configuration for %q was already given at %s. If multiple configurations are required, set the \"alias\" argument for alternative configurations.", existing.Name, existing.DeclRange),
Subject: &pc.DeclRange,
})
} else {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Duplicate provider configuration",
Detail: fmt.Sprintf("A provider configuration for %q with alias %q was already given at %s. Each configuration for the same provider must have a distinct alias.", existing.Name, existing.Alias, existing.DeclRange),
Subject: &pc.DeclRange,
})
}
continue
}
m.ProviderConfigs[key] = pc
}
for _, reqd := range file.ProviderRequirements {
m.ProviderRequirements[reqd.Name] = append(m.ProviderRequirements[reqd.Name], reqd.Requirement)
}
for _, v := range file.Variables {
if existing, exists := m.Variables[v.Name]; exists {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Duplicate variable declaration",
Detail: fmt.Sprintf("A variable named %q was already declared at %s. Variable names must be unique within a module.", existing.Name, existing.DeclRange),
Subject: &v.DeclRange,
})
}
m.Variables[v.Name] = v
}
for _, l := range file.Locals {
if existing, exists := m.Locals[l.Name]; exists {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Duplicate local value definition",
Detail: fmt.Sprintf("A local value named %q was already defined at %s. Local value names must be unique within a module.", existing.Name, existing.DeclRange),
Subject: &l.DeclRange,
})
}
m.Locals[l.Name] = l
}
for _, o := range file.Outputs {
if existing, exists := m.Outputs[o.Name]; exists {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Duplicate output definition",
Detail: fmt.Sprintf("An output named %q was already defined at %s. Output names must be unique within a module.", existing.Name, existing.DeclRange),
Subject: &o.DeclRange,
})
}
m.Outputs[o.Name] = o
}
for _, mc := range file.ModuleCalls {
if existing, exists := m.ModuleCalls[mc.Name]; exists {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Duplicate module call",
Detail: fmt.Sprintf("An module call named %q was already defined at %s. Module calls must have unique names within a module.", existing.Name, existing.DeclRange),
Subject: &mc.DeclRange,
})
}
m.ModuleCalls[mc.Name] = mc
}
for _, r := range file.ManagedResources {
key := r.moduleUniqueKey()
if existing, exists := m.ManagedResources[key]; exists {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: fmt.Sprintf("Duplicate resource %q configuration", existing.Type),
Detail: fmt.Sprintf("A %s resource named %q was already declared at %s. Resource names must be unique per type in each module.", existing.Type, existing.Name, existing.DeclRange),
Subject: &r.DeclRange,
})
continue
}
m.ManagedResources[key] = r
}
for _, r := range file.DataResources {
key := r.moduleUniqueKey()
if existing, exists := m.DataResources[key]; exists {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: fmt.Sprintf("Duplicate data %q configuration", existing.Type),
Detail: fmt.Sprintf("A %s data resource named %q was already declared at %s. Resource names must be unique per type in each module.", existing.Type, existing.Name, existing.DeclRange),
Subject: &r.DeclRange,
})
continue
}
m.DataResources[key] = r
}
return diags
}
func (m *Module) mergeFile(file *File) hcl.Diagnostics {
var diags hcl.Diagnostics
if len(file.CoreVersionConstraints) != 0 {
// This is a bit of a strange case for overriding since we normally
// would union together across multiple files anyway, but we'll
// allow it and have each override file clobber any existing list.
m.CoreVersionConstraints = nil
for _, constraint := range file.CoreVersionConstraints {
m.CoreVersionConstraints = append(m.CoreVersionConstraints, constraint)
}
}
if len(file.Backends) != 0 {
switch len(file.Backends) {
case 1:
m.Backend = file.Backends[0]
default:
// An override file with multiple backends is still invalid, even
// though it can override backends from _other_ files.
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Duplicate backend configuration",
Detail: fmt.Sprintf("Each override file may have only one backend configuration. A backend was previously configured at %s.", file.Backends[0].DeclRange),
Subject: &file.Backends[1].DeclRange,
})
}
}
for _, pc := range file.ProviderConfigs {
key := pc.moduleUniqueKey()
existing, exists := m.ProviderConfigs[key]
if pc.Alias == "" {
// We allow overriding a non-existing _default_ provider configuration
// because the user model is that an absent provider configuration
// implies an empty provider configuration, which is what the user
// is therefore overriding here.
if exists {
mergeDiags := existing.merge(pc)
diags = append(diags, mergeDiags...)
} else {
m.ProviderConfigs[key] = pc
}
} else {
// For aliased providers, there must be a base configuration to
// override. This allows us to detect and report alias typos
// that might otherwise cause the override to not apply.
if !exists {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Missing base provider configuration for override",
Detail: fmt.Sprintf("There is no %s provider configuration with the alias %q. An override file can only override an aliased provider configuration that was already defined in a primary configuration file.", pc.Name, pc.Alias),
Subject: &pc.DeclRange,
})
continue
}
mergeDiags := existing.merge(pc)
diags = append(diags, mergeDiags...)
}
}
if len(file.ProviderRequirements) != 0 {
mergeProviderVersionConstraints(m.ProviderRequirements, file.ProviderRequirements)
}
for _, v := range file.Variables {
existing, exists := m.Variables[v.Name]
if !exists {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Missing base variable declaration to override",
Detail: fmt.Sprintf("There is no variable named %q. An override file can only override a variable that was already declared in a primary configuration file.", v.Name),
Subject: &v.DeclRange,
})
continue
}
mergeDiags := existing.merge(v)
diags = append(diags, mergeDiags...)
}
for _, l := range file.Locals {
existing, exists := m.Locals[l.Name]
if !exists {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Missing base local value definition to override",
Detail: fmt.Sprintf("There is no local value named %q. An override file can only override a local value that was already defined in a primary configuration file.", l.Name),
Subject: &l.DeclRange,
})
continue
}
mergeDiags := existing.merge(l)
diags = append(diags, mergeDiags...)
}
for _, o := range file.Outputs {
existing, exists := m.Outputs[o.Name]
if !exists {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Missing base output definition to override",
Detail: fmt.Sprintf("There is no output named %q. An override file can only override an output that was already defined in a primary configuration file.", o.Name),
Subject: &o.DeclRange,
})
continue
}
mergeDiags := existing.merge(o)
diags = append(diags, mergeDiags...)
}
for _, mc := range file.ModuleCalls {
existing, exists := m.ModuleCalls[mc.Name]
if !exists {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Missing module call to override",
Detail: fmt.Sprintf("There is no module call named %q. An override file can only override a module call that was defined in a primary configuration file.", mc.Name),
Subject: &mc.DeclRange,
})
continue
}
mergeDiags := existing.merge(mc)
diags = append(diags, mergeDiags...)
}
for _, r := range file.ManagedResources {
key := r.moduleUniqueKey()
existing, exists := m.ManagedResources[key]
if !exists {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Missing resource to override",
Detail: fmt.Sprintf("There is no %s resource named %q. An override file can only override a resource block defined in a primary configuration file.", r.Type, r.Name),
Subject: &r.DeclRange,
})
continue
}
mergeDiags := existing.merge(r)
diags = append(diags, mergeDiags...)
}
for _, r := range file.DataResources {
key := r.moduleUniqueKey()
existing, exists := m.DataResources[key]
if !exists {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Missing data resource to override",
Detail: fmt.Sprintf("There is no %s data resource named %q. An override file can only override a data block defined in a primary configuration file.", r.Type, r.Name),
Subject: &r.DeclRange,
})
continue
}
mergeDiags := existing.merge(r)
diags = append(diags, mergeDiags...)
}
return diags
}

101
configs/module_call.go Normal file
View File

@ -0,0 +1,101 @@
package configs
import (
"github.com/hashicorp/hcl2/gohcl"
"github.com/hashicorp/hcl2/hcl"
"github.com/hashicorp/hcl2/hcl/hclsyntax"
)
// ModuleCall represents a "module" block in a module or file.
type ModuleCall struct {
Name string
SourceAddr string
SourceAddrRange hcl.Range
SourceSet bool
Config hcl.Body
Version VersionConstraint
Count hcl.Expression
ForEach hcl.Expression
DependsOn []hcl.Traversal
DeclRange hcl.Range
}
func decodeModuleBlock(block *hcl.Block, override bool) (*ModuleCall, hcl.Diagnostics) {
mc := &ModuleCall{
Name: block.Labels[0],
DeclRange: block.DefRange,
}
schema := moduleBlockSchema
if override {
schema = schemaForOverrides(schema)
}
content, remain, diags := block.Body.PartialContent(schema)
mc.Config = remain
if !hclsyntax.ValidIdentifier(mc.Name) {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid module instance name",
Detail: badIdentifierDetail,
Subject: &block.LabelRanges[0],
})
}
if attr, exists := content.Attributes["source"]; exists {
valDiags := gohcl.DecodeExpression(attr.Expr, nil, &mc.SourceAddr)
diags = append(diags, valDiags...)
mc.SourceAddrRange = attr.Expr.Range()
mc.SourceSet = true
}
if attr, exists := content.Attributes["version"]; exists {
var versionDiags hcl.Diagnostics
mc.Version, versionDiags = decodeVersionConstraint(attr)
diags = append(diags, versionDiags...)
}
if attr, exists := content.Attributes["count"]; exists {
mc.Count = attr.Expr
}
if attr, exists := content.Attributes["for_each"]; exists {
mc.ForEach = attr.Expr
}
if attr, exists := content.Attributes["depends_on"]; exists {
deps, depsDiags := decodeDependsOn(attr)
diags = append(diags, depsDiags...)
mc.DependsOn = append(mc.DependsOn, deps...)
}
return mc, diags
}
var moduleBlockSchema = &hcl.BodySchema{
Attributes: []hcl.AttributeSchema{
{
Name: "source",
Required: true,
},
{
Name: "version",
},
{
Name: "count",
},
{
Name: "for_each",
},
{
Name: "depends_on",
},
},
}

Some files were not shown because too many files have changed in this diff Show More