core: [refactor] store Deposed resource instances as a list

Deposed instances need to be stored as a list for certain pathological
cases where destroys fail for some reason (e.g. upstream API failure,
Terraform interrupted mid-run). Terraform needs to be able to remember
all Deposed nodes so that it can clean them up properly in subsequent
runs.

Deposed instances will now never touch the Tainted list - they're fully
managed from within their own list.

Added a "multiDepose" test case that walks through a scenario to
exercise this.
This commit is contained in:
Paul Hinze 2015-03-04 12:15:53 -06:00
parent 426f253085
commit 6c93fbb85d
10 changed files with 243 additions and 59 deletions

View File

@ -3845,6 +3845,136 @@ func TestContext2Apply_errorDestroy_createBeforeDestroy(t *testing.T) {
} }
} }
func TestContext2Apply_multiDepose_createBeforeDestroy(t *testing.T) {
m := testModule(t, "apply-multi-depose-create-before-destroy")
p := testProvider("aws")
p.DiffFn = testDiffFn
ps := map[string]ResourceProviderFactory{"aws": testProviderFuncFixed(p)}
state := &State{
Modules: []*ModuleState{
&ModuleState{
Path: rootModulePath,
Resources: map[string]*ResourceState{
"aws_instance.web": &ResourceState{
Type: "aws_instance",
Primary: &InstanceState{ID: "foo"},
},
},
},
},
}
ctx := testContext2(t, &ContextOpts{
Module: m,
Providers: ps,
State: state,
})
createdInstanceId := "bar"
// Create works
createFunc := func(is *InstanceState) (*InstanceState, error) {
return &InstanceState{ID: createdInstanceId}, nil
}
// Destroy starts broken
destroyFunc := func(is *InstanceState) (*InstanceState, error) {
return is, fmt.Errorf("destroy failed")
}
p.ApplyFn = func(info *InstanceInfo, is *InstanceState, id *InstanceDiff) (*InstanceState, error) {
if id.Destroy {
return destroyFunc(is)
} else {
return createFunc(is)
}
}
if _, err := ctx.Plan(nil); err != nil {
t.Fatalf("err: %s", err)
}
// Destroy is broken, so even though CBD successfully replaces the instance,
// we'll have to save the Deposed instance to destroy later
state, err := ctx.Apply()
if err == nil {
t.Fatal("should have error")
}
checkStateString(t, state, `
aws_instance.web: (1 deposed)
ID = bar
Deposed ID 1 = foo
`)
createdInstanceId = "baz"
ctx = testContext2(t, &ContextOpts{
Module: m,
Providers: ps,
State: state,
})
if _, err := ctx.Plan(nil); err != nil {
t.Fatalf("err: %s", err)
}
// We're replacing the primary instance once again. Destroy is _still_
// broken, so the Deposed list gets longer
state, err = ctx.Apply()
if err == nil {
t.Fatal("should have error")
}
checkStateString(t, state, `
aws_instance.web: (2 deposed)
ID = baz
Deposed ID 1 = foo
Deposed ID 2 = bar
`)
// Destroy partially fixed!
destroyFunc = func(is *InstanceState) (*InstanceState, error) {
if is.ID == "foo" || is.ID == "baz" {
return nil, nil
} else {
return is, fmt.Errorf("destroy partially failed")
}
}
createdInstanceId = "qux"
if _, err := ctx.Plan(nil); err != nil {
t.Fatalf("err: %s", err)
}
state, err = ctx.Apply()
// Expect error because 1/2 of Deposed destroys failed
if err == nil {
t.Fatal("should have error")
}
// foo and baz are now gone, bar sticks around
checkStateString(t, state, `
aws_instance.web: (1 deposed)
ID = qux
Deposed ID 1 = bar
`)
// Destroy working fully!
destroyFunc = func(is *InstanceState) (*InstanceState, error) {
return nil, nil
}
createdInstanceId = "quux"
if _, err := ctx.Plan(nil); err != nil {
t.Fatalf("err: %s", err)
}
state, err = ctx.Apply()
if err != nil {
t.Fatal("should not have error:", err)
}
// And finally the state is clean
checkStateString(t, state, `
aws_instance.web:
ID = quux
`)
}
func TestContext2Apply_provisionerResourceRef(t *testing.T) { func TestContext2Apply_provisionerResourceRef(t *testing.T) {
m := testModule(t, "apply-provisioner-resource-ref") m := testModule(t, "apply-provisioner-resource-ref")
p := testProvider("aws") p := testProvider("aws")
@ -5343,6 +5473,15 @@ func testProvisioner() *MockResourceProvisioner {
return p return p
} }
func checkStateString(t *testing.T, state *State, expected string) {
actual := strings.TrimSpace(state.String())
expected = strings.TrimSpace(expected)
if actual != expected {
t.Fatalf("state does not match! actual:\n%s\n\nexpected:\n%s", actual, expected)
}
}
const testContextGraph = ` const testContextGraph = `
root: root root: root
aws_instance.bar aws_instance.bar

View File

@ -12,5 +12,9 @@ type EvalReturnError struct {
} }
func (n *EvalReturnError) Eval(ctx EvalContext) (interface{}, error) { func (n *EvalReturnError) Eval(ctx EvalContext) (interface{}, error) {
if n.Error == nil {
return nil, nil
}
return nil, *n.Error return nil, *n.Error
} }

View File

@ -25,13 +25,13 @@ type EvalReadStateTainted struct {
// Tainted is a per-resource list, this index determines which item in the // Tainted is a per-resource list, this index determines which item in the
// list we are addressing // list we are addressing
TaintedIndex int Index int
} }
func (n *EvalReadStateTainted) Eval(ctx EvalContext) (interface{}, error) { func (n *EvalReadStateTainted) Eval(ctx EvalContext) (interface{}, error) {
return readInstanceFromState(ctx, n.Name, n.Output, func(rs *ResourceState) (*InstanceState, error) { return readInstanceFromState(ctx, n.Name, n.Output, func(rs *ResourceState) (*InstanceState, error) {
// Get the index. If it is negative, then we get the last one // Get the index. If it is negative, then we get the last one
idx := n.TaintedIndex idx := n.Index
if idx < 0 { if idx < 0 {
idx = len(rs.Tainted) - 1 idx = len(rs.Tainted) - 1
} }
@ -48,11 +48,21 @@ func (n *EvalReadStateTainted) Eval(ctx EvalContext) (interface{}, error) {
type EvalReadStateDeposed struct { type EvalReadStateDeposed struct {
Name string Name string
Output **InstanceState Output **InstanceState
Index int
} }
func (n *EvalReadStateDeposed) Eval(ctx EvalContext) (interface{}, error) { func (n *EvalReadStateDeposed) Eval(ctx EvalContext) (interface{}, error) {
return readInstanceFromState(ctx, n.Name, n.Output, func(rs *ResourceState) (*InstanceState, error) { return readInstanceFromState(ctx, n.Name, n.Output, func(rs *ResourceState) (*InstanceState, error) {
return rs.Deposed, nil // Get the index. If it is negative, then we get the last one
idx := n.Index
if idx < 0 {
idx = len(rs.Deposed) - 1
}
if idx >= 0 && idx < len(rs.Deposed) {
return rs.Deposed[idx], nil
} else {
return nil, fmt.Errorf("bad deposed index: %d, for resource: %#v", idx, rs)
}
}) })
} }
@ -183,7 +193,7 @@ type EvalWriteStateTainted struct {
ResourceType string ResourceType string
Dependencies []string Dependencies []string
State **InstanceState State **InstanceState
TaintedIndex int Index int
} }
func (n *EvalWriteStateTainted) Eval(ctx EvalContext) (interface{}, error) { func (n *EvalWriteStateTainted) Eval(ctx EvalContext) (interface{}, error) {
@ -212,10 +222,10 @@ func (n *EvalWriteStateTainted) Eval(ctx EvalContext) (interface{}, error) {
rs.Type = n.ResourceType rs.Type = n.ResourceType
rs.Dependencies = n.Dependencies rs.Dependencies = n.Dependencies
if n.TaintedIndex == -1 { if n.Index == -1 {
rs.Tainted = append(rs.Tainted, *n.State) rs.Tainted = append(rs.Tainted, *n.State)
} else { } else {
rs.Tainted[n.TaintedIndex] = *n.State rs.Tainted[n.Index] = *n.State
} }
return nil, nil return nil, nil
@ -226,6 +236,7 @@ type EvalWriteStateDeposed struct {
ResourceType string ResourceType string
Dependencies []string Dependencies []string
State **InstanceState State **InstanceState
Index int
} }
func (n *EvalWriteStateDeposed) Eval(ctx EvalContext) (interface{}, error) { func (n *EvalWriteStateDeposed) Eval(ctx EvalContext) (interface{}, error) {
@ -254,7 +265,11 @@ func (n *EvalWriteStateDeposed) Eval(ctx EvalContext) (interface{}, error) {
rs.Type = n.ResourceType rs.Type = n.ResourceType
rs.Dependencies = n.Dependencies rs.Dependencies = n.Dependencies
rs.Deposed = *n.State if n.Index == -1 {
rs.Deposed = append(rs.Deposed, *n.State)
} else {
rs.Deposed[n.Index] = *n.State
}
return nil, nil return nil, nil
} }
@ -324,7 +339,7 @@ func (n *EvalDeposeState) Eval(ctx EvalContext) (interface{}, error) {
} }
// Depose // Depose
rs.Deposed = rs.Primary rs.Deposed = append(rs.Deposed, rs.Primary)
rs.Primary = nil rs.Primary = nil
return nil, nil return nil, nil
@ -357,13 +372,13 @@ func (n *EvalUndeposeState) Eval(ctx EvalContext) (interface{}, error) {
} }
// If we don't have any desposed resource, then we don't have anything to do // If we don't have any desposed resource, then we don't have anything to do
if rs.Deposed == nil { if len(rs.Deposed) == 0 {
return nil, nil return nil, nil
} }
// Undepose // Undepose
rs.Primary = rs.Deposed rs.Primary = rs.Deposed[len(rs.Deposed)-1]
rs.Deposed = nil rs.Deposed = rs.Deposed[:len(rs.Deposed)-1]
return nil, nil return nil, nil
} }

View File

@ -99,19 +99,22 @@ func TestEvalReadState(t *testing.T) {
Node: &EvalReadStateTainted{ Node: &EvalReadStateTainted{
Name: "aws_instance.bar", Name: "aws_instance.bar",
Output: &output, Output: &output,
TaintedIndex: 0, Index: 0,
}, },
ExpectedInstanceId: "i-abc123", ExpectedInstanceId: "i-abc123",
}, },
"ReadStateDeposed gets deposed instance": { "ReadStateDeposed gets deposed instance": {
Resources: map[string]*ResourceState{ Resources: map[string]*ResourceState{
"aws_instance.bar": &ResourceState{ "aws_instance.bar": &ResourceState{
Deposed: &InstanceState{ID: "i-abc123"}, Deposed: []*InstanceState{
&InstanceState{ID: "i-abc123"},
},
}, },
}, },
Node: &EvalReadStateDeposed{ Node: &EvalReadStateDeposed{
Name: "aws_instance.bar", Name: "aws_instance.bar",
Output: &output, Output: &output,
Index: 0,
}, },
ExpectedInstanceId: "i-abc123", ExpectedInstanceId: "i-abc123",
}, },
@ -187,7 +190,7 @@ func TestEvalWriteStateTainted(t *testing.T) {
Name: "restype.resname", Name: "restype.resname",
ResourceType: "restype", ResourceType: "restype",
State: &is, State: &is,
TaintedIndex: -1, Index: -1,
} }
_, err := node.Eval(ctx) _, err := node.Eval(ctx)
if err != nil { if err != nil {
@ -195,7 +198,7 @@ func TestEvalWriteStateTainted(t *testing.T) {
} }
rs := state.ModuleByPath(ctx.Path()).Resources["restype.resname"] rs := state.ModuleByPath(ctx.Path()).Resources["restype.resname"]
if len(rs.Tainted) == 1 && rs.Tainted[0].ID != "i-abc123" { if len(rs.Tainted) != 1 || rs.Tainted[0].ID != "i-abc123" {
t.Fatalf("expected tainted instance to have ID 'i-abc123': %#v", rs) t.Fatalf("expected tainted instance to have ID 'i-abc123': %#v", rs)
} }
} }
@ -212,6 +215,7 @@ func TestEvalWriteStateDeposed(t *testing.T) {
Name: "restype.resname", Name: "restype.resname",
ResourceType: "restype", ResourceType: "restype",
State: &is, State: &is,
Index: -1,
} }
_, err := node.Eval(ctx) _, err := node.Eval(ctx)
if err != nil { if err != nil {
@ -219,7 +223,7 @@ func TestEvalWriteStateDeposed(t *testing.T) {
} }
rs := state.ModuleByPath(ctx.Path()).Resources["restype.resname"] rs := state.ModuleByPath(ctx.Path()).Resources["restype.resname"]
if rs.Deposed == nil || rs.Deposed.ID != "i-abc123" { if len(rs.Deposed) != 1 || rs.Deposed[0].ID != "i-abc123" {
t.Fatalf("expected deposed instance to have ID 'i-abc123': %#v", rs) t.Fatalf("expected deposed instance to have ID 'i-abc123': %#v", rs)
} }
} }

View File

@ -498,7 +498,7 @@ func (m *ModuleState) prune() {
for k, v := range m.Resources { for k, v := range m.Resources {
v.prune() v.prune()
if (v.Primary == nil || v.Primary.ID == "") && len(v.Tainted) == 0 { if (v.Primary == nil || v.Primary.ID == "") && len(v.Tainted) == 0 && len(v.Deposed) == 0 {
delete(m.Resources, k) delete(m.Resources, k)
} }
} }
@ -549,8 +549,8 @@ func (m *ModuleState) String() string {
} }
deposedStr := "" deposedStr := ""
if rs.Deposed != nil { if len(rs.Deposed) > 0 {
deposedStr = fmt.Sprintf(" (1 deposed)") deposedStr = fmt.Sprintf(" (%d deposed)", len(rs.Deposed))
} }
buf.WriteString(fmt.Sprintf("%s:%s%s\n", k, taintStr, deposedStr)) buf.WriteString(fmt.Sprintf("%s:%s%s\n", k, taintStr, deposedStr))
@ -579,8 +579,8 @@ func (m *ModuleState) String() string {
buf.WriteString(fmt.Sprintf(" Tainted ID %d = %s\n", idx+1, t.ID)) buf.WriteString(fmt.Sprintf(" Tainted ID %d = %s\n", idx+1, t.ID))
} }
if rs.Deposed != nil { for idx, t := range rs.Deposed {
buf.WriteString(fmt.Sprintf(" Deposed ID = %s\n", rs.Deposed.ID)) buf.WriteString(fmt.Sprintf(" Deposed ID %d = %s\n", idx+1, t.ID))
} }
if len(rs.Dependencies) > 0 { if len(rs.Dependencies) > 0 {
@ -660,7 +660,7 @@ type ResourceState struct {
// Deposed instance is cleaned up. If there were problems creating the // Deposed instance is cleaned up. If there were problems creating the
// replacement, we mark the replacement as Tainted and Undepose the former // replacement, we mark the replacement as Tainted and Undepose the former
// Primary. // Primary.
Deposed *InstanceState `json:"deposed,omitempty"` Deposed []*InstanceState `json:"deposed,omitempty"`
} }
// Equal tests whether two ResourceStates are equal. // Equal tests whether two ResourceStates are equal.
@ -762,7 +762,10 @@ func (r *ResourceState) deepcopy() *ResourceState {
} }
} }
if r.Deposed != nil { if r.Deposed != nil {
n.Deposed = r.Deposed.deepcopy() n.Deposed = make([]*InstanceState, 0, len(r.Deposed))
for _, inst := range r.Deposed {
n.Deposed = append(n.Deposed, inst.deepcopy())
}
} }
return n return n
@ -783,9 +786,18 @@ func (r *ResourceState) prune() {
r.Tainted = r.Tainted[:n] r.Tainted = r.Tainted[:n]
if r.Deposed != nil && r.Deposed.ID == "" { n = len(r.Deposed)
r.Deposed = nil for i := 0; i < n; i++ {
inst := r.Deposed[i]
if inst == nil || inst.ID == "" {
copy(r.Deposed[i:], r.Deposed[i+1:])
r.Deposed[n-1] = nil
n--
i--
} }
}
r.Deposed = r.Deposed[:n]
} }
func (r *ResourceState) sort() { func (r *ResourceState) sort() {

View File

@ -443,9 +443,9 @@ aws_instance.bar:
` `
const testTerraformApplyErrorDestroyCreateBeforeDestroyStr = ` const testTerraformApplyErrorDestroyCreateBeforeDestroyStr = `
aws_instance.bar: (1 tainted) aws_instance.bar: (1 deposed)
ID = foo ID = foo
Tainted ID 1 = bar Deposed ID 1 = bar
` `
const testTerraformApplyErrorPartialStr = ` const testTerraformApplyErrorPartialStr = `

View File

@ -0,0 +1,8 @@
resource "aws_instance" "web" {
// require_new is a special attribute recognized by testDiffFn that forces
// a new resource on every apply
require_new = "yes"
lifecycle {
create_before_destroy = true
}
}

View File

@ -29,27 +29,33 @@ func (t *DeposedTransformer) Transform(g *Graph) error {
// Go through all the resources in our state to look for deposed resources // Go through all the resources in our state to look for deposed resources
for k, rs := range state.Resources { for k, rs := range state.Resources {
if rs.Deposed == nil { // If we have no deposed resources, then move on
if len(rs.Deposed) == 0 {
continue continue
} }
deposed := rs.Deposed
for i, _ := range deposed {
g.Add(&graphNodeDeposedResource{ g.Add(&graphNodeDeposedResource{
Index: i,
ResourceName: k, ResourceName: k,
ResourceType: rs.Type, ResourceType: rs.Type,
}) })
} }
}
return nil return nil
} }
// graphNodeDeposedResource is the graph vertex representing a deposed resource. // graphNodeDeposedResource is the graph vertex representing a deposed resource.
type graphNodeDeposedResource struct { type graphNodeDeposedResource struct {
Index int
ResourceName string ResourceName string
ResourceType string ResourceType string
} }
func (n *graphNodeDeposedResource) Name() string { func (n *graphNodeDeposedResource) Name() string {
return fmt.Sprintf("%s (deposed)", n.ResourceName) return fmt.Sprintf("%s (deposed #%d)", n.ResourceName, n.Index)
} }
func (n *graphNodeDeposedResource) ProvidedBy() []string { func (n *graphNodeDeposedResource) ProvidedBy() []string {
@ -79,6 +85,7 @@ func (n *graphNodeDeposedResource) EvalTree() EvalNode {
&EvalReadStateDeposed{ &EvalReadStateDeposed{
Name: n.ResourceName, Name: n.ResourceName,
Output: &state, Output: &state,
Index: n.Index,
}, },
&EvalRefresh{ &EvalRefresh{
Info: info, Info: info,
@ -90,6 +97,7 @@ func (n *graphNodeDeposedResource) EvalTree() EvalNode {
Name: n.ResourceName, Name: n.ResourceName,
ResourceType: n.ResourceType, ResourceType: n.ResourceType,
State: &state, State: &state,
Index: n.Index,
}, },
}, },
}, },
@ -98,7 +106,6 @@ func (n *graphNodeDeposedResource) EvalTree() EvalNode {
// Apply // Apply
var diff *InstanceDiff var diff *InstanceDiff
var err error var err error
var emptyState *InstanceState
seq.Nodes = append(seq.Nodes, &EvalOpFilter{ seq.Nodes = append(seq.Nodes, &EvalOpFilter{
Ops: []walkOperation{walkApply}, Ops: []walkOperation{walkApply},
Node: &EvalSequence{ Node: &EvalSequence{
@ -110,6 +117,7 @@ func (n *graphNodeDeposedResource) EvalTree() EvalNode {
&EvalReadStateDeposed{ &EvalReadStateDeposed{
Name: n.ResourceName, Name: n.ResourceName,
Output: &state, Output: &state,
Index: n.Index,
}, },
&EvalDiffDestroy{ &EvalDiffDestroy{
Info: info, Info: info,
@ -124,20 +132,14 @@ func (n *graphNodeDeposedResource) EvalTree() EvalNode {
Output: &state, Output: &state,
Error: &err, Error: &err,
}, },
// Always write the resource back to the state tainted... if it // Always write the resource back to the state deposed... if it
// successfully destroyed it will be pruned. If it did not, it will // was successfully destroyed it will be pruned. If it was not, it will
// remain tainted. // be caught on the next run.
&EvalWriteStateTainted{
Name: n.ResourceName,
ResourceType: n.ResourceType,
State: &state,
TaintedIndex: -1,
},
// Then clear the deposed state.
&EvalWriteStateDeposed{ &EvalWriteStateDeposed{
Name: n.ResourceName, Name: n.ResourceName,
ResourceType: n.ResourceType, ResourceType: n.ResourceType,
State: &emptyState, State: &state,
Index: n.Index,
}, },
&EvalReturnError{ &EvalReturnError{
Error: &err, Error: &err,

View File

@ -406,7 +406,7 @@ func (n *graphNodeExpandedResource) EvalTree() EvalNode {
ResourceType: n.Resource.Type, ResourceType: n.Resource.Type,
Dependencies: n.DependentOn(), Dependencies: n.DependentOn(),
State: &state, State: &state,
TaintedIndex: -1, Index: -1,
}, },
&EvalIf{ &EvalIf{
If: func(ctx EvalContext) (bool, error) { If: func(ctx EvalContext) (bool, error) {
@ -515,7 +515,7 @@ func (n *graphNodeExpandedResourceDestroy) EvalTree() EvalNode {
Then: &EvalReadStateTainted{ Then: &EvalReadStateTainted{
Name: n.stateId(), Name: n.stateId(),
Output: &state, Output: &state,
TaintedIndex: -1, Index: -1,
}, },
Else: &EvalReadState{ Else: &EvalReadState{
Name: n.stateId(), Name: n.stateId(),

View File

@ -89,7 +89,7 @@ func (n *graphNodeTaintedResource) EvalTree() EvalNode {
}, },
&EvalReadStateTainted{ &EvalReadStateTainted{
Name: n.ResourceName, Name: n.ResourceName,
TaintedIndex: n.Index, Index: n.Index,
Output: &state, Output: &state,
}, },
&EvalRefresh{ &EvalRefresh{
@ -102,7 +102,7 @@ func (n *graphNodeTaintedResource) EvalTree() EvalNode {
Name: n.ResourceName, Name: n.ResourceName,
ResourceType: n.ResourceType, ResourceType: n.ResourceType,
State: &state, State: &state,
TaintedIndex: n.Index, Index: n.Index,
}, },
}, },
}, },
@ -120,7 +120,7 @@ func (n *graphNodeTaintedResource) EvalTree() EvalNode {
}, },
&EvalReadStateTainted{ &EvalReadStateTainted{
Name: n.ResourceName, Name: n.ResourceName,
TaintedIndex: n.Index, Index: n.Index,
Output: &state, Output: &state,
}, },
&EvalDiffDestroy{ &EvalDiffDestroy{
@ -139,7 +139,7 @@ func (n *graphNodeTaintedResource) EvalTree() EvalNode {
Name: n.ResourceName, Name: n.ResourceName,
ResourceType: n.ResourceType, ResourceType: n.ResourceType,
State: &state, State: &state,
TaintedIndex: n.Index, Index: n.Index,
}, },
&EvalUpdateStateHook{}, &EvalUpdateStateHook{},
}, },