Merge pull request #9527 from hashicorp/f-destroy-builder2

terraform: destroy graph builder based on state
This commit is contained in:
Mitchell Hashimoto 2016-10-26 12:53:20 -04:00 committed by GitHub
commit aed23a0a31
24 changed files with 386 additions and 47 deletions

View File

@ -115,6 +115,26 @@ func (c *ApplyCommand) Run(args []string) int {
}
}
// Check for the new destroy
if terraform.X_newDestroy {
desc := "Experimental new destroy graph has been enabled. This may still\n" +
"have bugs, and should be used with care. If you'd like to continue,\n" +
"you must enter exactly 'yes' as a response."
v, err := c.UIInput().Input(&terraform.InputOpts{
Id: "Xnew-destroy",
Query: "Experimental feature enabled: new destroy graph. Continue?",
Description: desc,
})
if err != nil {
c.Ui.Error(fmt.Sprintf("Error asking for confirmation: %s", err))
return 1
}
if v != "yes" {
c.Ui.Output("Apply cancelled.")
return 1
}
}
// Build the context based on the arguments given
ctx, planned, err := c.Context(contextOpts{
Destroy: c.Destroy,

View File

@ -337,6 +337,7 @@ func (m *Meta) flagSet(n string) *flag.FlagSet {
// Experimental features
f.BoolVar(&terraform.X_newApply, "Xnew-apply", false, "experiment: new apply")
f.BoolVar(&terraform.X_newDestroy, "Xnew-destroy", false, "experiment: new destroy")
// Create an io.Writer that writes to our Ui properly for errors.
// This is kind of a hack, but it does the job. Basically: create

2
go.sh
View File

@ -1 +1 @@
go test ./terraform | grep -E '(FAIL|panic)' | tee /dev/tty | wc -l
go test ./terraform -Xnew-apply -Xnew-destroy | grep -E '(FAIL|panic)' | tee /dev/tty | wc -l

View File

@ -21,6 +21,10 @@ var (
// X_newApply will enable the new apply graph. This will be removed
// and be on by default in 0.8.0.
X_newApply = false
// X_newDestroy will enable the new destroy graph. This will be removed
// and be on by default in 0.8.0.
X_newDestroy = false
)
// InputMode defines what sort of input will be asked for when Input
@ -371,6 +375,8 @@ func (c *Context) Apply() (*State, error) {
// Copy our own state
c.state = c.state.DeepCopy()
newGraphEnabled := (c.destroy && X_newDestroy) || (!c.destroy && X_newApply)
// Build the original graph. This is before the new graph builders
// coming in 0.8. We do this for shadow graphing.
oldGraph, err := c.Graph(&ContextGraphOpts{Validate: true})
@ -392,12 +398,13 @@ func (c *Context) Apply() (*State, error) {
State: c.state,
Providers: c.components.ResourceProviders(),
Provisioners: c.components.ResourceProvisioners(),
Destroy: c.destroy,
}).Build(RootModulePath)
if err != nil && !X_newApply {
if err != nil && !newGraphEnabled {
// If we had an error graphing but we're not using this graph, just
// set it to nil and record it as a shadow error.
c.shadowErr = multierror.Append(c.shadowErr, fmt.Errorf(
"Error building new apply graph: %s", err))
"Error building new graph: %s", err))
newGraph = nil
err = nil
@ -418,16 +425,11 @@ func (c *Context) Apply() (*State, error) {
//
real := oldGraph
shadow := newGraph
if c.destroy {
log.Printf("[WARN] terraform: real graph is original, shadow is nil")
shadow = nil
if newGraphEnabled {
log.Printf("[WARN] terraform: real graph is experiment, shadow is experiment")
real = shadow
} else {
if X_newApply {
log.Printf("[WARN] terraform: real graph is Xnew-apply, shadow is Xnew-apply")
real = shadow
} else {
log.Printf("[WARN] terraform: real graph is original, shadow is Xnew-apply")
}
log.Printf("[WARN] terraform: real graph is original, shadow is experiment")
}
// For now, always shadow with the real graph for verification. We don't
@ -505,8 +507,20 @@ func (c *Context) Plan() (*Plan, error) {
c.diff.init()
c.diffLock.Unlock()
// Build the graph
graph, err := c.Graph(&ContextGraphOpts{Validate: true})
// Build the graph. We have a branch here since for the pure-destroy
// plan (c.destroy) we use a much simpler graph builder that simply
// walks the state and reverses edges.
var graph *Graph
var err error
if c.destroy && X_newDestroy {
graph, err = (&DestroyPlanGraphBuilder{
Module: c.module,
State: c.state,
Targets: c.targets,
}).Build(RootModulePath)
} else {
graph, err = c.Graph(&ContextGraphOpts{Validate: true})
}
if err != nil {
return nil, err
}
@ -529,11 +543,16 @@ func (c *Context) Plan() (*Plan, error) {
p.Diff.DeepCopy()
}
// Now that we have a diff, we can build the exact graph that Apply will use
// and catch any possible cycles during the Plan phase.
if _, err := c.Graph(&ContextGraphOpts{Validate: true}); err != nil {
return nil, err
// We don't do the reverification during the new destroy plan because
// it will use a different apply process.
if !(c.destroy && X_newDestroy) {
// Now that we have a diff, we can build the exact graph that Apply will use
// and catch any possible cycles during the Plan phase.
if _, err := c.Graph(&ContextGraphOpts{Validate: true}); err != nil {
return nil, err
}
}
var errs error
if len(walker.ValidationErrors) > 0 {
errs = multierror.Append(errs, walker.ValidationErrors...)

View File

@ -889,7 +889,7 @@ func getContextForApply_destroyCrossProviders(
},
},
&ModuleState{
Path: []string{"root", "example"},
Path: []string{"root", "child"},
Resources: map[string]*ResourceState{
"aws_vpc.bar": &ResourceState{
Type: "aws_vpc",

View File

@ -1582,7 +1582,7 @@ func TestContext2Plan_moduleDestroy(t *testing.T) {
actual := strings.TrimSpace(plan.String())
expected := strings.TrimSpace(testTerraformPlanModuleDestroyStr)
if actual != expected {
t.Fatalf("bad:\n%s", actual)
t.Fatalf("bad:\n%s\n\nexpected:\n\n%s", actual, expected)
}
}
@ -1634,7 +1634,7 @@ func TestContext2Plan_moduleDestroyCycle(t *testing.T) {
actual := strings.TrimSpace(plan.String())
expected := strings.TrimSpace(testTerraformPlanModuleDestroyCycleStr)
if actual != expected {
t.Fatalf("bad:\n%s", actual)
t.Fatalf("bad:\n%s\n\nexpected:\n\n%s", actual, expected)
}
}
@ -1684,7 +1684,7 @@ func TestContext2Plan_moduleDestroyMultivar(t *testing.T) {
actual := strings.TrimSpace(plan.String())
expected := strings.TrimSpace(testTerraformPlanModuleDestroyMultivarStr)
if actual != expected {
t.Fatalf("bad:\n%s", actual)
t.Fatalf("bad:\n%s\n\nexpected:\n\n%s", actual, expected)
}
}

View File

@ -100,8 +100,20 @@ func (d *Diff) Equal(d2 *Diff) bool {
sort.Sort(moduleDiffSort(d.Modules))
sort.Sort(moduleDiffSort(d2.Modules))
// Copy since we have to modify the module destroy flag to false so
// we don't compare that. TODO: delete this when we get rid of the
// destroy flag on modules.
dCopy := d.DeepCopy()
d2Copy := d2.DeepCopy()
for _, m := range dCopy.Modules {
m.Destroy = false
}
for _, m := range d2Copy.Modules {
m.Destroy = false
}
// Use DeepEqual
return reflect.DeepEqual(d, d2)
return reflect.DeepEqual(dCopy, d2Copy)
}
// DeepCopy performs a deep copy of all parts of the Diff, making the
@ -238,10 +250,6 @@ func (d *ModuleDiff) IsRoot() bool {
func (d *ModuleDiff) String() string {
var buf bytes.Buffer
if d.Destroy {
buf.WriteString("DESTROY MODULE\n")
}
names := make([]string, 0, len(d.Resources))
for name, _ := range d.Resources {
names = append(names, name)

View File

@ -77,6 +77,20 @@ func TestDiffEqual(t *testing.T) {
},
true,
},
"different module diff destroys": {
&Diff{
Modules: []*ModuleDiff{
&ModuleDiff{Path: []string{"root", "foo"}, Destroy: true},
},
},
&Diff{
Modules: []*ModuleDiff{
&ModuleDiff{Path: []string{"root", "foo"}, Destroy: false},
},
},
true,
},
}
for name, tc := range cases {

View File

@ -26,6 +26,10 @@ type BasicGraphBuilder struct {
func (b *BasicGraphBuilder) Build(path []string) (*Graph, error) {
g := &Graph{Path: path}
for _, step := range b.Steps {
if step == nil {
continue
}
if err := step.Transform(g); err != nil {
return g, err
}

View File

@ -30,6 +30,9 @@ type ApplyGraphBuilder struct {
// DisableReduce, if true, will not reduce the graph. Great for testing.
DisableReduce bool
// Destroy, if true, represents a pure destroy operation
Destroy bool
}
// See GraphBuilder
@ -77,7 +80,10 @@ func (b *ApplyGraphBuilder) Steps() []GraphTransformer {
// Destruction ordering
&DestroyEdgeTransformer{Module: b.Module, State: b.State},
&CBDEdgeTransformer{Module: b.Module, State: b.State},
GraphTransformIf(
func() bool { return !b.Destroy },
&CBDEdgeTransformer{Module: b.Module, State: b.State},
),
// Create all the providers
&MissingProviderTransformer{Providers: b.Providers, Factory: providerFactory},
@ -87,8 +93,13 @@ func (b *ApplyGraphBuilder) Steps() []GraphTransformer {
&AttachProviderConfigTransformer{Module: b.Module},
// Provisioner-related transformations
&MissingProvisionerTransformer{Provisioners: b.Provisioners},
&ProvisionerTransformer{},
GraphTransformIf(
func() bool { return !b.Destroy },
GraphTransformMulti(
&MissingProvisionerTransformer{Provisioners: b.Provisioners},
&ProvisionerTransformer{},
),
),
// Add root variables
&RootVariableTransformer{Module: b.Module},

View File

@ -0,0 +1,58 @@
package terraform
import (
"github.com/hashicorp/terraform/config/module"
"github.com/hashicorp/terraform/dag"
)
// DestroyPlanGraphBuilder implements GraphBuilder and is responsible for
// planning a pure-destroy.
//
// Planning a pure destroy operation is simple because we can ignore most
// ordering configuration and simply reverse the state.
type DestroyPlanGraphBuilder struct {
// Module is the root module for the graph to build.
Module *module.Tree
// State is the current state
State *State
// Targets are resources to target
Targets []string
}
// See GraphBuilder
func (b *DestroyPlanGraphBuilder) Build(path []string) (*Graph, error) {
return (&BasicGraphBuilder{
Steps: b.Steps(),
Validate: true,
}).Build(path)
}
// See GraphBuilder
func (b *DestroyPlanGraphBuilder) Steps() []GraphTransformer {
concreteResource := func(a *NodeAbstractResource) dag.Vertex {
return &NodePlanDestroyableResource{
NodeAbstractResource: a,
}
}
steps := []GraphTransformer{
// Creates all the nodes represented in the state.
&StateTransformer{
Concrete: concreteResource,
State: b.State,
},
// Target
&TargetsTransformer{Targets: b.Targets},
// Attach the configuration to any resources
&AttachResourceConfigTransformer{Module: b.Module},
// Single root
&RootTransformer{},
}
return steps
}

View File

@ -24,8 +24,6 @@ type graphNodeConfig interface {
// configuration graph need to implement in order to be be addressed / targeted
// properly.
type GraphNodeAddressable interface {
graphNodeConfig
ResourceAddress() *ResourceAddress
}
@ -35,7 +33,5 @@ type GraphNodeAddressable interface {
// provided will contain every target provided, and each implementing graph
// node must filter this list to targets considered relevant.
type GraphNodeTargetable interface {
GraphNodeAddressable
SetTargets([]ResourceAddress)
}

View File

@ -59,7 +59,7 @@ func (n *GraphNodeConfigModule) Expand(b GraphBuilder) (GraphNodeSubgraph, error
{
// Add the destroy marker to the graph
t := &ModuleDestroyTransformer{}
t := &ModuleDestroyTransformerOld{}
if err := t.Transform(graph); err != nil {
return nil, err
}

View File

@ -0,0 +1,29 @@
package terraform
import (
"fmt"
)
// NodeDestroyableModule represents a module destruction.
type NodeDestroyableModuleVariable struct {
PathValue []string
}
func (n *NodeDestroyableModuleVariable) Name() string {
result := "plan-destroy"
if len(n.PathValue) > 1 {
result = fmt.Sprintf("%s.%s", modulePrefixStr(n.PathValue), result)
}
return result
}
// GraphNodeSubPath
func (n *NodeDestroyableModuleVariable) Path() []string {
return n.PathValue
}
// GraphNodeEvalable
func (n *NodeDestroyableModuleVariable) EvalTree() EvalNode {
return &EvalDiffDestroyModule{Path: n.PathValue}
}

View File

@ -28,6 +28,8 @@ type NodeAbstractResource struct {
Config *config.Resource // Config is the resource in the config
ResourceState *ResourceState // ResourceState is the ResourceState for this
Targets []ResourceAddress // Set from GraphNodeTargetable
}
func (n *NodeAbstractResource) Name() string {
@ -111,6 +113,16 @@ func (n *NodeAbstractResource) ResourceAddr() *ResourceAddress {
return n.Addr
}
// GraphNodeAddressable, TODO: remove, used by target, should unify
func (n *NodeAbstractResource) ResourceAddress() *ResourceAddress {
return n.ResourceAddr()
}
// GraphNodeTargetable
func (n *NodeAbstractResource) SetTargets(targets []ResourceAddress) {
n.Targets = targets
}
// GraphNodeAttachResourceState
func (n *NodeAbstractResource) AttachResourceState(s *ResourceState) {
n.ResourceState = s

View File

@ -2,11 +2,13 @@ package terraform
import (
"fmt"
"github.com/hashicorp/terraform/config"
)
// NodeDestroyResource represents a resource that is to be destroyed.
type NodeDestroyResource struct {
NodeAbstractResource
*NodeAbstractResource
}
func (n *NodeDestroyResource) Name() string {
@ -63,6 +65,11 @@ func (n *NodeDestroyResource) DynamicExpand(ctx EvalContext) (*Graph, error) {
View: n.Config.Id(),
})
// Target
steps = append(steps, &TargetsTransformer{
ParsedTargets: n.Targets,
})
// Always end with the root being added
steps = append(steps, &RootTransformer{})
@ -142,11 +149,13 @@ func (n *NodeDestroyResource) EvalTree() EvalNode {
// Make sure we handle data sources properly.
&EvalIf{
If: func(ctx EvalContext) (bool, error) {
/* TODO: data source
if n.Resource.Mode == config.DataResourceMode {
if n.Addr == nil {
return false, fmt.Errorf("nil address")
}
if n.Addr.Mode == config.DataResourceMode {
return true, nil
}
*/
return false, nil
},

View File

@ -0,0 +1,55 @@
package terraform
import (
"fmt"
)
// NodePlanDestroyableResource represents a resource that is "applyable":
// it is ready to be applied and is represented by a diff.
type NodePlanDestroyableResource struct {
*NodeAbstractResource
}
// GraphNodeEvalable
func (n *NodePlanDestroyableResource) EvalTree() EvalNode {
addr := n.NodeAbstractResource.Addr
// stateId is the ID to put into the state
stateId := addr.stateId()
if addr.Index > -1 {
stateId = fmt.Sprintf("%s.%d", stateId, addr.Index)
}
// Build the instance info. More of this will be populated during eval
info := &InstanceInfo{
Id: stateId,
Type: addr.Type,
}
// Declare a bunch of variables that are used for state during
// evaluation. Most of this are written to by-address below.
var diff *InstanceDiff
var state *InstanceState
return &EvalSequence{
Nodes: []EvalNode{
&EvalReadState{
Name: stateId,
Output: &state,
},
&EvalDiffDestroy{
Info: info,
State: &state,
Output: &diff,
},
&EvalCheckPreventDestroy{
Resource: n.Config,
Diff: &diff,
},
&EvalWriteDiff{
Name: stateId,
Diff: &diff,
},
},
}
}

View File

@ -24,6 +24,7 @@ const fixtureDir = "./test-fixtures"
func TestMain(m *testing.M) {
// Experimental features
xNewApply := flag.Bool("Xnew-apply", false, "Experiment: new apply graph")
xNewDestroy := flag.Bool("Xnew-destroy", false, "Experiment: new destroy graph")
// Normal features
shadow := flag.Bool("shadow", true, "Enable shadow graph")
@ -32,6 +33,7 @@ func TestMain(m *testing.M) {
// Setup experimental features
X_newApply = *xNewApply
X_newDestroy = *xNewDestroy
if testing.Verbose() {
// if we're verbose, use the logging requested by TF_LOG
@ -1137,7 +1139,6 @@ DIFF:
DESTROY: aws_instance.foo
module.child:
DESTROY MODULE
DESTROY: aws_instance.foo
STATE:
@ -1154,10 +1155,8 @@ const testTerraformPlanModuleDestroyCycleStr = `
DIFF:
module.a_module:
DESTROY MODULE
DESTROY: aws_instance.a
module.b_module:
DESTROY MODULE
DESTROY: aws_instance.b
STATE:
@ -1174,7 +1173,6 @@ const testTerraformPlanModuleDestroyMultivarStr = `
DIFF:
module.child:
DESTROY MODULE
DESTROY: aws_instance.foo.0
DESTROY: aws_instance.foo.1

View File

@ -19,3 +19,34 @@ type GraphTransformer interface {
type GraphVertexTransformer interface {
Transform(dag.Vertex) (dag.Vertex, error)
}
// GraphTransformIf is a helper function that conditionally returns a
// GraphTransformer given. This is useful for calling inline a sequence
// of transforms without having to split it up into multiple append() calls.
func GraphTransformIf(f func() bool, then GraphTransformer) GraphTransformer {
if f() {
return then
}
return nil
}
type graphTransformerMulti struct {
Transforms []GraphTransformer
}
func (t *graphTransformerMulti) Transform(g *Graph) error {
for _, t := range t.Transforms {
if err := t.Transform(g); err != nil {
return err
}
}
return nil
}
// GraphTransformMulti combines multiple graph transformers into a single
// GraphTransformer that runs all the individual graph transformers.
func GraphTransformMulti(ts ...GraphTransformer) GraphTransformer {
return &graphTransformerMulti{Transforms: ts}
}

View File

@ -41,7 +41,9 @@ func (t *AttachResourceConfigTransformer) Transform(g *Graph) error {
// Determine what we're looking for
addr := arn.ResourceAddr()
log.Printf("[TRACE] AttachResourceConfigTransformer: Attach resource request: %s", addr)
log.Printf(
"[TRACE] AttachResourceConfigTransformer: Attach resource "+
"config request: %s", addr)
// Get the configuration.
path := normalizeModulePath(addr.Path)

View File

@ -59,7 +59,7 @@ func (t *DiffTransformer) Transform(g *Graph) error {
// If we're destroying, add the destroy node
if inst.Destroy {
abstract := NodeAbstractResource{Addr: addr}
abstract := &NodeAbstractResource{Addr: addr}
g.Add(&NodeDestroyResource{NodeAbstractResource: abstract})
}

View File

@ -9,9 +9,9 @@ import (
// ModuleDestroyTransformer is a GraphTransformer that adds a node
// to the graph that will just mark the full module for destroy in
// the destroy scenario.
type ModuleDestroyTransformer struct{}
type ModuleDestroyTransformerOld struct{}
func (t *ModuleDestroyTransformer) Transform(g *Graph) error {
func (t *ModuleDestroyTransformerOld) Transform(g *Graph) error {
// Create the node
n := &graphNodeModuleDestroy{Path: g.Path}

View File

@ -0,0 +1,65 @@
package terraform
import (
"fmt"
"log"
"github.com/hashicorp/terraform/dag"
)
// StateTransformer is a GraphTransformer that adds the elements of
// the state to the graph.
//
// This transform is used for example by the DestroyPlanGraphBuilder to ensure
// that only resources that are in the state are represented in the graph.
type StateTransformer struct {
Concrete ConcreteResourceNodeFunc
State *State
}
func (t *StateTransformer) Transform(g *Graph) error {
// If the state is nil or empty (nil is empty) then do nothing
if t.State.Empty() {
return nil
}
// Go through all the modules in the diff.
log.Printf("[TRACE] StateTransformer: starting")
var nodes []dag.Vertex
for _, ms := range t.State.Modules {
log.Printf("[TRACE] StateTransformer: Module: %v", ms.Path)
// Go through all the resources in this module.
for name, rs := range ms.Resources {
log.Printf("[TRACE] StateTransformer: Resource %q: %#v", name, rs)
// Add the resource to the graph
addr, err := parseResourceAddressInternal(name)
if err != nil {
panic(fmt.Sprintf(
"Error parsing internal name, this is a bug: %q", name))
}
// Very important: add the module path for this resource to
// the address. Remove "root" from it.
addr.Path = ms.Path[1:]
// Add the resource to the graph
abstract := &NodeAbstractResource{Addr: addr}
var node dag.Vertex = abstract
if f := t.Concrete; f != nil {
node = f(abstract)
}
nodes = append(nodes, node)
}
}
// Add all the nodes to the graph
for _, n := range nodes {
g.Add(n)
}
return nil
}

View File

@ -28,8 +28,10 @@ func (t *TargetsTransformer) Transform(g *Graph) error {
if err != nil {
return err
}
t.ParsedTargets = addrs
}
if len(t.ParsedTargets) > 0 {
targetedNodes, err := t.selectTargetedNodes(g, t.ParsedTargets)
if err != nil {
@ -50,6 +52,7 @@ func (t *TargetsTransformer) Transform(g *Graph) error {
}
}
}
return nil
}
@ -62,6 +65,7 @@ func (t *TargetsTransformer) parseTargetAddresses() ([]ResourceAddress, error) {
}
addrs[i] = *ta
}
return addrs, nil
}
@ -107,6 +111,7 @@ func (t *TargetsTransformer) selectTargetedNodes(
}
}
}
return targetedNodes, nil
}
@ -116,12 +121,14 @@ func (t *TargetsTransformer) nodeIsTarget(
if !ok {
return false
}
addr := r.ResourceAddress()
for _, targetAddr := range addrs {
if targetAddr.Equals(addr) {
return true
}
}
return false
}