provider/aws: Use mutex & retry for WAF change operations (#13656)

builtin/providers/aws/resource_aws_waf_byte_match_set.go

@@ -69,24 +69,18 @@ func resourceAwsWafByteMatchSetCreate(d *schema.ResourceData, meta interface{})
 
 	log.Printf("[INFO] Creating ByteMatchSet: %s", d.Get("name").(string))
 
-	// ChangeToken
+	wr := newWafRetryer(conn, "global")
-	var ct *waf.GetChangeTokenInput
+	out, err := wr.RetryWithToken(func(token *string) (interface{}, error) {
-
-	res, err := conn.GetChangeToken(ct)
-	if err != nil {
-		return errwrap.Wrapf("[ERROR] Error getting change token: {{err}}", err)
-	}
-
 		params := &waf.CreateByteMatchSetInput{
-		ChangeToken: res.ChangeToken,
+			ChangeToken: token,
 			Name:        aws.String(d.Get("name").(string)),
 		}
-
+		return conn.CreateByteMatchSet(params)
-	resp, err := conn.CreateByteMatchSet(params)
+	})
-
 	if err != nil {
 		return errwrap.Wrapf("[ERROR] Error creating ByteMatchSet: {{err}}", err)
 	}
+	resp := out.(*waf.CreateByteMatchSetOutput)
 
 	d.SetId(*resp.ByteMatchSet.ByteMatchSetId)
 
@@ -134,17 +128,14 @@ func resourceAwsWafByteMatchSetDelete(d *schema.ResourceData, meta interface{})
 		return errwrap.Wrapf("[ERROR] Error deleting ByteMatchSet: {{err}}", err)
 	}
 
-	var ct *waf.GetChangeTokenInput
+	wr := newWafRetryer(conn, "global")
-
+	_, err = wr.RetryWithToken(func(token *string) (interface{}, error) {
-	resp, err := conn.GetChangeToken(ct)
-
 		req := &waf.DeleteByteMatchSetInput{
-		ChangeToken:    resp.ChangeToken,
+			ChangeToken:    token,
 			ByteMatchSetId: aws.String(d.Id()),
 		}
-
+		return conn.DeleteByteMatchSet(req)
-	_, err = conn.DeleteByteMatchSet(req)
+	})
-
 	if err != nil {
 		return errwrap.Wrapf("[ERROR] Error deleting ByteMatchSet: {{err}}", err)
 	}
@@ -155,15 +146,10 @@ func resourceAwsWafByteMatchSetDelete(d *schema.ResourceData, meta interface{})
 func updateByteMatchSetResource(d *schema.ResourceData, meta interface{}, ChangeAction string) error {
 	conn := meta.(*AWSClient).wafconn
 
-	var ct *waf.GetChangeTokenInput
+	wr := newWafRetryer(conn, "global")
-
+	_, err := wr.RetryWithToken(func(token *string) (interface{}, error) {
-	resp, err := conn.GetChangeToken(ct)
-	if err != nil {
-		return errwrap.Wrapf("[ERROR] Error getting change token: {{err}}", err)
-	}
-
 		req := &waf.UpdateByteMatchSetInput{
-		ChangeToken:    resp.ChangeToken,
+			ChangeToken:    token,
 			ByteMatchSetId: aws.String(d.Id()),
 		}
 
@@ -182,7 +168,8 @@ func updateByteMatchSetResource(d *schema.ResourceData, meta interface{}, Change
 			req.Updates = append(req.Updates, ByteMatchUpdate)
 		}
 
-	_, err = conn.UpdateByteMatchSet(req)
+		return conn.UpdateByteMatchSet(req)
+	})
 	if err != nil {
 		return errwrap.Wrapf("[ERROR] Error updating ByteMatchSet: {{err}}", err)
 	}

builtin/providers/aws/resource_aws_waf_byte_match_set_test.go

@@ -96,16 +96,10 @@ func testAccCheckAWSWafByteMatchSetDisappears(v *waf.ByteMatchSet) resource.Test
 	return func(s *terraform.State) error {
 		conn := testAccProvider.Meta().(*AWSClient).wafconn
 
-		// ChangeToken
+		wr := newWafRetryer(conn, "global")
-		var ct *waf.GetChangeTokenInput
+		_, err := wr.RetryWithToken(func(token *string) (interface{}, error) {
-
-		resp, err := conn.GetChangeToken(ct)
-		if err != nil {
-			return fmt.Errorf("Error getting change token: %s", err)
-		}
-
 			req := &waf.UpdateByteMatchSetInput{
-			ChangeToken:    resp.ChangeToken,
+				ChangeToken:    token,
 				ByteMatchSetId: v.ByteMatchSetId,
 			}
 
@@ -122,23 +116,23 @@ func testAccCheckAWSWafByteMatchSetDisappears(v *waf.ByteMatchSet) resource.Test
 				req.Updates = append(req.Updates, ByteMatchUpdate)
 			}
 
-		_, err = conn.UpdateByteMatchSet(req)
+			return conn.UpdateByteMatchSet(req)
+		})
 		if err != nil {
 			return errwrap.Wrapf("[ERROR] Error updating ByteMatchSet: {{err}}", err)
 		}
 
-		resp, err = conn.GetChangeToken(ct)
+		_, err = wr.RetryWithToken(func(token *string) (interface{}, error) {
-		if err != nil {
-			return errwrap.Wrapf("[ERROR] Error getting change token: {{err}}", err)
-		}
-
 			opts := &waf.DeleteByteMatchSetInput{
-			ChangeToken:    resp.ChangeToken,
+				ChangeToken:    token,
 				ByteMatchSetId: v.ByteMatchSetId,
 			}
-		if _, err := conn.DeleteByteMatchSet(opts); err != nil {
+			return conn.DeleteByteMatchSet(opts)
-			return err
+		})
+		if err != nil {
+			return errwrap.Wrapf("[ERROR] Error deleting ByteMatchSet: {{err}}", err)
 		}
+
 		return nil
 	}
 }

builtin/providers/aws/resource_aws_waf_ipset.go

@@ -46,23 +46,18 @@ func resourceAwsWafIPSet() *schema.Resource {
 func resourceAwsWafIPSetCreate(d *schema.ResourceData, meta interface{}) error {
 	conn := meta.(*AWSClient).wafconn
 
-	// ChangeToken
+	wr := newWafRetryer(conn, "global")
-	var ct *waf.GetChangeTokenInput
+	out, err := wr.RetryWithToken(func(token *string) (interface{}, error) {
-
-	res, err := conn.GetChangeToken(ct)
-	if err != nil {
-		return fmt.Errorf("Error getting change token: %s", err)
-	}
-
 		params := &waf.CreateIPSetInput{
-		ChangeToken: res.ChangeToken,
+			ChangeToken: token,
 			Name:        aws.String(d.Get("name").(string)),
 		}
-
+		return conn.CreateIPSet(params)
-	resp, err := conn.CreateIPSet(params)
+	})
 	if err != nil {
 		return err
 	}
+	resp := out.(*waf.CreateIPSetOutput)
 	d.SetId(*resp.IPSet.IPSetId)
 	return resourceAwsWafIPSetUpdate(d, meta)
 }
@@ -117,18 +112,15 @@ func resourceAwsWafIPSetDelete(d *schema.ResourceData, meta interface{}) error {
 		return fmt.Errorf("Error Removing IPSetDescriptors: %s", err)
 	}
 
-	// ChangeToken
+	wr := newWafRetryer(conn, "global")
-	var ct *waf.GetChangeTokenInput
+	_, err = wr.RetryWithToken(func(token *string) (interface{}, error) {
-
-	resp, err := conn.GetChangeToken(ct)
-
 		req := &waf.DeleteIPSetInput{
-		ChangeToken: resp.ChangeToken,
+			ChangeToken: token,
 			IPSetId:     aws.String(d.Id()),
 		}
 		log.Printf("[INFO] Deleting WAF IPSet")
-	_, err = conn.DeleteIPSet(req)
+		return conn.DeleteIPSet(req)
-
+	})
 	if err != nil {
 		return fmt.Errorf("Error Deleting WAF IPSet: %s", err)
 	}
@@ -139,16 +131,10 @@ func resourceAwsWafIPSetDelete(d *schema.ResourceData, meta interface{}) error {
 func updateIPSetResource(d *schema.ResourceData, meta interface{}, ChangeAction string) error {
 	conn := meta.(*AWSClient).wafconn
 
-	// ChangeToken
+	wr := newWafRetryer(conn, "global")
-	var ct *waf.GetChangeTokenInput
+	_, err := wr.RetryWithToken(func(token *string) (interface{}, error) {
-
-	resp, err := conn.GetChangeToken(ct)
-	if err != nil {
-		return fmt.Errorf("Error getting change token: %s", err)
-	}
-
 		req := &waf.UpdateIPSetInput{
-		ChangeToken: resp.ChangeToken,
+			ChangeToken: token,
 			IPSetId:     aws.String(d.Id()),
 		}
 
@@ -165,7 +151,8 @@ func updateIPSetResource(d *schema.ResourceData, meta interface{}, ChangeAction
 			req.Updates = append(req.Updates, IPSetUpdate)
 		}
 
-	_, err = conn.UpdateIPSet(req)
+		return conn.UpdateIPSet(req)
+	})
 	if err != nil {
 		return fmt.Errorf("Error Updating WAF IPSet: %s", err)
 	}

builtin/providers/aws/resource_aws_waf_ipset_test.go

@@ -100,16 +100,10 @@ func testAccCheckAWSWafIPSetDisappears(v *waf.IPSet) resource.TestCheckFunc {
 	return func(s *terraform.State) error {
 		conn := testAccProvider.Meta().(*AWSClient).wafconn
 
-		// ChangeToken
+		wr := newWafRetryer(conn, "global")
-		var ct *waf.GetChangeTokenInput
+		_, err := wr.RetryWithToken(func(token *string) (interface{}, error) {
-
-		resp, err := conn.GetChangeToken(ct)
-		if err != nil {
-			return fmt.Errorf("Error getting change token: %s", err)
-		}
-
 			req := &waf.UpdateIPSetInput{
-			ChangeToken: resp.ChangeToken,
+				ChangeToken: token,
 				IPSetId:     v.IPSetId,
 			}
 
@@ -124,22 +118,21 @@ func testAccCheckAWSWafIPSetDisappears(v *waf.IPSet) resource.TestCheckFunc {
 				req.Updates = append(req.Updates, IPSetUpdate)
 			}
 
-		_, err = conn.UpdateIPSet(req)
+			return conn.UpdateIPSet(req)
+		})
 		if err != nil {
 			return fmt.Errorf("Error Updating WAF IPSet: %s", err)
 		}
 
-		resp, err = conn.GetChangeToken(ct)
+		_, err = wr.RetryWithToken(func(token *string) (interface{}, error) {
-		if err != nil {
-			return fmt.Errorf("Error getting change token for waf IPSet: %s", err)
-		}
-
 			opts := &waf.DeleteIPSetInput{
-			ChangeToken: resp.ChangeToken,
+				ChangeToken: token,
 				IPSetId:     v.IPSetId,
 			}
-		if _, err := conn.DeleteIPSet(opts); err != nil {
+			return conn.DeleteIPSet(opts)
-			return err
+		})
+		if err != nil {
+			return fmt.Errorf("Error Deleting WAF IPSet: %s", err)
 		}
 		return nil
 	}

builtin/providers/aws/resource_aws_waf_rule.go

@@ -71,24 +71,20 @@ func resourceAwsWafRule() *schema.Resource {
 func resourceAwsWafRuleCreate(d *schema.ResourceData, meta interface{}) error {
 	conn := meta.(*AWSClient).wafconn
 
-	// ChangeToken
+	wr := newWafRetryer(conn, "global")
-	var ct *waf.GetChangeTokenInput
+	out, err := wr.RetryWithToken(func(token *string) (interface{}, error) {
-
-	res, err := conn.GetChangeToken(ct)
-	if err != nil {
-		return fmt.Errorf("Error getting change token: %s", err)
-	}
-
 		params := &waf.CreateRuleInput{
-		ChangeToken: res.ChangeToken,
+			ChangeToken: token,
 			MetricName:  aws.String(d.Get("metric_name").(string)),
 			Name:        aws.String(d.Get("name").(string)),
 		}
 
-	resp, err := conn.CreateRule(params)
+		return conn.CreateRule(params)
+	})
 	if err != nil {
 		return err
 	}
+	resp := out.(*waf.CreateRuleOutput)
 	d.SetId(*resp.Rule.RuleId)
 	return resourceAwsWafRuleUpdate(d, meta)
 }
@@ -143,18 +139,16 @@ func resourceAwsWafRuleDelete(d *schema.ResourceData, meta interface{}) error {
 	if err != nil {
 		return fmt.Errorf("Error Removing WAF Rule Predicates: %s", err)
 	}
-	// ChangeToken
-	var ct *waf.GetChangeTokenInput
-
-	resp, err := conn.GetChangeToken(ct)
 
+	wr := newWafRetryer(conn, "global")
+	_, err = wr.RetryWithToken(func(token *string) (interface{}, error) {
 		req := &waf.DeleteRuleInput{
-		ChangeToken: resp.ChangeToken,
+			ChangeToken: token,
 			RuleId:      aws.String(d.Id()),
 		}
 		log.Printf("[INFO] Deleting WAF Rule")
-	_, err = conn.DeleteRule(req)
+		return conn.DeleteRule(req)
-
+	})
 	if err != nil {
 		return fmt.Errorf("Error deleting WAF Rule: %s", err)
 	}
@@ -165,16 +159,10 @@ func resourceAwsWafRuleDelete(d *schema.ResourceData, meta interface{}) error {
 func updateWafRuleResource(d *schema.ResourceData, meta interface{}, ChangeAction string) error {
 	conn := meta.(*AWSClient).wafconn
 
-	// ChangeToken
+	wr := newWafRetryer(conn, "global")
-	var ct *waf.GetChangeTokenInput
+	_, err := wr.RetryWithToken(func(token *string) (interface{}, error) {
-
-	resp, err := conn.GetChangeToken(ct)
-	if err != nil {
-		return fmt.Errorf("Error getting change token: %s", err)
-	}
-
 		req := &waf.UpdateRuleInput{
-		ChangeToken: resp.ChangeToken,
+			ChangeToken: token,
 			RuleId:      aws.String(d.Id()),
 		}
 
@@ -192,7 +180,8 @@ func updateWafRuleResource(d *schema.ResourceData, meta interface{}, ChangeActio
 			req.Updates = append(req.Updates, updatePredicate)
 		}
 
-	_, err = conn.UpdateRule(req)
+		return conn.UpdateRule(req)
+	})
 	if err != nil {
 		return fmt.Errorf("Error Updating WAF Rule: %s", err)
 	}

builtin/providers/aws/resource_aws_waf_rule_test.go

@@ -99,16 +99,10 @@ func testAccCheckAWSWafRuleDisappears(v *waf.Rule) resource.TestCheckFunc {
 	return func(s *terraform.State) error {
 		conn := testAccProvider.Meta().(*AWSClient).wafconn
 
-		// ChangeToken
+		wr := newWafRetryer(conn, "global")
-		var ct *waf.GetChangeTokenInput
+		_, err := wr.RetryWithToken(func(token *string) (interface{}, error) {
-
-		resp, err := conn.GetChangeToken(ct)
-		if err != nil {
-			return fmt.Errorf("Error getting change token: %s", err)
-		}
-
 			req := &waf.UpdateRuleInput{
-			ChangeToken: resp.ChangeToken,
+				ChangeToken: token,
 				RuleId:      v.RuleId,
 			}
 
@@ -124,22 +118,21 @@ func testAccCheckAWSWafRuleDisappears(v *waf.Rule) resource.TestCheckFunc {
 				req.Updates = append(req.Updates, Predicate)
 			}
 
-		_, err = conn.UpdateRule(req)
+			return conn.UpdateRule(req)
+		})
 		if err != nil {
 			return fmt.Errorf("Error Updating WAF Rule: %s", err)
 		}
 
-		resp, err = conn.GetChangeToken(ct)
+		_, err = wr.RetryWithToken(func(token *string) (interface{}, error) {
-		if err != nil {
-			return fmt.Errorf("Error getting change token for waf Rule: %s", err)
-		}
-
 			opts := &waf.DeleteRuleInput{
-			ChangeToken: resp.ChangeToken,
+				ChangeToken: token,
 				RuleId:      v.RuleId,
 			}
-		if _, err := conn.DeleteRule(opts); err != nil {
+			return conn.DeleteRule(opts)
-			return err
+		})
+		if err != nil {
+			return fmt.Errorf("Error Deleting WAF Rule: %s", err)
 		}
 		return nil
 	}

builtin/providers/aws/resource_aws_waf_size_constraint_set.go

@@ -69,24 +69,19 @@ func resourceAwsWafSizeConstraintSetCreate(d *schema.ResourceData, meta interfac
 
 	log.Printf("[INFO] Creating SizeConstraintSet: %s", d.Get("name").(string))
 
-	// ChangeToken
+	wr := newWafRetryer(conn, "global")
-	var ct *waf.GetChangeTokenInput
+	out, err := wr.RetryWithToken(func(token *string) (interface{}, error) {
-
-	res, err := conn.GetChangeToken(ct)
-	if err != nil {
-		return errwrap.Wrapf("[ERROR] Error getting change token: {{err}}", err)
-	}
-
 		params := &waf.CreateSizeConstraintSetInput{
-		ChangeToken: res.ChangeToken,
+			ChangeToken: token,
 			Name:        aws.String(d.Get("name").(string)),
 		}
 
-	resp, err := conn.CreateSizeConstraintSet(params)
+		return conn.CreateSizeConstraintSet(params)
-
+	})
 	if err != nil {
 		return errwrap.Wrapf("[ERROR] Error creating SizeConstraintSet: {{err}}", err)
 	}
+	resp := out.(*waf.CreateSizeConstraintSetOutput)
 
 	d.SetId(*resp.SizeConstraintSet.SizeConstraintSetId)
 
@@ -134,17 +129,14 @@ func resourceAwsWafSizeConstraintSetDelete(d *schema.ResourceData, meta interfac
 		return errwrap.Wrapf("[ERROR] Error deleting SizeConstraintSet: {{err}}", err)
 	}
 
-	var ct *waf.GetChangeTokenInput
+	wr := newWafRetryer(conn, "global")
-
+	_, err = wr.RetryWithToken(func(token *string) (interface{}, error) {
-	resp, err := conn.GetChangeToken(ct)
-
 		req := &waf.DeleteSizeConstraintSetInput{
-		ChangeToken:         resp.ChangeToken,
+			ChangeToken:         token,
 			SizeConstraintSetId: aws.String(d.Id()),
 		}
-
+		return conn.DeleteSizeConstraintSet(req)
-	_, err = conn.DeleteSizeConstraintSet(req)
+	})
-
 	if err != nil {
 		return errwrap.Wrapf("[ERROR] Error deleting SizeConstraintSet: {{err}}", err)
 	}
@@ -155,15 +147,10 @@ func resourceAwsWafSizeConstraintSetDelete(d *schema.ResourceData, meta interfac
 func updateSizeConstraintSetResource(d *schema.ResourceData, meta interface{}, ChangeAction string) error {
 	conn := meta.(*AWSClient).wafconn
 
-	var ct *waf.GetChangeTokenInput
+	wr := newWafRetryer(conn, "global")
-
+	_, err := wr.RetryWithToken(func(token *string) (interface{}, error) {
-	resp, err := conn.GetChangeToken(ct)
-	if err != nil {
-		return errwrap.Wrapf("[ERROR] Error getting change token: {{err}}", err)
-	}
-
 		req := &waf.UpdateSizeConstraintSetInput{
-		ChangeToken:         resp.ChangeToken,
+			ChangeToken:         token,
 			SizeConstraintSetId: aws.String(d.Id()),
 		}
 
@@ -182,7 +169,8 @@ func updateSizeConstraintSetResource(d *schema.ResourceData, meta interface{}, C
 			req.Updates = append(req.Updates, sizeConstraintUpdate)
 		}
 
-	_, err = conn.UpdateSizeConstraintSet(req)
+		return conn.UpdateSizeConstraintSet(req)
+	})
 	if err != nil {
 		return errwrap.Wrapf("[ERROR] Error updating SizeConstraintSet: {{err}}", err)
 	}

builtin/providers/aws/resource_aws_waf_size_constraint_set_test.go

@@ -96,15 +96,10 @@ func testAccCheckAWSWafSizeConstraintSetDisappears(v *waf.SizeConstraintSet) res
 	return func(s *terraform.State) error {
 		conn := testAccProvider.Meta().(*AWSClient).wafconn
 
-		var ct *waf.GetChangeTokenInput
+		wr := newWafRetryer(conn, "global")
-
+		_, err := wr.RetryWithToken(func(token *string) (interface{}, error) {
-		resp, err := conn.GetChangeToken(ct)
-		if err != nil {
-			return fmt.Errorf("Error getting change token: %s", err)
-		}
-
 			req := &waf.UpdateSizeConstraintSetInput{
-			ChangeToken:         resp.ChangeToken,
+				ChangeToken:         token,
 				SizeConstraintSetId: v.SizeConstraintSetId,
 			}
 
@@ -120,21 +115,20 @@ func testAccCheckAWSWafSizeConstraintSetDisappears(v *waf.SizeConstraintSet) res
 				}
 				req.Updates = append(req.Updates, sizeConstraintUpdate)
 			}
-		_, err = conn.UpdateSizeConstraintSet(req)
+			return conn.UpdateSizeConstraintSet(req)
+		})
 		if err != nil {
 			return errwrap.Wrapf("[ERROR] Error updating SizeConstraintSet: {{err}}", err)
 		}
 
-		resp, err = conn.GetChangeToken(ct)
+		_, err = wr.RetryWithToken(func(token *string) (interface{}, error) {
-		if err != nil {
-			return errwrap.Wrapf("[ERROR] Error getting change token: {{err}}", err)
-		}
-
 			opts := &waf.DeleteSizeConstraintSetInput{
-			ChangeToken:         resp.ChangeToken,
+				ChangeToken:         token,
 				SizeConstraintSetId: v.SizeConstraintSetId,
 			}
-		if _, err := conn.DeleteSizeConstraintSet(opts); err != nil {
+			return conn.DeleteSizeConstraintSet(opts)
+		})
+		if err != nil {
 			return err
 		}
 		return nil

builtin/providers/aws/resource_aws_waf_sql_injection_match_set.go

@@ -61,25 +61,19 @@ func resourceAwsWafSqlInjectionMatchSetCreate(d *schema.ResourceData, meta inter
 
 	log.Printf("[INFO] Creating SqlInjectionMatchSet: %s", d.Get("name").(string))
 
-	// ChangeToken
+	wr := newWafRetryer(conn, "global")
-	var ct *waf.GetChangeTokenInput
+	out, err := wr.RetryWithToken(func(token *string) (interface{}, error) {
-
-	res, err := conn.GetChangeToken(ct)
-	if err != nil {
-		return errwrap.Wrapf("[ERROR] Error getting change token: {{err}}", err)
-	}
-
 		params := &waf.CreateSqlInjectionMatchSetInput{
-		ChangeToken: res.ChangeToken,
+			ChangeToken: token,
 			Name:        aws.String(d.Get("name").(string)),
 		}
 
-	resp, err := conn.CreateSqlInjectionMatchSet(params)
+		return conn.CreateSqlInjectionMatchSet(params)
-
+	})
 	if err != nil {
 		return errwrap.Wrapf("[ERROR] Error creating SqlInjectionMatchSet: {{err}}", err)
 	}
-
+	resp := out.(*waf.CreateSqlInjectionMatchSetOutput)
 	d.SetId(*resp.SqlInjectionMatchSet.SqlInjectionMatchSetId)
 
 	return resourceAwsWafSqlInjectionMatchSetUpdate(d, meta)
@@ -126,17 +120,15 @@ func resourceAwsWafSqlInjectionMatchSetDelete(d *schema.ResourceData, meta inter
 		return errwrap.Wrapf("[ERROR] Error deleting SqlInjectionMatchSet: {{err}}", err)
 	}
 
-	var ct *waf.GetChangeTokenInput
+	wr := newWafRetryer(conn, "global")
-
+	_, err = wr.RetryWithToken(func(token *string) (interface{}, error) {
-	resp, err := conn.GetChangeToken(ct)
-
 		req := &waf.DeleteSqlInjectionMatchSetInput{
-		ChangeToken:            resp.ChangeToken,
+			ChangeToken:            token,
 			SqlInjectionMatchSetId: aws.String(d.Id()),
 		}
 
-	_, err = conn.DeleteSqlInjectionMatchSet(req)
+		return conn.DeleteSqlInjectionMatchSet(req)
-
+	})
 	if err != nil {
 		return errwrap.Wrapf("[ERROR] Error deleting SqlInjectionMatchSet: {{err}}", err)
 	}
@@ -147,15 +139,10 @@ func resourceAwsWafSqlInjectionMatchSetDelete(d *schema.ResourceData, meta inter
 func updateSqlInjectionMatchSetResource(d *schema.ResourceData, meta interface{}, ChangeAction string) error {
 	conn := meta.(*AWSClient).wafconn
 
-	var ct *waf.GetChangeTokenInput
+	wr := newWafRetryer(conn, "global")
-
+	_, err := wr.RetryWithToken(func(token *string) (interface{}, error) {
-	resp, err := conn.GetChangeToken(ct)
-	if err != nil {
-		return errwrap.Wrapf("[ERROR] Error getting change token: {{err}}", err)
-	}
-
 		req := &waf.UpdateSqlInjectionMatchSetInput{
-		ChangeToken:            resp.ChangeToken,
+			ChangeToken:            token,
 			SqlInjectionMatchSetId: aws.String(d.Id()),
 		}
 
@@ -172,7 +159,8 @@ func updateSqlInjectionMatchSetResource(d *schema.ResourceData, meta interface{}
 			req.Updates = append(req.Updates, sizeConstraintUpdate)
 		}
 
-	_, err = conn.UpdateSqlInjectionMatchSet(req)
+		return conn.UpdateSqlInjectionMatchSet(req)
+	})
 	if err != nil {
 		return errwrap.Wrapf("[ERROR] Error updating SqlInjectionMatchSet: {{err}}", err)
 	}

builtin/providers/aws/resource_aws_waf_sql_injection_match_set_test.go

@@ -96,15 +96,10 @@ func testAccCheckAWSWafSqlInjectionMatchSetDisappears(v *waf.SqlInjectionMatchSe
 	return func(s *terraform.State) error {
 		conn := testAccProvider.Meta().(*AWSClient).wafconn
 
-		var ct *waf.GetChangeTokenInput
+		wr := newWafRetryer(conn, "global")
-
+		_, err := wr.RetryWithToken(func(token *string) (interface{}, error) {
-		resp, err := conn.GetChangeToken(ct)
-		if err != nil {
-			return fmt.Errorf("Error getting change token: %s", err)
-		}
-
 			req := &waf.UpdateSqlInjectionMatchSetInput{
-			ChangeToken:            resp.ChangeToken,
+				ChangeToken:            token,
 				SqlInjectionMatchSetId: v.SqlInjectionMatchSetId,
 			}
 
@@ -118,22 +113,21 @@ func testAccCheckAWSWafSqlInjectionMatchSetDisappears(v *waf.SqlInjectionMatchSe
 				}
 				req.Updates = append(req.Updates, sqlInjectionMatchTupleUpdate)
 			}
-		_, err = conn.UpdateSqlInjectionMatchSet(req)
+			return conn.UpdateSqlInjectionMatchSet(req)
+		})
 		if err != nil {
 			return errwrap.Wrapf("[ERROR] Error updating SqlInjectionMatchSet: {{err}}", err)
 		}
 
-		resp, err = conn.GetChangeToken(ct)
+		_, err = wr.RetryWithToken(func(token *string) (interface{}, error) {
-		if err != nil {
-			return errwrap.Wrapf("[ERROR] Error getting change token: {{err}}", err)
-		}
-
 			opts := &waf.DeleteSqlInjectionMatchSetInput{
-			ChangeToken:            resp.ChangeToken,
+				ChangeToken:            token,
 				SqlInjectionMatchSetId: v.SqlInjectionMatchSetId,
 			}
-		if _, err := conn.DeleteSqlInjectionMatchSet(opts); err != nil {
+			return conn.DeleteSqlInjectionMatchSet(opts)
-			return err
+		})
+		if err != nil {
+			return errwrap.Wrapf("[ERROR] Error deleting SqlInjectionMatchSet: {{err}}", err)
 		}
 		return nil
 	}

builtin/providers/aws/resource_aws_waf_web_acl.go

@@ -77,25 +77,21 @@ func resourceAwsWafWebAcl() *schema.Resource {
 func resourceAwsWafWebAclCreate(d *schema.ResourceData, meta interface{}) error {
 	conn := meta.(*AWSClient).wafconn
 
-	// ChangeToken
+	wr := newWafRetryer(conn, "global")
-	var ct *waf.GetChangeTokenInput
+	out, err := wr.RetryWithToken(func(token *string) (interface{}, error) {
-
-	res, err := conn.GetChangeToken(ct)
-	if err != nil {
-		return fmt.Errorf("Error getting change token: %s", err)
-	}
-
 		params := &waf.CreateWebACLInput{
-		ChangeToken:   res.ChangeToken,
+			ChangeToken:   token,
 			DefaultAction: expandDefaultAction(d),
 			MetricName:    aws.String(d.Get("metric_name").(string)),
 			Name:          aws.String(d.Get("name").(string)),
 		}
 
-	resp, err := conn.CreateWebACL(params)
+		return conn.CreateWebACL(params)
+	})
 	if err != nil {
 		return err
 	}
+	resp := out.(*waf.CreateWebACLOutput)
 	d.SetId(*resp.WebACL.WebACLId)
 	return resourceAwsWafWebAclUpdate(d, meta)
 }
@@ -144,18 +140,16 @@ func resourceAwsWafWebAclDelete(d *schema.ResourceData, meta interface{}) error
 		return fmt.Errorf("Error Removing WAF ACL Rules: %s", err)
 	}
 
-	var ct *waf.GetChangeTokenInput
+	wr := newWafRetryer(conn, "global")
-
+	_, err = wr.RetryWithToken(func(token *string) (interface{}, error) {
-	resp, err := conn.GetChangeToken(ct)
-
 		req := &waf.DeleteWebACLInput{
-		ChangeToken: resp.ChangeToken,
+			ChangeToken: token,
 			WebACLId:    aws.String(d.Id()),
 		}
 
 		log.Printf("[INFO] Deleting WAF ACL")
-	_, err = conn.DeleteWebACL(req)
+		return conn.DeleteWebACL(req)
-
+	})
 	if err != nil {
 		return fmt.Errorf("Error Deleting WAF ACL: %s", err)
 	}
@@ -164,16 +158,11 @@ func resourceAwsWafWebAclDelete(d *schema.ResourceData, meta interface{}) error
 
 func updateWebAclResource(d *schema.ResourceData, meta interface{}, ChangeAction string) error {
 	conn := meta.(*AWSClient).wafconn
-	// ChangeToken
-	var ct *waf.GetChangeTokenInput
-
-	resp, err := conn.GetChangeToken(ct)
-	if err != nil {
-		return fmt.Errorf("Error getting change token: %s", err)
-	}
 
+	wr := newWafRetryer(conn, "global")
+	_, err := wr.RetryWithToken(func(token *string) (interface{}, error) {
 		req := &waf.UpdateWebACLInput{
-		ChangeToken: resp.ChangeToken,
+			ChangeToken: token,
 			WebACLId:    aws.String(d.Id()),
 		}
 
@@ -195,7 +184,8 @@ func updateWebAclResource(d *schema.ResourceData, meta interface{}, ChangeAction
 			}
 			req.Updates = append(req.Updates, aclRuleUpdate)
 		}
-	_, err = conn.UpdateWebACL(req)
+		return conn.UpdateWebACL(req)
+	})
 	if err != nil {
 		return fmt.Errorf("Error Updating WAF ACL: %s", err)
 	}

builtin/providers/aws/resource_aws_waf_web_acl_test.go

@@ -159,16 +159,10 @@ func testAccCheckAWSWafWebAclDisappears(v *waf.WebACL) resource.TestCheckFunc {
 	return func(s *terraform.State) error {
 		conn := testAccProvider.Meta().(*AWSClient).wafconn
 
-		// ChangeToken
+		wr := newWafRetryer(conn, "global")
-		var ct *waf.GetChangeTokenInput
+		_, err := wr.RetryWithToken(func(token *string) (interface{}, error) {
-
-		resp, err := conn.GetChangeToken(ct)
-		if err != nil {
-			return fmt.Errorf("Error getting change token: %s", err)
-		}
-
 			req := &waf.UpdateWebACLInput{
-			ChangeToken: resp.ChangeToken,
+				ChangeToken: token,
 				WebACLId:    v.WebACLId,
 			}
 
@@ -184,22 +178,21 @@ func testAccCheckAWSWafWebAclDisappears(v *waf.WebACL) resource.TestCheckFunc {
 				req.Updates = append(req.Updates, WebACLUpdate)
 			}
 
-		_, err = conn.UpdateWebACL(req)
+			return conn.UpdateWebACL(req)
+		})
 		if err != nil {
 			return fmt.Errorf("Error Updating WAF ACL: %s", err)
 		}
 
-		resp, err = conn.GetChangeToken(ct)
+		_, err = wr.RetryWithToken(func(token *string) (interface{}, error) {
-		if err != nil {
-			return fmt.Errorf("Error getting change token for waf ACL: %s", err)
-		}
-
 			opts := &waf.DeleteWebACLInput{
-			ChangeToken: resp.ChangeToken,
+				ChangeToken: token,
 				WebACLId:    v.WebACLId,
 			}
-		if _, err := conn.DeleteWebACL(opts); err != nil {
+			return conn.DeleteWebACL(opts)
-			return err
+		})
+		if err != nil {
+			return fmt.Errorf("Error Deleting WAF ACL: %s", err)
 		}
 		return nil
 	}

builtin/providers/aws/resource_aws_waf_xss_match_set.go

@@ -61,24 +61,19 @@ func resourceAwsWafXssMatchSetCreate(d *schema.ResourceData, meta interface{}) e
 
 	log.Printf("[INFO] Creating XssMatchSet: %s", d.Get("name").(string))
 
-	// ChangeToken
+	wr := newWafRetryer(conn, "global")
-	var ct *waf.GetChangeTokenInput
+	out, err := wr.RetryWithToken(func(token *string) (interface{}, error) {
-
-	res, err := conn.GetChangeToken(ct)
-	if err != nil {
-		return errwrap.Wrapf("[ERROR] Error getting change token: {{err}}", err)
-	}
-
 		params := &waf.CreateXssMatchSetInput{
-		ChangeToken: res.ChangeToken,
+			ChangeToken: token,
 			Name:        aws.String(d.Get("name").(string)),
 		}
 
-	resp, err := conn.CreateXssMatchSet(params)
+		return conn.CreateXssMatchSet(params)
-
+	})
 	if err != nil {
 		return errwrap.Wrapf("[ERROR] Error creating XssMatchSet: {{err}}", err)
 	}
+	resp := out.(*waf.CreateXssMatchSetOutput)
 
 	d.SetId(*resp.XssMatchSet.XssMatchSetId)
 
@@ -126,17 +121,15 @@ func resourceAwsWafXssMatchSetDelete(d *schema.ResourceData, meta interface{}) e
 		return errwrap.Wrapf("[ERROR] Error deleting XssMatchSet: {{err}}", err)
 	}
 
-	var ct *waf.GetChangeTokenInput
+	wr := newWafRetryer(conn, "global")
-
+	_, err = wr.RetryWithToken(func(token *string) (interface{}, error) {
-	resp, err := conn.GetChangeToken(ct)
-
 		req := &waf.DeleteXssMatchSetInput{
-		ChangeToken:   resp.ChangeToken,
+			ChangeToken:   token,
 			XssMatchSetId: aws.String(d.Id()),
 		}
 
-	_, err = conn.DeleteXssMatchSet(req)
+		return conn.DeleteXssMatchSet(req)
-
+	})
 	if err != nil {
 		return errwrap.Wrapf("[ERROR] Error deleting XssMatchSet: {{err}}", err)
 	}
@@ -147,15 +140,10 @@ func resourceAwsWafXssMatchSetDelete(d *schema.ResourceData, meta interface{}) e
 func updateXssMatchSetResource(d *schema.ResourceData, meta interface{}, ChangeAction string) error {
 	conn := meta.(*AWSClient).wafconn
 
-	var ct *waf.GetChangeTokenInput
+	wr := newWafRetryer(conn, "global")
-
+	_, err := wr.RetryWithToken(func(token *string) (interface{}, error) {
-	resp, err := conn.GetChangeToken(ct)
-	if err != nil {
-		return errwrap.Wrapf("[ERROR] Error getting change token: {{err}}", err)
-	}
-
 		req := &waf.UpdateXssMatchSetInput{
-		ChangeToken:   resp.ChangeToken,
+			ChangeToken:   token,
 			XssMatchSetId: aws.String(d.Id()),
 		}
 
@@ -172,7 +160,8 @@ func updateXssMatchSetResource(d *schema.ResourceData, meta interface{}, ChangeA
 			req.Updates = append(req.Updates, xssMatchTupleUpdate)
 		}
 
-	_, err = conn.UpdateXssMatchSet(req)
+		return conn.UpdateXssMatchSet(req)
+	})
 	if err != nil {
 		return errwrap.Wrapf("[ERROR] Error updating XssMatchSet: {{err}}", err)
 	}

builtin/providers/aws/resource_aws_waf_xss_match_set_test.go

@@ -96,15 +96,10 @@ func testAccCheckAWSWafXssMatchSetDisappears(v *waf.XssMatchSet) resource.TestCh
 	return func(s *terraform.State) error {
 		conn := testAccProvider.Meta().(*AWSClient).wafconn
 
-		var ct *waf.GetChangeTokenInput
+		wr := newWafRetryer(conn, "global")
-
+		_, err := wr.RetryWithToken(func(token *string) (interface{}, error) {
-		resp, err := conn.GetChangeToken(ct)
-		if err != nil {
-			return fmt.Errorf("Error getting change token: %s", err)
-		}
-
 			req := &waf.UpdateXssMatchSetInput{
-			ChangeToken:   resp.ChangeToken,
+				ChangeToken:   token,
 				XssMatchSetId: v.XssMatchSetId,
 			}
 
@@ -118,22 +113,21 @@ func testAccCheckAWSWafXssMatchSetDisappears(v *waf.XssMatchSet) resource.TestCh
 				}
 				req.Updates = append(req.Updates, xssMatchTupleUpdate)
 			}
-		_, err = conn.UpdateXssMatchSet(req)
+			return conn.UpdateXssMatchSet(req)
+		})
 		if err != nil {
 			return errwrap.Wrapf("[ERROR] Error updating XssMatchSet: {{err}}", err)
 		}
 
-		resp, err = conn.GetChangeToken(ct)
+		_, err = wr.RetryWithToken(func(token *string) (interface{}, error) {
-		if err != nil {
-			return errwrap.Wrapf("[ERROR] Error getting change token: {{err}}", err)
-		}
-
 			opts := &waf.DeleteXssMatchSetInput{
-			ChangeToken:   resp.ChangeToken,
+				ChangeToken:   token,
 				XssMatchSetId: v.XssMatchSetId,
 			}
-		if _, err := conn.DeleteXssMatchSet(opts); err != nil {
+			return conn.DeleteXssMatchSet(opts)
-			return err
+		})
+		if err != nil {
+			return errwrap.Wrapf("[ERROR] Error deleting XssMatchSet: {{err}}", err)
 		}
 		return nil
 	}

builtin/providers/aws/waf_token_handlers.go

@@ -0,0 +1,49 @@
+package aws
+
+import (
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/service/waf"
+	"github.com/hashicorp/errwrap"
+	"github.com/hashicorp/terraform/helper/resource"
+)
+
+type WafRetryer struct {
+	Connection *waf.WAF
+	Region     string
+}
+
+type withTokenFunc func(token *string) (interface{}, error)
+
+func (t *WafRetryer) RetryWithToken(f withTokenFunc) (interface{}, error) {
+	awsMutexKV.Lock(t.Region)
+	defer awsMutexKV.Unlock(t.Region)
+
+	var out interface{}
+	err := resource.Retry(15*time.Minute, func() *resource.RetryError {
+		var err error
+		var tokenOut *waf.GetChangeTokenOutput
+
+		tokenOut, err = t.Connection.GetChangeToken(&waf.GetChangeTokenInput{})
+		if err != nil {
+			return resource.NonRetryableError(errwrap.Wrapf("Failed to acquire change token: {{err}}", err))
+		}
+
+		out, err = f(tokenOut.ChangeToken)
+		if err != nil {
+			awsErr, ok := err.(awserr.Error)
+			if ok && awsErr.Code() == "WAFStaleDataException" {
+				return resource.RetryableError(err)
+			}
+			return resource.NonRetryableError(err)
+		}
+		return nil
+	})
+
+	return out, err
+}
+
+func newWafRetryer(conn *waf.WAF, region string) *WafRetryer {
+	return &WafRetryer{Connection: conn, Region: region}
+}