backoff retries in remote-exec provisioner
Add a simple backoff to the remote-exec retryFunc. Backoff between tries, up to a 10s max.
This commit is contained in:
parent
8b3b678c2f
commit
db42015342
|
@ -19,6 +19,10 @@ import (
|
||||||
"github.com/mitchellh/go-linereader"
|
"github.com/mitchellh/go-linereader"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// maxBackoffDealy is the maximum delay between retry attempts
|
||||||
|
var maxBackoffDelay = 10 * time.Second
|
||||||
|
var initialBackoffDelay = time.Second
|
||||||
|
|
||||||
func Provisioner() terraform.ResourceProvisioner {
|
func Provisioner() terraform.ResourceProvisioner {
|
||||||
return &schema.Provisioner{
|
return &schema.Provisioner{
|
||||||
Schema: map[string]*schema.Schema{
|
Schema: map[string]*schema.Schema{
|
||||||
|
@ -246,7 +250,6 @@ func copyOutput(
|
||||||
}
|
}
|
||||||
|
|
||||||
// retryFunc is used to retry a function for a given duration
|
// retryFunc is used to retry a function for a given duration
|
||||||
// TODO: this should probably backoff too
|
|
||||||
func retryFunc(ctx context.Context, timeout time.Duration, f func() error) error {
|
func retryFunc(ctx context.Context, timeout time.Duration, f func() error) error {
|
||||||
// Build a new context with the timeout
|
// Build a new context with the timeout
|
||||||
ctx, done := context.WithTimeout(ctx, timeout)
|
ctx, done := context.WithTimeout(ctx, timeout)
|
||||||
|
@ -263,12 +266,13 @@ func retryFunc(ctx context.Context, timeout time.Duration, f func() error) error
|
||||||
go func() {
|
go func() {
|
||||||
defer close(doneCh)
|
defer close(doneCh)
|
||||||
|
|
||||||
|
delay := time.Duration(0)
|
||||||
for {
|
for {
|
||||||
// If our context ended, we want to exit right away.
|
// If our context ended, we want to exit right away.
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return
|
return
|
||||||
default:
|
case <-time.After(delay):
|
||||||
}
|
}
|
||||||
|
|
||||||
// Try the function call
|
// Try the function call
|
||||||
|
@ -279,7 +283,19 @@ func retryFunc(ctx context.Context, timeout time.Duration, f func() error) error
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Printf("Retryable error: %v", err)
|
log.Printf("[WARN] retryable error: %v", err)
|
||||||
|
|
||||||
|
delay *= 2
|
||||||
|
|
||||||
|
if delay == 0 {
|
||||||
|
delay = initialBackoffDelay
|
||||||
|
}
|
||||||
|
|
||||||
|
if delay > maxBackoffDelay {
|
||||||
|
delay = maxBackoffDelay
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("[INFO] sleeping for %s", delay)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
|
|
@ -211,6 +211,16 @@ func TestResourceProvider_CollectScripts_scriptsEmpty(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRetryFunc(t *testing.T) {
|
func TestRetryFunc(t *testing.T) {
|
||||||
|
origMax := maxBackoffDelay
|
||||||
|
maxBackoffDelay = time.Second
|
||||||
|
origStart := initialBackoffDelay
|
||||||
|
initialBackoffDelay = 10 * time.Millisecond
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
maxBackoffDelay = origMax
|
||||||
|
initialBackoffDelay = origStart
|
||||||
|
}()
|
||||||
|
|
||||||
// succeed on the third try
|
// succeed on the third try
|
||||||
errs := []error{io.EOF, &net.OpError{Err: errors.New("ERROR")}, nil}
|
errs := []error{io.EOF, &net.OpError{Err: errors.New("ERROR")}, nil}
|
||||||
count := 0
|
count := 0
|
||||||
|
@ -235,6 +245,29 @@ func TestRetryFunc(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRetryFuncBackoff(t *testing.T) {
|
||||||
|
origMax := maxBackoffDelay
|
||||||
|
maxBackoffDelay = time.Second
|
||||||
|
origStart := initialBackoffDelay
|
||||||
|
initialBackoffDelay = 100 * time.Millisecond
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
maxBackoffDelay = origMax
|
||||||
|
initialBackoffDelay = origStart
|
||||||
|
}()
|
||||||
|
|
||||||
|
count := 0
|
||||||
|
|
||||||
|
retryFunc(context.Background(), time.Second, func() error {
|
||||||
|
count++
|
||||||
|
return io.EOF
|
||||||
|
})
|
||||||
|
|
||||||
|
if count > 4 {
|
||||||
|
t.Fatalf("retry func failed to backoff. called %d times", count)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func testConfig(t *testing.T, c map[string]interface{}) *terraform.ResourceConfig {
|
func testConfig(t *testing.T, c map[string]interface{}) *terraform.ResourceConfig {
|
||||||
r, err := config.NewRawConfig(c)
|
r, err := config.NewRawConfig(c)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
Loading…
Reference in New Issue