-
Notifications
You must be signed in to change notification settings - Fork 14
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #82 from keep-network/balance-monitoring-retries
Balance monitoring retries We want to use a retry mechanism for single executions of the balance check. The retries will be performed during the period of retryTimeout, logging a warning on each error. When the timeout is hit an error will be logged and retries stopped. The next balance check will be triggered at the next tick.
- Loading branch information
Showing
6 changed files
with
522 additions
and
6 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,59 @@ | ||
| package ethlike | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "math/big" | ||
| "sync" | ||
| "time" | ||
|
|
||
| "testing" | ||
|
|
||
| "github.com/ipfs/go-log" | ||
| ) | ||
|
|
||
| func TestBalanceMonitor_Retries(t *testing.T) { | ||
| log.SetDebugLogging() | ||
|
|
||
| attemptsCount := 0 | ||
| expectedAttempts := 3 | ||
|
|
||
| wg := &sync.WaitGroup{} | ||
| wg.Add(expectedAttempts) | ||
|
|
||
| balanceSource := func(address Address) (*Token, error) { | ||
| attemptsCount++ | ||
| wg.Done() | ||
|
|
||
| if attemptsCount < expectedAttempts { | ||
| return nil, fmt.Errorf("not this time") | ||
| } | ||
|
|
||
| return &Token{big.NewInt(10)}, nil | ||
| } | ||
|
|
||
| balanceMonitor := NewBalanceMonitor(balanceSource) | ||
|
|
||
| address := Address{1, 2} | ||
| alertThreshold := &Token{big.NewInt(15)} | ||
| tick := 1 * time.Minute | ||
| retryTimeout := 5 * time.Second | ||
|
|
||
| balanceMonitor.Observe( | ||
| context.Background(), | ||
| address, | ||
| alertThreshold, | ||
| tick, | ||
| retryTimeout, | ||
| ) | ||
|
|
||
| wg.Wait() | ||
|
|
||
| if expectedAttempts != attemptsCount { | ||
| t.Errorf( | ||
| "unexpected retries count\nexpected: %d\nactual: %d", | ||
| expectedAttempts, | ||
| attemptsCount, | ||
| ) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,190 @@ | ||
| package wrappers | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "math/rand" | ||
| "time" | ||
| ) | ||
|
|
||
| // DoWithRetry executes the provided doFn as long as it returns an error or until | ||
| // a timeout is hit. It applies exponential backoff wait of backoffTime * 2^n | ||
| // before nth retry of doFn. In case the calculated backoff is longer than | ||
| // backoffMax, the backoffMax wait is applied. | ||
| func DoWithRetry( | ||
| backoffTime time.Duration, | ||
| backoffMax time.Duration, | ||
| timeout time.Duration, | ||
| doFn func(ctx context.Context) error, | ||
| ) error { | ||
| ctx, cancel := context.WithTimeout(context.Background(), timeout) | ||
| defer cancel() | ||
|
|
||
| var err error | ||
| for { | ||
| select { | ||
| case <-ctx.Done(): | ||
| return fmt.Errorf( | ||
| "retry timeout [%v] exceeded; most recent error: [%w]", | ||
| timeout, | ||
| err, | ||
| ) | ||
| default: | ||
| err = doFn(ctx) | ||
| if err == nil { | ||
| return nil | ||
| } | ||
|
|
||
| timedOut := backoffWait(ctx, backoffTime) | ||
| if timedOut { | ||
| return fmt.Errorf( | ||
| "retry timeout [%v] exceeded; most recent error: [%w]", | ||
| timeout, | ||
| err, | ||
| ) | ||
| } | ||
|
|
||
| backoffTime = calculateBackoff( | ||
| backoffTime, | ||
| backoffMax, | ||
| ) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| const ( | ||
| // DefaultDoBackoffTime is the default value of backoff time used by | ||
| // DoWithDefaultRetry function. | ||
| DefaultDoBackoffTime = 1 * time.Second | ||
|
|
||
| // DefaultDoMaxBackoffTime is the default value of max backoff time used by | ||
| // DoWithDefaultRetry function. | ||
| DefaultDoMaxBackoffTime = 120 * time.Second | ||
| ) | ||
|
|
||
| // DoWithDefaultRetry executes the provided doFn as long as it returns an error or | ||
| // until a timeout is hit. It applies exponential backoff wait of | ||
| // DefaultBackoffTime * 2^n before nth retry of doFn. In case the calculated | ||
| // backoff is longer than DefaultMaxBackoffTime, the DefaultMaxBackoffTime is | ||
| // applied. | ||
| func DoWithDefaultRetry( | ||
| timeout time.Duration, | ||
| doFn func(ctx context.Context) error, | ||
| ) error { | ||
| return DoWithRetry( | ||
| DefaultDoBackoffTime, | ||
| DefaultDoMaxBackoffTime, | ||
| timeout, | ||
| doFn, | ||
| ) | ||
| } | ||
|
|
||
| // ConfirmWithTimeout executes the provided confirmFn until it returns true or | ||
| // until it fails or until a timeout is hit. It applies exponential backoff wait | ||
| // of backoffTime * 2^n before nth execution of confirmFn. In case the | ||
| // calculated backoff is longer than backoffMax, the backoffMax is applied. | ||
| // In case confirmFn returns an error, ConfirmWithTimeout exits with the same | ||
| // error immediately. This is different from DoWithRetry behavior as the use | ||
| // case for this function is different. ConfirmWithTimeout is intended to be | ||
| // used to confirm a chain state and not to try to enforce a successful | ||
| // execution of some function. | ||
| func ConfirmWithTimeout( | ||
| backoffTime time.Duration, | ||
| backoffMax time.Duration, | ||
| timeout time.Duration, | ||
| confirmFn func(ctx context.Context) (bool, error), | ||
| ) (bool, error) { | ||
| ctx, cancel := context.WithTimeout(context.Background(), timeout) | ||
| defer cancel() | ||
|
|
||
| for { | ||
| select { | ||
| case <-ctx.Done(): | ||
| return false, nil | ||
| default: | ||
| ok, err := confirmFn(ctx) | ||
| if err == nil && ok { | ||
| return true, nil | ||
| } | ||
| if err != nil { | ||
| return false, err | ||
| } | ||
|
|
||
| timedOut := backoffWait(ctx, backoffTime) | ||
| if timedOut { | ||
| return false, nil | ||
| } | ||
|
|
||
| backoffTime = calculateBackoff( | ||
| backoffTime, | ||
| backoffMax, | ||
| ) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| const ( | ||
| // DefaultConfirmBackoffTime is the default value of backoff time used by | ||
| // ConfirmWithDefaultTimeout function. | ||
| DefaultConfirmBackoffTime = 5 * time.Second | ||
|
|
||
| // DefaultConfirmMaxBackoffTime is the default value of max backoff time | ||
| // used by ConfirmWithDefaultTimeout function. | ||
| DefaultConfirmMaxBackoffTime = 10 * time.Second | ||
| ) | ||
|
|
||
| // ConfirmWithTimeoutDefaultBackoff executed the provided confirmFn until it | ||
| // returns true or until it fails or until timeout is hit. It applies | ||
| // backoff wait of DefaultConfirmBackoffTime * 2^n before nth execution of | ||
| // confirmFn. In case the calculated backoff is longer than | ||
| // DefaultConfirmMaxBackoffTime, DefaultConfirmMaxBackoffTime is applied. | ||
| // In case confirmFn returns an error, ConfirmWithTimeoutDefaultBackoff exits | ||
| // with the same error immediately. This is different from DoWithDefaultRetry | ||
| // behavior as the use case for this function is different. | ||
| // ConfirmWithTimeoutDefaultBackoff is intended to be used to confirm a chain | ||
| // state and not to try to enforce a successful execution of some function. | ||
| func ConfirmWithTimeoutDefaultBackoff( | ||
| timeout time.Duration, | ||
| confirmFn func(ctx context.Context) (bool, error), | ||
| ) (bool, error) { | ||
| return ConfirmWithTimeout( | ||
| DefaultConfirmBackoffTime, | ||
| DefaultConfirmMaxBackoffTime, | ||
| timeout, | ||
| confirmFn, | ||
| ) | ||
| } | ||
|
|
||
| func calculateBackoff( | ||
| backoffPrev time.Duration, | ||
| backoffMax time.Duration, | ||
| ) time.Duration { | ||
| backoff := backoffPrev | ||
|
|
||
| backoff *= 2 | ||
|
|
||
| // #nosec G404 | ||
| // we are fine with not using cryptographically secure random integer, | ||
| // it is just exponential backoff jitter | ||
| r := rand.Int63n(backoff.Nanoseconds()/10 + 1) | ||
| jitter := time.Duration(r) * time.Nanosecond | ||
| backoff += jitter | ||
|
|
||
| if backoff > backoffMax { | ||
| backoff = backoffMax | ||
| } | ||
|
|
||
| return backoff | ||
| } | ||
|
|
||
| func backoffWait(ctx context.Context, waitTime time.Duration) bool { | ||
| timer := time.NewTimer(waitTime) | ||
| defer timer.Stop() | ||
|
|
||
| select { | ||
| case <-ctx.Done(): | ||
| return true | ||
| case <-timer.C: | ||
| return false | ||
| } | ||
| } |
Oops, something went wrong.