From 76dfaf7a597e66e4d2c44e9e9c2f49158956b81b Mon Sep 17 00:00:00 2001 From: Edward Date: Sun, 3 May 2020 10:52:05 +0800 Subject: [PATCH] add retry codes --- README.md | 4 +- gomore/retrier/backoffs.go | 24 ++++ gomore/retrier/backoffs_test.go | 55 +++++++++ gomore/retrier/classifier.go | 66 +++++++++++ gomore/retrier/classifier_test.go | 66 +++++++++++ gomore/retrier/retrier.go | 97 ++++++++++++++++ gomore/retrier/retrier_test.go | 179 ++++++++++++++++++++++++++++++ 7 files changed, 490 insertions(+), 1 deletion(-) create mode 100644 gomore/retrier/backoffs.go create mode 100644 gomore/retrier/backoffs_test.go create mode 100644 gomore/retrier/classifier.go create mode 100644 gomore/retrier/classifier_test.go create mode 100644 gomore/retrier/retrier.go create mode 100644 gomore/retrier/retrier_test.go diff --git a/README.md b/README.md index 326303e..c63979d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Go语言设计模式示例集合(Go Patterns Examples) -**包括了[go-patterns](https://github.com/tmrts/go-patterns) 和[golang-design-pattern](https://github.com/senghoo/golang-design-pattern)中的全部模式！！！** +**包括了[go-patterns](https://github.com/tmrts/go-patterns) 和[golang-design-pattern](https://github.com/senghoo/golang-design-pattern)中的全部模式** 目前包括了**39种Go中常用的、面向工程化和最佳实践的模式/套路**，自然也包含常见的23种设计模式,重点是这里全部是例子、通俗易懂，甚至每个模式下的例子，改一下名字，稍微再增加几行代码就可以直接用在项目和工程中了。 @@ -80,6 +80,8 @@ [design_pattern](http://c.biancheng.net/design_pattern) +[go-resiliency](https://github.com/eapache/go-resiliency) + ## 更多 diff --git a/gomore/retrier/backoffs.go b/gomore/retrier/backoffs.go new file mode 100644 index 0000000..faf6f8c --- /dev/null +++ b/gomore/retrier/backoffs.go @@ -0,0 +1,24 @@ +package retrier + +import "time" + +// ConstantBackoff generates a simple back-off strategy of retrying 'n' times, and waiting 'amount' time after each one. +func ConstantBackoff(n int, amount time.Duration) []time.Duration { + ret := make([]time.Duration, n) + for i := range ret { + ret[i] = amount + } + return ret +} + +// ExponentialBackoff generates a simple back-off strategy of retrying 'n' times, and doubling the amount of +// time waited after each one. +func ExponentialBackoff(n int, initialAmount time.Duration) []time.Duration { + ret := make([]time.Duration, n) + next := initialAmount + for i := range ret { + ret[i] = next + next *= 2 + } + return ret +} diff --git a/gomore/retrier/backoffs_test.go b/gomore/retrier/backoffs_test.go new file mode 100644 index 0000000..1168adf --- /dev/null +++ b/gomore/retrier/backoffs_test.go @@ -0,0 +1,55 @@ +package retrier + +import ( + "testing" + "time" +) + +func TestConstantBackoff(t *testing.T) { + b := ConstantBackoff(1, 10*time.Millisecond) + if len(b) != 1 { + t.Error("incorrect length") + } + for i := range b { + if b[i] != 10*time.Millisecond { + t.Error("incorrect value at", i) + } + } + + b = ConstantBackoff(10, 250*time.Hour) + if len(b) != 10 { + t.Error("incorrect length") + } + for i := range b { + if b[i] != 250*time.Hour { + t.Error("incorrect value at", i) + } + } +} + +func TestExponentialBackoff(t *testing.T) { + b := ExponentialBackoff(1, 10*time.Millisecond) + if len(b) != 1 { + t.Error("incorrect length") + } + if b[0] != 10*time.Millisecond { + t.Error("incorrect value") + } + + b = ExponentialBackoff(4, 1*time.Minute) + if len(b) != 4 { + t.Error("incorrect length") + } + if b[0] != 1*time.Minute { + t.Error("incorrect value") + } + if b[1] != 2*time.Minute { + t.Error("incorrect value") + } + if b[2] != 4*time.Minute { + t.Error("incorrect value") + } + if b[3] != 8*time.Minute { + t.Error("incorrect value") + } +} diff --git a/gomore/retrier/classifier.go b/gomore/retrier/classifier.go new file mode 100644 index 0000000..7dd71c7 --- /dev/null +++ b/gomore/retrier/classifier.go @@ -0,0 +1,66 @@ +package retrier + +// Action is the type returned by a Classifier to indicate how the Retrier should proceed. +type Action int + +const ( + Succeed Action = iota // Succeed indicates the Retrier should treat this value as a success. + Fail // Fail indicates the Retrier should treat this value as a hard failure and not retry. + Retry // Retry indicates the Retrier should treat this value as a soft failure and retry. +) + +// Classifier is the interface implemented by anything that can classify Errors for a Retrier. +type Classifier interface { + Classify(error) Action +} + +// DefaultClassifier classifies errors in the simplest way possible. If +// the error is nil, it returns Succeed, otherwise it returns Retry. +type DefaultClassifier struct{} + +// Classify implements the Classifier interface. +func (c DefaultClassifier) Classify(err error) Action { + if err == nil { + return Succeed + } + + return Retry +} + +// WhitelistClassifier classifies errors based on a whitelist. If the error is nil, it +// returns Succeed; if the error is in the whitelist, it returns Retry; otherwise, it returns Fail. +type WhitelistClassifier []error + +// Classify implements the Classifier interface. +func (list WhitelistClassifier) Classify(err error) Action { + if err == nil { + return Succeed + } + + for _, pass := range list { + if err == pass { + return Retry + } + } + + return Fail +} + +// BlacklistClassifier classifies errors based on a blacklist. If the error is nil, it +// returns Succeed; if the error is in the blacklist, it returns Fail; otherwise, it returns Retry. +type BlacklistClassifier []error + +// Classify implements the Classifier interface. +func (list BlacklistClassifier) Classify(err error) Action { + if err == nil { + return Succeed + } + + for _, pass := range list { + if err == pass { + return Fail + } + } + + return Retry +} diff --git a/gomore/retrier/classifier_test.go b/gomore/retrier/classifier_test.go new file mode 100644 index 0000000..953102f --- /dev/null +++ b/gomore/retrier/classifier_test.go @@ -0,0 +1,66 @@ +package retrier + +import ( + "errors" + "testing" +) + +var ( + errFoo = errors.New("FOO") + errBar = errors.New("BAR") + errBaz = errors.New("BAZ") +) + +func TestDefaultClassifier(t *testing.T) { + c := DefaultClassifier{} + + if c.Classify(nil) != Succeed { + t.Error("default misclassified nil") + } + + if c.Classify(errFoo) != Retry { + t.Error("default misclassified foo") + } + if c.Classify(errBar) != Retry { + t.Error("default misclassified bar") + } + if c.Classify(errBaz) != Retry { + t.Error("default misclassified baz") + } +} + +func TestWhitelistClassifier(t *testing.T) { + c := WhitelistClassifier{errFoo, errBar} + + if c.Classify(nil) != Succeed { + t.Error("whitelist misclassified nil") + } + + if c.Classify(errFoo) != Retry { + t.Error("whitelist misclassified foo") + } + if c.Classify(errBar) != Retry { + t.Error("whitelist misclassified bar") + } + if c.Classify(errBaz) != Fail { + t.Error("whitelist misclassified baz") + } +} + +func TestBlacklistClassifier(t *testing.T) { + c := BlacklistClassifier{errBar} + + if c.Classify(nil) != Succeed { + t.Error("blacklist misclassified nil") + } + + if c.Classify(errFoo) != Retry { + t.Error("blacklist misclassified foo") + } + if c.Classify(errBar) != Fail { + t.Error("blacklist misclassified bar") + } + if c.Classify(errBaz) != Retry { + t.Error("blacklist misclassified baz") + } +} diff --git a/gomore/retrier/retrier.go b/gomore/retrier/retrier.go new file mode 100644 index 0000000..6801c55 --- /dev/null +++ b/gomore/retrier/retrier.go @@ -0,0 +1,97 @@ +// Package retrier implements the "retriable" resiliency pattern for Go. +package retrier + +import ( + "context" + "math/rand" + "sync" + "time" +) + +// Retrier implements the "retriable" resiliency pattern, abstracting out the process of retrying a failed action +// a certain number of times with an optional back-off between each retry. +type Retrier struct { + backoff []time.Duration + class Classifier + jitter float64 + rand *rand.Rand + randMu sync.Mutex +} + +// New constructs a Retrier with the given backoff pattern and classifier. The length of the backoff pattern +// indicates how many times an action will be retried, and the value at each index indicates the amount of time +// waited before each subsequent retry. The classifier is used to determine which errors should be retried and +// which should cause the retrier to fail fast. The DefaultClassifier is used if nil is passed. +func New(backoff []time.Duration, class Classifier) *Retrier { + if class == nil { + class = DefaultClassifier{} + } + + return &Retrier{ + backoff: backoff, + class: class, + rand: rand.New(rand.NewSource(time.Now().UnixNano())), + } +} + +// Run executes the given work function by executing RunCtx without context.Context. +func (r *Retrier) Run(work func() error) error { + return r.RunCtx(context.Background(), func(ctx context.Context) error { + // never use ctx + return work() + }) +} + +// RunCtx executes the given work function, then classifies its return value based on the classifier used +// to construct the Retrier. If the result is Succeed or Fail, the return value of the work function is +// returned to the caller. If the result is Retry, then Run sleeps according to the its backoff policy +// before retrying. If the total number of retries is exceeded then the return value of the work function +// is returned to the caller regardless. +func (r *Retrier) RunCtx(ctx context.Context, work func(ctx context.Context) error) error { + retries := 0 + for { + ret := work(ctx) + + switch r.class.Classify(ret) { + case Succeed, Fail: + return ret + case Retry: + if retries >= len(r.backoff) { + return ret + } + + timeout := time.After(r.calcSleep(retries)) + if err := r.sleep(ctx, timeout); err != nil { + return err + } + + retries++ + } + } +} + +func (r *Retrier) sleep(ctx context.Context, t <-chan time.Time) error { + select { + case <-t: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (r *Retrier) calcSleep(i int) time.Duration { + // lock unsafe rand prng + r.randMu.Lock() + defer r.randMu.Unlock() + // take a random float in the range (-r.jitter, +r.jitter) and multiply it by the base amount + return r.backoff[i] + time.Duration(((r.rand.Float64()*2)-1)*r.jitter*float64(r.backoff[i])) +} + +// SetJitter sets the amount of jitter on each back-off to a factor between 0.0 and 1.0 (values outside this range +// are silently ignored). When a retry occurs, the back-off is adjusted by a random amount up to this value. +func (r *Retrier) SetJitter(jit float64) { + if jit < 0 || jit > 1 { + return + } + r.jitter = jit +} diff --git a/gomore/retrier/retrier_test.go b/gomore/retrier/retrier_test.go new file mode 100644 index 0000000..aaa1d51 --- /dev/null +++ b/gomore/retrier/retrier_test.go @@ -0,0 +1,179 @@ +package retrier + +import ( + "context" + "errors" + "testing" + "time" +) + +var i int + +func genWork(returns []error) func() error { + i = 0 + return func() error { + i++ + if i > len(returns) { + return nil + } + return returns[i-1] + } +} + +func genWorkWithCtx() func(ctx context.Context) error { + i = 0 + return func(ctx context.Context) error { + select { + case <-ctx.Done(): + return errFoo + default: + i++ + } + return nil + } +} + +func TestRetrier(t *testing.T) { + r := New([]time.Duration{0, 10 * time.Millisecond}, WhitelistClassifier{errFoo}) + + err := r.Run(genWork([]error{errFoo, errFoo})) + if err != nil { + t.Error(err) + } + if i != 3 { + t.Error("run wrong number of times") + } + + err = r.Run(genWork([]error{errFoo, errBar})) + if err != errBar { + t.Error(err) + } + if i != 2 { + t.Error("run wrong number of times") + } + + err = r.Run(genWork([]error{errBar, errBaz})) + if err != errBar { + t.Error(err) + } + if i != 1 { + t.Error("run wrong number of times") + } +} + +func TestRetrierCtx(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + + r := New([]time.Duration{0, 10 * time.Millisecond}, WhitelistClassifier{}) + + err := r.RunCtx(ctx, genWorkWithCtx()) + if err != nil { + t.Error(err) + } + if i != 1 { + t.Error("run wrong number of times") + } + + cancel() + + err = r.RunCtx(ctx, genWorkWithCtx()) + if err != errFoo { + t.Error("context must be cancelled") + } + if i != 0 { + t.Error("run wrong number of times") + } +} + +func TestRetrierNone(t *testing.T) { + r := New(nil, nil) + + i = 0 + err := r.Run(func() error { + i++ + return errFoo + }) + if err != errFoo { + t.Error(err) + } + if i != 1 { + t.Error("run wrong number of times") + } + + i = 0 + err = r.Run(func() error { + i++ + return nil + }) + if err != nil { + t.Error(err) + } + if i != 1 { + t.Error("run wrong number of times") + } +} + +func TestRetrierJitter(t *testing.T) { + r := New([]time.Duration{0, 10 * time.Millisecond, 4 * time.Hour}, nil) + + if r.calcSleep(0) != 0 { + t.Error("Incorrect sleep calculated") + } + if r.calcSleep(1) != 10*time.Millisecond { + t.Error("Incorrect sleep calculated") + } + if r.calcSleep(2) != 4*time.Hour { + t.Error("Incorrect sleep calculated") + } + + r.SetJitter(0.25) + for i := 0; i < 20; i++ { + if r.calcSleep(0) != 0 { + t.Error("Incorrect sleep calculated") + } + + slp := r.calcSleep(1) + if slp < 7500*time.Microsecond || slp > 12500*time.Microsecond { + t.Error("Incorrect sleep calculated") + } + + slp = r.calcSleep(2) + if slp < 3*time.Hour || slp > 5*time.Hour { + t.Error("Incorrect sleep calculated") + } + } + + r.SetJitter(-1) + if r.jitter != 0.25 { + t.Error("Invalid jitter value accepted") + } + + r.SetJitter(2) + if r.jitter != 0.25 { + t.Error("Invalid jitter value accepted") + } +} + +func TestRetrierThreadSafety(t *testing.T) { + r := New([]time.Duration{0}, nil) + for i := 0; i < 2; i++ { + go func() { + r.Run(func() error { + return errors.New("error") + }) + }() + } +} + +func ExampleRetrier() { + r := New(ConstantBackoff(3, 100*time.Millisecond), nil) + + err := r.Run(func() error { + // do some work + return nil + }) + + if err != nil { + // handle the case where the work failed three times + } +}