diff --git a/go.mod b/go.mod index e4b19aa3447..e0ca6dc5529 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/google/gofuzz v1.1.0 github.com/google/mako v0.0.0-20190821191249-122f8dcef9e3 github.com/google/uuid v1.1.1 + github.com/hashicorp/go-retryablehttp v0.6.6 github.com/influxdata/tdigest v0.0.0-20191024211133-5d87a7585faa // indirect github.com/json-iterator/go v1.1.9 // indirect github.com/kelseyhightower/envconfig v1.4.0 diff --git a/go.sum b/go.sum index 4c3d0e8a507..e87da3709c5 100644 --- a/go.sum +++ b/go.sum @@ -635,7 +635,9 @@ github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBt github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= @@ -643,6 +645,7 @@ github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1: github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-retryablehttp v0.6.4/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= +github.com/hashicorp/go-retryablehttp v0.6.6 h1:HJunrbHTDDbBb/ay4kxa1n+dLmttUlnP3V9oNE4hmsM= github.com/hashicorp/go-retryablehttp v0.6.6/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= diff --git a/pkg/channel/fanout/fanout_message_handler.go b/pkg/channel/fanout/fanout_message_handler.go index e19c1ba5ea0..855600a7bfe 100644 --- a/pkg/channel/fanout/fanout_message_handler.go +++ b/pkg/channel/fanout/fanout_message_handler.go @@ -23,7 +23,9 @@ package fanout import ( "context" + "encoding/json" "errors" + "fmt" nethttp "net/http" "net/url" "time" @@ -33,17 +35,32 @@ import ( "go.opencensus.io/trace" "go.uber.org/zap" + eventingduckv1 "knative.dev/eventing/pkg/apis/duck/v1" eventingduck "knative.dev/eventing/pkg/apis/duck/v1beta1" "knative.dev/eventing/pkg/channel" + "knative.dev/eventing/pkg/kncloudevents" ) const ( defaultTimeout = 15 * time.Minute ) +type Subscription struct { + eventingduck.SubscriberSpec + RetriesConfig kncloudevents.RetryConfig +} + +func (s *Subscription) MarshalJSON() ([]byte, error) { + return json.Marshal(s.SubscriberSpec) +} + +func (s *Subscription) UnmarshalJSON(bytes []byte) error { + return json.Unmarshal(bytes, &s.SubscriberSpec) +} + // Config for a fanout.MessageHandler. type Config struct { - Subscriptions []eventingduck.SubscriberSpec `json:"subscriptions"` + Subscriptions []Subscription `json:"subscriptions"` // AsyncHandler controls whether the Subscriptions are called synchronous or asynchronously. // It is expected to be false when used as a sidecar. AsyncHandler bool `json:"asyncHandler,omitempty"` @@ -78,9 +95,26 @@ func NewMessageHandler(logger *zap.Logger, messageDispatcher channel.MessageDisp return nil, err } handler.receiver = receiver + + for i := range config.Subscriptions { + retriesConfig, err := retriesOf(config.Subscriptions[i].SubscriberSpec) + if err != nil { + return nil, fmt.Errorf("failed to create retries config from SubscriberSpec: %w", err) + } + config.Subscriptions[i].RetriesConfig = retriesConfig + } + return handler, nil } +func retriesOf(spec eventingduck.SubscriberSpec) (kncloudevents.RetryConfig, error) { + delivery := &eventingduckv1.DeliverySpec{} + + _ = spec.ConvertTo(context.Background(), delivery) + + return kncloudevents.RetryConfigFromDeliverySpec(*delivery) +} + func createMessageReceiverFunction(f *MessageHandler) func(context.Context, channel.ChannelReference, binding.Message, []binding.Transformer, nethttp.Header) error { if f.config.AsyncHandler { return func(ctx context.Context, _ channel.ChannelReference, message binding.Message, transformers []binding.Transformer, additionalHeaders nethttp.Header) error { @@ -140,7 +174,7 @@ func (f *MessageHandler) dispatch(ctx context.Context, bufferedMessage binding.M errorCh := make(chan error, subs) for _, sub := range f.config.Subscriptions { - go func(s eventingduck.SubscriberSpec) { + go func(s Subscription) { errorCh <- f.makeFanoutRequest(ctx, bufferedMessage, additionalHeaders, s) }(sub) } @@ -163,18 +197,22 @@ func (f *MessageHandler) dispatch(ctx context.Context, bufferedMessage binding.M // makeFanoutRequest sends the request to exactly one subscription. It handles both the `call` and // the `sink` portions of the subscription. -func (f *MessageHandler) makeFanoutRequest(ctx context.Context, message binding.Message, additionalHeaders nethttp.Header, sub eventingduck.SubscriberSpec) error { +func (f *MessageHandler) makeFanoutRequest(ctx context.Context, message binding.Message, additionalHeaders nethttp.Header, sub Subscription) error { + var destination *url.URL if sub.SubscriberURI != nil { destination = sub.SubscriberURI.URL() } + var reply *url.URL if sub.ReplyURI != nil { reply = sub.ReplyURI.URL() } - var deadLetter *url.URL + + var deadLetterURL *url.URL if sub.Delivery != nil && sub.Delivery.DeadLetterSink != nil && sub.Delivery.DeadLetterSink.URI != nil { - deadLetter = sub.Delivery.DeadLetterSink.URI.URL() + deadLetterURL = sub.Delivery.DeadLetterSink.URI.URL() } - return f.dispatcher.DispatchMessage(ctx, message, additionalHeaders, destination, reply, deadLetter) + + return f.dispatcher.DispatchMessageWithRetries(ctx, message, additionalHeaders, destination, reply, deadLetterURL, &sub.RetriesConfig) } diff --git a/pkg/channel/fanout/fanout_message_handler_test.go b/pkg/channel/fanout/fanout_message_handler_test.go index a41bbf93fb7..4ff477f81a9 100644 --- a/pkg/channel/fanout/fanout_message_handler_test.go +++ b/pkg/channel/fanout/fanout_message_handler_test.go @@ -48,7 +48,7 @@ func TestFanoutMessageHandler_ServeHTTP(t *testing.T) { testCases := map[string]struct { receiverFunc channel.UnbufferedMessageReceiverFunc timeout time.Duration - subs []eventingduck.SubscriberSpec + subs []Subscription subscriber func(http.ResponseWriter, *http.Request) subscriberReqs int replier func(http.ResponseWriter, *http.Request) @@ -75,9 +75,11 @@ func TestFanoutMessageHandler_ServeHTTP(t *testing.T) { }, "fanout times out": { timeout: time.Millisecond, - subs: []eventingduck.SubscriberSpec{ + subs: []Subscription{ { - SubscriberURI: replaceSubscriber, + SubscriberSpec: eventingduck.SubscriberSpec{ + SubscriberURI: replaceSubscriber, + }, }, }, subscriber: func(writer http.ResponseWriter, _ *http.Request) { @@ -89,21 +91,23 @@ func TestFanoutMessageHandler_ServeHTTP(t *testing.T) { asyncExpectedStatus: http.StatusAccepted, }, "zero subs succeed": { - subs: []eventingduck.SubscriberSpec{}, + subs: []Subscription{}, expectedStatus: http.StatusAccepted, asyncExpectedStatus: http.StatusAccepted, }, "empty sub succeeds": { - subs: []eventingduck.SubscriberSpec{ + subs: []Subscription{ {}, }, expectedStatus: http.StatusAccepted, asyncExpectedStatus: http.StatusAccepted, }, "reply fails": { - subs: []eventingduck.SubscriberSpec{ + subs: []Subscription{ { - ReplyURI: replaceReplier, + SubscriberSpec: eventingduck.SubscriberSpec{ + ReplyURI: replaceReplier, + }, }, }, replier: func(writer http.ResponseWriter, _ *http.Request) { @@ -114,9 +118,11 @@ func TestFanoutMessageHandler_ServeHTTP(t *testing.T) { asyncExpectedStatus: http.StatusAccepted, }, "subscriber fails": { - subs: []eventingduck.SubscriberSpec{ + subs: []Subscription{ { - SubscriberURI: replaceSubscriber, + SubscriberSpec: eventingduck.SubscriberSpec{ + SubscriberURI: replaceSubscriber, + }, }, }, subscriber: func(writer http.ResponseWriter, _ *http.Request) { @@ -127,10 +133,12 @@ func TestFanoutMessageHandler_ServeHTTP(t *testing.T) { asyncExpectedStatus: http.StatusAccepted, }, "subscriber succeeds, result fails": { - subs: []eventingduck.SubscriberSpec{ + subs: []Subscription{ { - SubscriberURI: replaceSubscriber, - ReplyURI: replaceReplier, + SubscriberSpec: eventingduck.SubscriberSpec{ + SubscriberURI: replaceSubscriber, + ReplyURI: replaceReplier, + }, }, }, subscriber: callableSucceed, @@ -143,10 +151,12 @@ func TestFanoutMessageHandler_ServeHTTP(t *testing.T) { asyncExpectedStatus: http.StatusAccepted, }, "one sub succeeds": { - subs: []eventingduck.SubscriberSpec{ + subs: []Subscription{ { - SubscriberURI: replaceSubscriber, - ReplyURI: replaceReplier, + SubscriberSpec: eventingduck.SubscriberSpec{ + SubscriberURI: replaceSubscriber, + ReplyURI: replaceReplier, + }, }, }, subscriber: callableSucceed, @@ -159,14 +169,18 @@ func TestFanoutMessageHandler_ServeHTTP(t *testing.T) { asyncExpectedStatus: http.StatusAccepted, }, "one sub succeeds, one sub fails": { - subs: []eventingduck.SubscriberSpec{ + subs: []Subscription{ { - SubscriberURI: replaceSubscriber, - ReplyURI: replaceReplier, + SubscriberSpec: eventingduck.SubscriberSpec{ + SubscriberURI: replaceSubscriber, + ReplyURI: replaceReplier, + }, }, { - SubscriberURI: replaceSubscriber, - ReplyURI: replaceReplier, + SubscriberSpec: eventingduck.SubscriberSpec{ + SubscriberURI: replaceSubscriber, + ReplyURI: replaceReplier, + }, }, }, subscriber: callableSucceed, @@ -177,18 +191,24 @@ func TestFanoutMessageHandler_ServeHTTP(t *testing.T) { asyncExpectedStatus: http.StatusAccepted, }, "all subs succeed": { - subs: []eventingduck.SubscriberSpec{ + subs: []Subscription{ { - SubscriberURI: replaceSubscriber, - ReplyURI: replaceReplier, + SubscriberSpec: eventingduck.SubscriberSpec{ + SubscriberURI: replaceSubscriber, + ReplyURI: replaceReplier, + }, }, { - SubscriberURI: replaceSubscriber, - ReplyURI: replaceReplier, + SubscriberSpec: eventingduck.SubscriberSpec{ + SubscriberURI: replaceSubscriber, + ReplyURI: replaceReplier, + }, }, { - SubscriberURI: replaceSubscriber, - ReplyURI: replaceReplier, + SubscriberSpec: eventingduck.SubscriberSpec{ + SubscriberURI: replaceSubscriber, + ReplyURI: replaceReplier, + }, }, }, subscriber: callableSucceed, @@ -211,7 +231,7 @@ func TestFanoutMessageHandler_ServeHTTP(t *testing.T) { } } -func testFanoutMessageHandler(t *testing.T, async bool, receiverFunc channel.UnbufferedMessageReceiverFunc, timeout time.Duration, inSubs []eventingduck.SubscriberSpec, subscriberHandler func(http.ResponseWriter, *http.Request), subscriberReqs int, replierHandler func(http.ResponseWriter, *http.Request), replierReqs int, expectedStatus int) { +func testFanoutMessageHandler(t *testing.T, async bool, receiverFunc channel.UnbufferedMessageReceiverFunc, timeout time.Duration, inSubs []Subscription, subscriberHandler func(http.ResponseWriter, *http.Request), subscriberReqs int, replierHandler func(http.ResponseWriter, *http.Request), replierReqs int, expectedStatus int) { var subscriberServerWg *sync.WaitGroup if subscriberReqs != 0 { subscriberServerWg = &sync.WaitGroup{} @@ -235,7 +255,7 @@ func testFanoutMessageHandler(t *testing.T, async bool, receiverFunc channel.Unb defer replyServer.Close() // Rewrite the subs to use the servers we just started. - subs := make([]eventingduck.SubscriberSpec, 0) + subs := make([]Subscription, 0) for _, sub := range inSubs { if sub.SubscriberURI == replaceSubscriber { sub.SubscriberURI = apis.HTTP(subscriberServer.URL[7:]) // strip the leading 'http://' diff --git a/pkg/channel/message_dispatcher.go b/pkg/channel/message_dispatcher.go index c68c30e9cc8..76c59e808f7 100644 --- a/pkg/channel/message_dispatcher.go +++ b/pkg/channel/message_dispatcher.go @@ -39,6 +39,11 @@ type MessageDispatcher interface { // // The destination and reply are URLs. DispatchMessage(ctx context.Context, message cloudevents.Message, additionalHeaders nethttp.Header, destination *url.URL, reply *url.URL, deadLetter *url.URL) error + + // DispatchMessageWithRetries dispatches an event to a destination over HTTP. + // + // The destination and reply are URLs. + DispatchMessageWithRetries(ctx context.Context, message cloudevents.Message, additionalHeaders nethttp.Header, destination *url.URL, reply *url.URL, deadLetter *url.URL, config *kncloudevents.RetryConfig) error } // MessageDispatcherImpl is the 'real' MessageDispatcher used everywhere except unit tests. @@ -63,12 +68,11 @@ func NewMessageDispatcherFromConfig(logger *zap.Logger, config EventDispatcherCo sender, err := kncloudevents.NewHttpMessageSender(&config.ConnectionArgs, "") if err != nil { logger.Fatal("Unable to create cloudevents binding sender", zap.Error(err)) - return nil } return NewMessageDispatcherFromSender(logger, sender) } -// NewMessageDispatcherFromConfig creates a new Message dispatcher from a sender. +// NewMessageDispatcherFromConfig creates a new event dispatcher. func NewMessageDispatcherFromSender(logger *zap.Logger, sender *kncloudevents.HttpMessageSender) *MessageDispatcherImpl { return &MessageDispatcherImpl{ sender: sender, @@ -77,7 +81,12 @@ func NewMessageDispatcherFromSender(logger *zap.Logger, sender *kncloudevents.Ht } } -func (d *MessageDispatcherImpl) DispatchMessage(ctx context.Context, initialMessage cloudevents.Message, initialAdditionalHeaders nethttp.Header, destination *url.URL, reply *url.URL, deadLetter *url.URL) error { +func (d *MessageDispatcherImpl) DispatchMessage(ctx context.Context, message cloudevents.Message, additionalHeaders nethttp.Header, destination *url.URL, reply *url.URL, deadLetter *url.URL) error { + return d.DispatchMessageWithRetries(ctx, message, additionalHeaders, destination, reply, deadLetter, nil) +} + +func (d *MessageDispatcherImpl) DispatchMessageWithRetries(ctx context.Context, message cloudevents.Message, additionalHeaders nethttp.Header, destination *url.URL, reply *url.URL, deadLetter *url.URL, retriesConfig *kncloudevents.RetryConfig) error { + // All messages that should be finished at the end of this function // are placed in this slice var messagesToFinish []binding.Message @@ -100,13 +109,14 @@ func (d *MessageDispatcherImpl) DispatchMessage(ctx context.Context, initialMess if destination != nil { var err error // Try to send to destination - messagesToFinish = append(messagesToFinish, initialMessage) + messagesToFinish = append(messagesToFinish, message) - ctx, responseMessage, responseAdditionalHeaders, err = d.executeRequest(ctx, destination, initialMessage, initialAdditionalHeaders) + ctx, responseMessage, responseAdditionalHeaders, err = d.executeRequest(ctx, destination, message, additionalHeaders, retriesConfig) if err != nil { // DeadLetter is configured, send the message to it if deadLetter != nil { - _, deadLetterResponse, _, deadLetterErr := d.executeRequest(ctx, deadLetter, initialMessage, initialAdditionalHeaders) + + _, deadLetterResponse, _, deadLetterErr := d.executeRequest(ctx, deadLetter, message, additionalHeaders, retriesConfig) if deadLetterErr != nil { return fmt.Errorf("unable to complete request to either %s (%v) or %s (%v)", destination, err, deadLetter, deadLetterErr) } @@ -121,8 +131,8 @@ func (d *MessageDispatcherImpl) DispatchMessage(ctx context.Context, initialMess } } else { // No destination url, try to send to reply if available - responseMessage = initialMessage - responseAdditionalHeaders = initialAdditionalHeaders + responseMessage = message + responseAdditionalHeaders = additionalHeaders } // No response, dispatch completed @@ -137,11 +147,11 @@ func (d *MessageDispatcherImpl) DispatchMessage(ctx context.Context, initialMess return nil } - ctx, responseResponseMessage, _, err := d.executeRequest(ctx, reply, responseMessage, responseAdditionalHeaders) + ctx, responseResponseMessage, _, err := d.executeRequest(ctx, reply, responseMessage, responseAdditionalHeaders, retriesConfig) if err != nil { // DeadLetter is configured, send the message to it if deadLetter != nil { - _, deadLetterResponse, _, deadLetterErr := d.executeRequest(ctx, deadLetter, initialMessage, responseAdditionalHeaders) + _, deadLetterResponse, _, deadLetterErr := d.executeRequest(ctx, deadLetter, message, responseAdditionalHeaders, retriesConfig) if deadLetterErr != nil { return fmt.Errorf("failed to forward reply to %s (%v) and failed to send it to the dead letter sink %s (%v)", reply, err, deadLetter, deadLetterErr) } @@ -161,7 +171,7 @@ func (d *MessageDispatcherImpl) DispatchMessage(ctx context.Context, initialMess return nil } -func (d *MessageDispatcherImpl) executeRequest(ctx context.Context, url *url.URL, message cloudevents.Message, additionalHeaders nethttp.Header) (context.Context, cloudevents.Message, nethttp.Header, error) { +func (d *MessageDispatcherImpl) executeRequest(ctx context.Context, url *url.URL, message cloudevents.Message, additionalHeaders nethttp.Header, configs *kncloudevents.RetryConfig) (context.Context, cloudevents.Message, nethttp.Header, error) { d.logger.Debug("Dispatching event", zap.String("url", url.String())) ctx, span := trace.StartSpan(ctx, "knative.dev", trace.WithSpanKind(trace.SpanKindClient)) @@ -181,7 +191,7 @@ func (d *MessageDispatcherImpl) executeRequest(ctx context.Context, url *url.URL return ctx, nil, nil, err } - response, err := d.sender.Send(req) + response, err := d.sender.SendWithRetries(req, configs) if err != nil { return ctx, nil, nil, err } diff --git a/pkg/channel/multichannelfanout/multi_channel_fanout_message_handler.go b/pkg/channel/multichannelfanout/multi_channel_fanout_message_handler.go index 73acf48487c..9284e988a58 100644 --- a/pkg/channel/multichannelfanout/multi_channel_fanout_message_handler.go +++ b/pkg/channel/multichannelfanout/multi_channel_fanout_message_handler.go @@ -29,12 +29,15 @@ import ( "context" "fmt" "net/http" + "reflect" + "time" "github.com/google/go-cmp/cmp" "go.uber.org/zap" "knative.dev/eventing/pkg/channel" "knative.dev/eventing/pkg/channel/fanout" + "knative.dev/eventing/pkg/kncloudevents" ) // makeChannelKeyFromConfig creates the channel key for a given channelConfig. It is a helper around @@ -81,7 +84,16 @@ func NewMessageHandler(ctx context.Context, logger *zap.Logger, messageDispatche // empty string is returned. If there are differences, then a non-empty string is returned // describing the differences. func (h *MessageHandler) ConfigDiff(updated Config) string { - return cmp.Diff(h.config, updated) + return cmp.Diff(h.config, updated, ignoreCheckRetryAndBackFunctions()) +} + +func ignoreCheckRetryAndBackFunctions() cmp.Option { + return cmp.FilterPath(func(path cmp.Path) bool { + // is of type kncloudevents.Backoff? + return path.Last().Type().Kind() == reflect.TypeOf(kncloudevents.Backoff(func(attemptNum int, resp *http.Response) time.Duration { return 0 })).Kind() || + // is of type kncloudevents.CheckRetry? + path.Last().Type().Kind() == reflect.TypeOf(kncloudevents.CheckRetry(func(ctx context.Context, resp *http.Response, err error) (bool, error) { return false, nil })).Kind() + }, cmp.Ignore()) } // CopyWithNewConfig creates a new copy of this Handler with all the fields identical, except the diff --git a/pkg/channel/multichannelfanout/multi_channel_fanout_message_handler_test.go b/pkg/channel/multichannelfanout/multi_channel_fanout_message_handler_test.go index 9de7b0bfe1c..83850e2f9ad 100644 --- a/pkg/channel/multichannelfanout/multi_channel_fanout_message_handler_test.go +++ b/pkg/channel/multichannelfanout/multi_channel_fanout_message_handler_test.go @@ -92,9 +92,11 @@ func TestCopyMessageHandlerWithNewConfig(t *testing.T) { Namespace: "default", Name: "c1", FanoutConfig: fanout.Config{ - Subscriptions: []eventingduck.SubscriberSpec{ + Subscriptions: []fanout.Subscription{ { - SubscriberURI: apis.HTTP("subscriberdomain"), + SubscriberSpec: eventingduck.SubscriberSpec{ + SubscriberURI: apis.HTTP("subscriberdomain"), + }, }, }, }, @@ -107,9 +109,11 @@ func TestCopyMessageHandlerWithNewConfig(t *testing.T) { Namespace: "default", Name: "somethingdifferent", FanoutConfig: fanout.Config{ - Subscriptions: []eventingduck.SubscriberSpec{ + Subscriptions: []fanout.Subscription{ { - ReplyURI: apis.HTTP("replydomain"), + SubscriberSpec: eventingduck.SubscriberSpec{ + ReplyURI: apis.HTTP("replydomain"), + }, }, }, }, @@ -129,18 +133,23 @@ func TestCopyMessageHandlerWithNewConfig(t *testing.T) { if err != nil { t.Errorf("Unable to create handler, %v", err) } - if !cmp.Equal(h.config, orig) { - t.Errorf("Incorrect config. Expected '%v'. Actual '%v'", orig, h.config) + + option := ignoreCheckRetryAndBackFunctions() + if diff := cmp.Diff(h.config, orig, option); diff != "" { + t.Errorf("Incorrect config. Expected '%v'. Actual '%v' - diff: %s", orig, h.config, diff) } + newH, err := h.CopyWithNewConfig(context.TODO(), channel.EventDispatcherConfig{}, updated) if err != nil { t.Errorf("Unable to copy handler: %v", err) } + if h.logger != newH.logger { t.Errorf("Did not copy logger") } - if !cmp.Equal(newH.config, updated) { - t.Errorf("Incorrect copied config. Expected '%v'. Actual '%v'", updated, newH.config) + + if diff := cmp.Diff(newH.config, updated, option); diff != "" { + t.Errorf("Incorrect copied config. Expected '%v'. Actual '%v' - diff: %s", updated, newH.config, diff) } } @@ -151,9 +160,11 @@ func TestConfigDiffMessageHandler(t *testing.T) { Namespace: "default", Name: "c1", FanoutConfig: fanout.Config{ - Subscriptions: []eventingduck.SubscriberSpec{ + Subscriptions: []fanout.Subscription{ { - SubscriberURI: apis.HTTP("subscriberdomain"), + SubscriberSpec: eventingduck.SubscriberSpec{ + SubscriberURI: apis.HTTP("subscriberdomain"), + }, }, }, }, @@ -181,9 +192,11 @@ func TestConfigDiffMessageHandler(t *testing.T) { Namespace: "default", Name: "c1", FanoutConfig: fanout.Config{ - Subscriptions: []eventingduck.SubscriberSpec{ + Subscriptions: []fanout.Subscription{ { - SubscriberURI: apis.HTTP("different"), + SubscriberSpec: eventingduck.SubscriberSpec{ + SubscriberURI: apis.HTTP("different"), + }, }, }, }, @@ -203,7 +216,7 @@ func TestConfigDiffMessageHandler(t *testing.T) { diff := h.ConfigDiff(tc.updated) if hasDiff := diff != ""; hasDiff != tc.expectedDiff { - t.Errorf("Unexpected diff result. Expected %v. Actual %v", tc.expectedDiff, hasDiff) + t.Errorf("Unexpected diff result. Expected %v. Actual %v - diff: %s", tc.expectedDiff, hasDiff, diff) } }) } @@ -235,9 +248,11 @@ func TestServeHTTPMessageHandler(t *testing.T) { Name: "name", HostName: "first-channel.default", FanoutConfig: fanout.Config{ - Subscriptions: []eventingduck.SubscriberSpec{ + Subscriptions: []fanout.Subscription{ { - ReplyURI: replaceDomain, + SubscriberSpec: eventingduck.SubscriberSpec{ + ReplyURI: replaceDomain, + }, }, }, }, @@ -257,9 +272,11 @@ func TestServeHTTPMessageHandler(t *testing.T) { Name: "name", HostName: "first-channel.default", FanoutConfig: fanout.Config{ - Subscriptions: []eventingduck.SubscriberSpec{ + Subscriptions: []fanout.Subscription{ { - ReplyURI: apis.HTTP("first-to-domain"), + SubscriberSpec: eventingduck.SubscriberSpec{ + ReplyURI: apis.HTTP("first-to-domain"), + }, }, }, }, @@ -269,9 +286,11 @@ func TestServeHTTPMessageHandler(t *testing.T) { Name: "second-channel", HostName: "second-channel.default", FanoutConfig: fanout.Config{ - Subscriptions: []eventingduck.SubscriberSpec{ + Subscriptions: []fanout.Subscription{ { - SubscriberURI: replaceDomain, + SubscriberSpec: eventingduck.SubscriberSpec{ + SubscriberURI: replaceDomain, + }, }, }, }, diff --git a/pkg/channel/multichannelfanout/parse_test.go b/pkg/channel/multichannelfanout/parse_test.go index dbe83f862d1..511e52202d4 100644 --- a/pkg/channel/multichannelfanout/parse_test.go +++ b/pkg/channel/multichannelfanout/parse_test.go @@ -89,16 +89,22 @@ func TestConfigMapData(t *testing.T) { Namespace: "default", Name: "c1", FanoutConfig: fanout.Config{ - Subscriptions: []eventingduck.SubscriberSpec{ + Subscriptions: []fanout.Subscription{ { - SubscriberURI: apis.HTTP("event-changer.default.svc." + utils.GetClusterDomainName()), - ReplyURI: apis.HTTP("message-dumper-bar.default.svc." + utils.GetClusterDomainName()), + SubscriberSpec: eventingduck.SubscriberSpec{ + SubscriberURI: apis.HTTP("event-changer.default.svc." + utils.GetClusterDomainName()), + ReplyURI: apis.HTTP("message-dumper-bar.default.svc." + utils.GetClusterDomainName()), + }, }, { - SubscriberURI: apis.HTTP("message-dumper-foo.default.svc." + utils.GetClusterDomainName()), + SubscriberSpec: eventingduck.SubscriberSpec{ + SubscriberURI: apis.HTTP("message-dumper-foo.default.svc." + utils.GetClusterDomainName()), + }, }, { - ReplyURI: apis.HTTP("message-dumper-bar.default.svc." + utils.GetClusterDomainName()), + SubscriberSpec: eventingduck.SubscriberSpec{ + ReplyURI: apis.HTTP("message-dumper-bar.default.svc." + utils.GetClusterDomainName()), + }, }, }, }, @@ -107,9 +113,11 @@ func TestConfigMapData(t *testing.T) { Namespace: "default", Name: "c2", FanoutConfig: fanout.Config{ - Subscriptions: []eventingduck.SubscriberSpec{ + Subscriptions: []fanout.Subscription{ { - ReplyURI: apis.HTTP("message-dumper-foo.default.svc." + utils.GetClusterDomainName()), + SubscriberSpec: eventingduck.SubscriberSpec{ + ReplyURI: apis.HTTP("message-dumper-foo.default.svc." + utils.GetClusterDomainName()), + }, }, }, }, @@ -118,9 +126,11 @@ func TestConfigMapData(t *testing.T) { Namespace: "other", Name: "c3", FanoutConfig: fanout.Config{ - Subscriptions: []eventingduck.SubscriberSpec{ + Subscriptions: []fanout.Subscription{ { - ReplyURI: apis.HTTP("message-dumper-foo.default.svc." + utils.GetClusterDomainName()), + SubscriberSpec: eventingduck.SubscriberSpec{ + ReplyURI: apis.HTTP("message-dumper-foo.default.svc." + utils.GetClusterDomainName()), + }, }, }, }, @@ -141,8 +151,8 @@ func TestConfigMapData(t *testing.T) { } return } - if !cmp.Equal(c, tc.expected) { - t.Errorf("Unexpected config. Expected '%v'. Actual '%v'.", tc.expected, c) + if diff := cmp.Diff(c, tc.expected); diff != "" { + t.Errorf("Unexpected config. Expected '%v'. Actual '%v'. - diff: %s", tc.expected, c, diff) } }) } diff --git a/pkg/channel/swappable/swappable_message_handler_test.go b/pkg/channel/swappable/swappable_message_handler_test.go index 4f796ea7c06..9250de335cc 100644 --- a/pkg/channel/swappable/swappable_message_handler_test.go +++ b/pkg/channel/swappable/swappable_message_handler_test.go @@ -52,9 +52,11 @@ func TestMessageHandler(t *testing.T) { { HostName: hostName, FanoutConfig: fanout.Config{ - Subscriptions: []eventingduck.SubscriberSpec{ + Subscriptions: []fanout.Subscription{ { - SubscriberURI: replaceDomain, + SubscriberSpec: eventingduck.SubscriberSpec{ + SubscriberURI: replaceDomain, + }, }, }, }, @@ -66,9 +68,11 @@ func TestMessageHandler(t *testing.T) { { HostName: hostName, FanoutConfig: fanout.Config{ - Subscriptions: []eventingduck.SubscriberSpec{ + Subscriptions: []fanout.Subscription{ { - ReplyURI: replaceDomain, + SubscriberSpec: eventingduck.SubscriberSpec{ + ReplyURI: replaceDomain, + }, }, }, }, @@ -103,9 +107,11 @@ func TestMessageHandler_InvalidConfigChange(t *testing.T) { { HostName: hostName, FanoutConfig: fanout.Config{ - Subscriptions: []eventingduck.SubscriberSpec{ + Subscriptions: []fanout.Subscription{ { - SubscriberURI: replaceDomain, + SubscriberSpec: eventingduck.SubscriberSpec{ + SubscriberURI: replaceDomain, + }, }, }, }, diff --git a/pkg/inmemorychannel/message_dispatcher_benchmark_test.go b/pkg/inmemorychannel/message_dispatcher_benchmark_test.go index e44fe604c35..e9157dd7c3f 100644 --- a/pkg/inmemorychannel/message_dispatcher_benchmark_test.go +++ b/pkg/inmemorychannel/message_dispatcher_benchmark_test.go @@ -72,11 +72,13 @@ func BenchmarkDispatcher_dispatch_ok_through_2_channels(b *testing.B) { HostName: "channela.svc", FanoutConfig: fanout.Config{ AsyncHandler: false, - Subscriptions: []eventingduck.SubscriberSpec{{ - UID: "aaaa", - Generation: 1, - SubscriberURI: transformationsUrl, - ReplyURI: channelBUrl, + Subscriptions: []fanout.Subscription{{ + SubscriberSpec: eventingduck.SubscriberSpec{ + UID: "aaaa", + Generation: 1, + SubscriberURI: transformationsUrl, + ReplyURI: channelBUrl, + }, }}, }, }, @@ -86,10 +88,12 @@ func BenchmarkDispatcher_dispatch_ok_through_2_channels(b *testing.B) { HostName: "channelb.svc", FanoutConfig: fanout.Config{ AsyncHandler: false, - Subscriptions: []eventingduck.SubscriberSpec{{ - UID: "bbbb", - Generation: 1, - SubscriberURI: receiverUrl, + Subscriptions: []fanout.Subscription{{ + SubscriberSpec: eventingduck.SubscriberSpec{ + UID: "bbbb", + Generation: 1, + SubscriberURI: receiverUrl, + }, }}, }, }, diff --git a/pkg/inmemorychannel/message_dispatcher_test.go b/pkg/inmemorychannel/message_dispatcher_test.go index 25698991b5b..894c8f88996 100644 --- a/pkg/inmemorychannel/message_dispatcher_test.go +++ b/pkg/inmemorychannel/message_dispatcher_test.go @@ -231,18 +231,22 @@ func TestDispatcher_dispatch(t *testing.T) { HostName: "channela.svc", FanoutConfig: fanout.Config{ AsyncHandler: false, - Subscriptions: []eventingduck.SubscriberSpec{{ - UID: "aaaa", - Generation: 1, - SubscriberURI: mustParseUrl(t, transformationsServer.URL), - ReplyURI: mustParseUrl(t, channelBProxy.URL), + Subscriptions: []fanout.Subscription{{ + SubscriberSpec: eventingduck.SubscriberSpec{ + UID: "aaaa", + Generation: 1, + SubscriberURI: mustParseUrl(t, transformationsServer.URL), + ReplyURI: mustParseUrl(t, channelBProxy.URL), + }, }, { - UID: "cccc", - Generation: 1, - SubscriberURI: mustParseUrl(t, transformationsFailureServer.URL), - ReplyURI: mustParseUrl(t, channelBProxy.URL), - Delivery: &eventingduck.DeliverySpec{ - DeadLetterSink: &duckv1.Destination{URI: mustParseUrl(t, deadLetterServer.URL)}, + SubscriberSpec: eventingduck.SubscriberSpec{ + UID: "cccc", + Generation: 1, + SubscriberURI: mustParseUrl(t, transformationsFailureServer.URL), + ReplyURI: mustParseUrl(t, channelBProxy.URL), + Delivery: &eventingduck.DeliverySpec{ + DeadLetterSink: &duckv1.Destination{URI: mustParseUrl(t, deadLetterServer.URL)}, + }, }, }}, }, @@ -253,10 +257,12 @@ func TestDispatcher_dispatch(t *testing.T) { HostName: "channelb.svc", FanoutConfig: fanout.Config{ AsyncHandler: false, - Subscriptions: []eventingduck.SubscriberSpec{{ - UID: "bbbb", - Generation: 1, - SubscriberURI: mustParseUrl(t, receiverServer.URL), + Subscriptions: []fanout.Subscription{{ + SubscriberSpec: eventingduck.SubscriberSpec{ + UID: "bbbb", + Generation: 1, + SubscriberURI: mustParseUrl(t, receiverServer.URL), + }, }}, }, }, diff --git a/pkg/kncloudevents/message_sender.go b/pkg/kncloudevents/message_sender.go index cc6e757f3d7..96506b28aef 100644 --- a/pkg/kncloudevents/message_sender.go +++ b/pkg/kncloudevents/message_sender.go @@ -18,12 +18,34 @@ package kncloudevents import ( "context" + "fmt" + "math" nethttp "net/http" + "time" + "github.com/hashicorp/go-retryablehttp" + "github.com/rickb777/date/period" "go.opencensus.io/plugin/ochttp" "knative.dev/pkg/tracing/propagation/tracecontextb3" + + duckv1 "knative.dev/eventing/pkg/apis/duck/v1" +) + +const ( + defaultRetryWaitMin = 1 * time.Second + defaultRetryWaitMax = 30 * time.Second ) +var noRetries = RetryConfig{ + RetryMax: 0, + CheckRetry: func(ctx context.Context, resp *nethttp.Response, err error) (bool, error) { + return false, nil + }, + Backoff: func(attemptNum int, resp *nethttp.Response) time.Duration { + return 0 + }, +} + // ConnectionArgs allow to configure connection parameters to the underlying // HTTP Client transport. type ConnectionArgs struct { @@ -72,3 +94,85 @@ func (s *HttpMessageSender) NewCloudEventRequestWithTarget(ctx context.Context, func (s *HttpMessageSender) Send(req *nethttp.Request) (*nethttp.Response, error) { return s.Client.Do(req) } + +// CheckRetry specifies a policy for handling retries. It is called +// following each request with the response and error values returned by +// the http.Client. If CheckRetry returns false, the Client stops retrying +// and returns the response to the caller. If CheckRetry returns an error, +// that error value is returned in lieu of the error from the request. The +// Client will close any response body when retrying, but if the retry is +// aborted it is up to the CheckRetry callback to properly close any +// response body before returning. +type CheckRetry func(ctx context.Context, resp *nethttp.Response, err error) (bool, error) + +// Backoff specifies a policy for how long to wait between retries. +// It is called after a failing request to determine the amount of time +// that should pass before trying again. +type Backoff func(attemptNum int, resp *nethttp.Response) time.Duration + +type RetryConfig struct { + + // Maximum number of retries + RetryMax int + + CheckRetry CheckRetry + + Backoff Backoff +} + +func (s *HttpMessageSender) SendWithRetries(req *nethttp.Request, config *RetryConfig) (*nethttp.Response, error) { + if config == nil { + return s.Send(req) + } + + retryableClient := retryablehttp.Client{ + HTTPClient: s.Client, + RetryWaitMin: defaultRetryWaitMin, + RetryWaitMax: defaultRetryWaitMax, + RetryMax: config.RetryMax, + CheckRetry: retryablehttp.CheckRetry(config.CheckRetry), + Backoff: func(_, _ time.Duration, attemptNum int, resp *nethttp.Response) time.Duration { + return config.Backoff(attemptNum, resp) + }, + ErrorHandler: func(resp *nethttp.Response, err error, numTries int) (*nethttp.Response, error) { + return resp, err + }, + } + + return retryableClient.Do(&retryablehttp.Request{Request: req}) +} + +func NoRetries() RetryConfig { + return noRetries +} + +func RetryConfigFromDeliverySpec(spec duckv1.DeliverySpec) (RetryConfig, error) { + + retryConfig := NoRetries() + + if spec.Retry != nil { + retryConfig.RetryMax = int(*spec.Retry) + } + + if spec.BackoffPolicy != nil && spec.BackoffDelay != nil { + + delay, err := period.Parse(*spec.BackoffDelay) + if err != nil { + return retryConfig, fmt.Errorf("failed to parse Spec.BackoffDelay: %w", err) + } + + delayDuration, _ := delay.Duration() + switch *spec.BackoffPolicy { + case duckv1.BackoffPolicyExponential: + retryConfig.Backoff = func(attemptNum int, resp *nethttp.Response) time.Duration { + return time.Duration(math.Pow(float64(delayDuration*2), float64(attemptNum))) + } + case duckv1.BackoffPolicyLinear: + retryConfig.Backoff = func(attemptNum int, resp *nethttp.Response) time.Duration { + return delayDuration + } + } + } + + return retryConfig, nil +} diff --git a/pkg/reconciler/inmemorychannel/dispatcher/inmemorychannel.go b/pkg/reconciler/inmemorychannel/dispatcher/inmemorychannel.go index a8d097f2ab7..22079289e20 100644 --- a/pkg/reconciler/inmemorychannel/dispatcher/inmemorychannel.go +++ b/pkg/reconciler/inmemorychannel/dispatcher/inmemorychannel.go @@ -18,18 +18,21 @@ package dispatcher import ( "context" + "fmt" "knative.dev/pkg/reconciler" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/tools/cache" + eventingduckv1 "knative.dev/eventing/pkg/apis/duck/v1" "knative.dev/eventing/pkg/apis/messaging/v1beta1" "knative.dev/eventing/pkg/channel" "knative.dev/eventing/pkg/channel/fanout" "knative.dev/eventing/pkg/channel/multichannelfanout" listers "knative.dev/eventing/pkg/client/listers/messaging/v1beta1" "knative.dev/eventing/pkg/inmemorychannel" + "knative.dev/eventing/pkg/kncloudevents" "knative.dev/eventing/pkg/logging" ) @@ -59,7 +62,11 @@ func (r *Reconciler) ReconcileKind(ctx context.Context, imc *v1beta1.InMemoryCha } } - config := r.newConfigFromInMemoryChannels(inmemoryChannels) + config, err := r.newConfigFromInMemoryChannels(inmemoryChannels) + if err != nil { + logging.FromContext(ctx).Error("Error creating config from in memory channels") + return err + } err = r.dispatcher.UpdateConfig(ctx, r.eventDispatcherConfigStore.GetConfig(), config) if err != nil { logging.FromContext(ctx).Error("Error updating InMemory dispatcher config") @@ -70,21 +77,45 @@ func (r *Reconciler) ReconcileKind(ctx context.Context, imc *v1beta1.InMemoryCha } // newConfigFromInMemoryChannels creates a new Config from the list of inmemory channels. -func (r *Reconciler) newConfigFromInMemoryChannels(channels []*v1beta1.InMemoryChannel) *multichannelfanout.Config { +func (r *Reconciler) newConfigFromInMemoryChannels(channels []*v1beta1.InMemoryChannel) (*multichannelfanout.Config, error) { cc := make([]multichannelfanout.ChannelConfig, 0) for _, c := range channels { + + subs := make([]fanout.Subscription, len(c.Spec.Subscribers)) + for _, sub := range c.Spec.Subscribers { + + retriesConfig := kncloudevents.NoRetries() + if sub.Delivery != nil && sub.Delivery.Retry != nil { + delivery := eventingduckv1.DeliverySpec{} + err := sub.Delivery.ConvertTo(context.Background(), &delivery) + if err != nil { + return nil, err + } + _retriesConfig, err := kncloudevents.RetryConfigFromDeliverySpec(delivery) + if err != nil { + return nil, fmt.Errorf("failed to create retries config: %w", err) + } + retriesConfig = _retriesConfig + } + + subs = append(subs, fanout.Subscription{ + SubscriberSpec: sub, + RetriesConfig: retriesConfig, + }) + } + channelConfig := multichannelfanout.ChannelConfig{ Namespace: c.Namespace, Name: c.Name, HostName: c.Status.Address.URL.Host, FanoutConfig: fanout.Config{ AsyncHandler: true, - Subscriptions: c.Spec.Subscribers, + Subscriptions: subs, }, } cc = append(cc, channelConfig) } return &multichannelfanout.Config{ ChannelConfigs: cc, - } + }, nil } diff --git a/pkg/reconciler/inmemorychannel/dispatcher/inmemorychannel_test.go b/pkg/reconciler/inmemorychannel/dispatcher/inmemorychannel_test.go index a7e7f57bd93..1900d0f29d5 100644 --- a/pkg/reconciler/inmemorychannel/dispatcher/inmemorychannel_test.go +++ b/pkg/reconciler/inmemorychannel/dispatcher/inmemorychannel_test.go @@ -20,6 +20,9 @@ import ( "context" "testing" + "k8s.io/utils/pointer" + duckv1 "knative.dev/pkg/apis/duck/v1" + "knative.dev/eventing/pkg/channel" fakeeventingclient "knative.dev/eventing/pkg/client/injection/client/fake" "knative.dev/eventing/pkg/client/injection/reconciler/messaging/v1beta1/inmemorychannel" @@ -63,6 +66,8 @@ func TestAllCases(t *testing.T) { ReplyURI: apis.HTTP("sink2"), }} + backoffPolicy := duckv1beta1.BackoffPolicyLinear + imcKey := testNS + "/" + imcName table := TableTest{ { @@ -100,7 +105,36 @@ func TestAllCases(t *testing.T) { WithInMemoryChannelAddress(channelServiceAddress)), }, WantErr: false, - }, {}, + }, { + Name: "subscriber with delivery spec", + Key: imcKey, + Objects: []runtime.Object{ + NewInMemoryChannel(imcName, testNS, + WithInitInMemoryChannelConditions, + WithInMemoryChannelDeploymentReady(), + WithInMemoryChannelServiceReady(), + WithInMemoryChannelEndpointsReady(), + WithInMemoryChannelChannelServiceReady(), + WithInMemoryChannelSubscribers([]duckv1beta1.SubscriberSpec{ + { + UID: "2f9b5e8e-deb6-11e8-9f32-f2801f1b9fd1", + Generation: 1, + SubscriberURI: apis.HTTP("call1"), + ReplyURI: apis.HTTP("sink2"), + Delivery: &duckv1beta1.DeliverySpec{ + DeadLetterSink: &duckv1.Destination{ + URI: apis.HTTP("http://www.example.com"), + }, + Retry: pointer.Int32Ptr(10), + BackoffPolicy: &backoffPolicy, + BackoffDelay: pointer.StringPtr("PT1S"), + }, + }, + }), + WithInMemoryChannelAddress(channelServiceAddress)), + }, + WantErr: false, + }, } logger := logtesting.TestLogger(t) diff --git a/test/e2e/broker_redelivery_test.go b/test/e2e/broker_redelivery_test.go new file mode 100644 index 00000000000..ef0c6cc60b5 --- /dev/null +++ b/test/e2e/broker_redelivery_test.go @@ -0,0 +1,70 @@ +// +build e2e + +/* + * Copyright 2020 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package e2e + +import ( + "strings" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + + eventingduckv1beta1 "knative.dev/eventing/pkg/apis/duck/v1beta1" + "knative.dev/eventing/pkg/apis/eventing/v1beta1" + "knative.dev/eventing/test/e2e/helpers" + testlib "knative.dev/eventing/test/lib" + "knative.dev/eventing/test/lib/resources" +) + +// ChannelBasedBrokerCreator creates a BrokerCreator that creates a broker based on the channel parameter. +func ChannelBasedBrokerCreator(channel metav1.TypeMeta, brokerClass string) helpers.BrokerCreatorWithRetries { + return func(client *testlib.Client, numRetries int32) string { + brokerName := strings.ToLower(channel.Kind) + + // create a ConfigMap used by the broker. + config := client.CreateBrokerConfigMapOrFail("config-"+brokerName, &channel) + + backoff := eventingduckv1beta1.BackoffPolicyLinear + + // create a new broker. + client.CreateBrokerV1Beta1OrFail(brokerName, + resources.WithBrokerClassForBrokerV1Beta1(brokerClass), + resources.WithConfigForBrokerV1Beta1(config), + func(broker *v1beta1.Broker) { + broker.Spec.Delivery = &eventingduckv1beta1.DeliverySpec{ + Retry: &numRetries, + BackoffPolicy: &backoff, + BackoffDelay: pointer.StringPtr("PT1S"), + } + }, + ) + + return brokerName + } +} + +func TestBrokerRedelivery(t *testing.T) { + + channelTestRunner.RunTests(t, testlib.FeatureRedelivery, func(t *testing.T, component metav1.TypeMeta) { + + brokerCreator := ChannelBasedBrokerCreator(component, brokerClass) + + helpers.BrokerRedelivery(t, brokerCreator) + }) +} diff --git a/test/e2e/helpers/broker_redelivery_helper.go b/test/e2e/helpers/broker_redelivery_helper.go new file mode 100644 index 00000000000..32488302061 --- /dev/null +++ b/test/e2e/helpers/broker_redelivery_helper.go @@ -0,0 +1,123 @@ +/* + * Copyright 2020 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package helpers + +import ( + "fmt" + "testing" + + cloudevents "github.com/cloudevents/sdk-go/v2" + . "github.com/cloudevents/sdk-go/v2/test" + "github.com/google/uuid" + corev1 "k8s.io/api/core/v1" + + testlib "knative.dev/eventing/test/lib" + "knative.dev/eventing/test/lib/dropevents" + "knative.dev/eventing/test/lib/recordevents" + "knative.dev/eventing/test/lib/resources" +) + +// BrokerCreator creates a broker and returns its broker name. +type BrokerCreatorWithRetries func(client *testlib.Client, numRetries int32) string + +func BrokerRedelivery(t *testing.T, creator BrokerCreatorWithRetries) { + + numRetries := int32(5) + + t.Run(dropevents.Fibonacci, func(t *testing.T) { + brokerRedelivery(t, creator, numRetries, func(pod *corev1.Pod, client *testlib.Client) error { + container := pod.Spec.Containers[0] + container.Env = append(container.Env, + corev1.EnvVar{ + Name: dropevents.SkipAlgorithmKey, + Value: dropevents.Fibonacci, + }, + ) + return nil + }) + }) + + t.Run(dropevents.Sequence, func(t *testing.T) { + brokerRedelivery(t, creator, numRetries, func(pod *corev1.Pod, client *testlib.Client) error { + container := pod.Spec.Containers[0] + container.Env = append(container.Env, + corev1.EnvVar{ + Name: dropevents.SkipAlgorithmKey, + Value: dropevents.Sequence, + }, + corev1.EnvVar{ + Name: dropevents.NumberKey, + Value: fmt.Sprintf("%d", numRetries-1), + }, + ) + return nil + }) + }) +} + +func brokerRedelivery(t *testing.T, creator BrokerCreatorWithRetries, numRetries int32, options ...recordevents.EventRecordOption) { + + const ( + triggerName = "trigger" + eventRecord = "event-record" + senderName = "sender" + + eventType = "type" + eventSource = "http://source.com" + eventBody = `{"msg":"broker-redelivery"}` + ) + + client := testlib.Setup(t, true) + defer testlib.TearDown(client) + + // Create event tracker that should receive all events. + allEventTracker, _ := recordevents.StartEventRecordOrFail( + client, + eventRecord, + options..., + ) + defer allEventTracker.Cleanup() + + // Create a Broker. + brokerName := creator(client, numRetries) + + client.CreateTriggerV1OrFail( + triggerName, + resources.WithBrokerV1(brokerName), + resources.WithSubscriberServiceRefForTriggerV1(eventRecord), + ) + + client.WaitForAllTestResourcesReadyOrFail() + + // send CloudEvent to the broker + + eventToSend := cloudevents.NewEvent() + eventToSend.SetID(uuid.New().String()) + eventToSend.SetType(eventType) + eventToSend.SetSource(eventSource) + if err := eventToSend.SetData(cloudevents.ApplicationJSON, []byte(eventBody)); err != nil { + t.Fatalf("Cannot set the payload of the event: %s", err.Error()) + } + + client.SendEventToAddressable(senderName, brokerName, testlib.BrokerTypeMeta, eventToSend) + + allEventTracker.AssertAtLeast(1, recordevents.MatchEvent(AllOf( + HasSource(eventSource), + HasType(eventType), + HasData([]byte(eventBody)), + ))) +} diff --git a/test/lib/dropevents/dropeventsfibonacci/fibonacci.go b/test/lib/dropevents/dropeventsfibonacci/fibonacci.go new file mode 100644 index 00000000000..f0106cd2a06 --- /dev/null +++ b/test/lib/dropevents/dropeventsfibonacci/fibonacci.go @@ -0,0 +1,34 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dropeventsfibonacci + +type Fibonacci struct { + Prev uint64 + Current uint64 +} + +func (f *Fibonacci) Skip(counter uint64) bool { + if f.Current == counter { + f.Next() + return true + } + return false +} + +func (f *Fibonacci) Next() { + f.Prev, f.Current = f.Current, f.Prev+f.Current +} diff --git a/test/lib/dropevents/dropeventsfirst/sequence.go b/test/lib/dropevents/dropeventsfirst/sequence.go new file mode 100644 index 00000000000..090fa0ac308 --- /dev/null +++ b/test/lib/dropevents/dropeventsfirst/sequence.go @@ -0,0 +1,28 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dropeventsfirst + +type First struct { + N uint64 +} + +func (f First) Skip(counter uint64) bool { + if f.N >= counter { + return true + } + return false +} diff --git a/test/lib/dropevents/receiver.go b/test/lib/dropevents/receiver.go new file mode 100644 index 00000000000..4453b4f4714 --- /dev/null +++ b/test/lib/dropevents/receiver.go @@ -0,0 +1,75 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dropevents + +import ( + "os" + "strconv" + "sync" + + "knative.dev/eventing/test/lib/dropevents/dropeventsfibonacci" + "knative.dev/eventing/test/lib/dropevents/dropeventsfirst" +) + +const ( + Fibonacci = "fibonacci" + Sequence = "sequence" + SkipAlgorithmKey = "SKIP_ALGORITHM" + NumberKey = "NUMBER" +) + +func SkipperAlgorithm(algorithm string) Skipper { + + switch algorithm { + + case Fibonacci: + return &dropeventsfibonacci.Fibonacci{Prev: 1, Current: 1} + + case Sequence: + numberEnv, ok := os.LookupEnv(NumberKey) + var n int + if !ok { + n = 10 + } else { + n, _ = strconv.Atoi(numberEnv) + } + return dropeventsfirst.First{N: uint64(n)} + + default: + panic("unknown algorithm: " + algorithm) + } +} + +// Skipper represents the logic to apply to accept/reject events. +type Skipper interface { + // Skip returns true if the message must be rejected + Skip(counter uint64) bool +} + +type CounterHandler struct { + counter uint64 + Skipper Skipper + sync.Mutex +} + +func (h *CounterHandler) Skip() bool { + h.Lock() + defer h.Unlock() + + h.counter++ + return h.Skipper.Skip(h.counter) +} diff --git a/test/lib/recordevents/event_info_store.go b/test/lib/recordevents/event_info_store.go index 18cfc48d095..90af0ceaf3e 100644 --- a/test/lib/recordevents/event_info_store.go +++ b/test/lib/recordevents/event_info_store.go @@ -96,10 +96,12 @@ func NewEventInfoStore(client *testlib.Client, podName string) (*EventInfoStore, return ei, nil } +type EventRecordOption = func(*corev1.Pod, *testlib.Client) error + // Deploys a new recordevents pod and start the associated EventInfoStore -func StartEventRecordOrFail(client *testlib.Client, podName string) (*EventInfoStore, *corev1.Pod) { +func StartEventRecordOrFail(client *testlib.Client, podName string, options ...EventRecordOption) (*EventInfoStore, *corev1.Pod) { eventRecordPod := resources.EventRecordPod(podName) - client.CreatePodOrFail(eventRecordPod, testlib.WithService(podName)) + client.CreatePodOrFail(eventRecordPod, append(options, testlib.WithService(podName))...) err := pkgTest.WaitForPodRunning(client.Kube, podName, client.Namespace) if err != nil { client.T.Fatalf("Failed to start the recordevent pod '%s': %v", podName, errors.WithStack(err)) diff --git a/test/test_images/recordevents/main.go b/test/test_images/recordevents/main.go index b0bb15917a7..8dd74116338 100644 --- a/test/test_images/recordevents/main.go +++ b/test/test_images/recordevents/main.go @@ -22,6 +22,7 @@ import ( "fmt" "log" "net/http" + "os" "strconv" "strings" @@ -31,6 +32,7 @@ import ( "knative.dev/eventing/pkg/kncloudevents" testlib "knative.dev/eventing/test/lib" + "knative.dev/eventing/test/lib/dropevents" "knative.dev/eventing/test/lib/recordevents" "knative.dev/eventing/test/test_images" ) @@ -154,7 +156,24 @@ func main() { log.Fatalf("Unable to setup trace publishing: %v", err) } - err := http.ListenAndServe(":8080", kncloudevents.CreateHandler(er)) + algorithm, ok := os.LookupEnv(dropevents.SkipAlgorithmKey) + handler := kncloudevents.CreateHandler(er) + if ok { + skipper := dropevents.SkipperAlgorithm(algorithm) + counter := dropevents.CounterHandler{ + Skipper: skipper, + } + next := handler + handler = http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if counter.Skip() { + writer.WriteHeader(http.StatusConflict) + return + } + next.ServeHTTP(writer, request) + }) + } + + err := http.ListenAndServe(":8080", handler) if err != nil { panic(err) } diff --git a/third_party/VENDOR-LICENSE/github.com/hashicorp/go-cleanhttp/LICENSE b/third_party/VENDOR-LICENSE/github.com/hashicorp/go-cleanhttp/LICENSE new file mode 100644 index 00000000000..e87a115e462 --- /dev/null +++ b/third_party/VENDOR-LICENSE/github.com/hashicorp/go-cleanhttp/LICENSE @@ -0,0 +1,363 @@ +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. "Contributor" + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. "Contributor Version" + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms of + a Secondary License. + +1.6. "Executable Form" + + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + + means a work that combines Covered Software with other material, in a + separate file or files, that is not Covered Software. + +1.8. "License" + + means this document. + +1.9. "Licensable" + + means having the right to grant, to the maximum extent possible, whether + at the time of the initial grant or subsequently, any and all of the + rights conveyed by this License. + +1.10. "Modifications" + + means any of the following: + + a. any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. "Patent Claims" of a Contributor + + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the License, + by the making, using, selling, offering for sale, having made, import, + or transfer of either its Contributions or its Contributor Version. + +1.12. "Secondary License" + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. "Source Code Form" + + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, "control" means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor first + distributes such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under + this License. No additional rights or licenses will be implied from the + distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted by a + Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of + its Contributions. + + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License (if + permitted under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights to + grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, or other + equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under + the terms of this License. You must inform recipients that the Source + Code Form of the Covered Software is governed by the terms of this + License, and how they can obtain a copy of this License. You may not + attempt to alter or restrict the recipients' rights in the Source Code + Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter the + recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of Covered + Software with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the Covered + Software under the terms of either this License or such Secondary + License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, or + limitations of liability) contained within the Source Code Form of the + Covered Software, except that You may alter any license notices to the + extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on + behalf of any Contributor. You must make it absolutely clear that any + such warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, + judicial order, or regulation then You must: (a) comply with the terms of + this License to the maximum extent possible; and (b) describe the + limitations and the code they affect. Such description must be placed in a + text file included with all distributions of the Covered Software under + this License. Except to the extent prohibited by statute or regulation, + such description must be sufficiently detailed for a recipient of ordinary + skill to be able to understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing + basis, if such Contributor fails to notify You of the non-compliance by + some reasonable means prior to 60 days after You have come back into + compliance. Moreover, Your grants from a particular Contributor are + reinstated on an ongoing basis if such Contributor notifies You of the + non-compliance by some reasonable means, this is the first time You have + received notice of non-compliance with this License from such + Contributor, and You become compliant prior to 30 days after Your receipt + of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted to + You by any and all Contributors for the Covered Software under Section + 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an "as is" basis, + without warranty of any kind, either expressed, implied, or statutory, + including, without limitation, warranties that the Covered Software is free + of defects, merchantable, fit for a particular purpose or non-infringing. + The entire risk as to the quality and performance of the Covered Software + is with You. Should any Covered Software prove defective in any respect, + You (not any Contributor) assume the cost of any necessary servicing, + repair, or correction. This disclaimer of warranty constitutes an essential + part of this License. No use of any Covered Software is authorized under + this License except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from + such party's negligence to the extent applicable law prohibits such + limitation. Some jurisdictions do not allow the exclusion or limitation of + incidental or consequential damages, so this exclusion and limitation may + not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts + of a jurisdiction where the defendant maintains its principal place of + business and such litigation shall be governed by laws of that + jurisdiction, without reference to its conflict-of-law provisions. Nothing + in this Section shall prevent a party's ability to bring cross-claims or + counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. Any law or regulation which provides that + the language of a contract shall be construed against the drafter shall not + be used to construe this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a + modified version of this License if you rename the license and remove + any references to the name of the license steward (except to note that + such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary + Licenses If You choose to distribute Source Code Form that is + Incompatible With Secondary Licenses under the terms of this version of + the License, the notice described in Exhibit B of this License must be + attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, +then You may include the notice in a location (such as a LICENSE file in a +relevant directory) where a recipient would be likely to look for such a +notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice + + This Source Code Form is "Incompatible + With Secondary Licenses", as defined by + the Mozilla Public License, v. 2.0. + diff --git a/third_party/VENDOR-LICENSE/github.com/hashicorp/go-cleanhttp/README.md b/third_party/VENDOR-LICENSE/github.com/hashicorp/go-cleanhttp/README.md new file mode 100644 index 00000000000..036e5313fc8 --- /dev/null +++ b/third_party/VENDOR-LICENSE/github.com/hashicorp/go-cleanhttp/README.md @@ -0,0 +1,30 @@ +# cleanhttp + +Functions for accessing "clean" Go http.Client values + +------------- + +The Go standard library contains a default `http.Client` called +`http.DefaultClient`. It is a common idiom in Go code to start with +`http.DefaultClient` and tweak it as necessary, and in fact, this is +encouraged; from the `http` package documentation: + +> The Client's Transport typically has internal state (cached TCP connections), +so Clients should be reused instead of created as needed. Clients are safe for +concurrent use by multiple goroutines. + +Unfortunately, this is a shared value, and it is not uncommon for libraries to +assume that they are free to modify it at will. With enough dependencies, it +can be very easy to encounter strange problems and race conditions due to +manipulation of this shared value across libraries and goroutines (clients are +safe for concurrent use, but writing values to the client struct itself is not +protected). + +Making things worse is the fact that a bare `http.Client` will use a default +`http.Transport` called `http.DefaultTransport`, which is another global value +that behaves the same way. So it is not simply enough to replace +`http.DefaultClient` with `&http.Client{}`. + +This repository provides some simple functions to get a "clean" `http.Client` +-- one that uses the same default values as the Go standard library, but +returns a client that does not share any state with other clients. diff --git a/third_party/VENDOR-LICENSE/github.com/hashicorp/go-cleanhttp/cleanhttp.go b/third_party/VENDOR-LICENSE/github.com/hashicorp/go-cleanhttp/cleanhttp.go new file mode 100644 index 00000000000..8d306bf5134 --- /dev/null +++ b/third_party/VENDOR-LICENSE/github.com/hashicorp/go-cleanhttp/cleanhttp.go @@ -0,0 +1,57 @@ +package cleanhttp + +import ( + "net" + "net/http" + "runtime" + "time" +) + +// DefaultTransport returns a new http.Transport with similar default values to +// http.DefaultTransport, but with idle connections and keepalives disabled. +func DefaultTransport() *http.Transport { + transport := DefaultPooledTransport() + transport.DisableKeepAlives = true + transport.MaxIdleConnsPerHost = -1 + return transport +} + +// DefaultPooledTransport returns a new http.Transport with similar default +// values to http.DefaultTransport. Do not use this for transient transports as +// it can leak file descriptors over time. Only use this for transports that +// will be re-used for the same host(s). +func DefaultPooledTransport() *http.Transport { + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + MaxIdleConnsPerHost: runtime.GOMAXPROCS(0) + 1, + } + return transport +} + +// DefaultClient returns a new http.Client with similar default values to +// http.Client, but with a non-shared Transport, idle connections disabled, and +// keepalives disabled. +func DefaultClient() *http.Client { + return &http.Client{ + Transport: DefaultTransport(), + } +} + +// DefaultPooledClient returns a new http.Client with similar default values to +// http.Client, but with a shared Transport. Do not use this function for +// transient clients as it can leak file descriptors over time. Only use this +// for clients that will be re-used for the same host(s). +func DefaultPooledClient() *http.Client { + return &http.Client{ + Transport: DefaultPooledTransport(), + } +} diff --git a/third_party/VENDOR-LICENSE/github.com/hashicorp/go-cleanhttp/doc.go b/third_party/VENDOR-LICENSE/github.com/hashicorp/go-cleanhttp/doc.go new file mode 100644 index 00000000000..05841092a7b --- /dev/null +++ b/third_party/VENDOR-LICENSE/github.com/hashicorp/go-cleanhttp/doc.go @@ -0,0 +1,20 @@ +// Package cleanhttp offers convenience utilities for acquiring "clean" +// http.Transport and http.Client structs. +// +// Values set on http.DefaultClient and http.DefaultTransport affect all +// callers. This can have detrimental effects, esepcially in TLS contexts, +// where client or root certificates set to talk to multiple endpoints can end +// up displacing each other, leading to hard-to-debug issues. This package +// provides non-shared http.Client and http.Transport structs to ensure that +// the configuration will not be overwritten by other parts of the application +// or dependencies. +// +// The DefaultClient and DefaultTransport functions disable idle connections +// and keepalives. Without ensuring that idle connections are closed before +// garbage collection, short-term clients/transports can leak file descriptors, +// eventually leading to "too many open files" errors. If you will be +// connecting to the same hosts repeatedly from the same client, you can use +// DefaultPooledClient to receive a client that has connection pooling +// semantics similar to http.DefaultClient. +// +package cleanhttp diff --git a/third_party/VENDOR-LICENSE/github.com/hashicorp/go-cleanhttp/go.mod b/third_party/VENDOR-LICENSE/github.com/hashicorp/go-cleanhttp/go.mod new file mode 100644 index 00000000000..310f07569fc --- /dev/null +++ b/third_party/VENDOR-LICENSE/github.com/hashicorp/go-cleanhttp/go.mod @@ -0,0 +1 @@ +module github.com/hashicorp/go-cleanhttp diff --git a/third_party/VENDOR-LICENSE/github.com/hashicorp/go-cleanhttp/handlers.go b/third_party/VENDOR-LICENSE/github.com/hashicorp/go-cleanhttp/handlers.go new file mode 100644 index 00000000000..3c845dc0dc6 --- /dev/null +++ b/third_party/VENDOR-LICENSE/github.com/hashicorp/go-cleanhttp/handlers.go @@ -0,0 +1,48 @@ +package cleanhttp + +import ( + "net/http" + "strings" + "unicode" +) + +// HandlerInput provides input options to cleanhttp's handlers +type HandlerInput struct { + ErrStatus int +} + +// PrintablePathCheckHandler is a middleware that ensures the request path +// contains only printable runes. +func PrintablePathCheckHandler(next http.Handler, input *HandlerInput) http.Handler { + // Nil-check on input to make it optional + if input == nil { + input = &HandlerInput{ + ErrStatus: http.StatusBadRequest, + } + } + + // Default to http.StatusBadRequest on error + if input.ErrStatus == 0 { + input.ErrStatus = http.StatusBadRequest + } + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r != nil { + // Check URL path for non-printable characters + idx := strings.IndexFunc(r.URL.Path, func(c rune) bool { + return !unicode.IsPrint(c) + }) + + if idx != -1 { + w.WriteHeader(input.ErrStatus) + return + } + + if next != nil { + next.ServeHTTP(w, r) + } + } + + return + }) +} diff --git a/third_party/VENDOR-LICENSE/github.com/hashicorp/go-retryablehttp/.gitignore b/third_party/VENDOR-LICENSE/github.com/hashicorp/go-retryablehttp/.gitignore new file mode 100644 index 00000000000..4e309e0b326 --- /dev/null +++ b/third_party/VENDOR-LICENSE/github.com/hashicorp/go-retryablehttp/.gitignore @@ -0,0 +1,4 @@ +.idea/ +*.iml +*.test +.vscode/ \ No newline at end of file diff --git a/third_party/VENDOR-LICENSE/github.com/hashicorp/go-retryablehttp/.travis.yml b/third_party/VENDOR-LICENSE/github.com/hashicorp/go-retryablehttp/.travis.yml new file mode 100644 index 00000000000..c4fb6d6c8bb --- /dev/null +++ b/third_party/VENDOR-LICENSE/github.com/hashicorp/go-retryablehttp/.travis.yml @@ -0,0 +1,12 @@ +sudo: false + +language: go + +go: + - 1.12.4 + +branches: + only: + - master + +script: make updatedeps test diff --git a/third_party/VENDOR-LICENSE/github.com/hashicorp/go-retryablehttp/LICENSE b/third_party/VENDOR-LICENSE/github.com/hashicorp/go-retryablehttp/LICENSE new file mode 100644 index 00000000000..e87a115e462 --- /dev/null +++ b/third_party/VENDOR-LICENSE/github.com/hashicorp/go-retryablehttp/LICENSE @@ -0,0 +1,363 @@ +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. "Contributor" + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. "Contributor Version" + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms of + a Secondary License. + +1.6. "Executable Form" + + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + + means a work that combines Covered Software with other material, in a + separate file or files, that is not Covered Software. + +1.8. "License" + + means this document. + +1.9. "Licensable" + + means having the right to grant, to the maximum extent possible, whether + at the time of the initial grant or subsequently, any and all of the + rights conveyed by this License. + +1.10. "Modifications" + + means any of the following: + + a. any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. "Patent Claims" of a Contributor + + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the License, + by the making, using, selling, offering for sale, having made, import, + or transfer of either its Contributions or its Contributor Version. + +1.12. "Secondary License" + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. "Source Code Form" + + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, "control" means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor first + distributes such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under + this License. No additional rights or licenses will be implied from the + distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted by a + Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of + its Contributions. + + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License (if + permitted under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights to + grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, or other + equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under + the terms of this License. You must inform recipients that the Source + Code Form of the Covered Software is governed by the terms of this + License, and how they can obtain a copy of this License. You may not + attempt to alter or restrict the recipients' rights in the Source Code + Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter the + recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of Covered + Software with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the Covered + Software under the terms of either this License or such Secondary + License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, or + limitations of liability) contained within the Source Code Form of the + Covered Software, except that You may alter any license notices to the + extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on + behalf of any Contributor. You must make it absolutely clear that any + such warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, + judicial order, or regulation then You must: (a) comply with the terms of + this License to the maximum extent possible; and (b) describe the + limitations and the code they affect. Such description must be placed in a + text file included with all distributions of the Covered Software under + this License. Except to the extent prohibited by statute or regulation, + such description must be sufficiently detailed for a recipient of ordinary + skill to be able to understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing + basis, if such Contributor fails to notify You of the non-compliance by + some reasonable means prior to 60 days after You have come back into + compliance. Moreover, Your grants from a particular Contributor are + reinstated on an ongoing basis if such Contributor notifies You of the + non-compliance by some reasonable means, this is the first time You have + received notice of non-compliance with this License from such + Contributor, and You become compliant prior to 30 days after Your receipt + of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted to + You by any and all Contributors for the Covered Software under Section + 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an "as is" basis, + without warranty of any kind, either expressed, implied, or statutory, + including, without limitation, warranties that the Covered Software is free + of defects, merchantable, fit for a particular purpose or non-infringing. + The entire risk as to the quality and performance of the Covered Software + is with You. Should any Covered Software prove defective in any respect, + You (not any Contributor) assume the cost of any necessary servicing, + repair, or correction. This disclaimer of warranty constitutes an essential + part of this License. No use of any Covered Software is authorized under + this License except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from + such party's negligence to the extent applicable law prohibits such + limitation. Some jurisdictions do not allow the exclusion or limitation of + incidental or consequential damages, so this exclusion and limitation may + not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts + of a jurisdiction where the defendant maintains its principal place of + business and such litigation shall be governed by laws of that + jurisdiction, without reference to its conflict-of-law provisions. Nothing + in this Section shall prevent a party's ability to bring cross-claims or + counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. Any law or regulation which provides that + the language of a contract shall be construed against the drafter shall not + be used to construe this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a + modified version of this License if you rename the license and remove + any references to the name of the license steward (except to note that + such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary + Licenses If You choose to distribute Source Code Form that is + Incompatible With Secondary Licenses under the terms of this version of + the License, the notice described in Exhibit B of this License must be + attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, +then You may include the notice in a location (such as a LICENSE file in a +relevant directory) where a recipient would be likely to look for such a +notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice + + This Source Code Form is "Incompatible + With Secondary Licenses", as defined by + the Mozilla Public License, v. 2.0. + diff --git a/third_party/VENDOR-LICENSE/github.com/hashicorp/go-retryablehttp/Makefile b/third_party/VENDOR-LICENSE/github.com/hashicorp/go-retryablehttp/Makefile new file mode 100644 index 00000000000..da17640e644 --- /dev/null +++ b/third_party/VENDOR-LICENSE/github.com/hashicorp/go-retryablehttp/Makefile @@ -0,0 +1,11 @@ +default: test + +test: + go vet ./... + go test -race ./... + +updatedeps: + go get -f -t -u ./... + go get -f -u ./... + +.PHONY: default test updatedeps diff --git a/third_party/VENDOR-LICENSE/github.com/hashicorp/go-retryablehttp/README.md b/third_party/VENDOR-LICENSE/github.com/hashicorp/go-retryablehttp/README.md new file mode 100644 index 00000000000..30357c75668 --- /dev/null +++ b/third_party/VENDOR-LICENSE/github.com/hashicorp/go-retryablehttp/README.md @@ -0,0 +1,61 @@ +go-retryablehttp +================ + +[![Build Status](http://img.shields.io/travis/hashicorp/go-retryablehttp.svg?style=flat-square)][travis] +[![Go Documentation](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)][godocs] + +[travis]: http://travis-ci.org/hashicorp/go-retryablehttp +[godocs]: http://godoc.org/github.com/hashicorp/go-retryablehttp + +The `retryablehttp` package provides a familiar HTTP client interface with +automatic retries and exponential backoff. It is a thin wrapper over the +standard `net/http` client library and exposes nearly the same public API. This +makes `retryablehttp` very easy to drop into existing programs. + +`retryablehttp` performs automatic retries under certain conditions. Mainly, if +an error is returned by the client (connection errors, etc.), or if a 500-range +response code is received (except 501), then a retry is invoked after a wait +period. Otherwise, the response is returned and left to the caller to +interpret. + +The main difference from `net/http` is that requests which take a request body +(POST/PUT et. al) can have the body provided in a number of ways (some more or +less efficient) that allow "rewinding" the request body if the initial request +fails so that the full request can be attempted again. See the +[godoc](http://godoc.org/github.com/hashicorp/go-retryablehttp) for more +details. + +Version 0.6.0 and before are compatible with Go prior to 1.12. From 0.6.1 onward, Go 1.12+ is required. + +Example Use +=========== + +Using this library should look almost identical to what you would do with +`net/http`. The most simple example of a GET request is shown below: + +```go +resp, err := retryablehttp.Get("/foo") +if err != nil { + panic(err) +} +``` + +The returned response object is an `*http.Response`, the same thing you would +usually get from `net/http`. Had the request failed one or more times, the above +call would block and retry with exponential backoff. + +## Getting a stdlib `*http.Client` with retries + +It's possible to convert a `*retryablehttp.Client` directly to a `*http.Client`. +This makes use of retryablehttp broadly applicable with minimal effort. Simply +configure a `*retryablehttp.Client` as you wish, and then call `StandardClient()`: + +```go +retryClient := retryablehttp.NewClient() +retryClient.RetryMax = 10 + +standardClient := retryClient.StandardClient() // *http.Client +``` + +For more usage and examples see the +[godoc](http://godoc.org/github.com/hashicorp/go-retryablehttp). diff --git a/third_party/VENDOR-LICENSE/github.com/hashicorp/go-retryablehttp/client.go b/third_party/VENDOR-LICENSE/github.com/hashicorp/go-retryablehttp/client.go new file mode 100644 index 00000000000..f1ccd3df35c --- /dev/null +++ b/third_party/VENDOR-LICENSE/github.com/hashicorp/go-retryablehttp/client.go @@ -0,0 +1,705 @@ +// Package retryablehttp provides a familiar HTTP client interface with +// automatic retries and exponential backoff. It is a thin wrapper over the +// standard net/http client library and exposes nearly the same public API. +// This makes retryablehttp very easy to drop into existing programs. +// +// retryablehttp performs automatic retries under certain conditions. Mainly, if +// an error is returned by the client (connection errors etc), or if a 500-range +// response is received, then a retry is invoked. Otherwise, the response is +// returned and left to the caller to interpret. +// +// Requests which take a request body should provide a non-nil function +// parameter. The best choice is to provide either a function satisfying +// ReaderFunc which provides multiple io.Readers in an efficient manner, a +// *bytes.Buffer (the underlying raw byte slice will be used) or a raw byte +// slice. As it is a reference type, and we will wrap it as needed by readers, +// we can efficiently re-use the request body without needing to copy it. If an +// io.Reader (such as a *bytes.Reader) is provided, the full body will be read +// prior to the first request, and will be efficiently re-used for any retries. +// ReadSeeker can be used, but some users have observed occasional data races +// between the net/http library and the Seek functionality of some +// implementations of ReadSeeker, so should be avoided if possible. +package retryablehttp + +import ( + "bytes" + "context" + "crypto/x509" + "fmt" + "io" + "io/ioutil" + "log" + "math" + "math/rand" + "net/http" + "net/url" + "os" + "regexp" + "strings" + "sync" + "time" + + "github.com/hashicorp/go-cleanhttp" +) + +var ( + // Default retry configuration + defaultRetryWaitMin = 1 * time.Second + defaultRetryWaitMax = 30 * time.Second + defaultRetryMax = 4 + + // defaultLogger is the logger provided with defaultClient + defaultLogger = log.New(os.Stderr, "", log.LstdFlags) + + // defaultClient is used for performing requests without explicitly making + // a new client. It is purposely private to avoid modifications. + defaultClient = NewClient() + + // We need to consume response bodies to maintain http connections, but + // limit the size we consume to respReadLimit. + respReadLimit = int64(4096) + + // A regular expression to match the error returned by net/http when the + // configured number of redirects is exhausted. This error isn't typed + // specifically so we resort to matching on the error string. + redirectsErrorRe = regexp.MustCompile(`stopped after \d+ redirects\z`) + + // A regular expression to match the error returned by net/http when the + // scheme specified in the URL is invalid. This error isn't typed + // specifically so we resort to matching on the error string. + schemeErrorRe = regexp.MustCompile(`unsupported protocol scheme`) +) + +// ReaderFunc is the type of function that can be given natively to NewRequest +type ReaderFunc func() (io.Reader, error) + +// LenReader is an interface implemented by many in-memory io.Reader's. Used +// for automatically sending the right Content-Length header when possible. +type LenReader interface { + Len() int +} + +// Request wraps the metadata needed to create HTTP requests. +type Request struct { + // body is a seekable reader over the request body payload. This is + // used to rewind the request data in between retries. + body ReaderFunc + + // Embed an HTTP request directly. This makes a *Request act exactly + // like an *http.Request so that all meta methods are supported. + *http.Request +} + +// WithContext returns wrapped Request with a shallow copy of underlying *http.Request +// with its context changed to ctx. The provided ctx must be non-nil. +func (r *Request) WithContext(ctx context.Context) *Request { + r.Request = r.Request.WithContext(ctx) + return r +} + +// BodyBytes allows accessing the request body. It is an analogue to +// http.Request's Body variable, but it returns a copy of the underlying data +// rather than consuming it. +// +// This function is not thread-safe; do not call it at the same time as another +// call, or at the same time this request is being used with Client.Do. +func (r *Request) BodyBytes() ([]byte, error) { + if r.body == nil { + return nil, nil + } + body, err := r.body() + if err != nil { + return nil, err + } + buf := new(bytes.Buffer) + _, err = buf.ReadFrom(body) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// SetBody allows setting the request body. +// +// It is useful if a new body needs to be set without constructing a new Request. +func (r *Request) SetBody(rawBody interface{}) error { + bodyReader, contentLength, err := getBodyReaderAndContentLength(rawBody) + if err != nil { + return err + } + r.body = bodyReader + r.ContentLength = contentLength + return nil +} + +// WriteTo allows copying the request body into a writer. +// +// It writes data to w until there's no more data to write or +// when an error occurs. The return int64 value is the number of bytes +// written. Any error encountered during the write is also returned. +// The signature matches io.WriterTo interface. +func (r *Request) WriteTo(w io.Writer) (int64, error) { + body, err := r.body() + if err != nil { + return 0, err + } + if c, ok := body.(io.Closer); ok { + defer c.Close() + } + return io.Copy(w, body) +} + +func getBodyReaderAndContentLength(rawBody interface{}) (ReaderFunc, int64, error) { + var bodyReader ReaderFunc + var contentLength int64 + + switch body := rawBody.(type) { + // If they gave us a function already, great! Use it. + case ReaderFunc: + bodyReader = body + tmp, err := body() + if err != nil { + return nil, 0, err + } + if lr, ok := tmp.(LenReader); ok { + contentLength = int64(lr.Len()) + } + if c, ok := tmp.(io.Closer); ok { + c.Close() + } + + case func() (io.Reader, error): + bodyReader = body + tmp, err := body() + if err != nil { + return nil, 0, err + } + if lr, ok := tmp.(LenReader); ok { + contentLength = int64(lr.Len()) + } + if c, ok := tmp.(io.Closer); ok { + c.Close() + } + + // If a regular byte slice, we can read it over and over via new + // readers + case []byte: + buf := body + bodyReader = func() (io.Reader, error) { + return bytes.NewReader(buf), nil + } + contentLength = int64(len(buf)) + + // If a bytes.Buffer we can read the underlying byte slice over and + // over + case *bytes.Buffer: + buf := body + bodyReader = func() (io.Reader, error) { + return bytes.NewReader(buf.Bytes()), nil + } + contentLength = int64(buf.Len()) + + // We prioritize *bytes.Reader here because we don't really want to + // deal with it seeking so want it to match here instead of the + // io.ReadSeeker case. + case *bytes.Reader: + buf, err := ioutil.ReadAll(body) + if err != nil { + return nil, 0, err + } + bodyReader = func() (io.Reader, error) { + return bytes.NewReader(buf), nil + } + contentLength = int64(len(buf)) + + // Compat case + case io.ReadSeeker: + raw := body + bodyReader = func() (io.Reader, error) { + _, err := raw.Seek(0, 0) + return ioutil.NopCloser(raw), err + } + if lr, ok := raw.(LenReader); ok { + contentLength = int64(lr.Len()) + } + + // Read all in so we can reset + case io.Reader: + buf, err := ioutil.ReadAll(body) + if err != nil { + return nil, 0, err + } + bodyReader = func() (io.Reader, error) { + return bytes.NewReader(buf), nil + } + contentLength = int64(len(buf)) + + // No body provided, nothing to do + case nil: + + // Unrecognized type + default: + return nil, 0, fmt.Errorf("cannot handle type %T", rawBody) + } + return bodyReader, contentLength, nil +} + +// FromRequest wraps an http.Request in a retryablehttp.Request +func FromRequest(r *http.Request) (*Request, error) { + bodyReader, _, err := getBodyReaderAndContentLength(r.Body) + if err != nil { + return nil, err + } + // Could assert contentLength == r.ContentLength + return &Request{bodyReader, r}, nil +} + +// NewRequest creates a new wrapped request. +func NewRequest(method, url string, rawBody interface{}) (*Request, error) { + bodyReader, contentLength, err := getBodyReaderAndContentLength(rawBody) + if err != nil { + return nil, err + } + + httpReq, err := http.NewRequest(method, url, nil) + if err != nil { + return nil, err + } + httpReq.ContentLength = contentLength + + return &Request{bodyReader, httpReq}, nil +} + +// Logger interface allows to use other loggers than +// standard log.Logger. +type Logger interface { + Printf(string, ...interface{}) +} + +// LeveledLogger interface implements the basic methods that a logger library needs +type LeveledLogger interface { + Error(string, ...interface{}) + Info(string, ...interface{}) + Debug(string, ...interface{}) + Warn(string, ...interface{}) +} + +// hookLogger adapts an LeveledLogger to Logger for use by the existing hook functions +// without changing the API. +type hookLogger struct { + LeveledLogger +} + +func (h hookLogger) Printf(s string, args ...interface{}) { + h.Info(fmt.Sprintf(s, args...)) +} + +// RequestLogHook allows a function to run before each retry. The HTTP +// request which will be made, and the retry number (0 for the initial +// request) are available to users. The internal logger is exposed to +// consumers. +type RequestLogHook func(Logger, *http.Request, int) + +// ResponseLogHook is like RequestLogHook, but allows running a function +// on each HTTP response. This function will be invoked at the end of +// every HTTP request executed, regardless of whether a subsequent retry +// needs to be performed or not. If the response body is read or closed +// from this method, this will affect the response returned from Do(). +type ResponseLogHook func(Logger, *http.Response) + +// CheckRetry specifies a policy for handling retries. It is called +// following each request with the response and error values returned by +// the http.Client. If CheckRetry returns false, the Client stops retrying +// and returns the response to the caller. If CheckRetry returns an error, +// that error value is returned in lieu of the error from the request. The +// Client will close any response body when retrying, but if the retry is +// aborted it is up to the CheckRetry callback to properly close any +// response body before returning. +type CheckRetry func(ctx context.Context, resp *http.Response, err error) (bool, error) + +// Backoff specifies a policy for how long to wait between retries. +// It is called after a failing request to determine the amount of time +// that should pass before trying again. +type Backoff func(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration + +// ErrorHandler is called if retries are expired, containing the last status +// from the http library. If not specified, default behavior for the library is +// to close the body and return an error indicating how many tries were +// attempted. If overriding this, be sure to close the body if needed. +type ErrorHandler func(resp *http.Response, err error, numTries int) (*http.Response, error) + +// Client is used to make HTTP requests. It adds additional functionality +// like automatic retries to tolerate minor outages. +type Client struct { + HTTPClient *http.Client // Internal HTTP client. + Logger interface{} // Customer logger instance. Can be either Logger or LeveledLogger + + RetryWaitMin time.Duration // Minimum time to wait + RetryWaitMax time.Duration // Maximum time to wait + RetryMax int // Maximum number of retries + + // RequestLogHook allows a user-supplied function to be called + // before each retry. + RequestLogHook RequestLogHook + + // ResponseLogHook allows a user-supplied function to be called + // with the response from each HTTP request executed. + ResponseLogHook ResponseLogHook + + // CheckRetry specifies the policy for handling retries, and is called + // after each request. The default policy is DefaultRetryPolicy. + CheckRetry CheckRetry + + // Backoff specifies the policy for how long to wait between retries + Backoff Backoff + + // ErrorHandler specifies the custom error handler to use, if any + ErrorHandler ErrorHandler + + loggerInit sync.Once +} + +// NewClient creates a new Client with default settings. +func NewClient() *Client { + return &Client{ + HTTPClient: cleanhttp.DefaultPooledClient(), + Logger: defaultLogger, + RetryWaitMin: defaultRetryWaitMin, + RetryWaitMax: defaultRetryWaitMax, + RetryMax: defaultRetryMax, + CheckRetry: DefaultRetryPolicy, + Backoff: DefaultBackoff, + } +} + +func (c *Client) logger() interface{} { + c.loggerInit.Do(func() { + if c.Logger == nil { + return + } + + switch c.Logger.(type) { + case Logger, LeveledLogger: + // ok + default: + // This should happen in dev when they are setting Logger and work on code, not in prod. + panic(fmt.Sprintf("invalid logger type passed, must be Logger or LeveledLogger, was %T", c.Logger)) + } + }) + + return c.Logger +} + +// DefaultRetryPolicy provides a default callback for Client.CheckRetry, which +// will retry on connection errors and server errors. +func DefaultRetryPolicy(ctx context.Context, resp *http.Response, err error) (bool, error) { + // do not retry on context.Canceled or context.DeadlineExceeded + if ctx.Err() != nil { + return false, ctx.Err() + } + + if err != nil { + if v, ok := err.(*url.Error); ok { + // Don't retry if the error was due to too many redirects. + if redirectsErrorRe.MatchString(v.Error()) { + return false, nil + } + + // Don't retry if the error was due to an invalid protocol scheme. + if schemeErrorRe.MatchString(v.Error()) { + return false, nil + } + + // Don't retry if the error was due to TLS cert verification failure. + if _, ok := v.Err.(x509.UnknownAuthorityError); ok { + return false, nil + } + } + + // The error is likely recoverable so retry. + return true, nil + } + + // Check the response code. We retry on 500-range responses to allow + // the server time to recover, as 500's are typically not permanent + // errors and may relate to outages on the server side. This will catch + // invalid response codes as well, like 0 and 999. + if resp.StatusCode == 0 || (resp.StatusCode >= 500 && resp.StatusCode != 501) { + return true, nil + } + + return false, nil +} + +// DefaultBackoff provides a default callback for Client.Backoff which +// will perform exponential backoff based on the attempt number and limited +// by the provided minimum and maximum durations. +func DefaultBackoff(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration { + mult := math.Pow(2, float64(attemptNum)) * float64(min) + sleep := time.Duration(mult) + if float64(sleep) != mult || sleep > max { + sleep = max + } + return sleep +} + +// LinearJitterBackoff provides a callback for Client.Backoff which will +// perform linear backoff based on the attempt number and with jitter to +// prevent a thundering herd. +// +// min and max here are *not* absolute values. The number to be multiplied by +// the attempt number will be chosen at random from between them, thus they are +// bounding the jitter. +// +// For instance: +// * To get strictly linear backoff of one second increasing each retry, set +// both to one second (1s, 2s, 3s, 4s, ...) +// * To get a small amount of jitter centered around one second increasing each +// retry, set to around one second, such as a min of 800ms and max of 1200ms +// (892ms, 2102ms, 2945ms, 4312ms, ...) +// * To get extreme jitter, set to a very wide spread, such as a min of 100ms +// and a max of 20s (15382ms, 292ms, 51321ms, 35234ms, ...) +func LinearJitterBackoff(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration { + // attemptNum always starts at zero but we want to start at 1 for multiplication + attemptNum++ + + if max <= min { + // Unclear what to do here, or they are the same, so return min * + // attemptNum + return min * time.Duration(attemptNum) + } + + // Seed rand; doing this every time is fine + rand := rand.New(rand.NewSource(int64(time.Now().Nanosecond()))) + + // Pick a random number that lies somewhere between the min and max and + // multiply by the attemptNum. attemptNum starts at zero so we always + // increment here. We first get a random percentage, then apply that to the + // difference between min and max, and add to min. + jitter := rand.Float64() * float64(max-min) + jitterMin := int64(jitter) + int64(min) + return time.Duration(jitterMin * int64(attemptNum)) +} + +// PassthroughErrorHandler is an ErrorHandler that directly passes through the +// values from the net/http library for the final request. The body is not +// closed. +func PassthroughErrorHandler(resp *http.Response, err error, _ int) (*http.Response, error) { + return resp, err +} + +// Do wraps calling an HTTP method with retries. +func (c *Client) Do(req *Request) (*http.Response, error) { + if c.HTTPClient == nil { + c.HTTPClient = cleanhttp.DefaultPooledClient() + } + + logger := c.logger() + + if logger != nil { + switch v := logger.(type) { + case Logger: + v.Printf("[DEBUG] %s %s", req.Method, req.URL) + case LeveledLogger: + v.Debug("performing request", "method", req.Method, "url", req.URL) + } + } + + var resp *http.Response + var err error + + for i := 0; ; i++ { + var code int // HTTP response code + + // Always rewind the request body when non-nil. + if req.body != nil { + body, err := req.body() + if err != nil { + c.HTTPClient.CloseIdleConnections() + return resp, err + } + if c, ok := body.(io.ReadCloser); ok { + req.Body = c + } else { + req.Body = ioutil.NopCloser(body) + } + } + + if c.RequestLogHook != nil { + switch v := logger.(type) { + case Logger: + c.RequestLogHook(v, req.Request, i) + case LeveledLogger: + c.RequestLogHook(hookLogger{v}, req.Request, i) + default: + c.RequestLogHook(nil, req.Request, i) + } + } + + // Attempt the request + resp, err = c.HTTPClient.Do(req.Request) + if resp != nil { + code = resp.StatusCode + } + + // Check if we should continue with retries. + checkOK, checkErr := c.CheckRetry(req.Context(), resp, err) + + if err != nil { + switch v := logger.(type) { + case Logger: + v.Printf("[ERR] %s %s request failed: %v", req.Method, req.URL, err) + case LeveledLogger: + v.Error("request failed", "error", err, "method", req.Method, "url", req.URL) + } + } else { + // Call this here to maintain the behavior of logging all requests, + // even if CheckRetry signals to stop. + if c.ResponseLogHook != nil { + // Call the response logger function if provided. + switch v := logger.(type) { + case Logger: + c.ResponseLogHook(v, resp) + case LeveledLogger: + c.ResponseLogHook(hookLogger{v}, resp) + default: + c.ResponseLogHook(nil, resp) + } + } + } + + // Now decide if we should continue. + if !checkOK { + if checkErr != nil { + err = checkErr + } + c.HTTPClient.CloseIdleConnections() + return resp, err + } + + // We do this before drainBody because there's no need for the I/O if + // we're breaking out + remain := c.RetryMax - i + if remain <= 0 { + break + } + + // We're going to retry, consume any response to reuse the connection. + if err == nil && resp != nil { + c.drainBody(resp.Body) + } + + wait := c.Backoff(c.RetryWaitMin, c.RetryWaitMax, i, resp) + desc := fmt.Sprintf("%s %s", req.Method, req.URL) + if code > 0 { + desc = fmt.Sprintf("%s (status: %d)", desc, code) + } + if logger != nil { + switch v := logger.(type) { + case Logger: + v.Printf("[DEBUG] %s: retrying in %s (%d left)", desc, wait, remain) + case LeveledLogger: + v.Debug("retrying request", "request", desc, "timeout", wait, "remaining", remain) + } + } + select { + case <-req.Context().Done(): + c.HTTPClient.CloseIdleConnections() + return nil, req.Context().Err() + case <-time.After(wait): + } + } + + if c.ErrorHandler != nil { + c.HTTPClient.CloseIdleConnections() + return c.ErrorHandler(resp, err, c.RetryMax+1) + } + + // By default, we close the response body and return an error without + // returning the response + if resp != nil { + resp.Body.Close() + } + c.HTTPClient.CloseIdleConnections() + return nil, fmt.Errorf("%s %s giving up after %d attempts", + req.Method, req.URL, c.RetryMax+1) +} + +// Try to read the response body so we can reuse this connection. +func (c *Client) drainBody(body io.ReadCloser) { + defer body.Close() + _, err := io.Copy(ioutil.Discard, io.LimitReader(body, respReadLimit)) + if err != nil { + if c.logger() != nil { + switch v := c.logger().(type) { + case Logger: + v.Printf("[ERR] error reading response body: %v", err) + case LeveledLogger: + v.Error("error reading response body", "error", err) + } + } + } +} + +// Get is a shortcut for doing a GET request without making a new client. +func Get(url string) (*http.Response, error) { + return defaultClient.Get(url) +} + +// Get is a convenience helper for doing simple GET requests. +func (c *Client) Get(url string) (*http.Response, error) { + req, err := NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + return c.Do(req) +} + +// Head is a shortcut for doing a HEAD request without making a new client. +func Head(url string) (*http.Response, error) { + return defaultClient.Head(url) +} + +// Head is a convenience method for doing simple HEAD requests. +func (c *Client) Head(url string) (*http.Response, error) { + req, err := NewRequest("HEAD", url, nil) + if err != nil { + return nil, err + } + return c.Do(req) +} + +// Post is a shortcut for doing a POST request without making a new client. +func Post(url, bodyType string, body interface{}) (*http.Response, error) { + return defaultClient.Post(url, bodyType, body) +} + +// Post is a convenience method for doing simple POST requests. +func (c *Client) Post(url, bodyType string, body interface{}) (*http.Response, error) { + req, err := NewRequest("POST", url, body) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", bodyType) + return c.Do(req) +} + +// PostForm is a shortcut to perform a POST with form data without creating +// a new client. +func PostForm(url string, data url.Values) (*http.Response, error) { + return defaultClient.PostForm(url, data) +} + +// PostForm is a convenience method for doing simple POST operations using +// pre-filled url.Values form data. +func (c *Client) PostForm(url string, data url.Values) (*http.Response, error) { + return c.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode())) +} + +// StandardClient returns a stdlib *http.Client with a custom Transport, which +// shims in a *retryablehttp.Client for added retries. +func (c *Client) StandardClient() *http.Client { + return &http.Client{ + Transport: &RoundTripper{Client: c}, + } +} diff --git a/third_party/VENDOR-LICENSE/github.com/hashicorp/go-retryablehttp/go.mod b/third_party/VENDOR-LICENSE/github.com/hashicorp/go-retryablehttp/go.mod new file mode 100644 index 00000000000..7cc02b76fa4 --- /dev/null +++ b/third_party/VENDOR-LICENSE/github.com/hashicorp/go-retryablehttp/go.mod @@ -0,0 +1,8 @@ +module github.com/hashicorp/go-retryablehttp + +require ( + github.com/hashicorp/go-cleanhttp v0.5.1 + github.com/hashicorp/go-hclog v0.9.2 +) + +go 1.13 diff --git a/third_party/VENDOR-LICENSE/github.com/hashicorp/go-retryablehttp/go.sum b/third_party/VENDOR-LICENSE/github.com/hashicorp/go-retryablehttp/go.sum new file mode 100644 index 00000000000..71afe568227 --- /dev/null +++ b/third_party/VENDOR-LICENSE/github.com/hashicorp/go-retryablehttp/go.sum @@ -0,0 +1,10 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= diff --git a/third_party/VENDOR-LICENSE/github.com/hashicorp/go-retryablehttp/roundtripper.go b/third_party/VENDOR-LICENSE/github.com/hashicorp/go-retryablehttp/roundtripper.go new file mode 100644 index 00000000000..b841b4cfe53 --- /dev/null +++ b/third_party/VENDOR-LICENSE/github.com/hashicorp/go-retryablehttp/roundtripper.go @@ -0,0 +1,43 @@ +package retryablehttp + +import ( + "net/http" + "sync" +) + +// RoundTripper implements the http.RoundTripper interface, using a retrying +// HTTP client to execute requests. +// +// It is important to note that retryablehttp doesn't always act exactly as a +// RoundTripper should. This is highly dependent on the retryable client's +// configuration. +type RoundTripper struct { + // The client to use during requests. If nil, the default retryablehttp + // client and settings will be used. + Client *Client + + // once ensures that the logic to initialize the default client runs at + // most once, in a single thread. + once sync.Once +} + +// init initializes the underlying retryable client. +func (rt *RoundTripper) init() { + if rt.Client == nil { + rt.Client = NewClient() + } +} + +// RoundTrip satisfies the http.RoundTripper interface. +func (rt *RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + rt.once.Do(rt.init) + + // Convert the request to be retryable. + retryableReq, err := FromRequest(req) + if err != nil { + return nil, err + } + + // Execute the request. + return rt.Client.Do(retryableReq) +} diff --git a/vendor/github.com/hashicorp/go-cleanhttp/LICENSE b/vendor/github.com/hashicorp/go-cleanhttp/LICENSE new file mode 100644 index 00000000000..e87a115e462 --- /dev/null +++ b/vendor/github.com/hashicorp/go-cleanhttp/LICENSE @@ -0,0 +1,363 @@ +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. "Contributor" + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. "Contributor Version" + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms of + a Secondary License. + +1.6. "Executable Form" + + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + + means a work that combines Covered Software with other material, in a + separate file or files, that is not Covered Software. + +1.8. "License" + + means this document. + +1.9. "Licensable" + + means having the right to grant, to the maximum extent possible, whether + at the time of the initial grant or subsequently, any and all of the + rights conveyed by this License. + +1.10. "Modifications" + + means any of the following: + + a. any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. "Patent Claims" of a Contributor + + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the License, + by the making, using, selling, offering for sale, having made, import, + or transfer of either its Contributions or its Contributor Version. + +1.12. "Secondary License" + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. "Source Code Form" + + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, "control" means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor first + distributes such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under + this License. No additional rights or licenses will be implied from the + distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted by a + Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of + its Contributions. + + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License (if + permitted under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights to + grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, or other + equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under + the terms of this License. You must inform recipients that the Source + Code Form of the Covered Software is governed by the terms of this + License, and how they can obtain a copy of this License. You may not + attempt to alter or restrict the recipients' rights in the Source Code + Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter the + recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of Covered + Software with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the Covered + Software under the terms of either this License or such Secondary + License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, or + limitations of liability) contained within the Source Code Form of the + Covered Software, except that You may alter any license notices to the + extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on + behalf of any Contributor. You must make it absolutely clear that any + such warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, + judicial order, or regulation then You must: (a) comply with the terms of + this License to the maximum extent possible; and (b) describe the + limitations and the code they affect. Such description must be placed in a + text file included with all distributions of the Covered Software under + this License. Except to the extent prohibited by statute or regulation, + such description must be sufficiently detailed for a recipient of ordinary + skill to be able to understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing + basis, if such Contributor fails to notify You of the non-compliance by + some reasonable means prior to 60 days after You have come back into + compliance. Moreover, Your grants from a particular Contributor are + reinstated on an ongoing basis if such Contributor notifies You of the + non-compliance by some reasonable means, this is the first time You have + received notice of non-compliance with this License from such + Contributor, and You become compliant prior to 30 days after Your receipt + of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted to + You by any and all Contributors for the Covered Software under Section + 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an "as is" basis, + without warranty of any kind, either expressed, implied, or statutory, + including, without limitation, warranties that the Covered Software is free + of defects, merchantable, fit for a particular purpose or non-infringing. + The entire risk as to the quality and performance of the Covered Software + is with You. Should any Covered Software prove defective in any respect, + You (not any Contributor) assume the cost of any necessary servicing, + repair, or correction. This disclaimer of warranty constitutes an essential + part of this License. No use of any Covered Software is authorized under + this License except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from + such party's negligence to the extent applicable law prohibits such + limitation. Some jurisdictions do not allow the exclusion or limitation of + incidental or consequential damages, so this exclusion and limitation may + not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts + of a jurisdiction where the defendant maintains its principal place of + business and such litigation shall be governed by laws of that + jurisdiction, without reference to its conflict-of-law provisions. Nothing + in this Section shall prevent a party's ability to bring cross-claims or + counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. Any law or regulation which provides that + the language of a contract shall be construed against the drafter shall not + be used to construe this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a + modified version of this License if you rename the license and remove + any references to the name of the license steward (except to note that + such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary + Licenses If You choose to distribute Source Code Form that is + Incompatible With Secondary Licenses under the terms of this version of + the License, the notice described in Exhibit B of this License must be + attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, +then You may include the notice in a location (such as a LICENSE file in a +relevant directory) where a recipient would be likely to look for such a +notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice + + This Source Code Form is "Incompatible + With Secondary Licenses", as defined by + the Mozilla Public License, v. 2.0. + diff --git a/vendor/github.com/hashicorp/go-cleanhttp/README.md b/vendor/github.com/hashicorp/go-cleanhttp/README.md new file mode 100644 index 00000000000..036e5313fc8 --- /dev/null +++ b/vendor/github.com/hashicorp/go-cleanhttp/README.md @@ -0,0 +1,30 @@ +# cleanhttp + +Functions for accessing "clean" Go http.Client values + +------------- + +The Go standard library contains a default `http.Client` called +`http.DefaultClient`. It is a common idiom in Go code to start with +`http.DefaultClient` and tweak it as necessary, and in fact, this is +encouraged; from the `http` package documentation: + +> The Client's Transport typically has internal state (cached TCP connections), +so Clients should be reused instead of created as needed. Clients are safe for +concurrent use by multiple goroutines. + +Unfortunately, this is a shared value, and it is not uncommon for libraries to +assume that they are free to modify it at will. With enough dependencies, it +can be very easy to encounter strange problems and race conditions due to +manipulation of this shared value across libraries and goroutines (clients are +safe for concurrent use, but writing values to the client struct itself is not +protected). + +Making things worse is the fact that a bare `http.Client` will use a default +`http.Transport` called `http.DefaultTransport`, which is another global value +that behaves the same way. So it is not simply enough to replace +`http.DefaultClient` with `&http.Client{}`. + +This repository provides some simple functions to get a "clean" `http.Client` +-- one that uses the same default values as the Go standard library, but +returns a client that does not share any state with other clients. diff --git a/vendor/github.com/hashicorp/go-cleanhttp/cleanhttp.go b/vendor/github.com/hashicorp/go-cleanhttp/cleanhttp.go new file mode 100644 index 00000000000..8d306bf5134 --- /dev/null +++ b/vendor/github.com/hashicorp/go-cleanhttp/cleanhttp.go @@ -0,0 +1,57 @@ +package cleanhttp + +import ( + "net" + "net/http" + "runtime" + "time" +) + +// DefaultTransport returns a new http.Transport with similar default values to +// http.DefaultTransport, but with idle connections and keepalives disabled. +func DefaultTransport() *http.Transport { + transport := DefaultPooledTransport() + transport.DisableKeepAlives = true + transport.MaxIdleConnsPerHost = -1 + return transport +} + +// DefaultPooledTransport returns a new http.Transport with similar default +// values to http.DefaultTransport. Do not use this for transient transports as +// it can leak file descriptors over time. Only use this for transports that +// will be re-used for the same host(s). +func DefaultPooledTransport() *http.Transport { + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + MaxIdleConnsPerHost: runtime.GOMAXPROCS(0) + 1, + } + return transport +} + +// DefaultClient returns a new http.Client with similar default values to +// http.Client, but with a non-shared Transport, idle connections disabled, and +// keepalives disabled. +func DefaultClient() *http.Client { + return &http.Client{ + Transport: DefaultTransport(), + } +} + +// DefaultPooledClient returns a new http.Client with similar default values to +// http.Client, but with a shared Transport. Do not use this function for +// transient clients as it can leak file descriptors over time. Only use this +// for clients that will be re-used for the same host(s). +func DefaultPooledClient() *http.Client { + return &http.Client{ + Transport: DefaultPooledTransport(), + } +} diff --git a/vendor/github.com/hashicorp/go-cleanhttp/doc.go b/vendor/github.com/hashicorp/go-cleanhttp/doc.go new file mode 100644 index 00000000000..05841092a7b --- /dev/null +++ b/vendor/github.com/hashicorp/go-cleanhttp/doc.go @@ -0,0 +1,20 @@ +// Package cleanhttp offers convenience utilities for acquiring "clean" +// http.Transport and http.Client structs. +// +// Values set on http.DefaultClient and http.DefaultTransport affect all +// callers. This can have detrimental effects, esepcially in TLS contexts, +// where client or root certificates set to talk to multiple endpoints can end +// up displacing each other, leading to hard-to-debug issues. This package +// provides non-shared http.Client and http.Transport structs to ensure that +// the configuration will not be overwritten by other parts of the application +// or dependencies. +// +// The DefaultClient and DefaultTransport functions disable idle connections +// and keepalives. Without ensuring that idle connections are closed before +// garbage collection, short-term clients/transports can leak file descriptors, +// eventually leading to "too many open files" errors. If you will be +// connecting to the same hosts repeatedly from the same client, you can use +// DefaultPooledClient to receive a client that has connection pooling +// semantics similar to http.DefaultClient. +// +package cleanhttp diff --git a/vendor/github.com/hashicorp/go-cleanhttp/go.mod b/vendor/github.com/hashicorp/go-cleanhttp/go.mod new file mode 100644 index 00000000000..310f07569fc --- /dev/null +++ b/vendor/github.com/hashicorp/go-cleanhttp/go.mod @@ -0,0 +1 @@ +module github.com/hashicorp/go-cleanhttp diff --git a/vendor/github.com/hashicorp/go-cleanhttp/handlers.go b/vendor/github.com/hashicorp/go-cleanhttp/handlers.go new file mode 100644 index 00000000000..3c845dc0dc6 --- /dev/null +++ b/vendor/github.com/hashicorp/go-cleanhttp/handlers.go @@ -0,0 +1,48 @@ +package cleanhttp + +import ( + "net/http" + "strings" + "unicode" +) + +// HandlerInput provides input options to cleanhttp's handlers +type HandlerInput struct { + ErrStatus int +} + +// PrintablePathCheckHandler is a middleware that ensures the request path +// contains only printable runes. +func PrintablePathCheckHandler(next http.Handler, input *HandlerInput) http.Handler { + // Nil-check on input to make it optional + if input == nil { + input = &HandlerInput{ + ErrStatus: http.StatusBadRequest, + } + } + + // Default to http.StatusBadRequest on error + if input.ErrStatus == 0 { + input.ErrStatus = http.StatusBadRequest + } + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r != nil { + // Check URL path for non-printable characters + idx := strings.IndexFunc(r.URL.Path, func(c rune) bool { + return !unicode.IsPrint(c) + }) + + if idx != -1 { + w.WriteHeader(input.ErrStatus) + return + } + + if next != nil { + next.ServeHTTP(w, r) + } + } + + return + }) +} diff --git a/vendor/github.com/hashicorp/go-retryablehttp/.gitignore b/vendor/github.com/hashicorp/go-retryablehttp/.gitignore new file mode 100644 index 00000000000..4e309e0b326 --- /dev/null +++ b/vendor/github.com/hashicorp/go-retryablehttp/.gitignore @@ -0,0 +1,4 @@ +.idea/ +*.iml +*.test +.vscode/ \ No newline at end of file diff --git a/vendor/github.com/hashicorp/go-retryablehttp/.travis.yml b/vendor/github.com/hashicorp/go-retryablehttp/.travis.yml new file mode 100644 index 00000000000..c4fb6d6c8bb --- /dev/null +++ b/vendor/github.com/hashicorp/go-retryablehttp/.travis.yml @@ -0,0 +1,12 @@ +sudo: false + +language: go + +go: + - 1.12.4 + +branches: + only: + - master + +script: make updatedeps test diff --git a/vendor/github.com/hashicorp/go-retryablehttp/LICENSE b/vendor/github.com/hashicorp/go-retryablehttp/LICENSE new file mode 100644 index 00000000000..e87a115e462 --- /dev/null +++ b/vendor/github.com/hashicorp/go-retryablehttp/LICENSE @@ -0,0 +1,363 @@ +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. "Contributor" + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. "Contributor Version" + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms of + a Secondary License. + +1.6. "Executable Form" + + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + + means a work that combines Covered Software with other material, in a + separate file or files, that is not Covered Software. + +1.8. "License" + + means this document. + +1.9. "Licensable" + + means having the right to grant, to the maximum extent possible, whether + at the time of the initial grant or subsequently, any and all of the + rights conveyed by this License. + +1.10. "Modifications" + + means any of the following: + + a. any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. "Patent Claims" of a Contributor + + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the License, + by the making, using, selling, offering for sale, having made, import, + or transfer of either its Contributions or its Contributor Version. + +1.12. "Secondary License" + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. "Source Code Form" + + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, "control" means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor first + distributes such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under + this License. No additional rights or licenses will be implied from the + distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted by a + Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of + its Contributions. + + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License (if + permitted under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights to + grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, or other + equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under + the terms of this License. You must inform recipients that the Source + Code Form of the Covered Software is governed by the terms of this + License, and how they can obtain a copy of this License. You may not + attempt to alter or restrict the recipients' rights in the Source Code + Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter the + recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of Covered + Software with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the Covered + Software under the terms of either this License or such Secondary + License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, or + limitations of liability) contained within the Source Code Form of the + Covered Software, except that You may alter any license notices to the + extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on + behalf of any Contributor. You must make it absolutely clear that any + such warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, + judicial order, or regulation then You must: (a) comply with the terms of + this License to the maximum extent possible; and (b) describe the + limitations and the code they affect. Such description must be placed in a + text file included with all distributions of the Covered Software under + this License. Except to the extent prohibited by statute or regulation, + such description must be sufficiently detailed for a recipient of ordinary + skill to be able to understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing + basis, if such Contributor fails to notify You of the non-compliance by + some reasonable means prior to 60 days after You have come back into + compliance. Moreover, Your grants from a particular Contributor are + reinstated on an ongoing basis if such Contributor notifies You of the + non-compliance by some reasonable means, this is the first time You have + received notice of non-compliance with this License from such + Contributor, and You become compliant prior to 30 days after Your receipt + of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted to + You by any and all Contributors for the Covered Software under Section + 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an "as is" basis, + without warranty of any kind, either expressed, implied, or statutory, + including, without limitation, warranties that the Covered Software is free + of defects, merchantable, fit for a particular purpose or non-infringing. + The entire risk as to the quality and performance of the Covered Software + is with You. Should any Covered Software prove defective in any respect, + You (not any Contributor) assume the cost of any necessary servicing, + repair, or correction. This disclaimer of warranty constitutes an essential + part of this License. No use of any Covered Software is authorized under + this License except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from + such party's negligence to the extent applicable law prohibits such + limitation. Some jurisdictions do not allow the exclusion or limitation of + incidental or consequential damages, so this exclusion and limitation may + not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts + of a jurisdiction where the defendant maintains its principal place of + business and such litigation shall be governed by laws of that + jurisdiction, without reference to its conflict-of-law provisions. Nothing + in this Section shall prevent a party's ability to bring cross-claims or + counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. Any law or regulation which provides that + the language of a contract shall be construed against the drafter shall not + be used to construe this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a + modified version of this License if you rename the license and remove + any references to the name of the license steward (except to note that + such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary + Licenses If You choose to distribute Source Code Form that is + Incompatible With Secondary Licenses under the terms of this version of + the License, the notice described in Exhibit B of this License must be + attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, +then You may include the notice in a location (such as a LICENSE file in a +relevant directory) where a recipient would be likely to look for such a +notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice + + This Source Code Form is "Incompatible + With Secondary Licenses", as defined by + the Mozilla Public License, v. 2.0. + diff --git a/vendor/github.com/hashicorp/go-retryablehttp/Makefile b/vendor/github.com/hashicorp/go-retryablehttp/Makefile new file mode 100644 index 00000000000..da17640e644 --- /dev/null +++ b/vendor/github.com/hashicorp/go-retryablehttp/Makefile @@ -0,0 +1,11 @@ +default: test + +test: + go vet ./... + go test -race ./... + +updatedeps: + go get -f -t -u ./... + go get -f -u ./... + +.PHONY: default test updatedeps diff --git a/vendor/github.com/hashicorp/go-retryablehttp/README.md b/vendor/github.com/hashicorp/go-retryablehttp/README.md new file mode 100644 index 00000000000..30357c75668 --- /dev/null +++ b/vendor/github.com/hashicorp/go-retryablehttp/README.md @@ -0,0 +1,61 @@ +go-retryablehttp +================ + +[![Build Status](http://img.shields.io/travis/hashicorp/go-retryablehttp.svg?style=flat-square)][travis] +[![Go Documentation](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)][godocs] + +[travis]: http://travis-ci.org/hashicorp/go-retryablehttp +[godocs]: http://godoc.org/github.com/hashicorp/go-retryablehttp + +The `retryablehttp` package provides a familiar HTTP client interface with +automatic retries and exponential backoff. It is a thin wrapper over the +standard `net/http` client library and exposes nearly the same public API. This +makes `retryablehttp` very easy to drop into existing programs. + +`retryablehttp` performs automatic retries under certain conditions. Mainly, if +an error is returned by the client (connection errors, etc.), or if a 500-range +response code is received (except 501), then a retry is invoked after a wait +period. Otherwise, the response is returned and left to the caller to +interpret. + +The main difference from `net/http` is that requests which take a request body +(POST/PUT et. al) can have the body provided in a number of ways (some more or +less efficient) that allow "rewinding" the request body if the initial request +fails so that the full request can be attempted again. See the +[godoc](http://godoc.org/github.com/hashicorp/go-retryablehttp) for more +details. + +Version 0.6.0 and before are compatible with Go prior to 1.12. From 0.6.1 onward, Go 1.12+ is required. + +Example Use +=========== + +Using this library should look almost identical to what you would do with +`net/http`. The most simple example of a GET request is shown below: + +```go +resp, err := retryablehttp.Get("/foo") +if err != nil { + panic(err) +} +``` + +The returned response object is an `*http.Response`, the same thing you would +usually get from `net/http`. Had the request failed one or more times, the above +call would block and retry with exponential backoff. + +## Getting a stdlib `*http.Client` with retries + +It's possible to convert a `*retryablehttp.Client` directly to a `*http.Client`. +This makes use of retryablehttp broadly applicable with minimal effort. Simply +configure a `*retryablehttp.Client` as you wish, and then call `StandardClient()`: + +```go +retryClient := retryablehttp.NewClient() +retryClient.RetryMax = 10 + +standardClient := retryClient.StandardClient() // *http.Client +``` + +For more usage and examples see the +[godoc](http://godoc.org/github.com/hashicorp/go-retryablehttp). diff --git a/vendor/github.com/hashicorp/go-retryablehttp/client.go b/vendor/github.com/hashicorp/go-retryablehttp/client.go new file mode 100644 index 00000000000..f1ccd3df35c --- /dev/null +++ b/vendor/github.com/hashicorp/go-retryablehttp/client.go @@ -0,0 +1,705 @@ +// Package retryablehttp provides a familiar HTTP client interface with +// automatic retries and exponential backoff. It is a thin wrapper over the +// standard net/http client library and exposes nearly the same public API. +// This makes retryablehttp very easy to drop into existing programs. +// +// retryablehttp performs automatic retries under certain conditions. Mainly, if +// an error is returned by the client (connection errors etc), or if a 500-range +// response is received, then a retry is invoked. Otherwise, the response is +// returned and left to the caller to interpret. +// +// Requests which take a request body should provide a non-nil function +// parameter. The best choice is to provide either a function satisfying +// ReaderFunc which provides multiple io.Readers in an efficient manner, a +// *bytes.Buffer (the underlying raw byte slice will be used) or a raw byte +// slice. As it is a reference type, and we will wrap it as needed by readers, +// we can efficiently re-use the request body without needing to copy it. If an +// io.Reader (such as a *bytes.Reader) is provided, the full body will be read +// prior to the first request, and will be efficiently re-used for any retries. +// ReadSeeker can be used, but some users have observed occasional data races +// between the net/http library and the Seek functionality of some +// implementations of ReadSeeker, so should be avoided if possible. +package retryablehttp + +import ( + "bytes" + "context" + "crypto/x509" + "fmt" + "io" + "io/ioutil" + "log" + "math" + "math/rand" + "net/http" + "net/url" + "os" + "regexp" + "strings" + "sync" + "time" + + "github.com/hashicorp/go-cleanhttp" +) + +var ( + // Default retry configuration + defaultRetryWaitMin = 1 * time.Second + defaultRetryWaitMax = 30 * time.Second + defaultRetryMax = 4 + + // defaultLogger is the logger provided with defaultClient + defaultLogger = log.New(os.Stderr, "", log.LstdFlags) + + // defaultClient is used for performing requests without explicitly making + // a new client. It is purposely private to avoid modifications. + defaultClient = NewClient() + + // We need to consume response bodies to maintain http connections, but + // limit the size we consume to respReadLimit. + respReadLimit = int64(4096) + + // A regular expression to match the error returned by net/http when the + // configured number of redirects is exhausted. This error isn't typed + // specifically so we resort to matching on the error string. + redirectsErrorRe = regexp.MustCompile(`stopped after \d+ redirects\z`) + + // A regular expression to match the error returned by net/http when the + // scheme specified in the URL is invalid. This error isn't typed + // specifically so we resort to matching on the error string. + schemeErrorRe = regexp.MustCompile(`unsupported protocol scheme`) +) + +// ReaderFunc is the type of function that can be given natively to NewRequest +type ReaderFunc func() (io.Reader, error) + +// LenReader is an interface implemented by many in-memory io.Reader's. Used +// for automatically sending the right Content-Length header when possible. +type LenReader interface { + Len() int +} + +// Request wraps the metadata needed to create HTTP requests. +type Request struct { + // body is a seekable reader over the request body payload. This is + // used to rewind the request data in between retries. + body ReaderFunc + + // Embed an HTTP request directly. This makes a *Request act exactly + // like an *http.Request so that all meta methods are supported. + *http.Request +} + +// WithContext returns wrapped Request with a shallow copy of underlying *http.Request +// with its context changed to ctx. The provided ctx must be non-nil. +func (r *Request) WithContext(ctx context.Context) *Request { + r.Request = r.Request.WithContext(ctx) + return r +} + +// BodyBytes allows accessing the request body. It is an analogue to +// http.Request's Body variable, but it returns a copy of the underlying data +// rather than consuming it. +// +// This function is not thread-safe; do not call it at the same time as another +// call, or at the same time this request is being used with Client.Do. +func (r *Request) BodyBytes() ([]byte, error) { + if r.body == nil { + return nil, nil + } + body, err := r.body() + if err != nil { + return nil, err + } + buf := new(bytes.Buffer) + _, err = buf.ReadFrom(body) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// SetBody allows setting the request body. +// +// It is useful if a new body needs to be set without constructing a new Request. +func (r *Request) SetBody(rawBody interface{}) error { + bodyReader, contentLength, err := getBodyReaderAndContentLength(rawBody) + if err != nil { + return err + } + r.body = bodyReader + r.ContentLength = contentLength + return nil +} + +// WriteTo allows copying the request body into a writer. +// +// It writes data to w until there's no more data to write or +// when an error occurs. The return int64 value is the number of bytes +// written. Any error encountered during the write is also returned. +// The signature matches io.WriterTo interface. +func (r *Request) WriteTo(w io.Writer) (int64, error) { + body, err := r.body() + if err != nil { + return 0, err + } + if c, ok := body.(io.Closer); ok { + defer c.Close() + } + return io.Copy(w, body) +} + +func getBodyReaderAndContentLength(rawBody interface{}) (ReaderFunc, int64, error) { + var bodyReader ReaderFunc + var contentLength int64 + + switch body := rawBody.(type) { + // If they gave us a function already, great! Use it. + case ReaderFunc: + bodyReader = body + tmp, err := body() + if err != nil { + return nil, 0, err + } + if lr, ok := tmp.(LenReader); ok { + contentLength = int64(lr.Len()) + } + if c, ok := tmp.(io.Closer); ok { + c.Close() + } + + case func() (io.Reader, error): + bodyReader = body + tmp, err := body() + if err != nil { + return nil, 0, err + } + if lr, ok := tmp.(LenReader); ok { + contentLength = int64(lr.Len()) + } + if c, ok := tmp.(io.Closer); ok { + c.Close() + } + + // If a regular byte slice, we can read it over and over via new + // readers + case []byte: + buf := body + bodyReader = func() (io.Reader, error) { + return bytes.NewReader(buf), nil + } + contentLength = int64(len(buf)) + + // If a bytes.Buffer we can read the underlying byte slice over and + // over + case *bytes.Buffer: + buf := body + bodyReader = func() (io.Reader, error) { + return bytes.NewReader(buf.Bytes()), nil + } + contentLength = int64(buf.Len()) + + // We prioritize *bytes.Reader here because we don't really want to + // deal with it seeking so want it to match here instead of the + // io.ReadSeeker case. + case *bytes.Reader: + buf, err := ioutil.ReadAll(body) + if err != nil { + return nil, 0, err + } + bodyReader = func() (io.Reader, error) { + return bytes.NewReader(buf), nil + } + contentLength = int64(len(buf)) + + // Compat case + case io.ReadSeeker: + raw := body + bodyReader = func() (io.Reader, error) { + _, err := raw.Seek(0, 0) + return ioutil.NopCloser(raw), err + } + if lr, ok := raw.(LenReader); ok { + contentLength = int64(lr.Len()) + } + + // Read all in so we can reset + case io.Reader: + buf, err := ioutil.ReadAll(body) + if err != nil { + return nil, 0, err + } + bodyReader = func() (io.Reader, error) { + return bytes.NewReader(buf), nil + } + contentLength = int64(len(buf)) + + // No body provided, nothing to do + case nil: + + // Unrecognized type + default: + return nil, 0, fmt.Errorf("cannot handle type %T", rawBody) + } + return bodyReader, contentLength, nil +} + +// FromRequest wraps an http.Request in a retryablehttp.Request +func FromRequest(r *http.Request) (*Request, error) { + bodyReader, _, err := getBodyReaderAndContentLength(r.Body) + if err != nil { + return nil, err + } + // Could assert contentLength == r.ContentLength + return &Request{bodyReader, r}, nil +} + +// NewRequest creates a new wrapped request. +func NewRequest(method, url string, rawBody interface{}) (*Request, error) { + bodyReader, contentLength, err := getBodyReaderAndContentLength(rawBody) + if err != nil { + return nil, err + } + + httpReq, err := http.NewRequest(method, url, nil) + if err != nil { + return nil, err + } + httpReq.ContentLength = contentLength + + return &Request{bodyReader, httpReq}, nil +} + +// Logger interface allows to use other loggers than +// standard log.Logger. +type Logger interface { + Printf(string, ...interface{}) +} + +// LeveledLogger interface implements the basic methods that a logger library needs +type LeveledLogger interface { + Error(string, ...interface{}) + Info(string, ...interface{}) + Debug(string, ...interface{}) + Warn(string, ...interface{}) +} + +// hookLogger adapts an LeveledLogger to Logger for use by the existing hook functions +// without changing the API. +type hookLogger struct { + LeveledLogger +} + +func (h hookLogger) Printf(s string, args ...interface{}) { + h.Info(fmt.Sprintf(s, args...)) +} + +// RequestLogHook allows a function to run before each retry. The HTTP +// request which will be made, and the retry number (0 for the initial +// request) are available to users. The internal logger is exposed to +// consumers. +type RequestLogHook func(Logger, *http.Request, int) + +// ResponseLogHook is like RequestLogHook, but allows running a function +// on each HTTP response. This function will be invoked at the end of +// every HTTP request executed, regardless of whether a subsequent retry +// needs to be performed or not. If the response body is read or closed +// from this method, this will affect the response returned from Do(). +type ResponseLogHook func(Logger, *http.Response) + +// CheckRetry specifies a policy for handling retries. It is called +// following each request with the response and error values returned by +// the http.Client. If CheckRetry returns false, the Client stops retrying +// and returns the response to the caller. If CheckRetry returns an error, +// that error value is returned in lieu of the error from the request. The +// Client will close any response body when retrying, but if the retry is +// aborted it is up to the CheckRetry callback to properly close any +// response body before returning. +type CheckRetry func(ctx context.Context, resp *http.Response, err error) (bool, error) + +// Backoff specifies a policy for how long to wait between retries. +// It is called after a failing request to determine the amount of time +// that should pass before trying again. +type Backoff func(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration + +// ErrorHandler is called if retries are expired, containing the last status +// from the http library. If not specified, default behavior for the library is +// to close the body and return an error indicating how many tries were +// attempted. If overriding this, be sure to close the body if needed. +type ErrorHandler func(resp *http.Response, err error, numTries int) (*http.Response, error) + +// Client is used to make HTTP requests. It adds additional functionality +// like automatic retries to tolerate minor outages. +type Client struct { + HTTPClient *http.Client // Internal HTTP client. + Logger interface{} // Customer logger instance. Can be either Logger or LeveledLogger + + RetryWaitMin time.Duration // Minimum time to wait + RetryWaitMax time.Duration // Maximum time to wait + RetryMax int // Maximum number of retries + + // RequestLogHook allows a user-supplied function to be called + // before each retry. + RequestLogHook RequestLogHook + + // ResponseLogHook allows a user-supplied function to be called + // with the response from each HTTP request executed. + ResponseLogHook ResponseLogHook + + // CheckRetry specifies the policy for handling retries, and is called + // after each request. The default policy is DefaultRetryPolicy. + CheckRetry CheckRetry + + // Backoff specifies the policy for how long to wait between retries + Backoff Backoff + + // ErrorHandler specifies the custom error handler to use, if any + ErrorHandler ErrorHandler + + loggerInit sync.Once +} + +// NewClient creates a new Client with default settings. +func NewClient() *Client { + return &Client{ + HTTPClient: cleanhttp.DefaultPooledClient(), + Logger: defaultLogger, + RetryWaitMin: defaultRetryWaitMin, + RetryWaitMax: defaultRetryWaitMax, + RetryMax: defaultRetryMax, + CheckRetry: DefaultRetryPolicy, + Backoff: DefaultBackoff, + } +} + +func (c *Client) logger() interface{} { + c.loggerInit.Do(func() { + if c.Logger == nil { + return + } + + switch c.Logger.(type) { + case Logger, LeveledLogger: + // ok + default: + // This should happen in dev when they are setting Logger and work on code, not in prod. + panic(fmt.Sprintf("invalid logger type passed, must be Logger or LeveledLogger, was %T", c.Logger)) + } + }) + + return c.Logger +} + +// DefaultRetryPolicy provides a default callback for Client.CheckRetry, which +// will retry on connection errors and server errors. +func DefaultRetryPolicy(ctx context.Context, resp *http.Response, err error) (bool, error) { + // do not retry on context.Canceled or context.DeadlineExceeded + if ctx.Err() != nil { + return false, ctx.Err() + } + + if err != nil { + if v, ok := err.(*url.Error); ok { + // Don't retry if the error was due to too many redirects. + if redirectsErrorRe.MatchString(v.Error()) { + return false, nil + } + + // Don't retry if the error was due to an invalid protocol scheme. + if schemeErrorRe.MatchString(v.Error()) { + return false, nil + } + + // Don't retry if the error was due to TLS cert verification failure. + if _, ok := v.Err.(x509.UnknownAuthorityError); ok { + return false, nil + } + } + + // The error is likely recoverable so retry. + return true, nil + } + + // Check the response code. We retry on 500-range responses to allow + // the server time to recover, as 500's are typically not permanent + // errors and may relate to outages on the server side. This will catch + // invalid response codes as well, like 0 and 999. + if resp.StatusCode == 0 || (resp.StatusCode >= 500 && resp.StatusCode != 501) { + return true, nil + } + + return false, nil +} + +// DefaultBackoff provides a default callback for Client.Backoff which +// will perform exponential backoff based on the attempt number and limited +// by the provided minimum and maximum durations. +func DefaultBackoff(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration { + mult := math.Pow(2, float64(attemptNum)) * float64(min) + sleep := time.Duration(mult) + if float64(sleep) != mult || sleep > max { + sleep = max + } + return sleep +} + +// LinearJitterBackoff provides a callback for Client.Backoff which will +// perform linear backoff based on the attempt number and with jitter to +// prevent a thundering herd. +// +// min and max here are *not* absolute values. The number to be multiplied by +// the attempt number will be chosen at random from between them, thus they are +// bounding the jitter. +// +// For instance: +// * To get strictly linear backoff of one second increasing each retry, set +// both to one second (1s, 2s, 3s, 4s, ...) +// * To get a small amount of jitter centered around one second increasing each +// retry, set to around one second, such as a min of 800ms and max of 1200ms +// (892ms, 2102ms, 2945ms, 4312ms, ...) +// * To get extreme jitter, set to a very wide spread, such as a min of 100ms +// and a max of 20s (15382ms, 292ms, 51321ms, 35234ms, ...) +func LinearJitterBackoff(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration { + // attemptNum always starts at zero but we want to start at 1 for multiplication + attemptNum++ + + if max <= min { + // Unclear what to do here, or they are the same, so return min * + // attemptNum + return min * time.Duration(attemptNum) + } + + // Seed rand; doing this every time is fine + rand := rand.New(rand.NewSource(int64(time.Now().Nanosecond()))) + + // Pick a random number that lies somewhere between the min and max and + // multiply by the attemptNum. attemptNum starts at zero so we always + // increment here. We first get a random percentage, then apply that to the + // difference between min and max, and add to min. + jitter := rand.Float64() * float64(max-min) + jitterMin := int64(jitter) + int64(min) + return time.Duration(jitterMin * int64(attemptNum)) +} + +// PassthroughErrorHandler is an ErrorHandler that directly passes through the +// values from the net/http library for the final request. The body is not +// closed. +func PassthroughErrorHandler(resp *http.Response, err error, _ int) (*http.Response, error) { + return resp, err +} + +// Do wraps calling an HTTP method with retries. +func (c *Client) Do(req *Request) (*http.Response, error) { + if c.HTTPClient == nil { + c.HTTPClient = cleanhttp.DefaultPooledClient() + } + + logger := c.logger() + + if logger != nil { + switch v := logger.(type) { + case Logger: + v.Printf("[DEBUG] %s %s", req.Method, req.URL) + case LeveledLogger: + v.Debug("performing request", "method", req.Method, "url", req.URL) + } + } + + var resp *http.Response + var err error + + for i := 0; ; i++ { + var code int // HTTP response code + + // Always rewind the request body when non-nil. + if req.body != nil { + body, err := req.body() + if err != nil { + c.HTTPClient.CloseIdleConnections() + return resp, err + } + if c, ok := body.(io.ReadCloser); ok { + req.Body = c + } else { + req.Body = ioutil.NopCloser(body) + } + } + + if c.RequestLogHook != nil { + switch v := logger.(type) { + case Logger: + c.RequestLogHook(v, req.Request, i) + case LeveledLogger: + c.RequestLogHook(hookLogger{v}, req.Request, i) + default: + c.RequestLogHook(nil, req.Request, i) + } + } + + // Attempt the request + resp, err = c.HTTPClient.Do(req.Request) + if resp != nil { + code = resp.StatusCode + } + + // Check if we should continue with retries. + checkOK, checkErr := c.CheckRetry(req.Context(), resp, err) + + if err != nil { + switch v := logger.(type) { + case Logger: + v.Printf("[ERR] %s %s request failed: %v", req.Method, req.URL, err) + case LeveledLogger: + v.Error("request failed", "error", err, "method", req.Method, "url", req.URL) + } + } else { + // Call this here to maintain the behavior of logging all requests, + // even if CheckRetry signals to stop. + if c.ResponseLogHook != nil { + // Call the response logger function if provided. + switch v := logger.(type) { + case Logger: + c.ResponseLogHook(v, resp) + case LeveledLogger: + c.ResponseLogHook(hookLogger{v}, resp) + default: + c.ResponseLogHook(nil, resp) + } + } + } + + // Now decide if we should continue. + if !checkOK { + if checkErr != nil { + err = checkErr + } + c.HTTPClient.CloseIdleConnections() + return resp, err + } + + // We do this before drainBody because there's no need for the I/O if + // we're breaking out + remain := c.RetryMax - i + if remain <= 0 { + break + } + + // We're going to retry, consume any response to reuse the connection. + if err == nil && resp != nil { + c.drainBody(resp.Body) + } + + wait := c.Backoff(c.RetryWaitMin, c.RetryWaitMax, i, resp) + desc := fmt.Sprintf("%s %s", req.Method, req.URL) + if code > 0 { + desc = fmt.Sprintf("%s (status: %d)", desc, code) + } + if logger != nil { + switch v := logger.(type) { + case Logger: + v.Printf("[DEBUG] %s: retrying in %s (%d left)", desc, wait, remain) + case LeveledLogger: + v.Debug("retrying request", "request", desc, "timeout", wait, "remaining", remain) + } + } + select { + case <-req.Context().Done(): + c.HTTPClient.CloseIdleConnections() + return nil, req.Context().Err() + case <-time.After(wait): + } + } + + if c.ErrorHandler != nil { + c.HTTPClient.CloseIdleConnections() + return c.ErrorHandler(resp, err, c.RetryMax+1) + } + + // By default, we close the response body and return an error without + // returning the response + if resp != nil { + resp.Body.Close() + } + c.HTTPClient.CloseIdleConnections() + return nil, fmt.Errorf("%s %s giving up after %d attempts", + req.Method, req.URL, c.RetryMax+1) +} + +// Try to read the response body so we can reuse this connection. +func (c *Client) drainBody(body io.ReadCloser) { + defer body.Close() + _, err := io.Copy(ioutil.Discard, io.LimitReader(body, respReadLimit)) + if err != nil { + if c.logger() != nil { + switch v := c.logger().(type) { + case Logger: + v.Printf("[ERR] error reading response body: %v", err) + case LeveledLogger: + v.Error("error reading response body", "error", err) + } + } + } +} + +// Get is a shortcut for doing a GET request without making a new client. +func Get(url string) (*http.Response, error) { + return defaultClient.Get(url) +} + +// Get is a convenience helper for doing simple GET requests. +func (c *Client) Get(url string) (*http.Response, error) { + req, err := NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + return c.Do(req) +} + +// Head is a shortcut for doing a HEAD request without making a new client. +func Head(url string) (*http.Response, error) { + return defaultClient.Head(url) +} + +// Head is a convenience method for doing simple HEAD requests. +func (c *Client) Head(url string) (*http.Response, error) { + req, err := NewRequest("HEAD", url, nil) + if err != nil { + return nil, err + } + return c.Do(req) +} + +// Post is a shortcut for doing a POST request without making a new client. +func Post(url, bodyType string, body interface{}) (*http.Response, error) { + return defaultClient.Post(url, bodyType, body) +} + +// Post is a convenience method for doing simple POST requests. +func (c *Client) Post(url, bodyType string, body interface{}) (*http.Response, error) { + req, err := NewRequest("POST", url, body) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", bodyType) + return c.Do(req) +} + +// PostForm is a shortcut to perform a POST with form data without creating +// a new client. +func PostForm(url string, data url.Values) (*http.Response, error) { + return defaultClient.PostForm(url, data) +} + +// PostForm is a convenience method for doing simple POST operations using +// pre-filled url.Values form data. +func (c *Client) PostForm(url string, data url.Values) (*http.Response, error) { + return c.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode())) +} + +// StandardClient returns a stdlib *http.Client with a custom Transport, which +// shims in a *retryablehttp.Client for added retries. +func (c *Client) StandardClient() *http.Client { + return &http.Client{ + Transport: &RoundTripper{Client: c}, + } +} diff --git a/vendor/github.com/hashicorp/go-retryablehttp/go.mod b/vendor/github.com/hashicorp/go-retryablehttp/go.mod new file mode 100644 index 00000000000..7cc02b76fa4 --- /dev/null +++ b/vendor/github.com/hashicorp/go-retryablehttp/go.mod @@ -0,0 +1,8 @@ +module github.com/hashicorp/go-retryablehttp + +require ( + github.com/hashicorp/go-cleanhttp v0.5.1 + github.com/hashicorp/go-hclog v0.9.2 +) + +go 1.13 diff --git a/vendor/github.com/hashicorp/go-retryablehttp/go.sum b/vendor/github.com/hashicorp/go-retryablehttp/go.sum new file mode 100644 index 00000000000..71afe568227 --- /dev/null +++ b/vendor/github.com/hashicorp/go-retryablehttp/go.sum @@ -0,0 +1,10 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= diff --git a/vendor/github.com/hashicorp/go-retryablehttp/roundtripper.go b/vendor/github.com/hashicorp/go-retryablehttp/roundtripper.go new file mode 100644 index 00000000000..b841b4cfe53 --- /dev/null +++ b/vendor/github.com/hashicorp/go-retryablehttp/roundtripper.go @@ -0,0 +1,43 @@ +package retryablehttp + +import ( + "net/http" + "sync" +) + +// RoundTripper implements the http.RoundTripper interface, using a retrying +// HTTP client to execute requests. +// +// It is important to note that retryablehttp doesn't always act exactly as a +// RoundTripper should. This is highly dependent on the retryable client's +// configuration. +type RoundTripper struct { + // The client to use during requests. If nil, the default retryablehttp + // client and settings will be used. + Client *Client + + // once ensures that the logic to initialize the default client runs at + // most once, in a single thread. + once sync.Once +} + +// init initializes the underlying retryable client. +func (rt *RoundTripper) init() { + if rt.Client == nil { + rt.Client = NewClient() + } +} + +// RoundTrip satisfies the http.RoundTripper interface. +func (rt *RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + rt.once.Do(rt.init) + + // Convert the request to be retryable. + retryableReq, err := FromRequest(req) + if err != nil { + return nil, err + } + + // Execute the request. + return rt.Client.Do(retryableReq) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 64a86799453..9c74ca75213 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -182,6 +182,11 @@ github.com/googleapis/gnostic/extensions github.com/grpc-ecosystem/grpc-gateway/internal github.com/grpc-ecosystem/grpc-gateway/runtime github.com/grpc-ecosystem/grpc-gateway/utilities +# github.com/hashicorp/go-cleanhttp v0.5.1 +github.com/hashicorp/go-cleanhttp +# github.com/hashicorp/go-retryablehttp v0.6.6 +## explicit +github.com/hashicorp/go-retryablehttp # github.com/hashicorp/golang-lru v0.5.4 github.com/hashicorp/golang-lru github.com/hashicorp/golang-lru/simplelru