@@ -6,19 +6,26 @@ import (
66 "expvar"
77 "fmt"
88 "log"
9+ "net"
10+ "net/url"
911 "strings"
1012 "sync"
1113 "sync/atomic"
1214 "time"
1315
14- "github.com/aws/aws-sdk-go/aws"
15- "github.com/aws/aws-sdk-go/aws/awserr "
16- "github.com/aws/aws-sdk-go/aws/request "
17- "github.com/aws/aws-sdk-go/service/firehose "
18- "github.com/aws/aws-sdk- go/service/firehose/firehoseiface "
16+ "github.com/aws/aws-sdk-go-v2 /aws"
17+ "github.com/aws/aws-sdk-go-v2/service/firehose "
18+ "github.com/aws/aws-sdk-go-v2/service/firehose/types "
19+ "github.com/aws/smithy-go "
20+ smithyhttp "github.com/aws/smithy- go/transport/http "
1921 "github.com/cenkalti/backoff/v4"
2022)
2123
24+ // FirehoseAPI is the part of *firehose.Client that Queue uses.
25+ type FirehoseAPI interface {
26+ PutRecordBatch (ctx context.Context , params * firehose.PutRecordBatchInput , optFns ... func (* firehose.Options )) (* firehose.PutRecordBatchOutput , error )
27+ }
28+
2229// Option is a type for constructor options
2330type Option func (* Queue )
2431
@@ -30,7 +37,7 @@ func Parallel(i int) Option {
3037}
3138
3239// ErrorHandler is an option to specify the error handler
33- func ErrorHandler (fn func (error , []* firehose .Record )) Option {
40+ func ErrorHandler (fn func (error , []types .Record )) Option {
3441 return func (q * Queue ) {
3542 q .errorHandler = fn
3643 }
@@ -76,10 +83,10 @@ func BatchInterval(period time.Duration) Option {
7683}
7784
7885// New return new Queue
79- func New (fh firehoseiface. FirehoseAPI , DeliveryStreamName string , opts ... Option ) * Queue {
86+ func New (fh FirehoseAPI , DeliveryStreamName string , opts ... Option ) * Queue {
8087 q := & Queue {
81- queue : make ([]* firehose .Record , 0 ),
82- workerCh : make (chan []* firehose .Record ),
88+ queue : make ([]types .Record , 0 ),
89+ workerCh : make (chan []types .Record ),
8390 firehose : fh ,
8491 initialized : make (chan struct {}),
8592 DeliveryStreamName : aws .String (DeliveryStreamName ),
@@ -92,18 +99,18 @@ func New(fh firehoseiface.FirehoseAPI, DeliveryStreamName string, opts ...Option
9299
93100// Queue manages a sending list for firehose
94101type Queue struct {
95- queue []* firehose .Record
102+ queue []types .Record
96103 mu sync.RWMutex
97- workerCh chan []* firehose .Record
104+ workerCh chan []types .Record
98105
99106 initialized chan struct {}
100107 initMu sync.Mutex
101108
102109 para int
103110 inFlightBatchCounter int32
104111 maxQueueLength int
105- firehose firehoseiface. FirehoseAPI
106- errorHandler func (error , []* firehose .Record )
112+ firehose FirehoseAPI
113+ errorHandler func (error , []types .Record )
107114 batchSize int
108115 batchInterval time.Duration
109116 DeliveryStreamName * string
@@ -207,10 +214,14 @@ func (q *Queue) loop(ctx context.Context) {
207214 }
208215}
209216
217+ // isRetryable keeps the set of retryable errors that firequeue had with
218+ // aws-sdk-go v1 (request.IsErrorRetryable, request.IsErrorThrottle and the
219+ // HTTP status codes below).
210220func isRetryable (err error ) bool {
211- if reqfailure , ok := err .(awserr.RequestFailure ); ok {
221+ var respErr * smithyhttp.ResponseError
222+ if errors .As (err , & respErr ) {
212223 // ref. https://github.com/aws/aws-sdk-go/blob/fe72a52350a8962175bb71c531ec9724ce48abd8/aws/request/retryer.go#L228-L250
213- switch reqfailure . StatusCode () {
224+ switch respErr . HTTPStatusCode () {
214225 case
215226 429 ,
216227 500 ,
@@ -220,21 +231,114 @@ func isRetryable(err error) bool {
220231 return true
221232 }
222233 }
223- return request .IsErrorRetryable (err ) ||
224- request .IsErrorThrottle (err ) ||
225- isErrConnectionResetByPeer (err )
234+ return isErrorRetryable (err ) || isErrConnectionResetByPeer (err )
235+ }
236+
237+ // retryableErrorCodes are the API error codes that aws-sdk-go v1 retried:
238+ // retryableCodes, credsExpiredCodes and throttleCodes in aws/request/retryer.go.
239+ // v1 also had the client-side codes "RequestError" and "ResponseTimeout".
240+ // v2 does not use these codes. isErrorRetryable examines
241+ // smithyhttp.RequestSendError in place of "RequestError".
242+ var retryableErrorCodes = map [string ]struct {}{
243+ "RequestTimeout" : {},
244+ "RequestTimeoutException" : {},
245+
246+ "ExpiredToken" : {},
247+ "ExpiredTokenException" : {},
248+ "RequestExpired" : {},
249+
250+ "ProvisionedThroughputExceededException" : {},
251+ "ThrottledException" : {},
252+ "Throttling" : {},
253+ "ThrottlingException" : {},
254+ "RequestLimitExceeded" : {},
255+ "RequestThrottled" : {},
256+ "RequestThrottledException" : {},
257+ "TooManyRequestsException" : {},
258+ "PriorRequestNotComplete" : {},
259+ "TransactionInProgressException" : {},
260+ "EC2ThrottledException" : {},
261+ }
262+
263+ // isErrorRetryable follows request.IsErrorRetryable and
264+ // request.IsErrorThrottle of aws-sdk-go v1 for the errors that the v2 client
265+ // returns.
266+ func isErrorRetryable (err error ) bool {
267+ // v1 did not retry "RequestCanceled". v2 returns *smithy.CanceledError or
268+ // *aws.RequestCanceledError when the context is done.
269+ var canceledErr interface { CanceledError () bool }
270+ if errors .As (err , & canceledErr ) && canceledErr .CanceledError () {
271+ return false
272+ }
273+ var apiErr smithy.APIError
274+ if errors .As (err , & apiErr ) {
275+ _ , ok := retryableErrorCodes [apiErr .ErrorCode ()]
276+ return ok
277+ }
278+ // v1 "SerializationError": v1 retried it when it retried the cause,
279+ // and did not retry it without a cause.
280+ var deserializationErr * smithy.DeserializationError
281+ if errors .As (err , & deserializationErr ) {
282+ return deserializationErr .Err != nil && shouldRetryError (deserializationErr .Err )
283+ }
284+ // v1 "RequestError": v1 retried it when it retried the cause,
285+ // and retried it without a cause.
286+ var sendErr * smithyhttp.RequestSendError
287+ if errors .As (err , & sendErr ) {
288+ return shouldRetryError (sendErr .Err )
289+ }
290+ // v1 did not retry errors from parameter validation either.
291+ return false
292+ }
293+
294+ // shouldRetryError is the part of request.shouldRetryError in aws-sdk-go v1
295+ // that examines the causes of "RequestError" and "SerializationError".
296+ // ref. https://github.com/aws/aws-sdk-go/blob/v1.35.33/aws/request/retryer.go#L168-L226
297+ func shouldRetryError (err error ) bool {
298+ switch e := err .(type ) {
299+ case * url.Error :
300+ if strings .Contains (e .Error (), "connection refused" ) {
301+ return true
302+ }
303+ return shouldRetryError (e .Err )
304+ case interface { Temporary () bool }:
305+ if netErr , ok := e .(* net.OpError ); ok && netErr .Op == "dial" {
306+ return true
307+ }
308+ return e .Temporary () || isErrConnectionReset (err )
309+ case nil :
310+ return true
311+ default :
312+ switch err .Error () {
313+ case "net/http: request canceled" ,
314+ "net/http: request canceled while waiting for connection" :
315+ return false
316+ }
317+ return true
318+ }
226319}
227320
228- // request. isErrConnectionReset, which is used inside request.IsErrorRetryable,
229- // intentionally marks false (do not retry) if it contains "read: connection reset".
321+ // isErrConnectionReset is request.isErrConnectionReset in aws-sdk-go v1.
322+ // It intentionally marks false (do not retry) if it contains "read: connection reset".
230323// ref. https://github.com/aws/aws-sdk-go/blob/7814a7f61bf93cb54d00a5f97918c5501f07d351/aws/request/connection_reset_error.go#L7-L18
231- // However, in firehose, the connection reset by peer error often bursts, so it is to be retry
324+ func isErrConnectionReset (err error ) bool {
325+ msg := err .Error ()
326+ if strings .Contains (msg , "read: connection reset" ) {
327+ return false
328+ }
329+ return strings .Contains (msg , "use of closed network connection" ) ||
330+ strings .Contains (msg , "connection reset" ) ||
331+ strings .Contains (msg , "broken pipe" )
332+ }
333+
334+ // isErrConnectionResetByPeer catches "read: connection reset by peer", which isErrConnectionReset does not retry.
335+ // In firehose, the connection reset by peer error often bursts, so it is to be retry
232336func isErrConnectionResetByPeer (err error ) bool {
233337 return strings .Contains (err .Error (), "read: connection reset by peer" )
234338}
235339
236340// Enqueue add data to queue
237- func (q * Queue ) Enqueue (r * firehose .Record ) error {
341+ func (q * Queue ) Enqueue (r types .Record ) error {
238342 select {
239343 case <- q .initialized :
240344 // nop and continue to send
@@ -271,7 +375,7 @@ func (q *Queue) sendBatch(bf backoff.BackOff) time.Duration {
271375 return q .batchInterval
272376}
273377
274- func (q * Queue ) createBatch () []* firehose .Record {
378+ func (q * Queue ) createBatch () []types .Record {
275379 q .mu .Lock ()
276380 defer q .mu .Unlock ()
277381
@@ -280,13 +384,13 @@ func (q *Queue) createBatch() []*firehose.Record {
280384 batchSize = len (q .queue )
281385 }
282386
283- batch := make ([]* firehose .Record , batchSize )
387+ batch := make ([]types .Record , batchSize )
284388 copy (batch , q .queue )
285389 q .queue = q .queue [batchSize :]
286390 return batch
287391}
288392
289- func (q * Queue ) attemptToSendBatch (batch []* firehose .Record ) error {
393+ func (q * Queue ) attemptToSendBatch (batch []types .Record ) error {
290394 atomic .AddInt32 (& q .inFlightBatchCounter , 1 )
291395 defer atomic .AddInt32 (& q .inFlightBatchCounter , - 1 )
292396
@@ -297,7 +401,7 @@ func (q *Queue) attemptToSendBatch(batch []*firehose.Record) error {
297401 ctx , cancel := context .WithTimeout (context .Background (), 30 * time .Second )
298402 defer cancel ()
299403
300- resp , err := q .firehose .PutRecordBatchWithContext (ctx , input )
404+ resp , err := q .firehose .PutRecordBatch (ctx , input )
301405 if err != nil {
302406 // Handle non-retryable errors directly.
303407 if ! isRetryable (err ) {
@@ -318,18 +422,18 @@ func (q *Queue) attemptToSendBatch(batch []*firehose.Record) error {
318422 return nil
319423}
320424
321- func (q * Queue ) retryFailedRecords (batch []* firehose .Record , resp * firehose.PutRecordBatchOutput ) {
322- retryRecords := make ([]* firehose .Record , 0 , * resp .FailedPutCount )
425+ func (q * Queue ) retryFailedRecords (batch []types .Record , resp * firehose.PutRecordBatchOutput ) {
426+ retryRecords := make ([]types .Record , 0 , * resp .FailedPutCount )
323427
324428 for i , record := range batch {
325- if resp .RequestResponses != nil && resp .RequestResponses [i ] != nil && resp . RequestResponses [ i ] .ErrorCode != nil {
429+ if resp .RequestResponses != nil && resp .RequestResponses [i ].ErrorCode != nil {
326430 q .retryCount .Add (1 )
327431 retryRecords = append (retryRecords , record )
328432 }
329433 }
330434 q .push (retryRecords ... )
331435}
332- func (q * Queue ) handleError (err error , r []* firehose .Record ) {
436+ func (q * Queue ) handleError (err error , r []types .Record ) {
333437 if q .errorHandler != nil {
334438 q .errorHandler (err , r )
335439 return
@@ -352,7 +456,7 @@ func (q *Queue) remaining() bool {
352456 return q .len () > 0 || q .inFlightBatch ()
353457}
354458
355- func (q * Queue ) push (r ... * firehose .Record ) {
459+ func (q * Queue ) push (r ... types .Record ) {
356460 q .mu .Lock ()
357461 defer q .mu .Unlock ()
358462 q .queue = append (q .queue , r ... )
0 commit comments