Skip to content

Commit f5df875

Browse files
authored
Merge pull request #15 from natureglobal/refactor/aws-sdk-go-v2
Migrate to AWS SDK for Go v2
2 parents 567c240 + 0095552 commit f5df875

6 files changed

Lines changed: 279 additions & 110 deletions

File tree

‎README.md‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,19 +19,21 @@ firequeue is to ensure putting items to Amazon Kinesis Data Firehose with an in-
1919
import (
2020
"context"
2121

22-
"github.com/aws/aws-sdk-go/aws/session"
23-
"github.com/aws/aws-sdk-go/service/firehose"
22+
"github.com/aws/aws-sdk-go-v2/config"
23+
"github.com/aws/aws-sdk-go-v2/service/firehose"
24+
"github.com/aws/aws-sdk-go-v2/service/firehose/types"
2425
)
2526

2627
func main() {
27-
sess := session.Must(session.NewSession())
28-
fh := firehose.New(sess)
28+
ctx := context.Background()
29+
cfg, err := config.LoadDefaultConfig(ctx)
30+
...
31+
fh := firehose.NewFromConfig(cfg)
2932
fq := firequeue.New(fh, "DELIVERY_STREAM_NAME")
3033

31-
ctx := context.Background()
3234
go fq.Loop(ctx) // We should start looping before sending items
3335

34-
err := fq.Enqueue(&firehose.Record{...})
36+
err = fq.Enqueue(types.Record{...})
3537
...
3638
}
3739
```

‎firequeue.go‎

Lines changed: 135 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -6,19 +6,26 @@ import (
66
"expvar"
77
"fmt"
88
"log"
9+
"net"
10+
"net/url"
911
"strings"
1012
"sync"
1113
"sync/atomic"
1214
"time"
1315

14-
"github.com/aws/aws-sdk-go/aws"
15-
"github.com/aws/aws-sdk-go/aws/awserr"
16-
"github.com/aws/aws-sdk-go/aws/request"
17-
"github.com/aws/aws-sdk-go/service/firehose"
18-
"github.com/aws/aws-sdk-go/service/firehose/firehoseiface"
16+
"github.com/aws/aws-sdk-go-v2/aws"
17+
"github.com/aws/aws-sdk-go-v2/service/firehose"
18+
"github.com/aws/aws-sdk-go-v2/service/firehose/types"
19+
"github.com/aws/smithy-go"
20+
smithyhttp "github.com/aws/smithy-go/transport/http"
1921
"github.com/cenkalti/backoff/v4"
2022
)
2123

24+
// FirehoseAPI is the part of *firehose.Client that Queue uses.
25+
type FirehoseAPI interface {
26+
PutRecordBatch(ctx context.Context, params *firehose.PutRecordBatchInput, optFns ...func(*firehose.Options)) (*firehose.PutRecordBatchOutput, error)
27+
}
28+
2229
// Option is a type for constructor options
2330
type Option func(*Queue)
2431

@@ -30,7 +37,7 @@ func Parallel(i int) Option {
3037
}
3138

3239
// ErrorHandler is an option to specify the error handler
33-
func ErrorHandler(fn func(error, []*firehose.Record)) Option {
40+
func ErrorHandler(fn func(error, []types.Record)) Option {
3441
return func(q *Queue) {
3542
q.errorHandler = fn
3643
}
@@ -76,10 +83,10 @@ func BatchInterval(period time.Duration) Option {
7683
}
7784

7885
// New return new Queue
79-
func New(fh firehoseiface.FirehoseAPI, DeliveryStreamName string, opts ...Option) *Queue {
86+
func New(fh FirehoseAPI, DeliveryStreamName string, opts ...Option) *Queue {
8087
q := &Queue{
81-
queue: make([]*firehose.Record, 0),
82-
workerCh: make(chan []*firehose.Record),
88+
queue: make([]types.Record, 0),
89+
workerCh: make(chan []types.Record),
8390
firehose: fh,
8491
initialized: make(chan struct{}),
8592
DeliveryStreamName: aws.String(DeliveryStreamName),
@@ -92,18 +99,18 @@ func New(fh firehoseiface.FirehoseAPI, DeliveryStreamName string, opts ...Option
9299

93100
// Queue manages a sending list for firehose
94101
type Queue struct {
95-
queue []*firehose.Record
102+
queue []types.Record
96103
mu sync.RWMutex
97-
workerCh chan []*firehose.Record
104+
workerCh chan []types.Record
98105

99106
initialized chan struct{}
100107
initMu sync.Mutex
101108

102109
para int
103110
inFlightBatchCounter int32
104111
maxQueueLength int
105-
firehose firehoseiface.FirehoseAPI
106-
errorHandler func(error, []*firehose.Record)
112+
firehose FirehoseAPI
113+
errorHandler func(error, []types.Record)
107114
batchSize int
108115
batchInterval time.Duration
109116
DeliveryStreamName *string
@@ -207,10 +214,14 @@ func (q *Queue) loop(ctx context.Context) {
207214
}
208215
}
209216

217+
// isRetryable keeps the set of retryable errors that firequeue had with
218+
// aws-sdk-go v1 (request.IsErrorRetryable, request.IsErrorThrottle and the
219+
// HTTP status codes below).
210220
func isRetryable(err error) bool {
211-
if reqfailure, ok := err.(awserr.RequestFailure); ok {
221+
var respErr *smithyhttp.ResponseError
222+
if errors.As(err, &respErr) {
212223
// ref. https://github.com/aws/aws-sdk-go/blob/fe72a52350a8962175bb71c531ec9724ce48abd8/aws/request/retryer.go#L228-L250
213-
switch reqfailure.StatusCode() {
224+
switch respErr.HTTPStatusCode() {
214225
case
215226
429,
216227
500,
@@ -220,21 +231,114 @@ func isRetryable(err error) bool {
220231
return true
221232
}
222233
}
223-
return request.IsErrorRetryable(err) ||
224-
request.IsErrorThrottle(err) ||
225-
isErrConnectionResetByPeer(err)
234+
return isErrorRetryable(err) || isErrConnectionResetByPeer(err)
235+
}
236+
237+
// retryableErrorCodes are the API error codes that aws-sdk-go v1 retried:
238+
// retryableCodes, credsExpiredCodes and throttleCodes in aws/request/retryer.go.
239+
// v1 also had the client-side codes "RequestError" and "ResponseTimeout".
240+
// v2 does not use these codes. isErrorRetryable examines
241+
// smithyhttp.RequestSendError in place of "RequestError".
242+
var retryableErrorCodes = map[string]struct{}{
243+
"RequestTimeout": {},
244+
"RequestTimeoutException": {},
245+
246+
"ExpiredToken": {},
247+
"ExpiredTokenException": {},
248+
"RequestExpired": {},
249+
250+
"ProvisionedThroughputExceededException": {},
251+
"ThrottledException": {},
252+
"Throttling": {},
253+
"ThrottlingException": {},
254+
"RequestLimitExceeded": {},
255+
"RequestThrottled": {},
256+
"RequestThrottledException": {},
257+
"TooManyRequestsException": {},
258+
"PriorRequestNotComplete": {},
259+
"TransactionInProgressException": {},
260+
"EC2ThrottledException": {},
261+
}
262+
263+
// isErrorRetryable follows request.IsErrorRetryable and
264+
// request.IsErrorThrottle of aws-sdk-go v1 for the errors that the v2 client
265+
// returns.
266+
func isErrorRetryable(err error) bool {
267+
// v1 did not retry "RequestCanceled". v2 returns *smithy.CanceledError or
268+
// *aws.RequestCanceledError when the context is done.
269+
var canceledErr interface{ CanceledError() bool }
270+
if errors.As(err, &canceledErr) && canceledErr.CanceledError() {
271+
return false
272+
}
273+
var apiErr smithy.APIError
274+
if errors.As(err, &apiErr) {
275+
_, ok := retryableErrorCodes[apiErr.ErrorCode()]
276+
return ok
277+
}
278+
// v1 "SerializationError": v1 retried it when it retried the cause,
279+
// and did not retry it without a cause.
280+
var deserializationErr *smithy.DeserializationError
281+
if errors.As(err, &deserializationErr) {
282+
return deserializationErr.Err != nil && shouldRetryError(deserializationErr.Err)
283+
}
284+
// v1 "RequestError": v1 retried it when it retried the cause,
285+
// and retried it without a cause.
286+
var sendErr *smithyhttp.RequestSendError
287+
if errors.As(err, &sendErr) {
288+
return shouldRetryError(sendErr.Err)
289+
}
290+
// v1 did not retry errors from parameter validation either.
291+
return false
292+
}
293+
294+
// shouldRetryError is the part of request.shouldRetryError in aws-sdk-go v1
295+
// that examines the causes of "RequestError" and "SerializationError".
296+
// ref. https://github.com/aws/aws-sdk-go/blob/v1.35.33/aws/request/retryer.go#L168-L226
297+
func shouldRetryError(err error) bool {
298+
switch e := err.(type) {
299+
case *url.Error:
300+
if strings.Contains(e.Error(), "connection refused") {
301+
return true
302+
}
303+
return shouldRetryError(e.Err)
304+
case interface{ Temporary() bool }:
305+
if netErr, ok := e.(*net.OpError); ok && netErr.Op == "dial" {
306+
return true
307+
}
308+
return e.Temporary() || isErrConnectionReset(err)
309+
case nil:
310+
return true
311+
default:
312+
switch err.Error() {
313+
case "net/http: request canceled",
314+
"net/http: request canceled while waiting for connection":
315+
return false
316+
}
317+
return true
318+
}
226319
}
227320

228-
// request.isErrConnectionReset, which is used inside request.IsErrorRetryable,
229-
// intentionally marks false (do not retry) if it contains "read: connection reset".
321+
// isErrConnectionReset is request.isErrConnectionReset in aws-sdk-go v1.
322+
// It intentionally marks false (do not retry) if it contains "read: connection reset".
230323
// ref. https://github.com/aws/aws-sdk-go/blob/7814a7f61bf93cb54d00a5f97918c5501f07d351/aws/request/connection_reset_error.go#L7-L18
231-
// However, in firehose, the connection reset by peer error often bursts, so it is to be retry
324+
func isErrConnectionReset(err error) bool {
325+
msg := err.Error()
326+
if strings.Contains(msg, "read: connection reset") {
327+
return false
328+
}
329+
return strings.Contains(msg, "use of closed network connection") ||
330+
strings.Contains(msg, "connection reset") ||
331+
strings.Contains(msg, "broken pipe")
332+
}
333+
334+
// isErrConnectionResetByPeer catches "read: connection reset by peer", which isErrConnectionReset does not retry.
335+
// In firehose, the connection reset by peer error often bursts, so it is to be retry
232336
func isErrConnectionResetByPeer(err error) bool {
233337
return strings.Contains(err.Error(), "read: connection reset by peer")
234338
}
235339

236340
// Enqueue add data to queue
237-
func (q *Queue) Enqueue(r *firehose.Record) error {
341+
func (q *Queue) Enqueue(r types.Record) error {
238342
select {
239343
case <-q.initialized:
240344
// nop and continue to send
@@ -271,7 +375,7 @@ func (q *Queue) sendBatch(bf backoff.BackOff) time.Duration {
271375
return q.batchInterval
272376
}
273377

274-
func (q *Queue) createBatch() []*firehose.Record {
378+
func (q *Queue) createBatch() []types.Record {
275379
q.mu.Lock()
276380
defer q.mu.Unlock()
277381

@@ -280,13 +384,13 @@ func (q *Queue) createBatch() []*firehose.Record {
280384
batchSize = len(q.queue)
281385
}
282386

283-
batch := make([]*firehose.Record, batchSize)
387+
batch := make([]types.Record, batchSize)
284388
copy(batch, q.queue)
285389
q.queue = q.queue[batchSize:]
286390
return batch
287391
}
288392

289-
func (q *Queue) attemptToSendBatch(batch []*firehose.Record) error {
393+
func (q *Queue) attemptToSendBatch(batch []types.Record) error {
290394
atomic.AddInt32(&q.inFlightBatchCounter, 1)
291395
defer atomic.AddInt32(&q.inFlightBatchCounter, -1)
292396

@@ -297,7 +401,7 @@ func (q *Queue) attemptToSendBatch(batch []*firehose.Record) error {
297401
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
298402
defer cancel()
299403

300-
resp, err := q.firehose.PutRecordBatchWithContext(ctx, input)
404+
resp, err := q.firehose.PutRecordBatch(ctx, input)
301405
if err != nil {
302406
// Handle non-retryable errors directly.
303407
if !isRetryable(err) {
@@ -318,18 +422,18 @@ func (q *Queue) attemptToSendBatch(batch []*firehose.Record) error {
318422
return nil
319423
}
320424

321-
func (q *Queue) retryFailedRecords(batch []*firehose.Record, resp *firehose.PutRecordBatchOutput) {
322-
retryRecords := make([]*firehose.Record, 0, *resp.FailedPutCount)
425+
func (q *Queue) retryFailedRecords(batch []types.Record, resp *firehose.PutRecordBatchOutput) {
426+
retryRecords := make([]types.Record, 0, *resp.FailedPutCount)
323427

324428
for i, record := range batch {
325-
if resp.RequestResponses != nil && resp.RequestResponses[i] != nil && resp.RequestResponses[i].ErrorCode != nil {
429+
if resp.RequestResponses != nil && resp.RequestResponses[i].ErrorCode != nil {
326430
q.retryCount.Add(1)
327431
retryRecords = append(retryRecords, record)
328432
}
329433
}
330434
q.push(retryRecords...)
331435
}
332-
func (q *Queue) handleError(err error, r []*firehose.Record) {
436+
func (q *Queue) handleError(err error, r []types.Record) {
333437
if q.errorHandler != nil {
334438
q.errorHandler(err, r)
335439
return
@@ -352,7 +456,7 @@ func (q *Queue) remaining() bool {
352456
return q.len() > 0 || q.inFlightBatch()
353457
}
354458

355-
func (q *Queue) push(r ...*firehose.Record) {
459+
func (q *Queue) push(r ...types.Record) {
356460
q.mu.Lock()
357461
defer q.mu.Unlock()
358462
q.queue = append(q.queue, r...)

0 commit comments

Comments
 (0)