Skip to content

Commit e5dc2e6

Browse files
Move the LinkExtractor interface to the models package
1 parent 03a13e5 commit e5dc2e6

10 files changed

+45
-32
lines changed

crawl.go

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,6 @@ import (
99
syncutils "github.com/thewizardplusplus/go-sync-utils"
1010
)
1111

12-
//go:generate mockery --name=LinkExtractor --inpackage --case=underscore --testonly
13-
14-
// LinkExtractor ...
15-
type LinkExtractor interface {
16-
ExtractLinks(ctx context.Context, threadID int, link string) ([]string, error)
17-
}
18-
1912
//go:generate mockery --name=LinkChecker --inpackage --case=underscore --testonly
2013

2114
// LinkChecker ...
@@ -32,7 +25,7 @@ type LinkHandler interface {
3225

3326
// CrawlDependencies ...
3427
type CrawlDependencies struct {
35-
LinkExtractor LinkExtractor
28+
LinkExtractor models.LinkExtractor
3629
LinkChecker LinkChecker
3730
LinkHandler LinkHandler
3831
Logger log.Logger

crawl_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ func TestCrawl(test *testing.T) {
2929
bufferSize: 1000,
3030
links: []string{"http://example.com/"},
3131
dependencies: CrawlDependencies{
32-
LinkExtractor: func() LinkExtractor {
32+
LinkExtractor: func() models.LinkExtractor {
3333
threadIDChecker := mock.MatchedBy(func(threadID int) bool {
3434
return threadID >= 0 && threadID < 10
3535
})
@@ -108,7 +108,7 @@ func TestCrawl(test *testing.T) {
108108
bufferSize: 0,
109109
links: []string{"http://example.com/"},
110110
dependencies: CrawlDependencies{
111-
LinkExtractor: func() LinkExtractor {
111+
LinkExtractor: func() models.LinkExtractor {
112112
threadIDChecker := mock.MatchedBy(func(threadID int) bool {
113113
return threadID >= 0 && threadID < 10
114114
})

extractors/delayed_extractor.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,22 +5,22 @@ import (
55
"sync"
66
"time"
77

8-
crawler "github.com/thewizardplusplus/go-crawler"
8+
"github.com/thewizardplusplus/go-crawler/models"
99
)
1010

1111
// DelayingExtractor ...
1212
type DelayingExtractor struct {
1313
timestamps sync.Map // map[threadID]time.Time
1414
minimalDelay time.Duration
1515
sleeper SleepHandler
16-
linkExtractor crawler.LinkExtractor
16+
linkExtractor models.LinkExtractor
1717
}
1818

1919
// NewDelayingExtractor ...
2020
func NewDelayingExtractor(
2121
minimalDelay time.Duration,
2222
sleeper SleepHandler,
23-
linkExtractor crawler.LinkExtractor,
23+
linkExtractor models.LinkExtractor,
2424
) *DelayingExtractor {
2525
return &DelayingExtractor{
2626
minimalDelay: minimalDelay,

extractors/delayed_extractor_test.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import (
1010
"github.com/stretchr/testify/assert"
1111
"github.com/stretchr/testify/mock"
1212
"github.com/stretchr/testify/require"
13-
crawler "github.com/thewizardplusplus/go-crawler"
13+
"github.com/thewizardplusplus/go-crawler/models"
1414
)
1515

1616
func TestNewDelayingExtractor(test *testing.T) {
@@ -32,7 +32,7 @@ func TestDelayingExtractor_ExtractLinks(test *testing.T) {
3232
type fields struct {
3333
minimalDelay time.Duration
3434
sleeper Sleeper
35-
linkExtractor crawler.LinkExtractor
35+
linkExtractor models.LinkExtractor
3636
}
3737
type args struct {
3838
ctx context.Context
@@ -53,7 +53,7 @@ func TestDelayingExtractor_ExtractLinks(test *testing.T) {
5353
fields: fields{
5454
minimalDelay: 100 * time.Millisecond,
5555
sleeper: new(MockSleeper),
56-
linkExtractor: func() LinkExtractor {
56+
linkExtractor: func() models.LinkExtractor {
5757
extractor := new(MockLinkExtractor)
5858
extractor.
5959
On("ExtractLinks", context.Background(), 23, "http://example.com/").
@@ -85,7 +85,7 @@ func TestDelayingExtractor_ExtractLinks(test *testing.T) {
8585

8686
return sleeper
8787
}(),
88-
linkExtractor: func() LinkExtractor {
88+
linkExtractor: func() models.LinkExtractor {
8989
extractor := new(MockLinkExtractor)
9090
extractor.
9191
On("ExtractLinks", context.Background(), 23, "http://example.com/").
@@ -110,7 +110,7 @@ func TestDelayingExtractor_ExtractLinks(test *testing.T) {
110110
fields: fields{
111111
minimalDelay: 100 * time.Millisecond,
112112
sleeper: new(MockSleeper),
113-
linkExtractor: func() LinkExtractor {
113+
linkExtractor: func() models.LinkExtractor {
114114
extractor := new(MockLinkExtractor)
115115
extractor.
116116
On("ExtractLinks", context.Background(), 23, "http://example.com/").

extractors/mock_gen.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import (
44
"time"
55

66
"github.com/go-log/log"
7-
crawler "github.com/thewizardplusplus/go-crawler"
7+
"github.com/thewizardplusplus/go-crawler/models"
88
httputils "github.com/thewizardplusplus/go-http-utils"
99
)
1010

@@ -25,7 +25,7 @@ type HTTPClient interface {
2525
// It's used only for mock generating.
2626
//
2727
type LinkExtractor interface {
28-
crawler.LinkExtractor
28+
models.LinkExtractor
2929
}
3030

3131
//go:generate mockery --name=Logger --inpackage --case=underscore --testonly

extractors/repeating_extractor.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,15 @@ import (
55
"time"
66

77
"github.com/go-log/log"
8-
crawler "github.com/thewizardplusplus/go-crawler"
8+
"github.com/thewizardplusplus/go-crawler/models"
99
)
1010

1111
// SleepHandler ...
1212
type SleepHandler func(duration time.Duration)
1313

1414
// RepeatingExtractor ...
1515
type RepeatingExtractor struct {
16-
LinkExtractor crawler.LinkExtractor
16+
LinkExtractor models.LinkExtractor
1717
RepeatCount int
1818
RepeatDelay time.Duration
1919
Logger log.Logger

extractors/repeating_extractor_test.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,12 @@ import (
99
"github.com/go-log/log"
1010
"github.com/stretchr/testify/assert"
1111
"github.com/stretchr/testify/mock"
12-
crawler "github.com/thewizardplusplus/go-crawler"
12+
"github.com/thewizardplusplus/go-crawler/models"
1313
)
1414

1515
func TestRepeatingExtractor_ExtractLinks(test *testing.T) {
1616
type fields struct {
17-
LinkExtractor crawler.LinkExtractor
17+
LinkExtractor models.LinkExtractor
1818
RepeatCount int
1919
RepeatDelay time.Duration
2020
Logger log.Logger
@@ -36,7 +36,7 @@ func TestRepeatingExtractor_ExtractLinks(test *testing.T) {
3636
{
3737
name: "success on the first repeat",
3838
fields: fields{
39-
LinkExtractor: func() LinkExtractor {
39+
LinkExtractor: func() models.LinkExtractor {
4040
extractor := new(MockLinkExtractor)
4141
extractor.
4242
On("ExtractLinks", context.Background(), 23, "http://example.com/").
@@ -61,7 +61,7 @@ func TestRepeatingExtractor_ExtractLinks(test *testing.T) {
6161
{
6262
name: "success on the last repeat",
6363
fields: fields{
64-
LinkExtractor: func() LinkExtractor {
64+
LinkExtractor: func() models.LinkExtractor {
6565
var repeat int
6666

6767
extractor := new(MockLinkExtractor)
@@ -125,7 +125,7 @@ func TestRepeatingExtractor_ExtractLinks(test *testing.T) {
125125
{
126126
name: "error",
127127
fields: fields{
128-
LinkExtractor: func() LinkExtractor {
128+
LinkExtractor: func() models.LinkExtractor {
129129
extractor := new(MockLinkExtractor)
130130
extractor.
131131
On("ExtractLinks", context.Background(), 23, "http://example.com/").

handle_links_test.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ func TestHandleLinksConcurrently(test *testing.T) {
3737
}(),
3838
dependencies: HandleLinkDependencies{
3939
CrawlDependencies: CrawlDependencies{
40-
LinkExtractor: func() LinkExtractor {
40+
LinkExtractor: func() models.LinkExtractor {
4141
threadIDChecker := mock.MatchedBy(func(threadID int) bool {
4242
return threadID >= 0 && threadID < 10
4343
})
@@ -170,7 +170,7 @@ func TestHandleLinks(test *testing.T) {
170170
}(),
171171
dependencies: HandleLinkDependencies{
172172
CrawlDependencies: CrawlDependencies{
173-
LinkExtractor: func() LinkExtractor {
173+
LinkExtractor: func() models.LinkExtractor {
174174
extractor := new(MockLinkExtractor)
175175
extractor.
176176
On("ExtractLinks", context.Background(), 23, "http://example.com/").
@@ -281,7 +281,7 @@ func TestHandleLink(test *testing.T) {
281281
link: "http://example.com/",
282282
dependencies: HandleLinkDependencies{
283283
CrawlDependencies: CrawlDependencies{
284-
LinkExtractor: func() LinkExtractor {
284+
LinkExtractor: func() models.LinkExtractor {
285285
extractor := new(MockLinkExtractor)
286286
extractor.
287287
On("ExtractLinks", context.Background(), 23, "http://example.com/").
@@ -344,7 +344,7 @@ func TestHandleLink(test *testing.T) {
344344
link: "http://example.com/",
345345
dependencies: HandleLinkDependencies{
346346
CrawlDependencies: CrawlDependencies{
347-
LinkExtractor: func() LinkExtractor {
347+
LinkExtractor: func() models.LinkExtractor {
348348
extractor := new(MockLinkExtractor)
349349
extractor.
350350
On("ExtractLinks", context.Background(), 23, "http://example.com/").
@@ -407,7 +407,7 @@ func TestHandleLink(test *testing.T) {
407407
link: "http://example.com/",
408408
dependencies: HandleLinkDependencies{
409409
CrawlDependencies: CrawlDependencies{
410-
LinkExtractor: func() LinkExtractor {
410+
LinkExtractor: func() models.LinkExtractor {
411411
extractor := new(MockLinkExtractor)
412412
extractor.
413413
On("ExtractLinks", context.Background(), 23, "http://example.com/").

mock_gen.go

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,20 @@ package crawler
22

33
import (
44
"github.com/go-log/log"
5+
"github.com/thewizardplusplus/go-crawler/models"
56
syncutils "github.com/thewizardplusplus/go-sync-utils"
67
)
78

9+
//go:generate mockery --name=LinkExtractor --inpackage --case=underscore --testonly
10+
11+
// LinkExtractor ...
12+
//
13+
// It's used only for mock generating.
14+
//
15+
type LinkExtractor interface {
16+
models.LinkExtractor
17+
}
18+
819
//go:generate mockery --name=Waiter --inpackage --case=underscore --testonly
920

1021
// Waiter ...

models/models.go

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,16 @@
11
package models
22

3+
import (
4+
"context"
5+
)
6+
37
// SourcedLink ...
48
type SourcedLink struct {
59
SourceLink string
610
Link string
711
}
12+
13+
// LinkExtractor ...
14+
type LinkExtractor interface {
15+
ExtractLinks(ctx context.Context, threadID int, link string) ([]string, error)
16+
}

0 commit comments

Comments
 (0)