Skip to content

Commit 03a13e5

Browse files
thewizardplusplusthewizardplusplus
committed
Move the SourcedLink structure to the models package
1 parent a290f46 commit 03a13e5

24 files changed

+123
-103
lines changed

‎README.md

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ import (
8585
"github.com/thewizardplusplus/go-crawler/register"
8686
"github.com/thewizardplusplus/go-crawler/sanitizing"
8787
htmlselector "github.com/thewizardplusplus/go-html-selector"
88+
"github.com/thewizardplusplus/go-crawler/models"
8889
)
8990

9091
type LinkHandler struct {
@@ -93,7 +94,7 @@ type LinkHandler struct {
9394

9495
func (handler LinkHandler) HandleLink(
9596
ctx context.Context,
96-
link crawler.SourcedLink,
97+
link models.SourcedLink,
9798
) {
9899
fmt.Printf(
99100
"have got the link %q from the page %q\n",
@@ -231,6 +232,7 @@ import (
231232
"github.com/thewizardplusplus/go-crawler/registers"
232233
"github.com/thewizardplusplus/go-crawler/sanitizing"
233234
htmlselector "github.com/thewizardplusplus/go-html-selector"
235+
"github.com/thewizardplusplus/go-crawler/models"
234236
)
235237

236238
type LinkHandler struct {
@@ -239,7 +241,7 @@ type LinkHandler struct {
239241

240242
func (handler LinkHandler) HandleLink(
241243
ctx context.Context,
242-
link crawler.SourcedLink,
244+
link models.SourcedLink,
243245
) {
244246
fmt.Printf(
245247
"have got the link %q from the page %q\n",
@@ -393,6 +395,7 @@ import (
393395
"github.com/thewizardplusplus/go-crawler/checkers"
394396
"github.com/thewizardplusplus/go-crawler/extractors"
395397
htmlselector "github.com/thewizardplusplus/go-html-selector"
398+
"github.com/thewizardplusplus/go-crawler/models"
396399
)
397400

398401
type LinkHandler struct {
@@ -401,7 +404,7 @@ type LinkHandler struct {
401404

402405
func (handler LinkHandler) HandleLink(
403406
ctx context.Context,
404-
link crawler.SourcedLink,
407+
link models.SourcedLink,
405408
) {
406409
fmt.Printf(
407410
"have got the link %q from the page %q\n",
@@ -531,6 +534,7 @@ import (
531534
"github.com/thewizardplusplus/go-crawler/register"
532535
"github.com/thewizardplusplus/go-crawler/sanitizing"
533536
htmlselector "github.com/thewizardplusplus/go-html-selector"
537+
"github.com/thewizardplusplus/go-crawler/models"
534538
)
535539

536540
type LinkHandler struct {
@@ -539,7 +543,7 @@ type LinkHandler struct {
539543

540544
func (handler LinkHandler) HandleLink(
541545
ctx context.Context,
542-
link crawler.SourcedLink,
546+
link models.SourcedLink,
543547
) {
544548
fmt.Printf(
545549
"have got the link %q from the page %q\n",
@@ -676,6 +680,7 @@ import (
676680
"github.com/thewizardplusplus/go-crawler/register"
677681
"github.com/thewizardplusplus/go-crawler/sanitizing"
678682
htmlselector "github.com/thewizardplusplus/go-html-selector"
683+
"github.com/thewizardplusplus/go-crawler/models"
679684
)
680685

681686
type LinkHandler struct {
@@ -684,7 +689,7 @@ type LinkHandler struct {
684689

685690
func (handler LinkHandler) HandleLink(
686691
ctx context.Context,
687-
link crawler.SourcedLink,
692+
link models.SourcedLink,
688693
) {
689694
fmt.Printf(
690695
"have got the link %q from the page %q\n",
@@ -829,6 +834,7 @@ import (
829834
"github.com/thewizardplusplus/go-crawler/register"
830835
"github.com/thewizardplusplus/go-crawler/sanitizing"
831836
htmlselector "github.com/thewizardplusplus/go-html-selector"
837+
"github.com/thewizardplusplus/go-crawler/models"
832838
)
833839

834840
type LinkHandler struct {
@@ -837,7 +843,7 @@ type LinkHandler struct {
837843

838844
func (handler LinkHandler) HandleLink(
839845
ctx context.Context,
840-
link crawler.SourcedLink,
846+
link models.SourcedLink,
841847
) {
842848
fmt.Printf(
843849
"have got the link %q from the page %q\n",
@@ -984,6 +990,7 @@ import (
984990
"github.com/thewizardplusplus/go-crawler/extractors"
985991
"github.com/thewizardplusplus/go-crawler/registers"
986992
htmlselector "github.com/thewizardplusplus/go-html-selector"
993+
"github.com/thewizardplusplus/go-crawler/models"
987994
)
988995

989996
type LinkHandler struct {
@@ -992,7 +999,7 @@ type LinkHandler struct {
992999

9931000
func (handler LinkHandler) HandleLink(
9941001
ctx context.Context,
995-
link crawler.SourcedLink,
1002+
link models.SourcedLink,
9961003
) {
9971004
fmt.Printf(
9981005
"have got the link %q from the page %q\n",
@@ -1134,6 +1141,7 @@ import (
11341141
"github.com/thewizardplusplus/go-crawler/handlers"
11351142
"github.com/thewizardplusplus/go-crawler/registers"
11361143
htmlselector "github.com/thewizardplusplus/go-html-selector"
1144+
"github.com/thewizardplusplus/go-crawler/models"
11371145
)
11381146

11391147
type LinkHandler struct {
@@ -1142,7 +1150,7 @@ type LinkHandler struct {
11421150

11431151
func (handler LinkHandler) HandleLink(
11441152
ctx context.Context,
1145-
link crawler.SourcedLink,
1153+
link models.SourcedLink,
11461154
) {
11471155
fmt.Printf(
11481156
"have got the link %q from the page %q\n",

‎checkers/checker_group.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55

66
crawler "github.com/thewizardplusplus/go-crawler"
7+
"github.com/thewizardplusplus/go-crawler/models"
78
)
89

910
// CheckerGroup ...
@@ -12,7 +13,7 @@ type CheckerGroup []crawler.LinkChecker
1213
// CheckLink ...
1314
func (checkers CheckerGroup) CheckLink(
1415
ctx context.Context,
15-
link crawler.SourcedLink,
16+
link models.SourcedLink,
1617
) bool {
1718
for _, checker := range checkers {
1819
if !checker.CheckLink(ctx, link) {

‎checkers/checker_group_test.go

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,13 @@ import (
66

77
"github.com/stretchr/testify/assert"
88
"github.com/stretchr/testify/mock"
9-
crawler "github.com/thewizardplusplus/go-crawler"
9+
"github.com/thewizardplusplus/go-crawler/models"
1010
)
1111

1212
func TestCheckerGroup_CheckLink(test *testing.T) {
1313
type args struct {
1414
ctx context.Context
15-
link crawler.SourcedLink
15+
link models.SourcedLink
1616
}
1717

1818
for _, data := range []struct {
@@ -26,7 +26,7 @@ func TestCheckerGroup_CheckLink(test *testing.T) {
2626
checkers: nil,
2727
args: args{
2828
ctx: context.Background(),
29-
link: crawler.SourcedLink{
29+
link: models.SourcedLink{
3030
SourceLink: "http://example.com/",
3131
Link: "http://example.com/test",
3232
},
@@ -39,7 +39,7 @@ func TestCheckerGroup_CheckLink(test *testing.T) {
3939
func() LinkChecker {
4040
checker := new(MockLinkChecker)
4141
checker.
42-
On("CheckLink", context.Background(), crawler.SourcedLink{
42+
On("CheckLink", context.Background(), models.SourcedLink{
4343
SourceLink: "http://example.com/",
4444
Link: "http://example.com/test",
4545
}).
@@ -50,7 +50,7 @@ func TestCheckerGroup_CheckLink(test *testing.T) {
5050
func() LinkChecker {
5151
checker := new(MockLinkChecker)
5252
checker.
53-
On("CheckLink", context.Background(), crawler.SourcedLink{
53+
On("CheckLink", context.Background(), models.SourcedLink{
5454
SourceLink: "http://example.com/",
5555
Link: "http://example.com/test",
5656
}).
@@ -61,7 +61,7 @@ func TestCheckerGroup_CheckLink(test *testing.T) {
6161
},
6262
args: args{
6363
ctx: context.Background(),
64-
link: crawler.SourcedLink{
64+
link: models.SourcedLink{
6565
SourceLink: "http://example.com/",
6666
Link: "http://example.com/test",
6767
},
@@ -74,7 +74,7 @@ func TestCheckerGroup_CheckLink(test *testing.T) {
7474
func() LinkChecker {
7575
checker := new(MockLinkChecker)
7676
checker.
77-
On("CheckLink", context.Background(), crawler.SourcedLink{
77+
On("CheckLink", context.Background(), models.SourcedLink{
7878
SourceLink: "http://example.com/",
7979
Link: "http://example.com/test",
8080
}).
@@ -86,7 +86,7 @@ func TestCheckerGroup_CheckLink(test *testing.T) {
8686
},
8787
args: args{
8888
ctx: context.Background(),
89-
link: crawler.SourcedLink{
89+
link: models.SourcedLink{
9090
SourceLink: "http://example.com/",
9191
Link: "http://example.com/test",
9292
},

‎checkers/duplicate_checker.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ package checkers
33
import (
44
"context"
55

6-
crawler "github.com/thewizardplusplus/go-crawler"
6+
"github.com/thewizardplusplus/go-crawler/models"
77
"github.com/thewizardplusplus/go-crawler/registers"
88
)
99

@@ -15,7 +15,7 @@ type DuplicateChecker struct {
1515
// CheckLink ...
1616
func (checker DuplicateChecker) CheckLink(
1717
ctx context.Context,
18-
link crawler.SourcedLink,
18+
link models.SourcedLink,
1919
) bool {
2020
return checker.LinkRegister.RegisterLink(link.Link)
2121
}

‎checkers/duplicate_checker_test.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import (
55
"testing"
66

77
"github.com/stretchr/testify/assert"
8-
crawler "github.com/thewizardplusplus/go-crawler"
8+
"github.com/thewizardplusplus/go-crawler/models"
99
"github.com/thewizardplusplus/go-crawler/registers"
1010
"github.com/thewizardplusplus/go-crawler/sanitizing"
1111
)
@@ -16,7 +16,7 @@ func TestDuplicateChecker_CheckLink(test *testing.T) {
1616
}
1717
type args struct {
1818
ctx context.Context
19-
link crawler.SourcedLink
19+
link models.SourcedLink
2020
}
2121

2222
for _, data := range []struct {
@@ -33,7 +33,7 @@ func TestDuplicateChecker_CheckLink(test *testing.T) {
3333
},
3434
args: args{
3535
ctx: context.Background(),
36-
link: crawler.SourcedLink{
36+
link: models.SourcedLink{
3737
SourceLink: "http://example.com/",
3838
Link: "http://example.com/test",
3939
},
@@ -59,7 +59,7 @@ func TestDuplicateChecker_CheckLink(test *testing.T) {
5959
},
6060
args: args{
6161
ctx: context.Background(),
62-
link: crawler.SourcedLink{
62+
link: models.SourcedLink{
6363
SourceLink: "http://example.com/",
6464
Link: "http://example.com/test",
6565
},

‎checkers/host_checker.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import (
55
"net/url"
66

77
"github.com/go-log/log"
8-
crawler "github.com/thewizardplusplus/go-crawler"
8+
"github.com/thewizardplusplus/go-crawler/models"
99
)
1010

1111
// HostChecker ...
@@ -16,7 +16,7 @@ type HostChecker struct {
1616
// CheckLink ...
1717
func (checker HostChecker) CheckLink(
1818
ctx context.Context,
19-
link crawler.SourcedLink,
19+
link models.SourcedLink,
2020
) bool {
2121
parsedSourceLink, err := url.Parse(link.SourceLink)
2222
if err != nil {

‎checkers/host_checker_test.go

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import (
99
"github.com/go-log/log"
1010
"github.com/stretchr/testify/assert"
1111
"github.com/stretchr/testify/mock"
12-
crawler "github.com/thewizardplusplus/go-crawler"
12+
"github.com/thewizardplusplus/go-crawler/models"
1313
)
1414

1515
func TestHostChecker_CheckLink(test *testing.T) {
@@ -18,7 +18,7 @@ func TestHostChecker_CheckLink(test *testing.T) {
1818
}
1919
type args struct {
2020
ctx context.Context
21-
link crawler.SourcedLink
21+
link models.SourcedLink
2222
}
2323

2424
for _, data := range []struct {
@@ -34,7 +34,7 @@ func TestHostChecker_CheckLink(test *testing.T) {
3434
},
3535
args: args{
3636
ctx: context.Background(),
37-
link: crawler.SourcedLink{
37+
link: models.SourcedLink{
3838
SourceLink: "http://example1.com/",
3939
Link: "http://example2.com/test",
4040
},
@@ -48,7 +48,7 @@ func TestHostChecker_CheckLink(test *testing.T) {
4848
},
4949
args: args{
5050
ctx: context.Background(),
51-
link: crawler.SourcedLink{
51+
link: models.SourcedLink{
5252
SourceLink: "http://example.com/",
5353
Link: "http://example.com/test",
5454
},
@@ -70,7 +70,7 @@ func TestHostChecker_CheckLink(test *testing.T) {
7070
},
7171
args: args{
7272
ctx: context.Background(),
73-
link: crawler.SourcedLink{
73+
link: models.SourcedLink{
7474
SourceLink: ":",
7575
Link: "http://example.com/test",
7676
},
@@ -92,7 +92,7 @@ func TestHostChecker_CheckLink(test *testing.T) {
9292
},
9393
args: args{
9494
ctx: context.Background(),
95-
link: crawler.SourcedLink{
95+
link: models.SourcedLink{
9696
SourceLink: "http://example.com/",
9797
Link: ":",
9898
},

‎checkers/mock_link_checker_test.go

Lines changed: 3 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎checkers/robots_txt_checker.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import (
55
"net/url"
66

77
"github.com/go-log/log"
8-
crawler "github.com/thewizardplusplus/go-crawler"
8+
"github.com/thewizardplusplus/go-crawler/models"
99
"github.com/thewizardplusplus/go-crawler/registers"
1010
)
1111

@@ -19,7 +19,7 @@ type RobotsTXTChecker struct {
1919
// CheckLink ...
2020
func (checker RobotsTXTChecker) CheckLink(
2121
ctx context.Context,
22-
link crawler.SourcedLink,
22+
link models.SourcedLink,
2323
) bool {
2424
parsedLink, err := url.Parse(link.Link)
2525
if err != nil {

0 commit comments

Comments
 (0)