Use the crawler.ConcurrencyConfig structure in the crawler.CrawlByConcurrentHandler() function

Author: thewizardplusplus

README.md

@@ -308,10 +308,14 @@ func main() {
 
 	crawler.CrawlByConcurrentHandler(
 		context.Background(),
-		runtime.NumCPU(),
-		1000,
-		runtime.NumCPU(),
-		1000,
+		crawler.ConcurrencyConfig{
+			ConcurrencyFactor: runtime.NumCPU(),
+			BufferSize:        1000,
+		},
+		crawler.ConcurrencyConfig{
+			ConcurrencyFactor: runtime.NumCPU(),
+			BufferSize:        1000,
+		},
 		[]string{server.URL},
 		crawler.CrawlDependencies{
 			LinkExtractor: extractors.RepeatingExtractor{

crawl.go

@@ -58,22 +58,19 @@ func Crawl(
 // CrawlByConcurrentHandler ...
 func CrawlByConcurrentHandler(
 	ctx context.Context,
-	concurrencyFactor int,
-	bufferSize int,
-	handlerConcurrencyFactor int,
-	handlerBufferSize int,
+	concurrencyConfig ConcurrencyConfig,
+	handlerConcurrencyConfig ConcurrencyConfig,
 	links []string,
 	dependencies CrawlDependencies,
 ) {
+	handlerConcurrencyFactor, handlerBufferSize :=
+		handlerConcurrencyConfig.ConcurrencyFactor,
+		handlerConcurrencyConfig.BufferSize
 	concurrentHandler :=
 		handlers.NewConcurrentHandler(handlerBufferSize, dependencies.LinkHandler)
 	go concurrentHandler.RunConcurrently(ctx, handlerConcurrencyFactor)
 	defer concurrentHandler.Stop()
 
-	concurrencyConfig := ConcurrencyConfig{
-		ConcurrencyFactor: concurrencyFactor,
-		BufferSize:        bufferSize,
-	}
 	Crawl(ctx, concurrencyConfig, links, CrawlDependencies{
 		LinkExtractor: dependencies.LinkExtractor,
 		LinkChecker:   dependencies.LinkChecker,

crawl_test.go

@@ -205,10 +205,8 @@ func TestCrawl(test *testing.T) {
 func TestCrawlByConcurrentHandler(test *testing.T) {
 	type args struct {
 		ctx                      context.Context
-		concurrencyFactor        int
-		bufferSize               int
-		handlerConcurrencyFactor int
-		handlerBufferSize        int
+		concurrencyConfig        ConcurrencyConfig
+		handlerConcurrencyConfig ConcurrencyConfig
 		links                    []string
 		dependencies             CrawlDependencies
 	}
@@ -220,12 +218,16 @@ func TestCrawlByConcurrentHandler(test *testing.T) {
 		{
 			name: "success",
 			args: args{
-				ctx:                      context.Background(),
-				concurrencyFactor:        10,
-				bufferSize:               1000,
-				handlerConcurrencyFactor: 10,
-				handlerBufferSize:        1000,
-				links:                    []string{"http://example.com/"},
+				ctx: context.Background(),
+				concurrencyConfig: ConcurrencyConfig{
+					ConcurrencyFactor: 10,
+					BufferSize:        1000,
+				},
+				handlerConcurrencyConfig: ConcurrencyConfig{
+					ConcurrencyFactor: 10,
+					BufferSize:        1000,
+				},
+				links: []string{"http://example.com/"},
 				dependencies: CrawlDependencies{
 					LinkExtractor: func() models.LinkExtractor {
 						threadIDChecker := mock.MatchedBy(func(threadID int) bool {
@@ -302,10 +304,8 @@ func TestCrawlByConcurrentHandler(test *testing.T) {
 		test.Run(data.name, func(test *testing.T) {
 			CrawlByConcurrentHandler(
 				data.args.ctx,
-				data.args.concurrencyFactor,
-				data.args.bufferSize,
-				data.args.handlerConcurrencyFactor,
-				data.args.handlerBufferSize,
+				data.args.concurrencyConfig,
+				data.args.handlerConcurrencyConfig,
 				data.args.links,
 				data.args.dependencies,
 			)

examples_test.go

@@ -164,10 +164,14 @@ func ExampleCrawl_withConcurrentHandling() {
 
 	crawler.CrawlByConcurrentHandler(
 		context.Background(),
-		runtime.NumCPU(),
-		1000,
-		runtime.NumCPU(),
-		1000,
+		crawler.ConcurrencyConfig{
+			ConcurrencyFactor: runtime.NumCPU(),
+			BufferSize:        1000,
+		},
+		crawler.ConcurrencyConfig{
+			ConcurrencyFactor: runtime.NumCPU(),
+			BufferSize:        1000,
+		},
 		[]string{server.URL},
 		crawler.CrawlDependencies{
 			LinkExtractor: extractors.RepeatingExtractor{