feat: ✨ add concurrency support

AkashRajpurohit · Dec 4, 2024 · 130bd62 · 130bd62
1 parent bbd9873
commit 130bd62
Show file tree

Hide file tree

Showing 10 changed files with 194 additions and 135 deletions.
diff --git a/cmd/root.go b/cmd/root.go
@@ -44,6 +44,20 @@ var rootCmd = &cobra.Command{
 			}
 		}
 
+		// Backward Compatibility Starts
+		// TODO: Remove these before v1.0.0 release
+		// If concurrency is not set, set it to 5
+		if cfg.Concurrency == 0 {
+			cfg.Concurrency = 5
+		}
+
+		// If no clone_type is not set in the config file, set it to bare
+		if cfg.CloneType == "" {
+			cfg.CloneType = "bare"
+		}
+
+		// Backward Compatibility Ends
+
 		// If backupDir option is passed in the command line, use that instead of the one in the config file
 		if backupDir != "" {
 			cfg.BackupDir = config.GetBackupDir(backupDir)
@@ -54,11 +68,6 @@ var rootCmd = &cobra.Command{
 			cfg.Cron = cron
 		}
 
-		// If no clone_type is not set in the config file, set it to bare
-		if cfg.CloneType == "" {
-			cfg.CloneType = "bare"
-		}
-
 		logger.Info("Config loaded from: ", config.GetConfigFile(cfgFile))
 		logger.Info("Validating config ⏳")
 

diff --git a/pkg/bitbucket/bitbucket.go b/pkg/bitbucket/bitbucket.go
@@ -1,8 +1,6 @@
 package bitbucket
 
 import (
-	"sync"
-
 	"github.com/AkashRajpurohit/git-sync/pkg/config"
 	"github.com/AkashRajpurohit/git-sync/pkg/helpers"
 	"github.com/AkashRajpurohit/git-sync/pkg/logger"
@@ -29,27 +27,16 @@ func (c BitbucketClient) Sync(cfg config.Config) error {
 		return err
 	}
 
-	logger.Info("Total repositories: ", len(repos))
-
-	var wg sync.WaitGroup
-	sem := make(chan struct{}, 10) // Concurrency of 10
-
-	for _, repo := range repos {
-		wg.Add(1)
-		go func(repo *bb.Repository) {
-			defer wg.Done()
-			sem <- struct{}{}
-			gitSync.CloneOrUpdateRepo(cfg.Workspace, repo.Name, cfg)
-			if cfg.IncludeWiki && repo.Has_wiki {
-				gitSync.SyncWiki(cfg.Workspace, repo.Name, cfg)
-			}
-			<-sem
-		}(repo)
-	}
+	gitSync.LogRepoCount(len(repos), "Bitbucket")
 
-	wg.Wait()
-	logger.Info("All repositories synced ✅")
+	gitSync.SyncReposWithConcurrency(cfg, repos, func(repo *bb.Repository) {
+		gitSync.CloneOrUpdateRepo(cfg.Workspace, repo.Name, cfg)
+		if cfg.IncludeWiki && repo.Has_wiki {
+			gitSync.SyncWiki(cfg.Workspace, repo.Name, cfg)
+		}
+	})
 
+	gitSync.LogSyncComplete("Bitbucket")
 	return nil
 }
 

diff --git a/pkg/config/config.go b/pkg/config/config.go
@@ -30,6 +30,7 @@ type Config struct {
 	Cron         string   `mapstructure:"cron"`
 	CloneType    string   `mapstructure:"clone_type"`
 	RawGitURLs   []string `mapstructure:"raw_git_urls"`
+	Concurrency  int      `mapstructure:"concurrency"`
 }
 
 func expandPath(path string) string {
@@ -109,6 +110,7 @@ func SaveConfig(config Config, cfgFile string) error {
 	viper.Set("cron", config.Cron)
 	viper.Set("clone_type", config.CloneType)
 	viper.Set("raw_git_urls", config.RawGitURLs)
+	viper.Set("concurrency", config.Concurrency)
 
 	return viper.WriteConfig()
 }
@@ -133,5 +135,6 @@ func GetInitialConfig() Config {
 		BackupDir:    GetBackupDir(""),
 		CloneType:    "bare",
 		RawGitURLs:   []string{},
+		Concurrency:  5,
 	}
 }
diff --git a/pkg/config/validate.go b/pkg/config/validate.go
@@ -47,6 +47,11 @@ func ValidateConfig(cfg Config) error {
 		return fmt.Errorf("clone_type can only be `bare`, `full`, `mirror` or `shallow`")
 	}
 
+	// Validate concurrency
+	if cfg.Concurrency < 1 || cfg.Concurrency > 20 {
+		return fmt.Errorf("concurrency must be between 1 and 20")
+	}
+
 	// Validate cron if provided
 	if cfg.Cron != "" {
 		_, err := cron.ParseStandard(cfg.Cron)

diff --git a/pkg/config/validate_test.go b/pkg/config/validate_test.go
@@ -57,6 +57,51 @@ func TestValidateConfig(t *testing.T) {
 		cfg     Config
 		wantErr bool
 	}{
+		{
+			name: "Invalid Concurrency - Zero",
+			cfg: Config{
+				BackupDir:   "test",
+				CloneType:   "bare",
+				Concurrency: 0,
+				Server: Server{
+					Domain:   "test",
+					Protocol: "https",
+				},
+				Username: "test",
+				Token:    "test",
+			},
+			wantErr: true,
+		},
+		{
+			name: "Invalid Concurrency - Negative",
+			cfg: Config{
+				BackupDir:   "test",
+				CloneType:   "bare",
+				Concurrency: -1,
+				Server: Server{
+					Domain:   "test",
+					Protocol: "https",
+				},
+				Username: "test",
+				Token:    "test",
+			},
+			wantErr: true,
+		},
+		{
+			name: "Valid Concurrency - Custom",
+			cfg: Config{
+				BackupDir:   "test",
+				CloneType:   "bare",
+				Concurrency: 10,
+				Server: Server{
+					Domain:   "test",
+					Protocol: "https",
+				},
+				Username: "test",
+				Token:    "test",
+			},
+			wantErr: false,
+		},
 		{
 			name: "Empty BackupDir",
 			cfg: Config{
@@ -84,8 +129,9 @@ func TestValidateConfig(t *testing.T) {
 		{
 			name: "Valid Raw Git URLs Only",
 			cfg: Config{
-				BackupDir: "test",
-				CloneType: "bare",
+				BackupDir:   "test",
+				CloneType:   "bare",
+				Concurrency: 5,
 				RawGitURLs: []string{
 					"https://github.com/user/repo1.git",
 					"[email protected]:user/repo2.git",
@@ -96,8 +142,9 @@ func TestValidateConfig(t *testing.T) {
 		{
 			name: "Invalid Raw Git URL",
 			cfg: Config{
-				BackupDir: "test",
-				CloneType: "bare",
+				BackupDir:   "test",
+				CloneType:   "bare",
+				Concurrency: 5,
 				RawGitURLs: []string{
 					"https://github.com/user/repo1", // Missing .git
 					"[email protected]:user/repo2.git",
@@ -108,9 +155,10 @@ func TestValidateConfig(t *testing.T) {
 		{
 			name: "Empty Username with No Raw URLs",
 			cfg: Config{
-				BackupDir: "test",
-				CloneType: "bare",
-				Token:     "test",
+				BackupDir:   "test",
+				CloneType:   "bare",
+				Concurrency: 5,
+				Token:       "test",
 				Server: Server{
 					Domain:   "test",
 					Protocol: "https",
@@ -121,9 +169,10 @@ func TestValidateConfig(t *testing.T) {
 		{
 			name: "Empty Token with No Raw URLs",
 			cfg: Config{
-				Username:  "test",
-				BackupDir: "test",
-				CloneType: "bare",
+				Username:    "test",
+				BackupDir:   "test",
+				CloneType:   "bare",
+				Concurrency: 5,
 				Server: Server{
 					Domain:   "test",
 					Protocol: "https",
@@ -134,10 +183,11 @@ func TestValidateConfig(t *testing.T) {
 		{
 			name: "Empty Server Domain with No Raw URLs",
 			cfg: Config{
-				Username:  "test",
-				Token:     "test",
-				BackupDir: "test",
-				CloneType: "bare",
+				Username:    "test",
+				Token:       "test",
+				BackupDir:   "test",
+				CloneType:   "bare",
+				Concurrency: 5,
 				Server: Server{
 					Protocol: "https",
 				},
@@ -147,10 +197,11 @@ func TestValidateConfig(t *testing.T) {
 		{
 			name: "Invalid Server Protocol with No Raw URLs",
 			cfg: Config{
-				Username:  "test",
-				Token:     "test",
-				BackupDir: "test",
-				CloneType: "bare",
+				Username:    "test",
+				Token:       "test",
+				BackupDir:   "test",
+				CloneType:   "bare",
+				Concurrency: 5,
 				Server: Server{
 					Domain:   "test",
 					Protocol: "ftp",
@@ -161,10 +212,11 @@ func TestValidateConfig(t *testing.T) {
 		{
 			name: "Empty Workspace for Bitbucket with No Raw URLs",
 			cfg: Config{
-				Username:  "test",
-				Token:     "test",
-				BackupDir: "test",
-				CloneType: "bare",
+				Username:    "test",
+				Token:       "test",
+				BackupDir:   "test",
+				CloneType:   "bare",
+				Concurrency: 5,
 				Server: Server{
 					Domain:   "test",
 					Protocol: "https",
@@ -177,10 +229,11 @@ func TestValidateConfig(t *testing.T) {
 		{
 			name: "Valid Platform Config with No Raw URLs",
 			cfg: Config{
-				Username:  "test",
-				Token:     "test",
-				BackupDir: "test",
-				CloneType: "bare",
+				Username:    "test",
+				Token:       "test",
+				BackupDir:   "test",
+				CloneType:   "bare",
+				Concurrency: 5,
 				Server: Server{
 					Domain:   "test",
 					Protocol: "https",
@@ -191,10 +244,11 @@ func TestValidateConfig(t *testing.T) {
 		{
 			name: "Valid Mixed Config (Platform + Raw URLs)",
 			cfg: Config{
-				Username:  "test",
-				Token:     "test",
-				BackupDir: "test",
-				CloneType: "bare",
+				Username:    "test",
+				Token:       "test",
+				BackupDir:   "test",
+				CloneType:   "bare",
+				Concurrency: 5,
 				Server: Server{
 					Domain:   "test",
 					Protocol: "https",

diff --git a/pkg/forgejo/forgejo.go b/pkg/forgejo/forgejo.go
@@ -2,7 +2,6 @@ package forgejo
 
 import (
 	"fmt"
-	"sync"
 
 	fg "codeberg.org/mvdkleijn/forgejo-sdk/forgejo"
 	"github.com/AkashRajpurohit/git-sync/pkg/config"
@@ -39,27 +38,16 @@ func (c ForgejoClient) Sync(cfg config.Config) error {
 		return err
 	}
 
-	logger.Info("Total repositories: ", len(repos))
+	gitSync.LogRepoCount(len(repos), "Forgejo")
 
-	var wg sync.WaitGroup
-	sem := make(chan struct{}, 10) // Concurrency of 10
-
-	for _, repo := range repos {
-		wg.Add(1)
-		go func(repo *fg.Repository) {
-			defer wg.Done()
-			sem <- struct{}{}
-			gitSync.CloneOrUpdateRepo(repo.Owner.UserName, repo.Name, cfg)
-			if cfg.IncludeWiki && repo.HasWiki {
-				gitSync.SyncWiki(repo.Owner.UserName, repo.Name, cfg)
-			}
-			<-sem
-		}(repo)
-	}
-
-	wg.Wait()
-	logger.Info("All repositories synced ✅")
+	gitSync.SyncReposWithConcurrency(cfg, repos, func(repo *fg.Repository) {
+		gitSync.CloneOrUpdateRepo(repo.Owner.UserName, repo.Name, cfg)
+		if cfg.IncludeWiki && repo.HasWiki {
+			gitSync.SyncWiki(repo.Owner.UserName, repo.Name, cfg)
+		}
+	})
 
+	gitSync.LogSyncComplete("Forgejo")
 	return nil
 }
 

diff --git a/pkg/github/github.go b/pkg/github/github.go
@@ -2,7 +2,6 @@ package github
 
 import (
 	"context"
-	"sync"
 
 	"github.com/AkashRajpurohit/git-sync/pkg/config"
 	"github.com/AkashRajpurohit/git-sync/pkg/helpers"
@@ -38,27 +37,16 @@ func (c GitHubClient) Sync(cfg config.Config) error {
 		return err
 	}
 
-	logger.Info("Total repositories: ", len(repos))
+	gitSync.LogRepoCount(len(repos), "GitHub")
 
-	var wg sync.WaitGroup
-	sem := make(chan struct{}, 10) // Concurrency of 10
-
-	for _, repo := range repos {
-		wg.Add(1)
-		go func(repo *gh.Repository) {
-			defer wg.Done()
-			sem <- struct{}{}
-			gitSync.CloneOrUpdateRepo(repo.GetOwner().GetLogin(), repo.GetName(), cfg)
-			if cfg.IncludeWiki && repo.GetHasWiki() {
-				gitSync.SyncWiki(repo.GetOwner().GetLogin(), repo.GetName(), cfg)
-			}
-			<-sem
-		}(repo)
-	}
-
-	wg.Wait()
-	logger.Info("All repositories synced ✅")
+	gitSync.SyncReposWithConcurrency(cfg, repos, func(repo *gh.Repository) {
+		gitSync.CloneOrUpdateRepo(repo.GetOwner().GetLogin(), repo.GetName(), cfg)
+		if cfg.IncludeWiki && repo.GetHasWiki() {
+			gitSync.SyncWiki(repo.GetOwner().GetLogin(), repo.GetName(), cfg)
+		}
+	})
 
+	gitSync.LogSyncComplete("GitHub")
 	return nil
 }