-
Notifications
You must be signed in to change notification settings - Fork 0
/
user_agent.go
98 lines (84 loc) · 2.2 KB
/
user_agent.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
package twitterscraper
import (
"os"
"strings"
"sync"
"github.com/joho/godotenv"
"github.com/sirupsen/logrus"
"golang.org/x/exp/rand"
)
var (
userAgents []string
currentUserAgent string
userAgentMutex sync.Mutex
)
func init() {
if err := godotenv.Load(); err != nil {
logrus.WithError(err).Warn("Error loading .env file")
}
loadUserAgents()
}
func loadUserAgents() {
userAgentString := os.Getenv("USER_AGENTS")
logrus.WithField("raw_user_agents", userAgentString).Debug("Raw USER_AGENTS value")
if userAgentString == "" {
logrus.Warn("USER_AGENTS environment variable is not set. Using default user agent.")
userAgents = []string{"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"}
} else {
// Use a more robust splitting method
userAgents = splitUserAgents(userAgentString)
logrus.WithFields(logrus.Fields{
"count": len(userAgents),
"agents": userAgents,
}).Info("Loaded user agents from environment")
}
}
func splitUserAgents(s string) []string {
var result []string
var builder strings.Builder
inParentheses := false
for _, r := range s {
switch r {
case '(':
inParentheses = true
builder.WriteRune(r)
case ')':
inParentheses = false
builder.WriteRune(r)
case ',':
if inParentheses {
builder.WriteRune(r)
} else {
if builder.Len() > 0 {
result = append(result, strings.TrimSpace(builder.String()))
builder.Reset()
}
}
default:
builder.WriteRune(r)
}
}
if builder.Len() > 0 {
result = append(result, strings.TrimSpace(builder.String()))
}
return result
}
func GetUserAgent() string {
userAgentMutex.Lock()
defer userAgentMutex.Unlock()
if currentUserAgent == "" {
currentUserAgent = userAgents[0]
logrus.WithField("user_agent", currentUserAgent).Debug("Using first user agent from list")
}
return currentUserAgent
}
func GetRandomUserAgent() string {
userAgentMutex.Lock()
defer userAgentMutex.Unlock()
newAgent := userAgents[rand.Intn(len(userAgents))]
if newAgent != currentUserAgent {
currentUserAgent = newAgent
logrus.WithField("user_agent", currentUserAgent).Debug("Using new random user agent")
}
return currentUserAgent
}