diff --git a/README.md b/README.md index 7b9fd35..a4021d4 100644 --- a/README.md +++ b/README.md @@ -31,13 +31,14 @@ $ subjs -h | `-t` | Timeout (in seconds) for http client (default 15) | `subjs -t 20` | | `-ua` | User-Agent to send in requests | `subjs -ua "Chrome..."` | | `-version` | Show version number | `subjs -version"` | - +| `-jsl` | JSON Lines output (contains source url as well) | `subjs -jsl`| +| `-cookie` | Cookie to send in requests | `subjs -cookie "JSESSIONID...` | ## Installation ### From Source: ``` -$ GO111MODULE=on go get -u -v github.com/lc/subjs@latest +$ GO111MODULE=on go install -v github.com/lc/subjs@latest ``` ### From Binary diff --git a/runner/subjs/options.go b/runner/subjs/options.go index 73a3fe8..b1335e9 100644 --- a/runner/subjs/options.go +++ b/runner/subjs/options.go @@ -11,14 +11,18 @@ type Options struct { Workers int Timeout int UserAgent string + JSONLines bool + Cookie string } func ParseOptions() *Options { opts := &Options{} flag.StringVar(&opts.InputFile, "i", "", "Input file containing URLS") flag.StringVar(&opts.UserAgent, "ua", "", "User-Agent to send in requests") + flag.StringVar(&opts.Cookie, "cookie", "", "Cookie to send in requests") flag.IntVar(&opts.Workers, "c", 10, "Number of concurrent workers") flag.IntVar(&opts.Timeout, "t", 15, "Timeout (in seconds) for http client") + flag.BoolVar(&opts.JSONLines, "jsl", false, "JSON Lines output") showVersion := flag.Bool("version", false, "Show version number") flag.Parse() if *showVersion { diff --git a/runner/subjs/runner.go b/runner/subjs/runner.go index 4a1a3a6..92eee7b 100644 --- a/runner/subjs/runner.go +++ b/runner/subjs/runner.go @@ -15,7 +15,7 @@ import ( "github.com/PuerkitoBio/goquery" ) -const version = `1.0.2` +const version = `1.0.3` type SubJS struct { client *http.Client @@ -40,7 +40,7 @@ func (s *SubJS) Run() error { // otherwise read from file input, err = os.Open(s.opts.InputFile) if err != nil { - return fmt.Errorf("Could not open input file: %s", err) + return fmt.Errorf("Could not open input file: %s\"}", err) } defer input.Close() } @@ -89,6 +89,9 @@ func (s *SubJS) fetch(urls <-chan string, results chan string) { if s.opts.UserAgent != "" { req.Header.Add("User-Agent", s.opts.UserAgent) } + if s.opts.Cookie != "" { + req.Header.Add("Cookie", s.opts.Cookie) + } resp, err := s.client.Do(req) if err != nil { continue @@ -104,9 +107,23 @@ func (s *SubJS) fetch(urls <-chan string, results chan string) { //log.Fatalf("error parsing url: %v", err) return } - doc.Find("script").Each(func(index int, s *goquery.Selection) { - js, _ := s.Attr("src") - if js != "" { + doc.Find("script").Each(func(index int, z *goquery.Selection) { + js, _ := z.Attr("src") + if js != "" && s.opts.JSONLines { + if strings.HasPrefix(js, "http://") || strings.HasPrefix(js, "https://") { + js := fmt.Sprintf("{\"%s\": \"%s\"}", u, js) + results <- js + } else if strings.HasPrefix(js, "//") { + js := fmt.Sprintf("{\"%s\": \"%s:%s\"}", u, u.Scheme, js) + results <- js + } else if strings.HasPrefix(js, "/") { + js := fmt.Sprintf("{\"%s\": \"%s://%s%s\"}", u, u.Scheme, u.Host, js) + results <- js + } else { + js := fmt.Sprintf("{\"%s\": \"%s://%s/%s\"}", u, u.Scheme, u.Host, js) + results <- js + } + } else if js != "" { if strings.HasPrefix(js, "http://") || strings.HasPrefix(js, "https://") { results <- js } else if strings.HasPrefix(js, "//") { @@ -120,21 +137,43 @@ func (s *SubJS) fetch(urls <-chan string, results chan string) { results <- js } } - r := regexp.MustCompile(`[(\w./:)]*js`) - matches := r.FindAllString(s.Contents().Text(), -1) + r := regexp.MustCompile(`[(\w./:)]*\.js`) + matches := r.FindAllString(z.Contents().Text(), -1) for _, js := range matches { - if strings.HasPrefix(js, "//") { - js := fmt.Sprintf("%s:%s", u.Scheme, js) + if s.opts.JSONLines { + if strings.HasPrefix(js, "//") { + js := fmt.Sprintf("{\"%s\": \"%s:%s\"}", u, u.Scheme, js) + results <- js + } else if strings.HasPrefix(js, "/") { + js := fmt.Sprintf("{\"%s\": \"%s://%s%s\"}", u, u.Scheme, u.Host, js) + results <- js + } + } else { + if strings.HasPrefix(js, "//") { + js := fmt.Sprintf("%s:%s", u.Scheme, js) + results <- js + } else if strings.HasPrefix(js, "/") { + js := fmt.Sprintf("%s://%s%s", u.Scheme, u.Host, js) + results <- js + } + } + }}) + doc.Find("div").Each(func(index int, z *goquery.Selection) { + js, _ := z.Attr("data-script-src") + if js != "" && s.opts.JSONLines { + if strings.HasPrefix(js, "http://") || strings.HasPrefix(js, "https://") { + results <- js + } else if strings.HasPrefix(js, "//") { + js := fmt.Sprintf("{\"%s\": \"%s:%s\"}", u, u.Scheme, js) results <- js } else if strings.HasPrefix(js, "/") { - js := fmt.Sprintf("%s://%s%s", u.Scheme, u.Host, js) + js := fmt.Sprintf("{\"%s\": \"%s://%s%s\"}", u, u.Scheme, u.Host, js) + results <- js + } else { + js := fmt.Sprintf("{\"%s\": \"%s://%s/%s\"}", u, u.Scheme, u.Host, js) results <- js } - } - }) - doc.Find("div").Each(func(index int, s *goquery.Selection) { - js, _ := s.Attr("data-script-src") - if js != "" { + } else if js != "" { if strings.HasPrefix(js, "http://") || strings.HasPrefix(js, "https://") { results <- js } else if strings.HasPrefix(js, "//") {