Added JSON Lines and Cookies

README.md

@@ -31,13 +31,14 @@ $ subjs -h
 | `-t` | Timeout (in seconds) for http client (default 15) | `subjs -t 20` |
 | `-ua` | User-Agent to send in requests | `subjs -ua "Chrome..."` |
 | `-version` | Show version number | `subjs -version"` |
-
+| `-jsl` | JSON Lines output (contains source url as well) | `subjs -jsl`|
+| `-cookie` | Cookie to send in requests | `subjs -cookie "JSESSIONID...` |
 
 ## Installation
 ### From Source:
 
 ```
-$ GO111MODULE=on go get -u -v github.com/lc/subjs@latest
+$ GO111MODULE=on go install -v github.com/lc/subjs@latest
 ```
 
 ### From Binary

runner/subjs/options.go

@@ -11,14 +11,18 @@ type Options struct {
 	Workers   int
 	Timeout   int
 	UserAgent string
+	JSONLines bool
+	Cookie string
 }
 
 func ParseOptions() *Options {
 	opts := &Options{}
 	flag.StringVar(&opts.InputFile, "i", "", "Input file containing URLS")
 	flag.StringVar(&opts.UserAgent, "ua", "", "User-Agent to send in requests")
+	flag.StringVar(&opts.Cookie, "cookie", "", "Cookie to send in requests")
 	flag.IntVar(&opts.Workers, "c", 10, "Number of concurrent workers")
 	flag.IntVar(&opts.Timeout, "t", 15, "Timeout (in seconds) for http client")
+	flag.BoolVar(&opts.JSONLines, "jsl", false, "JSON Lines output")
 	showVersion := flag.Bool("version", false, "Show version number")
 	flag.Parse()
 	if *showVersion {

runner/subjs/runner.go

@@ -15,7 +15,7 @@ import (
 	"github.com/PuerkitoBio/goquery"
 )
 
-const version = `1.0.2`
+const version = `1.0.3`
 
 type SubJS struct {
 	client *http.Client
@@ -40,7 +40,7 @@ func (s *SubJS) Run() error {
 		// otherwise read from file
 		input, err = os.Open(s.opts.InputFile)
 		if err != nil {
-			return fmt.Errorf("Could not open input file: %s", err)
+			return fmt.Errorf("Could not open input file: %s\"}", err)
 		}
 		defer input.Close()
 	}
@@ -89,6 +89,9 @@ func (s *SubJS) fetch(urls <-chan string, results chan string) {
 		if s.opts.UserAgent != "" {
 			req.Header.Add("User-Agent", s.opts.UserAgent)
 		}
+		if s.opts.Cookie != "" {
+			req.Header.Add("Cookie", s.opts.Cookie)
+		}
 		resp, err := s.client.Do(req)
 		if err != nil {
 			continue
@@ -104,9 +107,23 @@ func (s *SubJS) fetch(urls <-chan string, results chan string) {
 			//log.Fatalf("error parsing url: %v", err)
 			return
 		}
-		doc.Find("script").Each(func(index int, s *goquery.Selection) {
-			js, _ := s.Attr("src")
-			if js != "" {
+		doc.Find("script").Each(func(index int, z *goquery.Selection) {
+			js, _ := z.Attr("src")
+			if js != "" && s.opts.JSONLines {
+				if strings.HasPrefix(js, "http://") || strings.HasPrefix(js, "https://") {
+					js := fmt.Sprintf("{\"%s\": \"%s\"}", u, js)
+					results <- js
+				} else if strings.HasPrefix(js, "//") {
+					js := fmt.Sprintf("{\"%s\": \"%s:%s\"}", u, u.Scheme, js)
+					results <- js
+				} else if strings.HasPrefix(js, "/") {
+					js := fmt.Sprintf("{\"%s\": \"%s://%s%s\"}", u, u.Scheme, u.Host, js)
+					results <- js
+				} else {
+					js := fmt.Sprintf("{\"%s\": \"%s://%s/%s\"}", u, u.Scheme, u.Host, js)
+					results <- js
+				}
+			} else if js != "" {
 				if strings.HasPrefix(js, "http://") || strings.HasPrefix(js, "https://") {
 					results <- js
 				} else if strings.HasPrefix(js, "//") {
@@ -120,21 +137,43 @@ func (s *SubJS) fetch(urls <-chan string, results chan string) {
 					results <- js
 				}
 			}
-			r := regexp.MustCompile(`[(\w./:)]*js`)
-			matches := r.FindAllString(s.Contents().Text(), -1)
+			r := regexp.MustCompile(`[(\w./:)]*\.js`)
+			matches := r.FindAllString(z.Contents().Text(), -1)
 			for _, js := range matches {
-				if strings.HasPrefix(js, "//") {
-					js := fmt.Sprintf("%s:%s", u.Scheme, js)
+				if s.opts.JSONLines {
+					if strings.HasPrefix(js, "//") {
+						js := fmt.Sprintf("{\"%s\": \"%s:%s\"}", u, u.Scheme, js)
+						results <- js
+					} else if strings.HasPrefix(js, "/") {
+						js := fmt.Sprintf("{\"%s\": \"%s://%s%s\"}", u, u.Scheme, u.Host, js)
+						results <- js
+					}
+				} else {
+					if strings.HasPrefix(js, "//") {
+						js := fmt.Sprintf("%s:%s", u.Scheme, js)
+						results <- js
+					} else if strings.HasPrefix(js, "/") {
+						js := fmt.Sprintf("%s://%s%s", u.Scheme, u.Host, js)
+						results <- js
+					}
+				}
+		}})
+		doc.Find("div").Each(func(index int, z *goquery.Selection) {
+			js, _ := z.Attr("data-script-src")
+			if js != "" && s.opts.JSONLines {
+				if strings.HasPrefix(js, "http://") || strings.HasPrefix(js, "https://") {
+					results <- js
+				} else if strings.HasPrefix(js, "//") {
+					js := fmt.Sprintf("{\"%s\": \"%s:%s\"}", u, u.Scheme, js)
 					results <- js
 				} else if strings.HasPrefix(js, "/") {
-					js := fmt.Sprintf("%s://%s%s", u.Scheme, u.Host, js)
+					js := fmt.Sprintf("{\"%s\": \"%s://%s%s\"}", u, u.Scheme, u.Host, js)
+					results <- js
+				} else {
+					js := fmt.Sprintf("{\"%s\": \"%s://%s/%s\"}", u, u.Scheme, u.Host, js)
 					results <- js
 				}
-			}
-		})
-		doc.Find("div").Each(func(index int, s *goquery.Selection) {
-			js, _ := s.Attr("data-script-src")
-			if js != "" {
+			} else if js != "" {
 				if strings.HasPrefix(js, "http://") || strings.HasPrefix(js, "https://") {
 					results <- js
 				} else if strings.HasPrefix(js, "//") {