Supports charset other than UTF-8 (#9)

soranoba · web-flow · commit 149059150a7b · 2022-06-30T11:43:01.000+09:00
diff --git a/data/6.html b/data/6.html
@@ -0,0 +1,12 @@
+<html xmlns:og="http://ogp.me/ns#">
+<head>
+    <meta charset="shift_jis" />
+    <title>�^�C�g��</title>
+    <meta property="og:title" content="ShiftJIS�^�C�g��" />
+    <meta property="og:type" content="website" />
+    <meta property="og:url" content="http://example.com" />
+    <meta property="og:image" content="http://example.com/image.png" />
+</head>
+<body>
+</body>
+</html>
diff --git a/go.mod b/go.mod
@@ -2,4 +2,4 @@ module github.com/soranoba/googp
 
 go 1.15
 
-require golang.org/x/net v0.0.0-20200904194848-62affa334b73
+require golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e
diff --git a/go.sum b/go.sum
@@ -1,9 +1,8 @@
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA=
-golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e h1:TsQ7F31D3bUCLeqPT0u+yjp1guoArKaNKmCr22PYgTQ=
+golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
diff --git a/googp.go b/googp.go
@@ -11,9 +11,13 @@
 package googp
 
 import (
+	"bufio"
 	"fmt"
+	"io"
 	"mime"
 	"net/http"
+
+	"golang.org/x/net/html/charset"
 )
 
 // Fetch the content from the URL and parse OGP information.
@@ -43,5 +47,11 @@ func Parse(res *http.Response, i interface{}, opts ...ParserOpts) error {
 		}
 	}
 
-	return NewParser(opts...).Parse(res.Body, i)
+	br := bufio.NewReader(res.Body)
+	var reader io.Reader = br
+	data, _ := br.Peek(1024)
+	enc, _, _ := charset.DetermineEncoding(data, ct)
+	reader = enc.NewDecoder().Reader(reader)
+
+	return NewParser(opts...).Parse(reader, i)
 }
diff --git a/googp_test.go b/googp_test.go
@@ -47,6 +47,26 @@ func TestParse(t *testing.T) {
 	assertEqual(t, ogp.Images[0].URL, "http://example.com/image.png")
 }
 
+func TestParseWithEncoding(t *testing.T) {
+	client := &http.Client{}
+	req, err := http.NewRequest("GET", endpoint()+"/6.html", nil)
+	if err != nil {
+		t.Error(err)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	res, err := client.Do(req.WithContext(ctx))
+	var ogp OGP
+	assertNoError(t, Parse(res, &ogp))
+
+	assertEqual(t, ogp.Title, "ShiftJISタイトル")
+	assertEqual(t, ogp.Type, "website")
+	assertEqual(t, ogp.URL, "http://example.com")
+	assertEqual(t, ogp.Images[0].URL, "http://example.com/image.png")
+}
+
 func ExampleFetch() {
 	var ogp OGP
 	if err := Fetch(endpoint()+"/5.html", &ogp); err != nil {

Original file line number	Diff line number	Diff line change
`@@ -2,4 +2,4 @@ module github.com/soranoba/googp`
`2`	`2`
`3`	`3`	`go 1.15`
`4`	`4`
`5`		`-require golang.org/x/net v0.0.0-20200904194848-62affa334b73`
	`5`	`+require golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e`
Original file line number	Diff line number	Diff line change
`@@ -11,9 +11,13 @@`
`11`	`11`	`package googp`
`12`	`12`
`13`	`13`	`import (`
	`14`	`+ "bufio"`
`14`	`15`	`"fmt"`
	`16`	`+ "io"`
`15`	`17`	`"mime"`
`16`	`18`	`"net/http"`
	`19`	`+`
	`20`	`+ "golang.org/x/net/html/charset"`
`17`	`21`	`)`
`18`	`22`
`19`	`23`	`// Fetch the content from the URL and parse OGP information.`
`@@ -43,5 +47,11 @@ func Parse(res *http.Response, i interface{}, opts ...ParserOpts) error {`
`43`	`47`	`}`
`44`	`48`	`}`
`45`	`49`
`46`		`- return NewParser(opts...).Parse(res.Body, i)`
	`50`	`+ br := bufio.NewReader(res.Body)`
	`51`	`+ var reader io.Reader = br`
	`52`	`+ data, _ := br.Peek(1024)`
	`53`	`+ enc, _, _ := charset.DetermineEncoding(data, ct)`
	`54`	`+ reader = enc.NewDecoder().Reader(reader)`
	`55`	`+`
	`56`	`+ return NewParser(opts...).Parse(reader, i)`
`47`	`57`	`}`