diff --git a/common/httpx/encodings.go b/common/httpx/encodings.go index 4ce49a35..df03348c 100644 --- a/common/httpx/encodings.go +++ b/common/httpx/encodings.go @@ -58,6 +58,8 @@ func DecodeData(data []byte, headers http.Header) ([]byte, error) { // Non UTF-8 if contentTypes, ok := headers["Content-Type"]; ok { contentType := strings.ToLower(strings.Join(contentTypes, ";")) + // the charset parameter value can be a quoted string (charset="gbk") + contentType = strings.ReplaceAll(contentType, `"`, "") switch { case stringsutil.ContainsAny(contentType, "charset=gb2312", "charset=gbk"): diff --git a/common/httpx/title.go b/common/httpx/title.go index e38ba7d3..35683798 100644 --- a/common/httpx/title.go +++ b/common/httpx/title.go @@ -50,9 +50,9 @@ func ExtractTitle(r *Response) (title string) { return title } -func CanHaveTitleTag(mimeType string) bool { - return slices.Contains(supportedTitleMimeTypes, mimeType) -} +func CanHaveTitleTag(mimeType string) bool { + return slices.Contains(supportedTitleMimeTypes, strings.ToLower(strings.TrimSpace(mimeType))) +} func getTitleWithDom(r *Response) (*html.Node, error) { var title *html.Node @@ -85,10 +85,10 @@ func renderNode(n *html.Node) string { } func trimTitleTags(title string) string { - titleBegin := strings.Index(title, ">") - titleEnd := strings.Index(title, "") + titleEnd := strings.Index(title, "中文 with the title text encoded in GBK + gbk := []byte{0xd6, 0xd0, 0xce, 0xc4} + body := append([]byte(""), gbk...) + body = append(body, []byte("")...) + + tests := []struct { + name string + contentType string + }{ + {"quoted charset", `text/html; charset="gbk"`}, + {"unquoted charset", "text/html; charset=gbk"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", tt.contentType) + _, _ = w.Write(body) + })) + defer srv.Close() + + req, err := retryablehttp.NewRequest(http.MethodGet, srv.URL, nil) + require.Nil(t, err) + resp, err := ht.Do(req, UnsafeOptions{}) + require.Nil(t, err) + require.Equal(t, "中文", ExtractTitle(resp)) + }) + } +}