Skip to content

fix(wechat): improve image extraction (5→12) and HTML entity cleanup #21

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
weitzu-com wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(wechat): improve image extraction (5→12) and HTML entity cleanup #21

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
153 changes: 126 additions & 27 deletions .claude/skills/news-extractor/scripts/crawlers/wechat.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from __future__ import annotations

import hashlib
import html
import json
import logging
import re
Expand Down Expand Up @@ -171,8 +172,54 @@ def parse_html_to_news_content(self, html_content: str) -> List[ContentItem]:
self._process_content_node(node)

contents = [item for item in self._contents if item.content.strip()]

# Also extract images from SSR data (image articles have most images there)
ssr_images = self._extract_ssr_images(html_content)
existing_image_urls = {item.content for item in contents if item.type == ContentType.IMAGE}
for ssr_img in ssr_images:
if ssr_img.content not in existing_image_urls:
contents.insert(0, ssr_img)
Comment on lines +179 to +181

@chatgpt-codex-connector chatgpt-codex-connector Bot Aug 8, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Preserve the SSR image order when merging

When a page has js_content and SSR contributes multiple images absent from the DOM, inserting every image at index zero reverses their source order: an SSR sequence [1, 2, 3] becomes [3, 2, 1, ...DOM]. Since both the Markdown body and NewsItem.images preserve this list order, affected image articles are exported with their images backwards.

Useful? React with 👍 / 👎.

existing_image_urls.add(ssr_img.content)

return self._remove_duplicate_contents(contents)

def _extract_ssr_images(self, html_content: str) -> List[ContentItem]:
"""Extract images from SSR/embedded data (covers image articles type 8).

WeChat image articles store most images inside inline <script> data blocks
rather than in the visible DOM. This method uses three fallback strategies
to find them all.
"""
images: List[ContentItem] = []

# Method 1: window.picture_page_info_list (primary source for image articles)
ssr_images = _parse_ssr_image_list(html_content)
images.extend(ssr_images)

# Method 2: __QMTPL_SSR_DATA__ / cgiDataNew (fallback)
ssr_data = _parse_ssr_data(html_content)
if ssr_data:
picture_list = ssr_data.get("picture_page_info_list", [])
for pic_info in picture_list:
pic_cdn = pic_info.get("cdn_url", "")
if pic_cdn:
pic_cdn = pic_cdn.replace("&amp;", "&")
images.append(ContentItem(type=ContentType.IMAGE, content=pic_cdn))

# Method 3: Global cdn_url scan across all <script> blocks.
# Some WeChat article types embed image lists as plain JS objects
# scattered across multiple inline scripts (e.g. type-8 image articles).
# We scan for cdn_url: '...' patterns and pick up only WeChat CDN JPEGs.
cdn_urls_seen = {item.content for item in images}
for m in re.finditer(r"""cdn_url\s*:\s*['"]([^'"]+)['"]""", html_content):
url = m.group(1)
if "mmbiz.qpic.cn" in url and ("wx_fmt=jpeg" in url or "wx_fmt=png" in url):
if url not in cdn_urls_seen:
images.append(ContentItem(type=ContentType.IMAGE, content=url))
Comment on lines +214 to +218

@chatgpt-codex-connector chatgpt-codex-connector Bot Aug 8, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Normalize fallback URLs before deduplicating them

When window.picture_page_info_list contains WeChat's escaped \x26amp; separators, _parse_ssr_image_list first normalizes those URLs, but this fallback scan compares and appends the raw escaped forms. Each image is therefore emitted twice—once correctly and once as an unusable escaped URL—and the later exact-string deduplication cannot remove either copy. Normalize the scanned URL before checking cdn_urls_seen or existing_urls.

Useful? React with 👍 / 👎.

cdn_urls_seen.add(url)

return images

def parse(self, html_content: str) -> List[ContentItem]:
return self.parse_html_to_news_content(html_content)

Expand Down Expand Up @@ -203,6 +250,13 @@ def _process_text_block(node: Selector) -> Optional[str]:
if node.root.tag in ["script", "style"]:
return None
text = node.xpath("string(.)").get("").strip()
if not text:
return None
# Unescape HTML entities (&lt;→<, &gt;→>, &quot;→", &amp;→&)
text = html.unescape(text)
# Strip residual HTML tags (e.g. topic links) but keep inner text
text = re.sub(r'<[^>]+>', '', text)
Comment on lines +255 to +258

@chatgpt-codex-connector chatgpt-codex-connector Bot Aug 8, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Avoid treating literal angle brackets as HTML tags

For article text containing legitimate angle-bracket content, such as 1 &lt; 2 and 3 &gt; 1, entity decoding produces 1 < 2 and 3 > 1 and this regex then deletes < 2 and 3 >, corrupting the paragraph to 1 1. The same occurs for code-like text and is repeated in the SSR path; strip only actual embedded markup rather than every substring delimited by < and >.

Useful? React with 👍 / 👎.

text = text.strip()
if not text:
return None
return text
Expand Down Expand Up @@ -297,37 +351,82 @@ def _process_content_node(self, node: Selector):
return

def parse_ssr_content(self, html_content: str) -> List[ContentItem]:
contents = []
"""Parse SSR/inline data when js_content div is not present.

WeChat articles rendered as image cards (type 8) or loaded dynamically
don't have a populated js_content div in the initial HTML. Instead the
data lives in inline <script> blocks and meta tags.
"""
contents: List[ContentItem] = []

# --- Images: try multiple sources ---
ssr_data_dict = _parse_ssr_data(html_content)

# Method 1: picture_page_info_list from SSR data
if ssr_data_dict:
picture_list = ssr_data_dict.get("picture_page_info_list", [])
for pic_info in picture_list:
pic_url = pic_info.get("cdn_url", "")
if pic_url:
pic_url = pic_url.replace("&amp;", "&")
contents.append(ContentItem(type=ContentType.IMAGE, content=pic_url))

# Method 2: window.picture_page_info_list (regex-based, works for type-8 articles)
ssr_image_list = _parse_ssr_image_list(html_content)
existing_urls = {item.content for item in contents if item.type == ContentType.IMAGE}
for img in ssr_image_list:
if img.content not in existing_urls:
contents.append(img)
existing_urls.add(img.content)

# Method 3: Global cdn_url scan across all <script> blocks
for m in re.finditer(r"""cdn_url\s*:\s*['"]([^'"]+)['"]""", html_content):
url = m.group(1)
if "mmbiz.qpic.cn" in url and ("wx_fmt=jpeg" in url or "wx_fmt=png" in url):
if url not in existing_urls:
contents.append(ContentItem(type=ContentType.IMAGE, content=url))
existing_urls.add(url)

# --- Text: try multiple sources ---
text_source = ""

# Source 1: SSR data desc / content_noencode
if ssr_data_dict:
try:
picture_list = ssr_data_dict.get("picture_page_info_list", [])
if picture_list:
for pic_info in picture_list:
cdn_url = pic_info.get("cdn_url", "")
if cdn_url:
cdn_url = cdn_url.replace("&amp;", "&")
contents.append(ContentItem(type=ContentType.IMAGE, content=cdn_url))

if not picture_list:
contents.extend(_parse_ssr_image_list(html_content))

desc = ssr_data_dict.get("desc") or ssr_data_dict.get("content_noencode")
title = ssr_data_dict.get("title")
final_desc = desc or title
if final_desc:
desc_list = final_desc.split("\n")
for desc_item in desc_list:
if not desc_item:
continue
contents.append(
ContentItem(type=ContentType.TEXT, content=desc_item.strip())
)
except (json.JSONDecodeError, Exception) as e:
logger.error(f"Failed to parse SSR data: {str(e)}")
desc = ssr_data_dict.get("desc") or ssr_data_dict.get("content_noencode")
if desc:
text_source = desc

# Source 2: Meta description (contains article body for image articles)
if not text_source:
meta_match = re.search(
r'<meta[^>]*name="description"[^>]*content="([^"]+)"',
html_content,
)
if meta_match:
text_source = meta_match.group(1)

# Source 3: og:title (bare minimum)
if not text_source:
og_match = re.search(
r'<meta[^>]*property="og:title"[^>]*content="([^"]+)"',
html_content,
)
if og_match:
text_source = og_match.group(1)

if text_source:
# Decode JS-style escape sequences and HTML entities
text_source = _js_decode(text_source)
text_source = html.unescape(text_source)
# Strip topic-link HTML tags, keep hashtag text
text_source = re.sub(r'<[^>]+>', '', text_source)
# Split into paragraphs
for line in text_source.split('\n'):
line = line.strip()
if line:
contents.append(ContentItem(type=ContentType.TEXT, content=line))

return contents
return self._remove_duplicate_contents(contents)


class WeChatNewsCrawler(BaseNewsCrawler):
Expand Down