From 46f83cdc16a9c096024fb5cada5555a19f7ebfe1 Mon Sep 17 00:00:00 2001 From: poiuy Date: Mon, 13 Jul 2026 01:31:47 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- my-daily/Dockerfile | 3 + my-daily/config.json.example | 5 + my-daily/docker-compose.nas.yml | 2 + my-daily/docker-compose.yml | 2 + my-daily/resources/rss_feeds copy 2.opml | 144 ++++++++++++ my-daily/resources/rss_feeds.opml | 4 + my-daily/src/config.py | 56 +++-- my-daily/src/content_fetcher.py | 280 +++++++++++++++++++++++ my-daily/src/fetch_pipeline.py | 40 ++++ 9 files changed, 522 insertions(+), 14 deletions(-) create mode 100644 my-daily/resources/rss_feeds copy 2.opml create mode 100644 my-daily/src/content_fetcher.py diff --git a/my-daily/Dockerfile b/my-daily/Dockerfile index c216a89..79adcac 100644 --- a/my-daily/Dockerfile +++ b/my-daily/Dockerfile @@ -29,6 +29,9 @@ RUN pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple \ # ── 复制项目代码 ────────────────────────────────────────────── COPY . . +# ── 备份默认订阅源（用于 docker-compose 未挂载 resources 时 fallback）── +RUN mkdir -p /app/resources_default && cp /app/resources/rss_feeds.opml /app/resources_default/ + # ── 创建运行时目录 ──────────────────────────────────────────── RUN mkdir -p /app/data/fetch /app/data/hotalert /app/data/cluster /app/data/digest /app/data/images /app/data/cache /app/data/output /app/logs /app/cache diff --git a/my-daily/config.json.example b/my-daily/config.json.example index abb1969..409a068 100644 --- a/my-daily/config.json.example +++ b/my-daily/config.json.example @@ -17,6 +17,11 @@ "max_workers": 10, "timeout": 15 }, + "content_fetch": { + "enabled": false, + "max_workers": 5, + "timeout": 15 + }, "translate": { "max_concurrent": 6 }, diff --git a/my-daily/docker-compose.nas.yml b/my-daily/docker-compose.nas.yml index 260fa2b..08ee7fe 100644 --- a/my-daily/docker-compose.nas.yml +++ b/my-daily/docker-compose.nas.yml @@ -11,6 +11,8 @@ services: - /volume2/webdav/code/my-daily/docker/config.json:/app/config.json:ro # 提示词文件 - /volume2/webdav/code/my-daily/docker/prompts:/app/prompts:ro + # 订阅源文件（持久化，便于管理 OPML 文件） + - /volume2/webdav/code/my-daily/resources:/app/resources:ro # 运行时数据 - /volume2/webdav/code/my-daily/docker/data:/app/data # 运行日志 diff --git a/my-daily/docker-compose.yml b/my-daily/docker-compose.yml index 8b49dc9..f47fa72 100644 --- a/my-daily/docker-compose.yml +++ b/my-daily/docker-compose.yml @@ -11,6 +11,8 @@ services: - ./docker/config.json:/app/config.json:ro # 提示词文件（在 NAS 上直接修改，容器内只读） - ./docker/prompts:/app/prompts:ro + # 订阅源文件（持久化，便于管理 OPML 文件） + - ./resources:/app/resources:ro # 运行时数据（持久化保存，含 fetch/hotalert/cluster/digest/images/cache 子目录） - ./docker/data:/app/data # 运行日志（持久化，方便排查问题） diff --git a/my-daily/resources/rss_feeds copy 2.opml b/my-daily/resources/rss_feeds copy 2.opml new file mode 100644 index 0000000..abf9e06 --- /dev/null +++ b/my-daily/resources/rss_feeds copy 2.opml @@ -0,0 +1,144 @@ + + + + 军事科技RSS订阅源 + Mon, 29 May 2026 12:00:00 +0800 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/my-daily/resources/rss_feeds.opml b/my-daily/resources/rss_feeds.opml index abf9e06..b816ec2 100644 --- a/my-daily/resources/rss_feeds.opml +++ b/my-daily/resources/rss_feeds.opml @@ -10,6 +10,10 @@ + + + + diff --git a/my-daily/src/config.py b/my-daily/src/config.py index 67c222c..e79785f 100644 --- a/my-daily/src/config.py +++ b/my-daily/src/config.py @@ -78,21 +78,49 @@ def load_env_config() -> Dict: def parse_opml(opml_path: str) -> List[Dict]: + """解析 OPML 文件，支持 fallback 到默认订阅源 + + 优先级： + 1. 配置的 opml_path（通常是挂载的 resources/rss_feeds.opml） + 2. /app/resources_default/rss_feeds.opml（镜像内默认文件） + """ path = Path(opml_path) - if not path.exists(): - return [] - - tree = ET.parse(path) - root = tree.getroot() - - feeds = [] - for outline in root.findall(".//outline[@type='rss']"): - feeds.append({ - "title": outline.get("title", ""), - "xmlUrl": outline.get("xmlUrl", ""), - }) - - return feeds + + # 尝试原始路径 + if path.exists() and path.stat().st_size > 0: + try: + tree = ET.parse(path) + root = tree.getroot() + feeds = [] + for outline in root.findall(".//outline[@type='rss']"): + feeds.append({ + "title": outline.get("title", ""), + "xmlUrl": outline.get("xmlUrl", ""), + }) + if feeds: + return feeds + except Exception as e: + print(f"⚠️ 解析 {opml_path} 失败: {e}") + + # Fallback 到镜像内默认文件 + default_path = Path("/app/resources_default/rss_feeds.opml") + if default_path.exists() and default_path.stat().st_size > 0: + try: + tree = ET.parse(default_path) + root = tree.getroot() + feeds = [] + for outline in root.findall(".//outline[@type='rss']"): + feeds.append({ + "title": outline.get("title", ""), + "xmlUrl": outline.get("xmlUrl", ""), + }) + if feeds: + print(f" 使用镜像默认订阅源: {default_path}") + return feeds + except Exception as e: + print(f"⚠️ 解析默认订阅源失败: {e}") + + return [] def merge_sources(sources_config: Dict) -> List[Dict]: diff --git a/my-daily/src/content_fetcher.py b/my-daily/src/content_fetcher.py new file mode 100644 index 0000000..4247e4a --- /dev/null +++ b/my-daily/src/content_fetcher.py @@ -0,0 +1,280 @@ +"""全文内容抓取模块 + +从RSS条目的链接地址异步获取完整文章内容。 +使用 aiohttp 并发请求 + BeautifulSoup 提取正文。 +""" +import asyncio +import re +from typing import List, Dict, Optional +from urllib.parse import urlparse + +import aiohttp +from bs4 import BeautifulSoup + +from src.fetcher import DEFAULT_HEADERS + +# 文章正文常见容器选择器 +CONTENT_SELECTORS = [ + "article", + "[role='main']", + "main", + ".article-body", + ".article-content", + ".post-content", + ".entry-content", + ".content", + ".article", + "#article-body", + "#content", + ".field-body", + ".news-content", +] + +# 需要移除的元素选择器（导航、广告、侧边栏等） +# 注意：不包含 main/article 等可能作为正文容器的标签 +# 注意：避免使用 [class*='ad'] 这样过于宽泛的选择器，会误伤正常内容 +REMOVE_SELECTORS = [ + "nav", "script", "style", + ".sidebar", ".navigation", ".menu", + ".ad-container", ".advertisement", ".ads", ".ad-banner", # 精确的广告类名 + ".share-buttons", ".related-posts", ".comments", ".tags", + "[class*='sidebar']", "[class*='navigation']", "[class*='menu']", + "[class*='share']", "[class*='related']", "[class*='comment']", + "[id*='sidebar']", "[id*='ad-container']", "[id*='advertisement']", +] + +# 常见的时间/作者元数据标记 +META_INDICATORS = [ + "published", "updated", "author", "byline", "dateline", + "read time", "min read", "阅读时间", "作者", "发布时间", +] + +FETCH_TIMEOUT = 15 # 单篇文章抓取超时(秒) + + +def _is_likely_body_text(text: str) -> bool: + """判断文本段落是否可能是正文内容""" + text = text.strip() + if len(text) < 20: + return False + # 过滤掉明显的元数据 + lower = text.lower() + if any(indicator in lower for indicator in META_INDICATORS): + return False + # 过滤掉太短的行（可能是标题或标签） + if len(text) < 50 and not text.endswith(('.', '!', '?', '。', '！', '？')): + return False + return True + + +def _extract_main_content(soup: BeautifulSoup, url: str) -> Optional[str]: + """从HTML中提取正文内容 + + 策略： + 1. 先尝试常见文章容器选择器 + 2. 如果没有找到，尝试查找包含最多段落的

的父元素 + 3. 移除导航、广告等干扰元素 + 4. 过滤掉过短的段落（可能是元数据） + """ + # 策略1: 尝试常见选择器 + for selector in CONTENT_SELECTORS: + element = soup.select_one(selector) + if element and len(element.get_text(strip=True)) > 100: + # 移除干扰元素 + for remove_sel in REMOVE_SELECTORS: + for bad in element.select(remove_sel): + bad.decompose() + return element + + # 策略2: 查找包含最多段落的容器 + body = soup.find('body') + if not body: + body = soup + + # 找到包含最多

标签的元素 + best_element = None + max_p_count = 0 + + for elem in body.find_all(True): + p_count = len(elem.find_all('p')) + if p_count > max_p_count: + max_p_count = p_count + best_element = elem + + if best_element and max_p_count >= 2: + # 移除干扰元素 + for remove_sel in REMOVE_SELECTORS: + for bad in best_element.select(remove_sel): + bad.decompose() + return best_element + + # 策略3: 直接使用 body + if body and len(body.get_text(strip=True)) > 100: + for remove_sel in REMOVE_SELECTORS: + for bad in body.select(remove_sel): + bad.decompose() + return body + + return None + + +def _clean_content(element) -> str: + """清理提取的正文内容""" + # 过滤掉过短的段落（可能是元数据） + paragraphs = element.find_all('p') + valid_paragraphs = [] + + for p in paragraphs: + text = p.get_text(strip=True) + if _is_likely_body_text(text): + valid_paragraphs.append(p) + + # 如果过滤后还有有效段落，使用它们 + if valid_paragraphs: + # 用有效段落替换原内容 + new_div = element.find_parent().new_tag('div') + for p in valid_paragraphs: + new_div.append(p) + return str(new_div) + + # 如果没有有效段落（可能段落都很短或者是其他结构），保留原始内容 + return str(element) + + +def extract_text_from_html(html: str) -> str: + """从HTML中提取纯文本正文""" + if not html or len(html.strip()) < 50: + return "" + + try: + soup = BeautifulSoup(html, 'html.parser') + + # 移除所有链接的 href（避免干扰） + for a in soup.find_all('a'): + a.unwrap() + + # 移除图片 + for img in soup.find_all('img'): + img.decompose() + + # 保留段落结构，转换为文本 + text_parts = [] + for elem in soup.find_all(['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li', 'blockquote']): + text = elem.get_text(strip=True) + if text and len(text) > 10: + text_parts.append(text) + + if not text_parts: + # 如果没有找到结构化内容，尝试获取所有文本 + text_parts = [soup.get_text(separator='\n', strip=True)] + + return '\n\n'.join(text_parts) + + except Exception as e: + print(f" ⚠️ 内容提取失败: {e}") + return "" + + +async def fetch_article_content( + session: aiohttp.ClientSession, + url: str, + timeout: int = FETCH_TIMEOUT +) -> Optional[str]: + """异步抓取单篇文章的完整内容 + + Returns: + 提取后的纯文本正文，失败返回 None + """ + try: + client_timeout = aiohttp.ClientTimeout(total=timeout) + headers = { + **DEFAULT_HEADERS, + "Referer": url, + } + + async with session.get(url, headers=headers, timeout=client_timeout) as resp: + if resp.status != 200: + print(f" ⚠️ HTTP {resp.status}: {url}") + return None + + content_type = resp.headers.get('Content-Type', '') + if 'text/html' not in content_type and 'application/xhtml' not in content_type: + # 不是HTML内容（可能是PDF等） + return None + + html = await resp.text() + + # 提取正文 + soup = BeautifulSoup(html, 'html.parser') + main_content = _extract_main_content(soup, url) + + if not main_content: + return None + + # 清理并提取文本 + cleaned = _clean_content(main_content) if hasattr(main_content, 'find_all') else str(main_content) + text = extract_text_from_html(cleaned) + + return text if text else None + + except asyncio.TimeoutError: + print(f" ⚠️ 抓取超时: {url}") + return None + except Exception as e: + print(f" ⚠️ 抓取异常: {url} - {e}") + return None + + +async def fetch_contents_batch( + entries: List[Dict], + max_workers: int = 5, + timeout: int = FETCH_TIMEOUT +) -> List[Dict]: + """批量抓取文章完整内容 + + Args: + entries: RSS条目列表，每个条目需包含 'link' 字段 + max_workers: 最大并发数 + timeout: 单篇文章超时时间 + + Returns: + 更新后的条目列表，新增 'full_content' 字段 + """ + if not entries: + return entries + + print(f" 📄 开始抓取全文内容: {len(entries)} 篇文章 (并发={max_workers})") + + semaphore = asyncio.Semaphore(max_workers) + + async def fetch_one(entry, session): + async with semaphore: + link = entry.get("link", "") + if not link: + return entry + + content = await fetch_article_content(session, link, timeout) + if content: + entry["full_content"] = content + # 如果原文太短（摘要），用全文替换 + original_content = entry.get("content", "") + if len(original_content) < 200 and len(content) > 500: + entry["content"] = content + print(f" ✅ {entry.get('title', '')[:40]}... (摘要→全文)") + else: + print(f" ✅ {entry.get('title', '')[:40]}...") + else: + print(f" ⚠️ {entry.get('title', '')[:40]}... (抓取失败)") + return entry + + # 创建共享的 session + async with aiohttp.ClientSession() as session: + tasks = [asyncio.create_task(fetch_one(e, session)) for e in entries] + results = await asyncio.gather(*tasks, return_exceptions=True) + + # 处理结果 + success_count = sum(1 for e in entries if e.get("full_content")) + fail_count = len(entries) - success_count + print(f" 全文抓取完成: 成功 {success_count} 条 | 失败 {fail_count} 条") + + return entries diff --git a/my-daily/src/fetch_pipeline.py b/my-daily/src/fetch_pipeline.py index a725780..8199a66 100644 --- a/my-daily/src/fetch_pipeline.py +++ b/my-daily/src/fetch_pipeline.py @@ -8,6 +8,7 @@ from typing import Dict, List, Tuple from src.cache import get_cache from src.config import get_merged_sources, get_timezone +from src.content_fetcher import fetch_contents_batch from src.fetcher import fetch_all_feeds from src.llm import ( generate_immediate_push, @@ -127,6 +128,42 @@ async def fetch_rss_entries( return entries, lookback +async def fetch_full_contents( + entries: List[Dict], + config: Dict, + monitor, +) -> List[Dict]: + """阶段1.5: 全文内容抓取 + + 从RSS条目的链接地址获取完整文章内容。 + 仅当配置启用且条目内容较短（摘要）时触发。 + + Args: + entries: RSS条目列表 + config: 配置字典 + monitor: 监控器实例 + + Returns: + 更新后的条目列表 + """ + content_config = config.get("content_fetch", {}) + if not content_config.get("enabled", False): + return entries + + # 检查是否有需要抓取全文的条目（内容较短的视为摘要） + needs_fetch = [e for e in entries if len(e.get("content", "")) < 200 and e.get("link")] + if not needs_fetch: + print(f" ℹ️ 所有条目内容完整，跳过全文抓取") + return entries + + with monitor.stage("content_fetch"): + max_workers = content_config.get("max_workers", 5) + timeout = content_config.get("timeout", 15) + entries = await fetch_contents_batch(entries, max_workers=max_workers, timeout=timeout) + + return entries + + def deduplicate_entries( entries: List[Dict], config: Dict, @@ -542,6 +579,9 @@ async def run_fetch_pipeline( if not entries: return + # 阶段1.5: 全文抓取（如启用） + entries = await fetch_full_contents(entries, config, monitor) + # 阶段2: 去重 new_entries, cache_hits = deduplicate_entries(entries, config, monitor, lookback) if not new_entries: