feat: 军事科技每日资讯推送系统 - Docker部署 + 日志系统 + 数据目录重组

ai-daily-main/src/config.py

@@ -0,0 +1,115 @@
+"""配置加载和源管理"""
+import fnmatch
+import json
+import xml.etree.ElementTree as ET
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Dict, List
+from urllib.parse import urlparse
+
+
+def _get_local_timezone() -> timezone:
+    """自动检测本地时区"""
+    return datetime.now().astimezone().tzinfo
+
+
+def get_timezone(config: Dict = None) -> timezone:
+    """
+    获取配置时区，用于推送消息展示本地化时间
+    读取信息源统一使用 UTC 时间
+    如果 config 中没有 timezone_hours，则自动检测本地时区
+    """
+    if config is None:
+        try:
+            config = load_config()
+        except Exception:
+            return _get_local_timezone()
+
+    hours = config.get("schedule", {}).get("timezone_hours")
+    if hours is None:
+        return _get_local_timezone()
+
+    return timezone(timedelta(hours=hours))
+
+
+# 向后兼容的别名
+def get_cst(config: Dict = None) -> timezone:
+    """向后兼容，使用 get_timezone"""
+    return get_timezone(config)
+
+
+def load_config(config_path: str = "config.json") -> Dict:
+    """加载配置文件"""
+    path = Path(config_path)
+    if not path.exists():
+        raise FileNotFoundError(f"配置文件不存在: {config_path}")
+
+    with open(path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def parse_opml(opml_path: str) -> List[Dict]:
+    """解析OPML文件获取订阅源列表"""
+    path = Path(opml_path)
+    if not path.exists():
+        return []
+
+    tree = ET.parse(path)
+    root = tree.getroot()
+
+    feeds = []
+    for outline in root.findall(".//outline[@type='rss']"):
+        feeds.append({
+            "title": outline.get("title", ""),
+            "xmlUrl": outline.get("xmlUrl", ""),
+            "category": outline.get("category", "未分类"),
+        })
+
+    return feeds
+
+
+def merge_sources(sources_config: Dict) -> List[Dict]:
+    """合并base_opml + add - block，以xmlUrl为key去重"""
+    # 1. 解析base OPML
+    base = parse_opml(sources_config.get("base_opml", ""))
+
+    # 2. 添加自定义源
+    add_list = sources_config.get("add", [])
+    all_sources = base + add_list
+
+    # 3. 应用block (以xmlUrl匹配)
+    block_list = sources_config.get("block", [])
+    block_urls = {b.get("xmlUrl", "") for b in block_list}
+    filtered = [s for s in all_sources if s.get("xmlUrl", "") not in block_urls]
+
+    # 4. 应用block_domains (域名屏蔽，支持通配符 *.substack.com)
+    block_domains = sources_config.get("block_domains", [])
+    if block_domains:
+        def is_domain_blocked(url: str) -> bool:
+            try:
+                domain = urlparse(url).netloc.lower()
+                for pattern in block_domains:
+                    # 转换通配符模式为匹配格式
+                    if pattern.startswith("*."):
+                        # *.substack.com 匹配 substack.com 和 addyo.substack.com
+                        suffix = pattern[2:]  # substack.com
+                        if domain == suffix or domain.endswith("." + suffix):
+                            return True
+                    elif fnmatch.fnmatch(domain, pattern):
+                        return True
+                return False
+            except Exception:
+                return False
+
+        filtered = [s for s in filtered if not is_domain_blocked(s.get("xmlUrl", ""))]
+
+    # 5. 去重 (以xmlUrl为key)
+    seen = set()
+    result = []
+    for s in filtered:
+        url = s.get("xmlUrl", "")
+        if url and url not in seen:
+            seen.add(url)
+            result.append(s)
+
+    return result

ai-daily-main/src/fetcher.py

@@ -0,0 +1,213 @@
+"""RSS抓取模块"""
+
+import asyncio
+from datetime import datetime, timezone
+from typing import Dict, List, Optional
+
+import aiohttp
+import feedparser
+import requests
+
+# 默认超时配置（秒）
+DEFAULT_FEED_TIMEOUT = 5
+
+# title 截断阈值：nitter 会把整条推文塞进 <title>，需要截断
+TITLE_MAX_CHARS = 200
+
+# nitter / xcancel 实例：必须用白名单 UA + requests 客户端（aiohttp 的 TLS
+# 指纹过不了），详见 nitter-practice.md
+NITTER_HOSTS = (
+    "xcancel.com",
+    "nitter.net",
+    "nuku.trabun.org",
+)
+NITTER_HEADERS = {
+    "User-Agent": "Inoreader",
+    "Accept": "application/rss+xml, application/atom+xml, application/xml;q=0.9",
+}
+# 公益实例，独立的低并发池 + 每次抓完 sleep，避免给上游施压
+NITTER_MAX_CONCURRENCY = 2
+NITTER_REQUEST_DELAY = 1.0
+
+DEFAULT_HEADERS = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.5",
+    "Accept-Encoding": "gzip, deflate",
+    "Connection": "keep-alive",
+}
+
+
+def is_nitter_url(url: str) -> bool:
+    """判断是否为 nitter / xcancel 实例 URL"""
+    return any(host in url for host in NITTER_HOSTS)
+
+
+def parse_entry_time(entry) -> Optional[datetime]:
+    """解析条目的发布时间 (返回带 UTC 时区的 datetime)"""
+    published_parsed = getattr(entry, "published_parsed", None)
+    if published_parsed is not None:
+        return datetime(*published_parsed[:6], tzinfo=timezone.utc)
+
+    updated_parsed = getattr(entry, "updated_parsed", None)
+    if updated_parsed is not None:
+        return datetime(*updated_parsed[:6], tzinfo=timezone.utc)
+
+    return None
+
+
+def _extract_body(entry) -> str:
+    """提取条目正文：优先 content（含 <content:encoded>），其次 description，最后 summary"""
+    content_list = getattr(entry, "content", None)
+    if content_list:
+        value = content_list[0].get("value", "")
+        if value:
+            return value
+    description = getattr(entry, "description", "")
+    if description:
+        return description
+    return getattr(entry, "summary", "") or ""
+
+
+def _truncate_title(title: str) -> str:
+    if len(title) <= TITLE_MAX_CHARS:
+        return title
+    return title[:TITLE_MAX_CHARS].rstrip() + "…"
+
+
+def _parse_feed_entries(content, feed_info: Dict, cutoff_time: datetime) -> List[Dict]:
+    """把 feed 字节/字符串解析为条目列表，按 cutoff 时间过滤"""
+    feed = feedparser.parse(content)
+    entries = []
+
+    for entry in feed.entries:
+        pub_date = parse_entry_time(entry)
+
+        # RSS 通常按时间倒序排列，一旦发现过期直接跳出
+        if pub_date and pub_date < cutoff_time:
+            break
+
+        entries.append(
+            {
+                "title": _truncate_title(entry.get("title", "无标题")),
+                "link": entry.get("link", ""),
+                "published": pub_date,
+                "source": feed_info["title"],
+                "content": _extract_body(entry),
+                "tags": [],
+                "score": 0,
+                "summary": "",
+            }
+        )
+
+    return entries
+
+
+async def _fetch_nitter_content(url: str, timeout: int) -> Optional[bytes]:
+    """nitter / xcancel 专用：requests + Inoreader UA，丢线程池避免阻塞 loop"""
+
+    def _sync():
+        try:
+            r = requests.get(url, headers=NITTER_HEADERS, timeout=timeout)
+            if r.status_code != 200:
+                print(f"⚠️ HTTP {r.status_code}: {url}")
+                return None
+            return r.content
+        except Exception as e:
+            print(f"⚠️ nitter 抓取失败 {url}: {e}")
+            return None
+
+    return await asyncio.to_thread(_sync)
+
+
+async def _fetch_aiohttp_content(
+    url: str, timeout: int, session: aiohttp.ClientSession = None
+) -> Optional[str]:
+    """普通 RSS 源：aiohttp + 浏览器 UA"""
+    client_timeout = aiohttp.ClientTimeout(total=timeout)
+
+    if session is not None:
+        async with session.get(
+            url, headers=DEFAULT_HEADERS, timeout=client_timeout
+        ) as resp:
+            if resp.status != 200:
+                print(f"⚠️ HTTP {resp.status}: {url}")
+                return None
+            return await resp.text()
+
+    async with aiohttp.ClientSession() as sess:
+        async with sess.get(
+            url, headers=DEFAULT_HEADERS, timeout=client_timeout
+        ) as resp:
+            if resp.status != 200:
+                print(f"⚠️ HTTP {resp.status}: {url}")
+                return None
+            return await resp.text()
+
+
+async def fetch_single_feed_async(
+    feed_info: Dict,
+    cutoff_time: datetime,
+    timeout: int = 5,
+    session: aiohttp.ClientSession = None,
+) -> List[Dict]:
+    """异步获取单个源的条目"""
+    try:
+        if timeout is None:
+            timeout = DEFAULT_FEED_TIMEOUT
+
+        url = feed_info["xmlUrl"]
+
+        if is_nitter_url(url):
+            content = await _fetch_nitter_content(url, timeout)
+        else:
+            content = await _fetch_aiohttp_content(url, timeout, session)
+
+        if content is None:
+            return []
+
+        return _parse_feed_entries(content, feed_info, cutoff_time)
+    except Exception as e:
+        print(f"⚠️ 获取失败 {feed_info['title']}: {e}")
+        return []
+
+
+async def fetch_all_feeds(
+    feeds: List[Dict], cutoff_time: datetime, max_workers: int = 10, timeout: int = None
+) -> List[Dict]:
+    """并发获取所有源的条目；nitter/xcancel 走独立的低并发池"""
+    if timeout is None:
+        timeout = DEFAULT_FEED_TIMEOUT
+
+    nitter_feeds = [f for f in feeds if is_nitter_url(f.get("xmlUrl", ""))]
+    normal_feeds = [f for f in feeds if not is_nitter_url(f.get("xmlUrl", ""))]
+
+    normal_sem = asyncio.Semaphore(max_workers)
+    nitter_sem = asyncio.Semaphore(NITTER_MAX_CONCURRENCY)
+
+    async def fetch_normal(feed):
+        async with normal_sem:
+            return await fetch_single_feed_async(feed, cutoff_time, timeout)
+
+    async def fetch_nitter(feed):
+        async with nitter_sem:
+            result = await fetch_single_feed_async(feed, cutoff_time, timeout)
+            # 公益实例：抓完 sleep，把同一 worker 串内的请求拉开
+            await asyncio.sleep(NITTER_REQUEST_DELAY)
+            return result
+
+    ordered_feeds = normal_feeds + nitter_feeds
+    tasks = [fetch_normal(f) for f in normal_feeds] + [
+        fetch_nitter(f) for f in nitter_feeds
+    ]
+
+    results = await asyncio.gather(*tasks, return_exceptions=True)
+
+    all_entries = []
+    for feed, result in zip(ordered_feeds, results):
+        if isinstance(result, Exception):
+            print(f"⚠️ 获取失败 {feed['title']}: {result}")
+        else:
+            all_entries.extend(result)
+
+    return all_entries

ai-daily-main/src/llm.py

@@ -0,0 +1,634 @@
+"""LLM模块 - 评分和汇总"""
+
+import asyncio
+import json
+import os
+import re
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+from src.markdown_utils import normalize_str_list, parse_frontmatter
+
+
+def load_prompt(prompt_path: str, **kwargs) -> str:
+    """加载提示词模板并填充变量"""
+    path = Path(prompt_path)
+    if not path.exists():
+        raise FileNotFoundError(f"提示词文件不存在: {prompt_path}")
+
+    with open(path, "r", encoding="utf-8") as f:
+        template = f.read()
+
+    # 先把模板中的 {{ 和 }} 替换成占位符，避免与format冲突
+    template = template.replace("{{", "\x00LEFT_BRACE\x00").replace(
+        "}}", "\x00RIGHT_BRACE\x00"
+    )
+
+    # 替换变量
+    for key, value in kwargs.items():
+        template = template.replace(f"{{{key}}}", str(value))
+
+    # 恢复 {{ 和 }}
+    template = template.replace("\x00LEFT_BRACE\x00", "{").replace(
+        "\x00RIGHT_BRACE\x00", "}"
+    )
+
+    return template
+
+
+async def call_llm(
+    prompt: str, config: Dict, response_format: Optional[Dict] = None
+) -> str:
+    """调用LLM API - 统一使用OpenAI兼容接口"""
+    model = config.get("model", "gpt-4o-mini")
+    base_url = config.get("baseUrl", "https://api.openai.com/v1")
+    api_key_name = config.get("apiKeyName", "OPENAI_API_KEY")
+
+    api_key = os.environ.get(api_key_name)
+    if not api_key:
+        raise ValueError(f"未设置{api_key_name}环境变量")
+
+    import aiohttp
+
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json",
+    }
+
+    payload = {
+        "model": model,
+        "messages": [{"role": "user", "content": prompt}],
+        "temperature": 0.3,
+    }
+    if response_format is not None:
+        payload["response_format"] = response_format
+
+    url = f"{base_url}/chat/completions"
+
+    async with aiohttp.ClientSession() as session:
+        async with session.post(url, headers=headers, json=payload) as resp:
+            if resp.status != 200:
+                text = await resp.text()
+                raise RuntimeError(f"LLM API错误: {resp.status} - {text}")
+
+            data = await resp.json()
+            return data["choices"][0]["message"]["content"]
+
+
+async def check_llm_available(config: Dict, timeout_seconds: int = 15) -> str:
+    """启动时检查 LLM 接口可用性"""
+    prompt = "Reply with OK only."
+
+    try:
+        response = await asyncio.wait_for(
+            call_llm(prompt, config), timeout=timeout_seconds
+        )
+    except asyncio.TimeoutError as exc:
+        raise RuntimeError(f"LLM可用性检查超时({timeout_seconds}s)") from exc
+    except Exception as exc:
+        raise RuntimeError(f"LLM可用性检查失败: {exc}") from exc
+
+    response_text = response.strip()
+    if not response_text:
+        raise RuntimeError("LLM可用性检查返回空响应")
+
+    return response_text
+
+
+def _build_batch_prompt(entries: List[Dict], prompt_path: str = None) -> str:
+    """构建批量评分prompt"""
+    # 构建entries JSON列表（只包含必要字段）
+    entries_for_llm = []
+    for e in entries:
+        entries_for_llm.append(
+            {
+                "link": e.get("link", ""),
+                "title": e.get("title", "无标题"),
+                "source": e.get("source", "未知来源"),
+                "published": e.get("published", ""),
+                "content": e.get("content", "")[:2000],  # 限制内容长度
+            }
+        )
+
+    entries_json = json.dumps(entries_for_llm, ensure_ascii=False, indent=2)
+
+    # 从文件加载提示词模板，如果未指定则使用默认路径
+    if prompt_path is None:
+        prompt_path = "prompts/score_batch.md"
+
+    return load_prompt(prompt_path, entries_json=entries_json)
+
+
+def _parse_llm_json_response(response: str) -> List[Dict]:
+    """解析LLM返回的JSON响应"""
+    text = response.strip()
+
+    # 尝试去除markdown代码块
+    if text.startswith("```json"):
+        text = text[7:]
+    elif text.startswith("```"):
+        text = text[3:]
+
+    if text.endswith("```"):
+        text = text[:-3]
+
+    text = text.strip()
+
+    # 尝试查找JSON数组
+    if text.startswith("[") and text.endswith("]"):
+        try:
+            return json.loads(text)
+        except json.JSONDecodeError:
+            print("⚠️ 直接解析JSON失败，尝试从文本中提取JSON数组")
+            pass
+
+    # 尝试从文本中提取JSON数组
+    match = re.search(r"\[.*\]", text, re.DOTALL)
+    if match:
+        try:
+            return json.loads(match.group())
+        except json.JSONDecodeError:
+            print("⚠️ 从文本中提取JSON数组失败:", text)
+            pass
+
+    raise ValueError(f"无法从响应中解析JSON: {response[:200]}...")
+
+
+def _parse_score_response(response: str) -> List[Dict]:
+    """解析评分LLM响应。
+
+    json_object 模式下应返回 {"items": [...]} 形式的对象;
+    兼容直接数组与 markdown 包裹作为兜底路径。
+    """
+    text = response.strip()
+
+    if text.startswith("```json"):
+        text = text[7:]
+    elif text.startswith("```"):
+        text = text[3:]
+    if text.endswith("```"):
+        text = text[:-3]
+    text = text.strip()
+
+    parsed = None
+    try:
+        parsed = json.loads(text)
+    except json.JSONDecodeError:
+        for pattern in (r"\{.*\}", r"\[.*\]"):
+            match = re.search(pattern, text, re.DOTALL)
+            if match:
+                try:
+                    parsed = json.loads(match.group())
+                    break
+                except json.JSONDecodeError:
+                    continue
+
+    if parsed is None:
+        print(f"无法从响应中解析JSON: {response}")
+        raise ValueError(f"无法从响应中解析JSON: {response[:200]}...")
+
+    if isinstance(parsed, list):
+        return parsed
+
+    if isinstance(parsed, dict):
+        for key in ("items", "results", "data", "scores"):
+            if isinstance(parsed.get(key), list):
+                return parsed[key]
+        list_values = [v for v in parsed.values() if isinstance(v, list)]
+        if len(list_values) == 1:
+            return list_values[0]
+
+    print(f"无法从响应中提取评分数组: {response}")
+    raise ValueError(f"无法从响应中提取评分数组: {response[:200]}...")
+
+
+def _split_entries_for_batch(
+    entries: List[Dict], max_prompt_chars: int = 10000
+) -> List[List[Dict]]:
+    """将entries分成多个批次，每批不超过max_prompt_chars字符"""
+    if not entries:
+        return []
+
+    batches = []
+    current_batch = []
+    current_chars = 0
+
+    # 预留prompt模板和JSON包装的空间
+    overhead = len(_build_batch_prompt([])) + 500
+
+    for entry in entries:
+        # 估算该entry在JSON中的字符数
+        entry_chars = len(
+            json.dumps(
+                {
+                    "link": entry.get("link", ""),
+                    "title": entry.get("title", "")[:100],
+                    "source": entry.get("source", ""),
+                    "published": entry.get("published", ""),
+                    "content": entry.get("content", "")[:2000],
+                },
+                ensure_ascii=False,
+            )
+        )
+
+        # 如果当前批次加上这个entry会超出限制，且当前批次不为空，则创建新批次
+        if current_chars + entry_chars + overhead > max_prompt_chars and current_batch:
+            batches.append(current_batch)
+            current_batch = [entry]
+            current_chars = entry_chars
+        else:
+            current_batch.append(entry)
+            current_chars += entry_chars
+
+    # 添加最后一个批次
+    if current_batch:
+        batches.append(current_batch)
+
+    return batches
+
+
+def _reconcile_batch_results(
+    entries: List[Dict], results: List[Dict], batch_index: int
+) -> Tuple[List[Dict], List[str]]:
+    """对单批评分结果按 link 过滤，保留可回收结果"""
+    entry_links = {entry.get("link") for entry in entries if entry.get("link")}
+    matched_results = []
+    result_links = set()
+
+    for item in results:
+        if not isinstance(item, dict):
+            continue
+
+        link = item.get("link")
+        if link:
+            result_links.add(link)
+            if link in entry_links:
+                matched_results.append(item)
+
+    errors = []
+    if len(results) != len(entries) or len(matched_results) != len(entries):
+        missing_links = sorted(entry_links - result_links)
+        error_message = (
+            "批次{batch} 评分结果异常: 输入{input_count}, 返回{output_count}, "
+            "匹配{matched_count}, 未评分链接({missing_count}): {missing}"
+        ).format(
+            batch=batch_index + 1,
+            input_count=len(entries),
+            output_count=len(results),
+            matched_count=len(matched_results),
+            missing_count=len(missing_links),
+            missing=missing_links,
+        )
+        print(f"⚠️ {error_message}")
+        errors.append(error_message)
+
+    return matched_results, errors
+
+
+async def _score_single_batch(
+    entries: List[Dict], config: Dict, batch_index: int = 0
+) -> Tuple[List[Dict], List[str]]:
+    """对单批entries进行评分"""
+    # 从config获取批量评分提示词路径
+    prompt_path = config.get("prompts", {}).get("score_batch", "prompts/score_batch.md")
+    prompt = _build_batch_prompt(entries, prompt_path)
+
+    try:
+        response = await call_llm(
+            prompt, config, response_format={"type": "json_object"}
+        )
+        results = _parse_score_response(response)
+
+        if not isinstance(results, list):
+            raise ValueError(f"LLM返回的不是数组: {type(results)}")
+
+        return _reconcile_batch_results(entries, results, batch_index)
+
+    except Exception as e:
+        error_message = f"批次{batch_index + 1} 评分失败: {e}"
+        print(f"⚠️ {error_message}")
+        return [], [error_message]
+
+
+async def score_batch(
+    entries: List[Dict], config: Dict
+) -> Tuple[List[Dict], List[str]]:
+    """
+    批量评分 - 智能分批处理
+
+    根据数据量自动决定分批策略：
+    - 小批量：一次性发送
+    - 大批量：分成多个批次并行处理
+    """
+    if not entries:
+        return [], []
+
+    # 获取分批配置
+    max_prompt_chars = config.get("max_prompt_chars", 10000)
+    max_concurrent_batches = config.get("max_concurrent_batches", 3)
+
+    # 分批
+    batches = _split_entries_for_batch(entries, max_prompt_chars)
+    print(f"📦 分成 {len(batches)} 个批次评分 (共 {len(entries)} 条)")
+
+    # 如果只有一批，直接处理
+    if len(batches) == 1:
+        scores, errors = await _score_single_batch(batches[0], config, batch_index=0)
+        return _merge_scores(entries, scores), errors
+
+    # 多批并行处理（限制并发数）
+    semaphore = asyncio.Semaphore(max_concurrent_batches)
+
+    async def score_with_limit(batch_index: int, batch: List[Dict]):
+        async with semaphore:
+            return await _score_single_batch(batch, config, batch_index=batch_index)
+
+    # 并发处理所有批次
+    batch_tasks = [
+        score_with_limit(batch_index, batch)
+        for batch_index, batch in enumerate(batches)
+    ]
+    batch_results = await asyncio.gather(*batch_tasks)
+
+    # 合并所有评分结果
+    all_scores = []
+    all_errors = []
+    for scores, errors in batch_results:
+        all_scores.extend(scores)
+        all_errors.extend(errors)
+
+    return _merge_scores(entries, all_scores), all_errors
+
+
+def _merge_scores(entries: List[Dict], scores: List[Dict]) -> List[Dict]:
+    """将评分结果合并到原始entries中"""
+    # 构建link到score的映射
+    score_map = {s.get("link"): s for s in scores if s.get("link")}
+
+    merged = []
+    for entry in entries:
+        link = entry.get("link")
+        score_data = score_map.get(link, {})
+
+        # 确保 score 为整数类型
+        score_value = score_data.get("score", entry.get("score"))
+        if isinstance(score_value, str):
+            try:
+                score_value = int(score_value)
+            except (ValueError, TypeError):
+                score_value = 0
+
+        merged.append(
+            {
+                **entry,
+                "tags": score_data.get("tags", entry.get("tags", [])),
+                "score": score_value,
+                "summary": score_data.get("summary", entry.get("summary", "")),
+            }
+        )
+
+    return merged
+
+
+async def generate_immediate_push(
+    entries: List[Dict], config: Dict, recent_push_context: str = ""
+) -> Tuple[str, Optional[str]]:
+    """生成即时推送内容
+
+    Args:
+        entries: 原始entries列表（调用方已筛选好高分条目）
+        config: LLM配置
+        recent_push_context: 近期推送上下文，用于去重
+    """
+    prompt_path = config.get("prompts", {}).get(
+        "immediate_push", "prompts/immediate_push.txt"
+    )
+
+    # 直接使用传入的entries，转为JSON格式传给prompt
+    prompt = load_prompt(
+        prompt_path,
+        count=len(entries),
+        entries=json.dumps(entries, ensure_ascii=False, indent=2),
+        recent_push_context=recent_push_context,
+    )
+
+    try:
+        return await call_llm(prompt, config), None
+    except Exception as e:
+        error_message = f"生成即时推送失败: {e}"
+        print(f"⚠️ {error_message}")
+        return "", error_message
+
+
+async def compose_digest(
+    entries: List[Dict],
+    context: List[Dict],
+    config: Dict,
+    recent_push_context: str = "",
+) -> str:
+    """生成定时汇总推送内容
+
+    Args:
+        entries: 原始entries列表
+        context: 历史碎片化信息（用于去重参考），只保留 title, published, tags, summary, source
+        config: LLM配置
+        recent_push_context: 近期汇总推送上下文，用于去重
+    """
+    prompt_path = config.get("prompts", {}).get("digest", "prompts/digest.md")
+
+    # context 只保留必要字段，拼接成字符串
+    context_text = []
+    for c in context:
+        tags_str = ", ".join(c.get("tags", [])) if c.get("tags") else ""
+        context_text.append(
+            f"[score: {c.get('score', 0)}] title:{c.get('title', '')}\n"
+            f"published: {c.get('published', '')}\n"
+            f"tags: {tags_str}\n"
+            f"source: {c.get('source', '')}\n"
+            f"summary: {c.get('summary', '')}"
+        )
+
+    prompt = load_prompt(
+        prompt_path,
+        count=len(entries),
+        entries=json.dumps(entries, ensure_ascii=False, indent=2),
+        context="\n\n".join(context_text),
+        recent_push_context=recent_push_context,
+        date=datetime.now().strftime("%Y-%m-%d"),
+    )
+
+    try:
+        return await call_llm(prompt, config)
+    except Exception:
+        raise
+
+
+async def summarize_github_trending(
+    enriched_repos: List[Dict], config: Dict
+) -> Tuple[str, Optional[str]]:
+    """GH 板块总结:从 enriched 候选中选 1-max_items + 写 markdown。不传历史上下文。"""
+    prompt_path = config.get("prompts", {}).get(
+        "section_github", "prompts/section_github.md"
+    )
+    max_items = (
+        config.get("sections", {}).get("github_trending", {}).get("max_items", 3)
+    )
+    prompt = load_prompt(
+        prompt_path,
+        repos_json=json.dumps(enriched_repos, ensure_ascii=False, indent=2),
+        max_items=max_items,
+    )
+    try:
+        return await call_llm(prompt, config), None
+    except Exception as e:
+        msg = f"summarize_github_trending 失败: {e}"
+        print(f"⚠️ {msg}")
+        return "", msg
+
+
+async def select_ai_related_hn(
+    candidates: List[Dict], k: int, config: Dict
+) -> Tuple[List[str], Optional[str]]:
+    """轻 LLM:从 HN 首页候选元数据中挑 k 个 AI 相关 id。
+
+    输入候选只含 id/title/site/points/comments 字段(不含正文)。
+    """
+    prompt_path = config.get("prompts", {}).get(
+        "section_hackernews_select", "prompts/section_hackernews_select.md"
+    )
+    slim = [
+        {
+            "id": c.get("id"),
+            "title": c.get("title", ""),
+            "site": c.get("site", ""),
+            "points": c.get("points", 0),
+            "comments": c.get("comments", 0),
+        }
+        for c in candidates
+    ]
+    prompt = load_prompt(
+        prompt_path,
+        k=k,
+        candidates_json=json.dumps(slim, ensure_ascii=False, indent=2),
+    )
+    try:
+        response = await call_llm(prompt, config)
+    except Exception as e:
+        msg = f"select_ai_related_hn 失败: {e}"
+        print(f"⚠️ {msg}")
+        return [], msg
+
+    try:
+        ids = _parse_llm_json_response(response)
+    except ValueError as e:
+        msg = f"select_ai_related_hn 解析失败: {e}"
+        print(f"⚠️ {msg}")
+        return [], msg
+
+    if not isinstance(ids, list):
+        return [], "select_ai_related_hn 返回非数组"
+    return [str(x) for x in ids][:k], None
+
+
+async def summarize_hackernews(
+    enriched_stories: List[Dict], config: Dict
+) -> Tuple[str, Optional[str]]:
+    """对输入的 K 个 enriched stories 行文(K 由 select_k 决定)。不传历史上下文。"""
+    prompt_path = config.get("prompts", {}).get(
+        "section_hackernews", "prompts/section_hackernews.md"
+    )
+    prompt = load_prompt(
+        prompt_path,
+        stories_json=json.dumps(enriched_stories, ensure_ascii=False, indent=2),
+    )
+    try:
+        return await call_llm(prompt, config), None
+    except Exception as e:
+        msg = f"summarize_hackernews 失败: {e}"
+        print(f"⚠️ {msg}")
+        return "", msg
+
+
+async def generate_trend_insights(
+    sections: Dict[str, str], config: Dict
+) -> Tuple[str, Optional[str]]:
+    """输入三段成品,返回洞察段 markdown(含 frontmatter)。"""
+    prompt_path = config.get("prompts", {}).get("insights", "prompts/insights.md")
+    prompt = load_prompt(
+        prompt_path,
+        rss=sections.get("rss", ""),
+        github=sections.get("github", ""),
+        hackernews=sections.get("hackernews", ""),
+    )
+    try:
+        return await call_llm(prompt, config), None
+    except Exception as e:
+        msg = f"generate_trend_insights 失败: {e}"
+        print(f"⚠️ {msg}")
+        return "", msg
+
+
+def parse_insights_with_metadata(llm_output: str, date: str) -> Tuple[str, Dict]:
+    """解析 insights LLM 输出,返回 (insights_md, metadata)。
+
+    metadata 字段:title / excerpt / seotitle / seodescription / lead / highlights /
+    profile / date。缺失字段补默认值。
+    """
+    meta, body = parse_frontmatter(llm_output)
+    insights_md = body if meta else llm_output
+
+    metadata = {
+        "title": meta.get("title") or f"📰 AI Daily 每日精选 | {date}",
+        "excerpt": meta.get("excerpt", ""),
+        "seotitle": meta.get("seotitle", ""),
+        "seodescription": meta.get("seodescription", ""),
+        "lead": meta.get("lead", ""),
+        "highlights": normalize_str_list(meta.get("highlights")),
+        "profile": "morning",
+        "date": date,
+    }
+    return insights_md, metadata
+
+
+def parse_digest_with_metadata(llm_output: str, date: str) -> Tuple[str, Dict]:
+    """解析 digest LLM 输出,返回 (digest_md, metadata)。
+
+    metadata 字段:title / lead / highlights / profile / date。
+    无 frontmatter 时回退到 "🌙 AI Daily 晚报 | {date}" 标题。
+    """
+    meta, body = parse_frontmatter(llm_output)
+    digest_md = body if meta else llm_output
+
+    metadata = {
+        "title": meta.get("title") or f"🌙 AI Daily 晚报 | {date}",
+        "lead": meta.get("lead", ""),
+        "highlights": normalize_str_list(meta.get("highlights")),
+        "profile": "default",
+        "date": date,
+    }
+    return digest_md, metadata
+
+
+def parse_immediate_push_with_metadata(
+    llm_output: str, default_title: str
+) -> Tuple[str, Dict]:
+    """解析即时推送 LLM 输出,返回 (body, metadata)。
+
+    metadata 仅含 title / profile。无 frontmatter 时降级到旧式 `# ` 标题提取,
+    再降级到 default_title。
+    """
+    meta, body = parse_frontmatter(llm_output)
+
+    if meta and meta.get("title"):
+        return body, {"title": meta["title"], "profile": "hotspot"}
+
+    # 兼容旧格式:从正文一级标题提取
+    match = re.search(r"^\s*#\s+(.+?)\s*\n(.*)$", llm_output, re.DOTALL | re.MULTILINE)
+    if match:
+        return match.group(2).rstrip(), {
+            "title": match.group(1).strip(),
+            "profile": "hotspot",
+        }
+
+    return llm_output, {"title": default_title, "profile": "hotspot"}

ai-daily-main/src/main.py

@@ -0,0 +1,714 @@
+"""AI每日资讯推送系统 - 主程序"""
+
+import argparse
+import asyncio
+import os
+import sys
+from datetime import date, datetime, timedelta, timezone
+from typing import Dict, List, Optional
+
+# 加载 .env 文件
+from dotenv import load_dotenv
+
+load_dotenv()
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from croniter import croniter
+
+from src.config import get_timezone, load_config, merge_sources
+from src.fetcher import fetch_all_feeds
+from src.llm import (
+    check_llm_available,
+    generate_immediate_push,
+    parse_immediate_push_with_metadata,
+    score_batch,
+)
+from src.processor import html_to_markdown
+from src.push import send_to_platforms
+from src.sections.github.section import run_github_section
+from src.sections.hackernews.section import run_hackernews_section
+from src.sections.insights.section import run_insights_section
+from src.sections.rss.section import run_rss_section
+from src.storage import (
+    append_entries,
+    assemble_with_sentinels,
+    cleanup_old_files,
+    get_fetch_file,
+    get_notify_file,
+    get_push_file,
+    load_existing_links,
+    load_recent_notify_content,
+    load_recent_push_content,
+    read_entries,
+    save_notify_file,
+    save_push_file,
+)
+
+
+async def notify_llm_errors(stage: str, errors: List[str], config: Dict):
+    """发送简单的 LLM 异常通知"""
+    if not errors:
+        return
+
+    lines = [
+        "## LLM异常",
+        "",
+        f"stage: {stage}",
+        f"time: {now_local(config).strftime('%Y-%m-%d %H:%M:%S')}",
+        "",
+    ]
+    lines.extend(f"- {error}" for error in errors)
+
+    try:
+        await send_to_platforms("\n".join(lines), config["push"])
+    except Exception as e:
+        print(f"⚠️ LLM异常通知发送失败: {e}")
+
+
+def now_local(config: Dict = None) -> datetime:
+    """获取配置时区的当前时间"""
+    return datetime.now(get_timezone(config))
+
+
+def parse_time_to_local(time_str: str, config: Dict = None) -> Optional[datetime]:
+    """解析时间字符串为配置时区的datetime"""
+    try:
+        dt = datetime.fromisoformat(time_str.replace("Z", "+00:00"))
+        return dt.astimezone(get_timezone(config))
+    except (ValueError, TypeError):
+        return None
+
+
+def calculate_push_times(
+    cron_list: List[str], offset_days: int = 0, config: Dict = None
+) -> List[datetime]:
+    base_date = datetime.now(get_timezone(config)).date() + timedelta(days=offset_days)
+    times = []
+    for cron in cron_list:
+        try:
+            minute, hour, _, _, _ = cron.split()
+            t = datetime.combine(
+                base_date,
+                datetime.strptime(f"{hour}:{minute}", "%H:%M").time(),
+                tzinfo=get_timezone(config),
+            )
+            times.append(t)
+        except ValueError:
+            continue
+    return sorted(times)
+
+
+def is_morning_push(now: datetime, config: Dict) -> bool:
+    """判定当前 push 是否为「早报」(触发 GH/HN/insights 三段)。
+
+    规则:在 `schedule.push_cron` 列表里,now 离哪条 cron 最近,就归为那条;
+    最近的那条若是当天最早的 cron,则视为早报。
+
+    特例:
+    - `push_cron` 为空 → 不视为早报
+    - `push_cron` 只有一条 → 该条即"最早"也即"最近",任何触发都视为早报
+    """
+    cron_list = config.get("schedule", {}).get("push_cron", [])
+    if not cron_list:
+        return False
+    if len(cron_list) == 1:
+        return True
+
+    base = now.replace(hour=0, minute=0, second=0, microsecond=0)
+    today_fires = [croniter(c, base).get_next(datetime) for c in cron_list]
+    closest = min(today_fires, key=lambda f: abs(now - f))
+    return closest == min(today_fires)
+
+
+def collect_entries_for_push(
+    last_push_time: Optional[datetime],
+    context_days: int = 2,
+    min_score: int = 60,
+    data_dir: str = "news-data",
+) -> tuple[List[Dict], List[Dict]]:
+    """
+    收集推送所需的条目，返回 (待推送条目, 上下文条目)
+
+    逻辑：
+    1. 获取 context_days 天内的所有条目
+    2. 按 min_score 过滤
+    3. push_time = max(last_push_time, now - 24h)
+    4. 晚于 push_time 的 → 待推送条目
+    5. 早于 push_time 的 → 上下文条目（用于LLM去重参考）
+    """
+    tz = get_timezone()
+    now = datetime.now(tz)
+
+    # 获取 context_days 天的所有条目
+    all_entries = []
+    today = now.date()
+    for i in range(context_days):
+        d = today - timedelta(days=i)
+        fetch_file = get_fetch_file(d, data_dir)
+        for entry in read_entries(fetch_file):
+            all_entries.append(entry)
+
+    print(
+        f"📋 收集总条目: {len(all_entries)} 条 , context_days: {context_days}, min_score:{min_score}"
+    )
+
+    # 按 min_score 过滤
+    qualified_entries = [e for e in all_entries if (e.get("score") or 0) >= min_score]
+    print(f"📋 过滤后条目: {len(qualified_entries)} 条 ")
+
+    # 计算推送时间边界：max(last_push_time, now - 24h)
+    past_24h = now - timedelta(hours=24)
+    push_cutoff = (
+        last_push_time if last_push_time and last_push_time > past_24h else past_24h
+    )
+
+    print(f"推送时间边界: {push_cutoff.strftime('%Y-%m-%d %H:%M:%S')}")
+
+    # 分割条目
+    to_push = []
+    context = []
+    # context 只供 LLM 做去重/历史参考,不需要 content/link/fetched_at 等大字段
+    CONTEXT_FIELDS = ("title", "source", "score", "summary", "tags", "published")
+
+    for entry in qualified_entries:
+        entry_time = parse_time_to_local(entry.get("fetched_at", ""))
+        if entry_time and entry_time > push_cutoff:
+            to_push.append(entry)
+        else:
+            context.append({k: entry.get(k) for k in CONTEXT_FIELDS})
+
+    # 上下文按分数排序，取前50
+    context = sorted(context, key=lambda x: x.get("score", 0), reverse=True)[:50]
+
+    return to_push, context
+
+
+async def run_fetch_job(config: Dict):
+    print(f"\n{'=' * 50}")
+    print(f"🔄 Fetch Job | {now_local().strftime('%Y-%m-%d %H:%M:%S')}")
+    print(f"{'=' * 50}")
+
+    interval = config["schedule"]["fetch_interval_minutes"]
+    lookback = config["schedule"].get("fetch_lookback_minutes", 120)
+    lookback = max(lookback, interval)
+    threshold = lookback + interval
+    cutoff = datetime.now(timezone.utc) - timedelta(minutes=lookback)
+
+    sources = merge_sources(config["sources"])
+    print(f"📂 共 {len(sources)} 个订阅源")
+
+    if not sources:
+        print("⚠️ 没有可用的订阅源")
+        return
+
+    max_workers = config.get("fetch", {}).get("max_workers", 20)
+    timeout = config.get("fetch", {}).get("timeout", 30)
+    entries = await fetch_all_feeds(
+        sources, cutoff, max_workers=max_workers, timeout=timeout
+    )
+    print(f"📥 抓取到 {len(entries)} 条原始消息")
+
+    if not entries:
+        return
+
+    for entry in entries:
+        entry["content"] = html_to_markdown(
+            entry.get("content", ""), entry.get("link", "")
+        )
+
+    fetch_file = get_fetch_file()
+    existing_links = load_existing_links(fetch_file, threshold)
+    new_entries = [
+        e for e in entries if e.get("link") and e["link"] not in existing_links
+    ]
+    print(f"🆕 新消息 {len(new_entries)} 条 | 链接数：{len(existing_links)}")
+
+    if not new_entries:
+        return
+
+    print("🤖 LLM评分中...")
+    # 预处理：将所有 datetime 转换为字符串，避免 JSON 序列化错误
+    for entry in new_entries:
+        if isinstance(entry.get("published"), datetime):
+            entry["published"] = (
+                entry["published"].astimezone(get_timezone(config)).isoformat()
+            )
+
+    scored, score_errors = await score_batch(new_entries, config["llm"])
+    if score_errors:
+        print(f"⚠️ [score_batch] {len(score_errors)} 个错误: {score_errors[0]}")
+        await notify_llm_errors("score_batch", score_errors, config)
+
+    is_new_file = not os.path.exists(fetch_file)
+    if is_new_file:
+        cleanup_old_files(days=config["filter"]["keep_days"])
+
+    # 添加 fetched_at 时间戳
+    for entry in scored:
+        entry["fetched_at"] = now_local().isoformat()
+        if isinstance(entry.get("published"), datetime):
+            entry["published"] = (
+                entry["published"].astimezone(get_timezone(config)).isoformat()
+            )
+
+    # 批量保存到 JSON 文件
+    from datetime import date
+
+    meta = {"date": date.today().isoformat()}
+    append_entries(fetch_file, scored, meta)
+
+    print(f"💾 已保存到 {fetch_file}")
+
+    hot_threshold = config["filter"]["hot_threshold"]
+    no_content_marker = config["filter"].get("no_content_marker", "[NO_NEW_CONTENT]")
+    hot_entries = [e for e in scored if (e.get("score") or 0) >= hot_threshold]
+    if hot_entries:
+        print(f"🔥 发现 {len(hot_entries)} 条热点消息，即时推送...")
+
+        # 加载近期已推送内容（仅供 LLM 查重，避免风格趋同）
+        context_days = config["filter"]["context_days"]
+        recent_notify = load_recent_notify_content(context_days)
+        recent_push = load_recent_push_content(context_days)
+        recent_context = (
+            f"=== 近期即时推送 ===\n{recent_notify}\n\n"
+            f"=== 近期汇总推送 ===\n{recent_push}"
+        )
+
+        push_content, immediate_push_error = await generate_immediate_push(
+            hot_entries, config["llm"], recent_push_context=recent_context
+        )
+
+        if immediate_push_error:
+            print(f"⚠️ [generate_immediate_push] {immediate_push_error}")
+            await notify_llm_errors(
+                "generate_immediate_push", [immediate_push_error], config
+            )
+
+        if not push_content:
+            print("⚠️ 即时推送内容生成失败，跳过本次热点推送")
+            print(
+                f"✅ Fetch Job 完成 | 新消息: {len(scored)} 条 | 热点: {len(hot_entries)} 条"
+            )
+            return
+
+        # 检查是否有实际内容需要推送
+        if no_content_marker in push_content:
+            print(f"ℹ️ 无新内容需要推送 (LLM判定为重复内容)")
+        else:
+            # 提取标题并构建 metadata
+            now = now_local(config)
+            timestamp = now.strftime("%Y-%m-%d %H:%M")
+            content_without_title, metadata = parse_immediate_push_with_metadata(
+                push_content, f"🚨 AI Daily 快讯 | {timestamp}"
+            )
+            metadata["pushTime"] = now.isoformat()
+
+            await send_to_platforms(
+                content_without_title,
+                config["push"],
+                "🚨 AI Daily 快讯 | " + metadata["title"],
+                metadata=metadata,
+            )
+            # 保存即时推送内容到notify文件
+            notify_file = get_notify_file()
+            save_notify_file(notify_file, content_without_title, metadata)
+            print(f"💾 已保存即时推送到 {notify_file}")
+
+    print(f"✅ Fetch Job 完成 | 新消息: {len(scored)} 条 | 热点: {len(hot_entries)} 条")
+
+
+async def run_push_job(config: Dict):
+    print(f"\n{'=' * 50}")
+    print(f"📤 Push Job | {now_local().strftime('%Y-%m-%d %H:%M:%S')}")
+    print(f"{'=' * 50}")
+
+    if is_morning_push(now_local(config), config):
+        await _run_morning_push(config)
+    else:
+        await _run_default_push(config)
+
+
+async def _run_default_push(config: Dict):
+    """晚报或非早报时段:沿用原有纯 RSS digest 流程,委托给 run_rss_section"""
+    now = now_local(config)
+    rss_md, metadata, rss_err = await run_rss_section(config, now)
+
+    if rss_err and not rss_md:
+        print(f"⚠️ [compose_digest] {rss_err}")
+        await notify_llm_errors("compose_digest", [rss_err], config)
+        raise RuntimeError(f"RSS section failed: {rss_err}")
+
+    if not rss_md:
+        # run_rss_section 在无新消息时已打印 "ℹ️ RSS: 无新消息"
+        return
+
+    # metadata 缺失兜底:parse_digest_with_metadata 失败 / LLM 输出无 frontmatter 时可能返回 None
+    if not metadata:
+        date_str = now.strftime("%Y-%m-%d")
+        metadata = {
+            "title": f"🌙 AI Daily 晚报 | {date_str}",
+            "lead": "",
+            "highlights": [],
+            "profile": "default",
+            "date": date_str,
+        }
+    metadata.setdefault("pushTime", now.isoformat())
+
+    await send_to_platforms(
+        rss_md,
+        config["push"],
+        title="📰 AI Daily 每日精选 | " + metadata["title"],
+        metadata=metadata,
+    )
+    push_file = get_push_file()
+    rss_count = rss_md.count("###")
+    save_push_file(
+        push_file,
+        rss_md,
+        rss_count,
+        rss_count,
+        profile="default",
+        metadata=metadata,
+    )
+    print(f"💾 已保存到 {push_file}")
+    print(f"✅ Push Job 完成 | 推送条目: {rss_count}")
+
+
+async def _run_morning_push(config: Dict):
+    """早报四模块编排:RSS/GH/HN 并发 → insights 串行 → sentinel 拼装 → 推送 → 落盘。
+
+    失败语义:
+    - RSS 失败 → 整体抛 RuntimeError (核心承诺不变)
+    - GH/HN/insights 失败 → 该段省略 + 告警,其他段照推
+    """
+    now = now_local(config)
+
+    rss_result, gh_result, hn_result = await asyncio.gather(
+        run_rss_section(config, now),
+        run_github_section(config, now),
+        run_hackernews_section(config, now),
+    )
+
+    # 早报场景:digest 的 metadata 通常会被 insights 段覆盖,
+    # 但保留以便在 insights 失败时作为兜底来源(title 关键词 / lead / highlights)
+    rss_md, digest_meta, rss_err = rss_result
+    gh_md, gh_err = gh_result
+    hn_md, hn_err = hn_result
+
+    if gh_err:
+        print(f"⚠️ [section_github] {gh_err}")
+        await notify_llm_errors("section_github", [gh_err], config)
+    if hn_err:
+        print(f"⚠️ [section_hackernews] {hn_err}")
+        await notify_llm_errors("section_hackernews", [hn_err], config)
+
+    if rss_err and not rss_md:
+        print(f"⚠️ [compose_digest] {rss_err}")
+        await notify_llm_errors("compose_digest", [rss_err], config)
+        raise RuntimeError(f"RSS section failed: {rss_err}")
+
+    insights_md, metadata, insights_err = await run_insights_section(
+        rss_md, gh_md, hn_md, config, now
+    )
+    if insights_err:
+        print(f"⚠️ [insights] {insights_err}")
+        await notify_llm_errors("insights", [insights_err], config)
+
+    # 如果 insights 失败,优先用 digest metadata 兜底;两者都缺再走默认
+    if not metadata:
+        date_str = now.strftime("%Y-%m-%d")
+        fallback = digest_meta or {}
+        digest_title = fallback.get("title", "")
+
+        title = digest_title if digest_title else f"📰 AI Daily 每日精选 | {date_str}"
+        metadata = {
+            "date": date_str,
+            "pushTime": now.isoformat(),
+            "title": title,
+            "excerpt": "",
+            "seotitle": "",
+            "seodescription": "",
+            "lead": fallback.get("lead", ""),
+            "highlights": fallback.get("highlights", []),
+            "profile": "morning",
+        }
+    else:
+        metadata.setdefault("pushTime", now.isoformat())
+
+    final = assemble_with_sentinels(
+        {
+            "rss": rss_md,
+            "github": gh_md,
+            "hackernews": hn_md,
+            "insights": insights_md,
+        }
+    )
+
+    if not final.strip():
+        print("ℹ️ 早报无任何段输出,跳过推送")
+        return
+
+    await send_to_platforms(
+        final,
+        config["push"],
+        title="📰 AI Daily 每日精选 | " + metadata["title"],
+        metadata=metadata,
+    )
+    push_file = get_push_file()
+    rss_count = rss_md.count("###") if rss_md else 0
+    save_push_file(
+        push_file, final, rss_count, rss_count, profile="morning", metadata=metadata
+    )
+    print(f"💾 已保存早报到 {push_file}")
+
+
+async def fetch_loop(config: Dict):
+    """Fetch循环 - 修复时间漂移并支持优雅退出"""
+    import time
+
+    interval_seconds = config["schedule"]["fetch_interval_minutes"] * 60
+    print(f"🔄 Fetch循环已启动 | 严格间隔: {interval_seconds / 60}分钟")
+
+    while True:
+        start_time = time.monotonic()  # 使用 monotonic 避免系统时间修改影响
+
+        try:
+            await run_fetch_job(config)
+        except asyncio.CancelledError:
+            print("⚠️ Fetch循环被外部取消，正在安全退出...")
+            break  # 允许外部取消任务
+        except Exception as e:
+            print(f"❌ Fetch Job 失败: {e}")
+
+        # 计算任务耗时
+        elapsed = time.monotonic() - start_time
+        # 计算还需要睡多久（如果任务耗时超过间隔，则不睡，立刻进入下一次）
+        sleep_time = max(0.0, interval_seconds - elapsed)
+
+        if sleep_time > 0:
+            print(f"⏰ 下次抓取: {sleep_time / 60:.1f}分钟后")
+
+        try:
+            await asyncio.sleep(sleep_time)
+        except asyncio.CancelledError:
+            print("⚠️ 睡眠被中断，Fetch循环安全退出...")
+            break
+
+
+async def push_loop(config: Dict):
+    """Push循环 - 无状态 croniter + 原生异步睡眠"""
+    cron_list = config["schedule"]["push_cron"]
+    tz = get_timezone(config)
+
+    # 1. 启动前预校验 cron 表达式，过滤掉无效配置
+    valid_crons = []
+    for cron in cron_list:
+        if croniter.is_valid(cron):
+            valid_crons.append(cron)
+        else:
+            print(f"⚠️ 忽略无效的 cron 表达式: '{cron}'")
+
+    if not valid_crons:
+        print("❌ 没有有效的推送时间配置，Push循环退出")
+        return
+
+    print(f"📤 Push循环已启动 | 定时: {', '.join(valid_crons)} | 时区: {tz}")
+
+    # 2. 主循环
+    while True:
+        try:
+            now = datetime.now(tz)
+
+            # 💡 核心优化：无状态计算。
+            # 每次都基于此刻的真实时间，动态计算所有有效 cron 的下一次时间，取最近的一个。
+            # 这样无论 run_push_job 执行多久，或者系统休眠过，永远都不会算错。
+            next_push = min(
+                croniter(cron, now).get_next(datetime) for cron in valid_crons
+            )
+
+            wait_seconds = (next_push - datetime.now(tz)).total_seconds()
+
+            if wait_seconds > 0:
+                print(
+                    f"⏰ 下次推送: {next_push.strftime('%Y-%m-%d %H:%M:%S')} (等待 {wait_seconds / 60:.1f} 分钟)"
+                )
+
+                # 💡 核心优化：直接 Sleep。asyncio 天生支持被 CancelledError 瞬间打断
+                await asyncio.sleep(wait_seconds)
+
+            # 到达推送时间，执行推送
+            print(f"📤 执行推送: {datetime.now(tz).strftime('%Y-%m-%d %H:%M:%S')}")
+            await run_push_job(config)
+
+            # 增加 1 秒缓冲：防止 run_push_job 执行过快（不到 1 秒），
+            # 导致下一个循环的 now 仍停留在当前秒，croniter 算出重复的时间点。
+            await asyncio.sleep(1)
+
+        except asyncio.CancelledError:
+            print("⚠️ Push循环收到取消信号，安全退出...")
+            break  # 直接 break 退出循环即可
+        except Exception as e:
+            print(f"❌ Push 循环异常: {e}")
+            # 遇到未知异常时休眠 60 秒，防止死循环疯狂报错打满日志
+            await asyncio.sleep(60)
+
+
+async def cmd_check(config: Dict) -> int:
+    """校验 LLM 接口可达性（部署期使用，运行期不再校验）"""
+    print("🔍 校验 LLM 接口...")
+    try:
+        await check_llm_available(config["llm"])
+    except Exception as e:
+        print(f"❌ LLM 接口不可用: {e}")
+        return 1
+    print("✅ LLM 接口可用")
+    return 0
+
+
+async def cmd_fetch(config: Dict) -> int:
+    """单次抓取（systemd timer 调用）"""
+    try:
+        await run_fetch_job(config)
+        return 0
+    except Exception as e:
+        print(f"❌ Fetch 任务失败: {e}")
+        return 1
+
+
+async def cmd_push(config: Dict) -> int:
+    """单次推送（systemd timer 调用）"""
+    try:
+        await run_push_job(config)
+        return 0
+    except Exception as e:
+        print(f"❌ Push 任务失败: {e}")
+        return 1
+
+
+async def cmd_loop(config: Dict) -> int:
+    """长跑模式（本地开发/调试用）"""
+    print("🔍 检查 LLM 接口可用性...")
+    try:
+        await check_llm_available(config["llm"])
+        print("✅ LLM 接口可用")
+    except Exception as e:
+        print(f"❌ LLM 接口不可用: {e}")
+        return 1
+    await asyncio.gather(fetch_loop(config), push_loop(config))
+    return 0
+
+
+async def cmd_rss(config: Dict) -> int:
+    """单独跑一次 RSS digest 板块,打印结果不推送"""
+    print("📰 RSS Digest 单板块运行")
+    try:
+        md, meta, err = await run_rss_section(config, now=now_local(config))
+    except Exception as e:
+        print(f"❌ RSS 板块失败: {e}")
+        return 1
+    if err:
+        print(f"❌ {err}")
+        return 1
+    if not md:
+        print("ℹ️ 本次无内容")
+        return 0
+    print("\n" + "=" * 60)
+    print("📑 metadata:")
+    if meta:
+        import json as _json
+
+        print(_json.dumps(meta, ensure_ascii=False, indent=2))
+    else:
+        print("(none)")
+    print("=" * 60)
+    print(md)
+    print("=" * 60)
+    return 0
+
+
+async def cmd_github(config: Dict) -> int:
+    """单独跑一次 GitHub trending 板块,打印结果不推送"""
+    print("⭐ GitHub Trending 单板块运行")
+    try:
+        md, err = await run_github_section(config, now=now_local(config))
+    except Exception as e:
+        print(f"❌ GitHub 板块失败: {e}")
+        return 1
+    if err:
+        print(f"❌ {err}")
+        return 1
+    if not md:
+        print("ℹ️ 本次无内容")
+        return 0
+    print("\n" + "=" * 60)
+    print(md)
+    print("=" * 60)
+    return 0
+
+
+async def cmd_hackernews(config: Dict) -> int:
+    """单独跑一次 Hacker News 板块,打印结果不推送"""
+    print("🟧 Hacker News 单板块运行")
+    try:
+        md, err = await run_hackernews_section(config, now=now_local(config))
+    except Exception as e:
+        print(f"❌ Hacker News 板块失败: {e}")
+        return 1
+    if err:
+        print(f"❌ {err}")
+        return 1
+    if not md:
+        print("ℹ️ 本次无内容")
+        return 0
+    print("\n" + "=" * 60)
+    print(md)
+    print("=" * 60)
+    return 0
+
+
+def _parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        prog="daily-news",
+        description="AI 每日资讯推送系统",
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+    sub.add_parser("check", help="校验 LLM 接口可达性")
+    sub.add_parser("fetch", help="单次抓取并退出")
+    sub.add_parser("push", help="单次推送并退出")
+    sub.add_parser("loop", help="长跑模式（开发/调试用）")
+    sub.add_parser("rss", help="单独跑一次 RSS Digest 板块（仅打印,不推送）")
+    sub.add_parser("github", help="单独跑一次 GitHub Trending 板块（仅打印,不推送）")
+    sub.add_parser("hackernews", help="单独跑一次 Hacker News 板块（仅打印,不推送）")
+    return parser.parse_args()
+
+
+def main() -> int:
+    print("🚀 AI每日资讯推送系统")
+    args = _parse_args()
+
+    try:
+        config = load_config()
+        print("✅ 配置加载成功")
+    except Exception as e:
+        print(f"❌ 加载配置失败: {e}")
+        return 1
+
+    handlers = {
+        "check": cmd_check,
+        "fetch": cmd_fetch,
+        "push": cmd_push,
+        "loop": cmd_loop,
+        "rss": cmd_rss,
+        "github": cmd_github,
+        "hackernews": cmd_hackernews,
+    }
+    return asyncio.run(handlers[args.command](config))
+
+
+if __name__ == "__main__":
+    try:
+        sys.exit(main())
+    except KeyboardInterrupt:
+        print("\n👋 程序已退出")
+        sys.exit(0)

ai-daily-main/src/markdown_utils.py

@@ -0,0 +1,83 @@
+"""Markdown / YAML frontmatter 公共辅助。
+
+供 storage、llm 等模块复用,避免重复实现。本模块零业务依赖,仅依赖 yaml/json/re。
+"""
+
+import json
+import re
+from typing import Any, Dict, List, Tuple
+
+import yaml
+
+__all__ = [
+    "yaml_value",
+    "dump_frontmatter",
+    "parse_frontmatter",
+    "normalize_str_list",
+]
+
+
+def yaml_value(v: Any) -> str:
+    """把单个值序列化为 YAML 合法的 token,借道 JSON 语法。
+
+    依据:JSON 是 YAML 1.2 的真子集,任何 json.dumps 的输出都是合法 YAML 标量/序列/映射。
+    始终带引号的字符串可以避免 PyYAML 的若干怪癖(折行、未引号字符串歧义、unicode 转义)。
+    """
+    if isinstance(v, (dict, list, str)):
+        return json.dumps(v, ensure_ascii=False)
+    if isinstance(v, bool):
+        return "true" if v else "false"
+    if v is None:
+        return ""
+    return str(v)
+
+
+def dump_frontmatter(meta: Dict) -> str:
+    """把扁平 metadata dict 序列化为 frontmatter 文本(不含包围的 `---`)。
+
+    保留插入顺序(title 在前,bookkeeping 字段在后)。仅支持扁平结构 —— 当前所有
+    metadata 都是扁平的,无需处理嵌套。
+    """
+    return "".join(f"{k}: {yaml_value(v)}\n" for k, v in meta.items())
+
+
+_FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n?(.*)$", re.DOTALL)
+
+
+def parse_frontmatter(text: str) -> Tuple[Dict, str]:
+    """从 markdown 文本中分离 YAML frontmatter 与正文。
+
+    返回 (metadata_dict, body)。无 frontmatter / YAML 解析失败 / 非 dict 时返回
+    ({}, 原文) —— 保证降级路径不丢内容,调用方可凭 metadata_dict 是否为空判定。
+    """
+    match = _FRONTMATTER_RE.match(text.strip())
+    if not match:
+        return {}, text
+
+    try:
+        meta = yaml.safe_load(match.group(1)) or {}
+    except yaml.YAMLError:
+        print("frontmatter 数据段解析失败")
+        return {}, text
+
+    if not isinstance(meta, dict):
+        return {}, text
+
+    return meta, match.group(2).strip()
+
+
+def normalize_str_list(value: Any) -> List[str]:
+    """将 str/list/None 规整为非空字符串列表。
+
+    用于兜底 LLM 输出的列表型字段(如 frontmatter 中的 highlights):单字符串包裹为单元素列表,
+    null/非列表/非字符串返回空列表,列表中夹杂的空白项过滤掉。不做数量截断。
+    """
+    if not value:
+        return []
+    if isinstance(value, str):
+        items = [value]
+    elif isinstance(value, list):
+        items = value
+    else:
+        return []
+    return [str(x).strip() for x in items if str(x).strip()]

ai-daily-main/src/processor.py

@@ -0,0 +1,37 @@
+"""内容处理模块 - HTML转Markdown"""
+
+import re
+from urllib.parse import urljoin
+
+from markdownify import markdownify as md
+
+
+def html_to_markdown(html: str, base_url: str = "") -> str:
+    """
+    将HTML转换为Markdown，保留链接和图片
+    使用markdownify库，并进行后处理优化
+    """
+    # markdownify会自动处理<img>为![](url)，<a>为[text](url)
+    markdown = md(html, heading_style="ATX")
+
+    # 处理相对链接
+    if base_url:
+
+        def replace_rel_link(m):
+            prefix, path, suffix = m.groups()
+            if path.startswith(("http://", "https://", "data:")):
+                return m.group(0)
+            abs_url = urljoin(base_url, path)
+            return f"{prefix}{abs_url}{suffix}"
+
+        markdown = re.sub(r"(!?\[.*?\]\()(.*?)(\))", replace_rel_link, markdown)
+
+    # 后处理优化
+    # 1. 直接匹配移除 xgo.ing 推广链接
+    markdown = markdown.replace("[⚡ Powered by xgo.ing](https://xgo.ing)", "")
+    markdown = markdown.replace("[⚡ Powered by xgo.ing](https://xgo.ing/)", "")
+
+    # 2. 清理多余空行
+    markdown = re.sub(r"\n{3,}", "\n\n", markdown)
+
+    return markdown.strip()

ai-daily-main/src/push/__init__.py

@@ -0,0 +1,42 @@
+"""推送平台模块"""
+
+from typing import Dict, Optional
+
+from .base import PushPlatform
+from .discord import DiscordPlatform
+from .feishu import FeishuPlatform
+from .custom import CustomPlatform
+
+
+def create_platform(name: str, config: Dict) -> Optional[PushPlatform]:
+    """工厂函数，创建推送平台实例"""
+    platforms = {
+        "discord": DiscordPlatform,
+        "feishu": FeishuPlatform,
+        "custom": CustomPlatform,
+    }
+
+    if name not in platforms:
+        raise ValueError(f"未知推送平台: {name}")
+
+    platform_class = platforms[name]
+    platform = platform_class(config)
+
+    if not platform.validate_config(config):
+        return None
+
+    return platform
+
+
+async def send_to_platforms(content: str, push_config: Dict, title: str = None, metadata: Optional[Dict] = None):
+    """发送内容到所有已启用且配置有效的平台"""
+    for platform_name, platform_conf in push_config.items():
+        platform = create_platform(platform_name, platform_conf)
+        if platform is None:
+            continue
+
+        try:
+            await platform.send(content, title, metadata)
+            print(f"✅ 已推送到 {platform_name}")
+        except Exception as e:
+            print(f"❌ 推送到 {platform_name} 失败: {e}")

ai-daily-main/src/push/base.py

@@ -0,0 +1,26 @@
+"""推送平台基类"""
+from abc import ABC, abstractmethod
+from typing import Dict
+
+
+class PushPlatform(ABC):
+    """推送平台抽象基类"""
+
+    def __init__(self, config: Dict):
+        self.config = config
+
+    @abstractmethod
+    def validate_config(self, config: Dict) -> bool:
+        """验证配置是否有效"""
+        pass
+
+    @abstractmethod
+    async def send(self, content: str, title: str = None, metadata: Dict = None):
+        """发送内容
+
+        Args:
+            content: 正文内容
+            title: 标题（可选，兼容旧接口）
+            metadata: 元信息（可选，新增参数）
+        """
+        pass

ai-daily-main/src/push/custom.py

@@ -0,0 +1,54 @@
+"""自定义 API 推送平台"""
+import os
+from typing import Dict, Optional
+import aiohttp
+from .base import PushPlatform
+
+
+class CustomPlatform(PushPlatform):
+    """自定义 API 推送平台"""
+
+    def validate_config(self, config: Dict) -> bool:
+        """验证配置"""
+        if not config.get("enabled", False):
+            return False
+
+        api_key_name = config.get("apiKeyName")
+        token_key_name = config.get("tokenKeyName")
+
+        if not api_key_name or not token_key_name:
+            print("❌ Custom 平台配置缺少 apiKeyName 或 tokenKeyName")
+            return False
+
+        url = os.getenv(api_key_name)
+        token = os.getenv(token_key_name)
+
+        if not url or not token:
+            print(f"❌ 环境变量 {api_key_name} 或 {token_key_name} 未设置")
+            return False
+
+        return True
+
+    async def send(self, content: str, title: str = None, metadata: Optional[Dict] = None):
+        """发送到自定义 API"""
+        api_key_name = self.config.get("apiKeyName")
+        token_key_name = self.config.get("tokenKeyName")
+
+        url = os.getenv(api_key_name)
+        token = os.getenv(token_key_name)
+
+        payload = {
+            "content": content,
+            "metadata": metadata
+        }
+
+        headers = {
+            "Authorization": f"Bearer {token}",
+            "Content-Type": "application/json"
+        }
+
+        async with aiohttp.ClientSession() as session:
+            async with session.post(url, json=payload, headers=headers) as resp:
+                if resp.status != 200:
+                    error_text = await resp.text()
+                    raise Exception(f"API 返回错误 {resp.status}: {error_text}")

ai-daily-main/src/push/discord.py

@@ -0,0 +1,60 @@
+"""Discord推送平台"""
+
+import os
+from typing import Dict
+
+import aiohttp
+
+from .base import PushPlatform
+
+
+class DiscordPlatform(PushPlatform):
+    """Discord Webhook推送"""
+
+    def __init__(self, config: Dict):
+        super().__init__(config)
+        self.api_key_name = config.get("apiKeyName", "DISCORD_WEBHOOK_URL")
+        self.webhook_url = os.environ.get(self.api_key_name, "")
+
+    def validate_config(self, config: Dict) -> bool:
+        """检查Discord配置是否有效"""
+        if not config.get("enabled", False):
+            return False
+        api_key_name = config.get("apiKeyName", "DISCORD_WEBHOOK_URL")
+        webhook = os.environ.get(api_key_name, "")
+        return bool(webhook and webhook.startswith("https://discord.com/api/webhooks/"))
+
+    async def send(self, content: str, title: str = None, metadata: Dict = None):
+        """发送到Discord（忽略 metadata）"""
+        full_content = f"# {title}\n\n{content}" if title else content
+        chunks = self._split_content(full_content, limit=2000)
+
+        async with aiohttp.ClientSession() as session:
+            for chunk in chunks:
+                payload = {"content": chunk}
+                async with session.post(self.webhook_url, json=payload) as resp:
+                    if resp.status != 204:
+                        text = await resp.text()
+                        raise RuntimeError(f"Discord推送失败: {resp.status} - {text}")
+
+    def _split_content(self, content: str, limit: int = 2000) -> list:
+        """Discord限制2000字符，需要分割"""
+        if len(content) <= limit:
+            return [content]
+
+        chunks = []
+        lines = content.split("\n")
+        current = ""
+
+        for line in lines:
+            if len(current) + len(line) + 1 > limit:
+                if current:
+                    chunks.append(current)
+                current = line
+            else:
+                current += "\n" + line if current else line
+
+        if current:
+            chunks.append(current)
+
+        return chunks

ai-daily-main/src/push/feishu.py

@@ -0,0 +1,92 @@
+"""飞书推送平台"""
+
+import os
+from typing import Dict
+
+import aiohttp
+
+from .base import PushPlatform
+
+
+class FeishuPlatform(PushPlatform):
+    """飞书 Webhook 推送"""
+
+    def __init__(self, config: Dict):
+        super().__init__(config)
+        self.api_key_name = config.get("apiKeyName", "FEISHU_WEBHOOK_URL")
+        self.webhook_url = os.environ.get(self.api_key_name, "")
+
+    def validate_config(self, config: Dict) -> bool:
+        """检查飞书配置是否有效"""
+        if not config.get("enabled", False):
+            return False
+        api_key_name = config.get("apiKeyName", "FEISHU_WEBHOOK_URL")
+        webhook = os.environ.get(api_key_name, "")
+        return bool(webhook)
+
+    async def send(self, content: str, title: str = None, metadata: Dict = None):
+        """发送到飞书（忽略 metadata）"""
+        chunks = self._split_content(content, limit=8000)
+
+        async with aiohttp.ClientSession() as session:
+            for chunk in chunks:
+                payload = self._build_payload(chunk, title)
+                async with session.post(self.webhook_url, json=payload) as resp:
+                    if resp.status != 200:
+                        text = await resp.text()
+                        raise RuntimeError(f"飞书推送失败: {resp.status} - {text}")
+                    data = await resp.json()
+                    if data.get("code") != 0:
+                        raise RuntimeError(f"飞书推送失败: {data.get('msg')}")
+
+    def _build_payload(self, content: str, title: str = None) -> Dict:
+        """
+        构建飞书卡片消息 payload，支持 Markdown，
+        参考  https://open.feishu.cn/document/feishu-cards/card-json-v2-structure
+        """
+
+        header = {}
+        if title:
+            header = {
+                "title": {"content": title, "tag": "plain_text"},
+                "template": "blue",
+            }
+
+        return {
+            "msg_type": "interactive",
+            "card": {
+                "schema": "2.0",  # 【重点1】显式声明使用 V2 版本结构
+                "header": header,
+                "body": {  # 【重点2】V2 中，所有的内容元素都必须放在 body 里面
+                    "elements": [
+                        {
+                            "tag": "markdown",
+                            "content": content,
+                            "text_align": "left",  # 可选：left / center / right
+                        },
+                    ],
+                },
+            },
+        }
+
+    def _split_content(self, content: str, limit: int = 8000) -> list:
+        """飞书卡片消息 markdown 元素限制 8000 字符"""
+        if len(content) <= limit:
+            return [content]
+
+        chunks = []
+        lines = content.split("\n")
+        current = ""
+
+        for line in lines:
+            if len(current) + len(line) + 1 > limit:
+                if current:
+                    chunks.append(current)
+                current = line
+            else:
+                current += "\n" + line if current else line
+
+        if current:
+            chunks.append(current)
+
+        return chunks

ai-daily-main/src/sections/__init__.py

@@ -0,0 +1 @@
+"""板块模块包。每个子模块导出 run_<board>_section(config, now) -> (markdown, error)"""

ai-daily-main/src/sections/github/repo_enricher.py

@@ -0,0 +1,111 @@
+"""GitHub REST API enrich:metadata + README → enriched repo dict
+
+匿名调用受 60 req/hr 限,设置 GITHUB_TOKEN 环境变量后走 5000 req/hr。
+"""
+
+import asyncio
+import base64
+import os
+from typing import Dict, List, Optional, Tuple
+
+import aiohttp
+
+API_BASE = "https://api.github.com"
+
+
+def _auth_headers(token_env: str = "GITHUB_TOKEN") -> Dict[str, str]:
+    headers = {"Accept": "application/vnd.github+json"}
+    token = os.environ.get(token_env)
+    if token:
+        headers["Authorization"] = f"Bearer {token}"
+        print(f"🔑 GH: 已配置 {token_env},鉴权调用 (5000 req/hr)")
+    else:
+        print(f"⚠️ GH: 未配置 {token_env},匿名调用 (60 req/hr)")
+    return headers
+
+
+async def _get_json(
+    session: aiohttp.ClientSession, url: str, timeout: int = 10
+) -> Optional[Dict]:
+    async with session.get(
+        url, timeout=aiohttp.ClientTimeout(total=timeout)
+    ) as resp:
+        if resp.status == 404:
+            return None
+        if resp.status != 200:
+            raise RuntimeError(f"GitHub API {resp.status} for {url}")
+        return await resp.json()
+
+
+async def enrich_repo(
+    session: aiohttp.ClientSession,
+    repo: Dict,
+    token_env: str = "GITHUB_TOKEN",
+    readme_max_chars: int = 3000,
+    timeout: int = 10,
+) -> Optional[Dict]:
+    """单 repo 双调用 enrich。返回 None 表示该 repo 应剔除(archived 或 metadata 不可达)。
+
+    任一调用失败 raise → 调用方按 return_exceptions 模式聚合错误。
+    """
+    full_name = repo["full_name"]
+    meta_url = f"{API_BASE}/repos/{full_name}"
+    readme_url = f"{API_BASE}/repos/{full_name}/readme"
+
+    meta, readme = await asyncio.gather(
+        _get_json(session, meta_url, timeout=timeout),
+        _get_json(session, readme_url, timeout=timeout),
+    )
+
+    if meta is None:
+        return None
+    if meta.get("archived"):
+        return None
+
+    license_spdx = ""
+    if isinstance(meta.get("license"), dict):
+        license_spdx = meta["license"].get("spdx_id") or ""
+
+    readme_excerpt = ""
+    if readme and readme.get("content"):
+        try:
+            raw = base64.b64decode(readme["content"]).decode("utf-8", errors="replace")
+            readme_excerpt = raw[:readme_max_chars]
+        except Exception:
+            readme_excerpt = ""
+
+    return {
+        **repo,
+        "topics": meta.get("topics") or [],
+        "license": license_spdx,
+        "pushed_at": meta.get("pushed_at") or "",
+        "readme_excerpt": readme_excerpt,
+    }
+
+
+async def enrich_repos(
+    candidates: List[Dict],
+    token_env: str = "GITHUB_TOKEN",
+    readme_max_chars: int = 3000,
+    timeout: int = 10,
+) -> Tuple[List[Dict], List[str]]:
+    """并发 enrich 多个 repo。返回 (enriched_list_with_archived_filtered, errors)"""
+    errors: List[str] = []
+    headers = _auth_headers(token_env)
+
+    async with aiohttp.ClientSession(headers=headers) as session:
+        results = await asyncio.gather(
+            *[
+                enrich_repo(session, r, token_env, readme_max_chars, timeout)
+                for r in candidates
+            ],
+            return_exceptions=True,
+        )
+
+    enriched: List[Dict] = []
+    for r, candidate in zip(results, candidates):
+        if isinstance(r, Exception):
+            errors.append(f"enrich {candidate['full_name']} 失败: {r}")
+        elif r is not None:
+            enriched.append(r)
+    return enriched, errors

ai-daily-main/src/sections/github/section.py

@@ -0,0 +1,104 @@
+"""GitHub Trending 板块入口。
+
+流程:trending 抓取 → history 过滤 → 候选写回 history → deep-dive → LLM 总结
+"""
+
+from datetime import datetime
+from typing import Dict, Optional, Tuple
+
+from src.config import get_timezone
+from src.sections.github.repo_enricher import enrich_repos
+from src.sections.github.trending_scraper import (
+    fetch_trending_page,
+    parse_trending_html,
+)
+from src.storage import load_trending_history
+
+
+async def run_github_section(
+    config: Dict, now: Optional[datetime] = None
+) -> Tuple[str, Optional[str]]:
+    cfg = config.get("sections", {}).get("github_trending", {})
+    if not cfg.get("enabled", False):
+        return "", None
+
+    # 延迟 import,Task 11 才提供 summarize_github_trending
+    from src.llm import summarize_github_trending
+
+    today = (now or datetime.now(get_timezone())).date()
+    keep_days = config["filter"]["keep_days"]
+    timeout = cfg.get("request_timeout", 10)
+    max_deep_dive = cfg.get("max_deep_dive", 10)
+    readme_max_chars = cfg.get("readme_max_chars", 3000)
+    history_path = cfg.get("history_file", "news-data/trending-history.json")
+    token_env = cfg.get("tokenName", "GITHUB_TOKEN")
+
+    # 1. 抓取
+    print("📥 GH: 抓取 trending 页...")
+    try:
+        html = await fetch_trending_page(timeout=timeout)
+    except Exception as e:
+        return "", f"GH 抓取失败: {e}"
+
+    all_repos = parse_trending_html(html)
+    print(f"📋 GH: 解析 {len(all_repos)} 个 repo")
+    if not all_repos:
+        return "", None
+
+    # 2. history 加载 + 清理
+    history = load_trending_history(history_path)
+    before_cleanup = len(history.repos)
+    history.cleanup(today=today, keep_days=keep_days)
+    after_cleanup = len(history.repos)
+    if before_cleanup != after_cleanup:
+        print(
+            f"🧹 GH: history 清理过期 {before_cleanup - after_cleanup} 条 (剩 {after_cleanup})"
+        )
+
+    # 3. 候选筛选(按 spec §4.3 语义)
+    candidates = []
+    already_seen = 0
+    for repo in all_repos:
+        if repo["url"] in history:
+            history.touch(repo["url"], today)
+            already_seen += 1
+        else:
+            candidates.append(repo)
+    print(f"🔍 GH: history 过滤掉 {already_seen} 条已见,新候选 {len(candidates)} 条")
+
+    if not candidates:
+        print("ℹ️ GH: 无新候选,跳过")
+        return "", None
+    if len(candidates) > max_deep_dive:
+        print(f"✂️ GH: 候选 {len(candidates)} 超 max_deep_dive={max_deep_dive},截断")
+        candidates = candidates[:max_deep_dive]
+
+    # 4. 候选写回 history + 持久化
+    for repo in candidates:
+        history.touch(repo["url"], today)
+    history.save()
+
+    # 5. 并发 enrich
+    print(f"🌐 GH: 并发 enrich {len(candidates)} 个 repo (REST API)...")
+    enriched, enrich_errors = await enrich_repos(
+        candidates,
+        token_env=token_env,
+        readme_max_chars=readme_max_chars,
+        timeout=timeout,
+    )
+    for e in enrich_errors:
+        print(f"⚠️ GH enrich: {e}")
+    print(
+        f"✅ GH: enrich 成功 {len(enriched)} / 失败 {len(enrich_errors)} / "
+        f"输入 {len(candidates)}"
+    )
+    if not enriched:
+        return "", None
+
+    # 6. LLM 总结
+    print(f"🤖 GH: summarize {len(enriched)} 个候选...")
+    md, err = await summarize_github_trending(enriched, config["llm"])
+    if err:
+        return "", f"summarize_github_trending: {err}"
+    print(f"✅ GH: 板块输出 {len(md or '')} chars")
+    return md or "", None

ai-daily-main/src/sections/github/trending_scraper.py

@@ -0,0 +1,98 @@
+"""GitHub Trending 单页 HTML 抓取与解析。
+
+数据源: https://github.com/trending (无 language / since 过滤)
+"""
+
+import re
+from typing import Dict, List
+
+import aiohttp
+from bs4 import BeautifulSoup
+
+TRENDING_URL = "https://github.com/trending"
+USER_AGENT = (
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+)
+_NUM_RE = re.compile(r"[\d,]+")
+
+
+def _parse_int(s: str) -> int:
+    m = _NUM_RE.search(s or "")
+    if not m:
+        return 0
+    return int(m.group(0).replace(",", ""))
+
+
+async def fetch_trending_page(timeout: int = 10) -> str:
+    """抓取 trending 页 HTML;非 200 抛 RuntimeError"""
+    async with aiohttp.ClientSession(
+        headers={"User-Agent": USER_AGENT}
+    ) as session:
+        async with session.get(
+            TRENDING_URL, timeout=aiohttp.ClientTimeout(total=timeout)
+        ) as resp:
+            if resp.status != 200:
+                text = await resp.text()
+                raise RuntimeError(
+                    f"GitHub trending 返回 {resp.status}: {text[:200]}"
+                )
+            return await resp.text()
+
+
+def parse_trending_html(html: str) -> List[Dict]:
+    """解析 trending HTML,返回去重后的 repo 字典数组。"""
+    if not html:
+        return []
+
+    soup = BeautifulSoup(html, "html.parser")
+    seen_urls = set()
+    repos: List[Dict] = []
+
+    for article in soup.select("article.Box-row"):
+        h2 = article.find("h2")
+        a = h2.find("a") if h2 else None
+        if not a or not a.get("href"):
+            continue
+
+        href = a["href"].strip()
+        full_name = href.lstrip("/")
+        url = f"https://github.com/{full_name}"
+        if url in seen_urls:
+            continue
+        seen_urls.add(url)
+
+        # description
+        desc_tag = article.find("p")
+        description = (desc_tag.get_text(strip=True) if desc_tag else "") or ""
+
+        # language
+        lang_tag = article.find("span", attrs={"itemprop": "programmingLanguage"})
+        language = (lang_tag.get_text(strip=True) if lang_tag else "") or ""
+
+        # stars_total: 第一个指向 /stargazers 的链接
+        stars_total = 0
+        star_a = article.find("a", href=re.compile(r"/stargazers$"))
+        if star_a:
+            stars_total = _parse_int(star_a.get_text(strip=True))
+
+        # stars_today: 末尾的 "N stars today" span
+        stars_today = 0
+        for span in article.find_all("span"):
+            t = span.get_text(strip=True)
+            if "stars today" in t or "stars this week" in t or "stars this month" in t:
+                stars_today = _parse_int(t)
+                break
+
+        repos.append(
+            {
+                "url": url,
+                "full_name": full_name,
+                "description": description,
+                "language": language,
+                "stars_today": stars_today,
+                "stars_total": stars_total,
+            }
+        )
+
+    return repos

ai-daily-main/src/sections/hackernews/frontpage_scraper.py

@@ -0,0 +1,91 @@
+"""HN 首页 HTML 抓取与解析。
+
+数据源: https://news.ycombinator.com/news (30 条)
+"""
+
+import re
+from typing import Dict, List
+
+import aiohttp
+from bs4 import BeautifulSoup
+
+FRONTPAGE_URL = "https://news.ycombinator.com/news"
+USER_AGENT = (
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+)
+_NUM_RE = re.compile(r"\d+")
+
+
+def _first_int(text: str) -> int:
+    m = _NUM_RE.search(text or "")
+    return int(m.group(0)) if m else 0
+
+
+async def fetch_frontpage(timeout: int = 10) -> str:
+    async with aiohttp.ClientSession(headers={"User-Agent": USER_AGENT}) as session:
+        async with session.get(
+            FRONTPAGE_URL, timeout=aiohttp.ClientTimeout(total=timeout)
+        ) as resp:
+            if resp.status != 200:
+                raise RuntimeError(f"HN frontpage 返回 {resp.status}")
+            return await resp.text()
+
+
+def parse_frontpage_html(html: str) -> List[Dict]:
+    """解析首页 HTML,返回 [{id, title, url, site, points, comments, comments_url}]
+
+    注:HN frontpage 的 athing 行的实际 class 是 'athing submission' (多类),
+    用 CSS 选择器 'tr.athing' 仍然匹配。
+    """
+    if not html:
+        return []
+    soup = BeautifulSoup(html, "html.parser")
+    stories: List[Dict] = []
+
+    for athing in soup.select("tr.athing"):
+        item_id = athing.get("id")
+        if not item_id:
+            continue
+
+        title_a = athing.select_one("span.titleline > a")
+        if not title_a:
+            continue
+        title = title_a.get_text(strip=True)
+        href = title_a.get("href", "")
+        if href.startswith("item?id="):
+            url = f"https://news.ycombinator.com/{href}"
+            site = ""
+        else:
+            url = href
+            site_tag = athing.select_one("span.sitestr")
+            site = site_tag.get_text(strip=True) if site_tag else ""
+
+        sub_tr = athing.find_next_sibling("tr")
+        points = 0
+        comments = 0
+        comments_url = f"https://news.ycombinator.com/item?id={item_id}"
+        if sub_tr:
+            score = sub_tr.select_one("span.score")
+            if score:
+                points = _first_int(score.get_text(strip=True))
+            comment_a = None
+            for a in sub_tr.find_all("a", href=re.compile(r"^item\?id=")):
+                comment_a = a
+            if comment_a:
+                comments = _first_int(comment_a.get_text(strip=True))
+                comments_url = f"https://news.ycombinator.com/{comment_a['href']}"
+
+        stories.append(
+            {
+                "id": item_id,
+                "title": title,
+                "url": url,
+                "site": site,
+                "points": points,
+                "comments": comments,
+                "comments_url": comments_url,
+            }
+        )
+
+    return stories

ai-daily-main/src/sections/hackernews/item_enricher.py

@@ -0,0 +1,237 @@
+"""HN 单 story enrich:Algolia 评论树 + 外链正文。
+
+Algolia API: GET /api/v1/items/{id}
+- root.text 是 Show HN / Ask HN 的 post 正文
+- root.children[] 是顶层评论(按 HN ranking 排序)
+- 每条 child 自己还有 children[],承载嵌套回复
+
+enrich 策略:取 L1 + 每个 L1 下前 N 条 L2 回复,合并为 tree JSON:
+    [{"l1": "顶层评论", "replies": ["回复 1", "回复 2"]}, ...]
+
+外链正文优先走 Jina Reader (https://r.jina.ai/<url>, 返回 markdown),
+失败回退到直接 GET + html_to_markdown。Jina API key 可选(配置后免费额度更高),
+环境变量名通过 `sections.hackernews.jinaTokenName` 配置(默认 JINA_API_KEY)。
+"""
+
+import asyncio
+import os
+from typing import Dict, List, Optional, Tuple
+
+import aiohttp
+
+from src.processor import html_to_markdown
+
+USER_AGENT = (
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+)
+
+JINA_READER_BASE = "https://r.jina.ai"
+
+
+def _is_internal_hn_url(url: str) -> bool:
+    return url.startswith("https://news.ycombinator.com/item?id=")
+
+
+async def _fetch_algolia_item(
+    session: aiohttp.ClientSession, item_id: str, algolia_base: str, timeout: int
+) -> Dict:
+    url = f"{algolia_base}/items/{item_id}"
+    async with session.get(
+        url, timeout=aiohttp.ClientTimeout(total=timeout)
+    ) as resp:
+        if resp.status != 200:
+            raise RuntimeError(f"Algolia /items/{item_id} 返回 {resp.status}")
+        return await resp.json()
+
+
+async def _fetch_url_html(
+    session: aiohttp.ClientSession, url: str, timeout: int
+) -> str:
+    async with session.get(
+        url, timeout=aiohttp.ClientTimeout(total=timeout)
+    ) as resp:
+        if resp.status != 200:
+            raise RuntimeError(f"外链 {url} 返回 {resp.status}")
+        return await resp.text()
+
+
+async def _fetch_via_jina(
+    session: aiohttp.ClientSession,
+    url: str,
+    timeout: int,
+    jina_token_env: str = "JINA_API_KEY",
+) -> str:
+    """通过 Jina Reader 拉取外链 markdown。`jina_token_env` 指定 API key 环境变量名,配置后免费额度更高。"""
+    jina_url = f"{JINA_READER_BASE}/{url}"
+    headers = {"Accept": "text/markdown"}
+    api_key = os.environ.get(jina_token_env) if jina_token_env else None
+    if api_key:
+        headers["Authorization"] = f"Bearer {api_key}"
+    async with session.get(
+        jina_url, headers=headers, timeout=aiohttp.ClientTimeout(total=timeout)
+    ) as resp:
+        if resp.status != 200:
+            raise RuntimeError(f"Jina Reader {url} 返回 {resp.status}")
+        return await resp.text()
+
+
+async def _fetch_external_markdown(
+    session: aiohttp.ClientSession,
+    url: str,
+    timeout: int,
+    jina_token_env: str = "JINA_API_KEY",
+) -> str:
+    """获取外链正文 markdown:先走 Jina Reader,失败回退到直接 GET + html_to_markdown。"""
+    try:
+        return await _fetch_via_jina(session, url, timeout, jina_token_env=jina_token_env)
+    except Exception:
+        html = await _fetch_url_html(session, url, timeout)
+        return html_to_markdown(html, base_url=url)
+
+
+def _collect_comments_tree(
+    root: Dict,
+    top_comments: int,
+    top_l2_per_l1: int,
+    comment_max_chars: int,
+    comments_total_chars: int,
+) -> List[Dict]:
+    """从 Algolia 根节点提取评论树,返回 [{l1, replies}]。
+
+    规则:
+    - L1 上限 `top_comments`,每个 L1 下取前 `top_l2_per_l1` 条 L2 作为 replies
+    - 每条 text 过 html_to_markdown,单条截断到 `comment_max_chars`
+    - 累计字符达 `comments_total_chars` 时立即停止(防离群 story 撑爆 prompt)
+    - 跳过空 text;空 replies 仍保留 `replies: []`,schema 一致
+    """
+    out: List[Dict] = []
+    consumed = 0
+    l1_children = (root.get("children") or [])[:top_comments]
+    for l1 in l1_children:
+        if consumed >= comments_total_chars:
+            break
+        l1_raw = (l1 or {}).get("text") or ""
+        if not l1_raw:
+            continue
+        l1_md = html_to_markdown(l1_raw)[:comment_max_chars]
+        consumed += len(l1_md)
+        replies: List[str] = []
+        for l2 in ((l1 or {}).get("children") or [])[:top_l2_per_l1]:
+            if consumed >= comments_total_chars:
+                break
+            l2_raw = (l2 or {}).get("text") or ""
+            if not l2_raw:
+                continue
+            l2_md = html_to_markdown(l2_raw)[:comment_max_chars]
+            consumed += len(l2_md)
+            replies.append(l2_md)
+        out.append({"l1": l1_md, "replies": replies})
+    return out
+
+
+async def enrich_story(
+    session: aiohttp.ClientSession,
+    story: Dict,
+    top_comments: int,
+    top_l2_per_l1: int,
+    comment_max_chars: int,
+    comments_total_chars: int,
+    link_content_max_chars: int,
+    algolia_base: str,
+    timeout: int,
+    jina_token_env: str = "JINA_API_KEY",
+) -> Dict:
+    """对单 story enrich。任一子任务失败 → 对应字段留空,不抛。"""
+    item_id = story["id"]
+    is_internal = _is_internal_hn_url(story["url"])
+
+    tasks = [
+        _fetch_algolia_item(
+            session, item_id, algolia_base=algolia_base, timeout=timeout
+        )
+    ]
+    if not is_internal:
+        tasks.append(
+            _fetch_external_markdown(
+                session, story["url"], timeout=timeout, jina_token_env=jina_token_env
+            )
+        )
+
+    results = await asyncio.gather(*tasks, return_exceptions=True)
+    algolia_result = results[0]
+    external_markdown_result = results[1] if not is_internal else None
+
+    comments_tree: List[Dict] = []
+    post_text = ""
+    if not isinstance(algolia_result, Exception) and algolia_result:
+        post_text = algolia_result.get("text") or ""
+        comments_tree = _collect_comments_tree(
+            algolia_result,
+            top_comments=top_comments,
+            top_l2_per_l1=top_l2_per_l1,
+            comment_max_chars=comment_max_chars,
+            comments_total_chars=comments_total_chars,
+        )
+
+    link_content = ""
+    if is_internal:
+        if post_text:
+            link_content = html_to_markdown(post_text)[:link_content_max_chars]
+    else:
+        if (
+            not isinstance(external_markdown_result, Exception)
+            and external_markdown_result
+        ):
+            link_content = external_markdown_result[:link_content_max_chars]
+
+    return {
+        **story,
+        "link_content": link_content,
+        "top_comments": comments_tree,
+    }
+
+
+async def enrich_stories(
+    stories: List[Dict],
+    top_comments: int,
+    top_l2_per_l1: int,
+    comment_max_chars: int,
+    comments_total_chars: int,
+    link_content_max_chars: int,
+    algolia_base: str = "https://hn.algolia.com/api/v1",
+    timeout: int = 10,
+    jina_token_env: str = "JINA_API_KEY",
+) -> Tuple[List[Dict], List[str]]:
+    """并发 enrich 多个 stories。"""
+    errors: List[str] = []
+    if os.environ.get(jina_token_env):
+        print(f"🔑 HN: 已配置 {jina_token_env},Jina Reader 鉴权调用")
+    else:
+        print(f"⚠️ HN: 未配置 {jina_token_env},Jina Reader 匿名调用 (额度受限)")
+    async with aiohttp.ClientSession(headers={"User-Agent": USER_AGENT}) as session:
+        results = await asyncio.gather(
+            *[
+                enrich_story(
+                    session,
+                    s,
+                    top_comments=top_comments,
+                    top_l2_per_l1=top_l2_per_l1,
+                    comment_max_chars=comment_max_chars,
+                    comments_total_chars=comments_total_chars,
+                    link_content_max_chars=link_content_max_chars,
+                    algolia_base=algolia_base,
+                    timeout=timeout,
+                    jina_token_env=jina_token_env,
+                )
+                for s in stories
+            ],
+            return_exceptions=True,
+        )
+    enriched: List[Dict] = []
+    for r, src in zip(results, stories):
+        if isinstance(r, Exception):
+            errors.append(f"enrich story {src['id']} 失败: {r}")
+        else:
+            enriched.append(r)
+    return enriched, errors

ai-daily-main/src/sections/hackernews/section.py

@@ -0,0 +1,90 @@
+"""HN 板块入口。流程:首页 → 轻 LLM 选 K → enrich → 最终 LLM 行文"""
+
+from datetime import datetime
+from typing import Dict, Optional, Tuple
+
+from src.sections.hackernews.frontpage_scraper import (
+    fetch_frontpage,
+    parse_frontpage_html,
+)
+from src.sections.hackernews.item_enricher import enrich_stories
+
+
+async def run_hackernews_section(
+    config: Dict, now: Optional[datetime] = None
+) -> Tuple[str, Optional[str]]:
+    cfg = config.get("sections", {}).get("hackernews", {})
+    if not cfg.get("enabled", False):
+        return "", None
+
+    # 延迟 import (Task 16 提供这两个函数)
+    from src.llm import select_ai_related_hn, summarize_hackernews
+
+    timeout = cfg.get("request_timeout", 10)
+    select_k = cfg.get("select_k", 1)
+    top_comments = cfg.get("top_comments", 30)
+    top_l2_per_l1 = cfg.get("top_l2_per_l1", 3)
+    comment_max_chars = cfg.get("comment_max_chars", 2000)
+    comments_total_chars = cfg.get("comments_total_chars", 60000)
+    link_content_max_chars = cfg.get("link_content_max_chars", 50000)
+    algolia_base = cfg.get("algolia_base", "https://hn.algolia.com/api/v1")
+    jina_token_env = cfg.get("jinaTokenName", "JINA_API_KEY")
+
+    # 1. 抓首页
+    print("📥 HN: 抓取首页...")
+    try:
+        html = await fetch_frontpage(timeout=timeout)
+    except Exception as e:
+        return "", f"HN 首页抓取失败: {e}"
+
+    front = parse_frontpage_html(html)
+    print(f"📋 HN: 解析 {len(front)} 条 frontpage stories")
+    if not front:
+        return "", None
+
+    # 2. 轻 LLM 初筛
+    print(f"🤖 HN: 轻 LLM 初筛 (k={select_k})...")
+    selected_ids, select_err = await select_ai_related_hn(front, k=select_k, config=config["llm"])
+    if select_err:
+        return "", f"select_ai_related_hn: {select_err}"
+    if not selected_ids:
+        print("ℹ️ HN: 初筛未挑出 AI 相关内容,跳过")
+        return "", None
+
+    selected = [s for s in front if s["id"] in set(selected_ids)]
+    if not selected:
+        return "", None
+    print(
+        f"🎯 HN: 初筛选出 {len(selected)} 个: "
+        f"{', '.join(s['id'] for s in selected)}"
+    )
+
+    # 3. enrich
+    print(f"🌐 HN: enrich {len(selected)} 个 story (Algolia + 外链)...")
+    enriched, enrich_errors = await enrich_stories(
+        selected,
+        top_comments=top_comments,
+        top_l2_per_l1=top_l2_per_l1,
+        comment_max_chars=comment_max_chars,
+        comments_total_chars=comments_total_chars,
+        link_content_max_chars=link_content_max_chars,
+        algolia_base=algolia_base,
+        timeout=timeout,
+        jina_token_env=jina_token_env,
+    )
+    for e in enrich_errors:
+        print(f"⚠️ HN enrich: {e}")
+    print(
+        f"✅ HN: enrich 成功 {len(enriched)} / 失败 {len(enrich_errors)} / "
+        f"输入 {len(selected)}"
+    )
+    if not enriched:
+        return "", None
+
+    # 4. LLM 总结
+    print(f"🤖 HN: summarize {len(enriched)} 个 enriched story...")
+    md, err = await summarize_hackernews(enriched, config["llm"])
+    if err:
+        return "", f"summarize_hackernews: {err}"
+    print(f"✅ HN: 板块输出 {len(md or '')} chars")
+    return md or "", None

ai-daily-main/src/sections/insights/section.py

@@ -0,0 +1,37 @@
+"""Insights 板块:基于 RSS/GH/HN 三段成品做跨板块小结"""
+
+from datetime import datetime
+from typing import Dict, Optional, Tuple
+
+EMPTY_MARKER = "(本次无内容)"
+
+
+async def run_insights_section(
+    rss_md: str,
+    gh_md: str,
+    hn_md: str,
+    config: Dict,
+    now: Optional[datetime] = None,
+) -> Tuple[str, Optional[Dict], Optional[str]]:
+    cfg = config.get("sections", {}).get("insights", {})
+    if not cfg.get("enabled", False):
+        return "", None, None
+
+    from src.llm import generate_trend_insights, parse_insights_with_metadata
+
+    sections = {
+        "rss": rss_md or EMPTY_MARKER,
+        "github": gh_md or EMPTY_MARKER,
+        "hackernews": hn_md or EMPTY_MARKER,
+    }
+
+    md, err = await generate_trend_insights(sections, config["llm"])
+    if err:
+        return "", None, f"generate_trend_insights: {err}"
+
+    if now is None:
+        now = datetime.now()
+    date_str = now.strftime("%Y-%m-%d")
+    insights_md, metadata = parse_insights_with_metadata(md or "", date_str)
+
+    return insights_md, metadata, None

ai-daily-main/src/sections/rss/__init__.py

@@ -0,0 +1,3 @@
+from src.sections.rss.section import run_rss_section
+
+__all__ = ["run_rss_section"]

ai-daily-main/src/sections/rss/section.py

@@ -0,0 +1,62 @@
+"""RSS 板块:沿用现有 collect_entries_for_push + compose_digest 流程
+
+迁移自 src/main.py::run_push_job 中 RSS digest 部分,行为保持一致。
+"""
+
+from datetime import datetime
+from typing import Dict, Optional, Tuple
+
+from src.llm import compose_digest, parse_digest_with_metadata
+from src.storage import (
+    extract_push_time,
+    get_last_push_file,
+    load_recent_push_content,
+)
+
+
+async def run_rss_section(
+    config: Dict, now: Optional[datetime] = None
+) -> Tuple[str, Optional[Dict], Optional[str]]:
+    """生成 RSS digest markdown 段(不含 sentinel)。
+
+    返回:
+        (markdown_body, metadata, error)
+        - 无新内容时返回 ("", None, None)
+        - compose_digest 失败时返回 ("", None, error_message)
+        - metadata 字段:title / lead / highlights / profile=default / date
+          早报场景下调用方可丢弃 metadata(由 insights 段覆盖)
+    """
+    # 延迟 import 避免循环:Task 20-21 后 main.py 会反向 import run_rss_section
+    from src.main import collect_entries_for_push
+
+    last_push_file = get_last_push_file()
+    last_push_time = extract_push_time(last_push_file) if last_push_file else None
+
+    min_score = config["filter"]["min_score"]
+    context_days = config["filter"]["context_days"]
+
+    to_push, context = collect_entries_for_push(
+        last_push_time=last_push_time,
+        context_days=context_days,
+        min_score=min_score,
+    )
+
+    if not to_push:
+        print("ℹ️ RSS: 无新消息")
+        return "", None, None
+
+    push_context_days = config["filter"].get("push_context_days", 5)
+    recent = load_recent_push_content(push_context_days)
+
+    try:
+        raw = await compose_digest(
+            to_push, context, config["llm"], recent_push_context=recent
+        )
+    except Exception as e:
+        msg = f"compose_digest 失败: {e}"
+        print(f"⚠️ RSS: {msg}")
+        return "", None, msg
+
+    date_str = (now or datetime.now()).strftime("%Y-%m-%d")
+    body, metadata = parse_digest_with_metadata(raw or "", date_str)
+    return body, metadata, None

ai-daily-main/src/storage.py

@@ -0,0 +1,546 @@
+"""数据存储模块 - JSON文件读写"""
+
+import json
+import re
+from datetime import date, datetime, timedelta
+from pathlib import Path
+from typing import Dict, List, Optional
+
+from src.config import get_timezone
+from src.markdown_utils import dump_frontmatter, parse_frontmatter
+
+
+def get_fetch_file(d: date = None, data_dir: str = "news-data") -> str:
+    """获取fetch文件路径 (使用配置时区)"""
+    if d is None:
+        d = datetime.now(get_timezone()).date()
+    return f"{data_dir}/fetch-{d.isoformat()}.json"
+
+
+def get_push_file(push_time: datetime = None, data_dir: str = "news-data") -> str:
+    """生成push文件路径"""
+    if push_time is None:
+        push_time = datetime.now(get_timezone())
+    time_str = push_time.strftime("%Y-%m-%d-%H-%M-%S")
+    return f"{data_dir}/push-{time_str}.md"
+
+
+def get_notify_file(d: date = None, data_dir: str = "news-data") -> str:
+    """获取notify文件路径 (使用配置时区)"""
+    if d is None:
+        d = datetime.now(get_timezone()).date()
+    return f"{data_dir}/notify-{d.isoformat()}.md"
+
+
+def save_notify_file(
+    filepath: str,
+    content: str,
+    metadata: Dict = None,
+):
+    """保存即时推送文件（Markdown格式），同一天的内容追加到同一文件"""
+    path = Path(filepath)
+    path.parent.mkdir(parents=True, exist_ok=True)
+
+    notify_time = datetime.now(get_timezone()).isoformat()
+
+    if metadata:
+        frontmatter_dict = metadata.copy()
+    else:
+        frontmatter_dict = {"pushTime": notify_time}
+
+    frontmatter = dump_frontmatter(frontmatter_dict)
+
+    new_content = f"---\n{frontmatter}---\n\n{content}\n\n------\n"
+
+    with open(path, "a", encoding="utf-8") as f:
+        f.write(new_content)
+
+
+_SECTION_RE_CACHE: Dict[str, re.Pattern] = {}
+
+
+def _section_re(section: str) -> re.Pattern:
+    """获取/缓存 sentinel 正则。section 名做转义,允许字母数字下划线"""
+    if section not in _SECTION_RE_CACHE:
+        s = re.escape(section)
+        pattern = (
+            rf"<!--\s*SECTION:{s}\s*BEGIN\s*-->(.*?)<!--\s*SECTION:{s}\s*END\s*-->"
+        )
+        _SECTION_RE_CACHE[section] = re.compile(pattern, flags=re.DOTALL)
+    return _SECTION_RE_CACHE[section]
+
+
+def extract_section(push_md: str, section: str) -> str:
+    """从 push 文件内容中切出 <!-- SECTION:{section} BEGIN/END --> 之间的 markdown。
+
+    向后兼容:
+    - 新文件(带 sentinel): 返回 sentinel 边界内的原文(不去边界空行)
+    - 老文件(无 sentinel) 且 section == 'rss': 返回整个 push_md
+    - 老文件(无 sentinel) 且 section != 'rss': 返回空字符串
+    - sentinel 残缺(只有 BEGIN 没有 END): 返回空字符串
+    """
+    match = _section_re(section).search(push_md)
+    if match:
+        return match.group(1)
+
+    # 老文件兜底:rss 段视为整个 body
+    has_any_sentinel = "<!-- SECTION:" in push_md
+    if section == "rss" and not has_any_sentinel:
+        return push_md
+    return ""
+
+
+def load_recent_notify_content(
+    context_days: int = 3, data_dir: str = "news-data"
+) -> str:
+    """加载最近 context_days 天 notify 文件正文（去除 frontmatter，仅供 LLM 查重）
+
+    notify 文件由多个推送块用 `------` 分隔，每块带各自 frontmatter；这里逐块剥离
+    frontmatter 后用 `------` 重新拼接，保留事件全文。
+    """
+    data_path = Path(data_dir)
+    if not data_path.exists():
+        return ""
+
+    tz = get_timezone()
+    today = datetime.now(tz).date()
+
+    blocks: List[str] = []
+    loaded_files = []
+    for i in range(context_days):
+        d = today - timedelta(days=i)
+        notify_file = data_path / f"notify-{d.isoformat()}.md"
+        if not notify_file.exists() or notify_file.stat().st_size == 0:
+            continue
+        try:
+            with open(notify_file, "r", encoding="utf-8") as f:
+                content = f.read()
+        except Exception:
+            continue
+        for block in content.split("------"):
+            if not block.strip():
+                continue
+            _, body = parse_frontmatter(block)
+            if body:
+                blocks.append(body)
+        loaded_files.append(notify_file.name)
+
+    if loaded_files:
+        print(
+            f"   📂 已加载 {len(loaded_files)} 个 notify 文件: {', '.join(loaded_files)}"
+        )
+
+    return "\n\n------\n\n".join(blocks)
+
+
+def load_recent_push_content(
+    context_days: int = 3, data_dir: str = "news-data", section: str = "rss"
+) -> str:
+    """加载最近 context_days 天 push 文件中指定 section 的正文(去除 frontmatter,仅供 LLM 查重)。
+
+    Args:
+        section: sentinel 段名,默认 "rss"。老文件(无 sentinel) 且 section == "rss"
+                 时会兜底返回整个 body(由 extract_section 处理),其它 section 在
+                 老文件上返回空。
+    """
+    data_path = Path(data_dir)
+    if not data_path.exists():
+        return ""
+
+    tz = get_timezone()
+    today = datetime.now(tz).date()
+
+    bodies: List[str] = []
+    loaded_files = []
+    for i in range(context_days):
+        d = today - timedelta(days=i)
+        pattern = f"push-{d.isoformat()}-*.md"
+        for push_file in sorted(data_path.glob(pattern)):
+            if push_file.stat().st_size == 0:
+                continue
+            try:
+                with open(push_file, "r", encoding="utf-8") as f:
+                    content = f.read()
+            except Exception:
+                continue
+            section_md = extract_section(content, section)
+            if not section_md:
+                continue
+            # 老文件兜底路径会把整篇文件还回来,此时仍需剥离 frontmatter;
+            # 新文件 sentinel 内不含 frontmatter,parse_frontmatter 会原样返回。
+            _, body = parse_frontmatter(section_md)
+            body = body or section_md
+            body = body.strip()
+            if body:
+                bodies.append(body)
+                loaded_files.append(push_file.name)
+
+    if loaded_files:
+        print(
+            f"   📂 已加载 {len(loaded_files)} 个 push 文件 (section={section}): "
+            f"{', '.join(loaded_files)}"
+        )
+
+    return "\n\n------\n\n".join(bodies)
+
+
+def get_last_push_file(data_dir: str = "news-data") -> Optional[str]:
+    """从news-data目录找到最新的push文件"""
+    data_path = Path(data_dir)
+    if not data_path.exists():
+        return None
+
+    push_files = sorted(data_path.glob("push-*.md"))
+    return str(push_files[-1]) if push_files else None
+
+
+def extract_push_time(filepath: str) -> Optional[datetime]:
+    """从push文件名提取时间"""
+    try:
+        basename = Path(filepath).name
+        time_str = basename.replace("push-", "").replace(".md", "")
+        dt = datetime.strptime(time_str, "%Y-%m-%d-%H-%M-%S")
+        return dt.replace(tzinfo=get_timezone())
+    except (ValueError, AttributeError):
+        return None
+
+
+def read_entries(filepath: str) -> List[Dict]:
+    """读取fetch文件，返回entries列表"""
+    path = Path(filepath)
+    if not path.exists():
+        return []
+
+    with open(path, "r", encoding="utf-8") as f:
+        data = json.load(f)
+
+    return data.get("entries", [])
+
+
+def read_fetch_data(filepath: str) -> Dict:
+    """读取完整的fetch文件数据（包含meta和entries）"""
+    path = Path(filepath)
+    if not path.exists():
+        return {"meta": {}, "entries": []}
+
+    # 检查文件是否为空
+    if path.stat().st_size == 0:
+        return {"meta": {}, "entries": []}
+
+    with open(path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def save_fetch_file(filepath: str, meta: Dict, entries: List[Dict]):
+    """保存fetch文件（JSON格式）"""
+    path = Path(filepath)
+    path.parent.mkdir(parents=True, exist_ok=True)
+
+    data = {"meta": meta, "entries": entries}
+
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(data, f, ensure_ascii=False, indent=2)
+
+
+def append_entries(filepath: str, new_entries: List[Dict], meta: Dict = None):
+    """追加条目到fetch文件"""
+    path = Path(filepath)
+
+    # 读取现有数据
+    if path.exists():
+        data = read_fetch_data(filepath)
+    else:
+        data = {"meta": meta or {}, "entries": []}
+
+    # 更新meta（如果提供了）
+    if meta:
+        data["meta"].update(meta)
+
+    # 去重：基于link字段
+    existing_links = {e.get("link") for e in data["entries"]}
+    for entry in new_entries:
+        if entry.get("link") not in existing_links:
+            data["entries"].append(entry)
+            existing_links.add(entry.get("link"))
+
+    # 保存
+    save_fetch_file(filepath, data["meta"], data["entries"])
+    return len(new_entries)
+
+
+def format_entry(entry: Dict) -> str:
+    """格式化单条条目为Markdown字符串"""
+    tags = entry.get("tags", [])
+    tags_str = json.dumps(tags, ensure_ascii=False) if tags else "[]"
+    score = entry.get("score", "")
+    summary = entry.get("summary", "")
+
+    return f"""## {entry["title"]}
+
+---
+source: {entry["source"]}
+link: {entry["link"]}
+published: {entry["published"]}
+fetched_at: {entry["fetched_at"]}
+tags: {tags_str}
+score: {score}
+summary: {summary}
+---
+
+{entry["content"]}
+
+------
+"""
+
+
+def json_to_md(data: Dict) -> str:
+    """
+    将JSON格式的fetch数据转换为Markdown格式，便于阅读
+
+    Args:
+        data: {"meta": {...}, "entries": [...]}
+
+    Returns:
+        Markdown格式的字符串
+    """
+    meta = data.get("meta", {})
+    entries = data.get("entries", [])
+
+    lines = []
+
+    # 文件头部YAML frontmatter
+    if meta.get("date"):
+        lines.append("---")
+        lines.append(f'date: "{meta["date"]}"')
+        lines.append("---")
+        lines.append("")
+
+    # 条目
+    for entry in entries:
+        lines.append(format_entry(entry))
+
+    return "\n".join(lines)
+
+
+def convert_fetch_json_to_md(json_filepath: str, md_filepath: str = None) -> str:
+    """
+    将fetch JSON文件转换为Markdown文件
+
+    Args:
+        json_filepath: JSON文件路径
+        md_filepath: 输出MD文件路径，默认为同名.md
+
+    Returns:
+        生成的Markdown内容
+    """
+    data = read_fetch_data(json_filepath)
+    md_content = json_to_md(data)
+
+    if md_filepath:
+        path = Path(md_filepath)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        with open(path, "w", encoding="utf-8") as f:
+            f.write(md_content)
+
+    return md_content
+
+
+def save_push_file(
+    filepath: str,
+    content: str,
+    source_count: int,
+    total_entries: int,
+    profile: str = "default",
+    metadata: Dict = None,
+):
+    """保存推送文件（Markdown格式）
+
+    Args:
+        profile: "morning" | "default"  ← 早报或常规;写入 frontmatter,便于按 profile 分析
+        metadata: 元信息（可选），如果提供则使用 metadata，否则使用默认格式
+    """
+    path = Path(filepath)
+    path.parent.mkdir(parents=True, exist_ok=True)
+
+    if metadata:
+        # 使用提供的 metadata
+        frontmatter_dict = metadata.copy()
+        # 添加推送时间和统计信息
+        frontmatter_dict["pushDate"] = datetime.now(get_timezone()).isoformat()
+        frontmatter_dict["sourceCount"] = source_count
+        frontmatter_dict["totalEntries"] = total_entries
+    else:
+        # 降级：使用默认格式
+        push_time = datetime.now(get_timezone())
+        frontmatter_dict = {
+            "profile": profile,
+            "pushDate": push_time.isoformat(),
+            "sourceCount": source_count,
+            "totalEntries": total_entries,
+        }
+
+    frontmatter = dump_frontmatter(frontmatter_dict)
+    full_content = f"---\n{frontmatter}---\n\n{content}"
+
+    with open(path, "w", encoding="utf-8") as f:
+        f.write(full_content)
+
+
+def load_existing_links(filepath: str, threshold: int = 150) -> set:
+    """加载文件中已有的链接（用于去重）
+
+    如果当天时间已超过 threshold 分钟，则只需加载当天文件；
+    否则需要同时加载当天和昨天的文件（用于处理跨天边界情况）。
+
+    Args:
+        filepath: 当天的 fetch 文件路径
+        threshold: 阈值（分钟），超过此时间只加载当天文件
+    """
+    tz = get_timezone()
+    now = datetime.now(tz)
+    current_minutes = now.hour * 60 + now.minute
+
+    need_yesterday = current_minutes < threshold
+
+    if not need_yesterday:
+        if not filepath or not Path(filepath).exists():
+            return set()
+        entries = read_entries(filepath)
+        return {e.get("link") for e in entries if e.get("link")}
+
+    all_links = set()
+    if filepath and Path(filepath).exists():
+        all_links.update(
+            {e.get("link") for e in read_entries(filepath) if e.get("link")}
+        )
+
+    yesterday = (now - timedelta(days=1)).date()
+    yesterday_file = get_fetch_file(yesterday)
+    if Path(yesterday_file).exists():
+        all_links.update(
+            {e.get("link") for e in read_entries(yesterday_file) if e.get("link")}
+        )
+
+    return all_links
+
+
+def cleanup_old_files(days: int = 7, data_dir: str = "news-data"):
+    """清理超过days天的旧文件"""
+    data_path = Path(data_dir)
+    if not data_path.exists():
+        return
+
+    cutoff = datetime.now() - timedelta(days=days)
+    deleted_count = 0
+
+    for pattern in ["fetch-*.json", "fetch-*.md", "push-*.md", "notify-*.md"]:
+        for file in data_path.glob(pattern):
+            try:
+                date_str = (
+                    file.name.replace("fetch-", "")
+                    .replace("push-", "")
+                    .replace("notify-", "")
+                    .replace(".json", "")
+                    .replace(".md", "")
+                )
+                date_parts = date_str.split("-")
+                if len(date_parts) >= 3:
+                    file_date = date(
+                        int(date_parts[0]), int(date_parts[1]), int(date_parts[2])
+                    )
+                    if file_date < cutoff.date():
+                        file.unlink()
+                        deleted_count += 1
+                        print(f"   🗑️ 删除旧文件: {file.name}")
+            except (ValueError, OSError):
+                continue
+
+    # trending-history.json: 剪枝过期条目,保留文件本身
+    trending_path = data_path / "trending-history.json"
+    if trending_path.exists() and trending_path.stat().st_size > 0:
+        try:
+            history = load_trending_history(str(trending_path))
+            before = len(history.repos)
+            history.cleanup(today=datetime.now().date(), keep_days=days)
+            after = len(history.repos)
+            if after < before:
+                history.save()
+                print(f"   ✂️ trending-history 剪枝: {before} → {after} 条")
+        except Exception as e:
+            print(f"   ⚠️ trending-history 剪枝失败: {e}")
+
+    if deleted_count > 0:
+        print(f"   ✅ 清理完成: 删除了 {deleted_count} 个旧文件")
+
+
+class TrendingHistory:
+    """GitHub trending 已查阅 repo 索引。
+
+    repos 字段:url → last_seen_date (ISO YYYY-MM-DD)。
+    每次早报 cleanup 一次,touch 完所有今日 URL 后 save。
+    """
+
+    def __init__(self, path: str, repos: Dict[str, str]):
+        self._path = path
+        self.repos: Dict[str, str] = dict(repos)
+
+    def __contains__(self, url: str) -> bool:
+        return url in self.repos
+
+    def touch(self, url: str, today: date) -> None:
+        self.repos[url] = today.isoformat()
+
+    def cleanup(self, today: date, keep_days: int) -> None:
+        cutoff = today - timedelta(days=keep_days)
+        self.repos = {
+            url: d
+            for url, d in self.repos.items()
+            if _parse_iso_date_safe(d) is not None and _parse_iso_date_safe(d) >= cutoff
+        }
+
+    def save(self) -> None:
+        path = Path(self._path)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        payload = {
+            "repos": self.repos,
+            "updated_at": datetime.now(get_timezone()).isoformat(),
+        }
+        with open(path, "w", encoding="utf-8") as f:
+            json.dump(payload, f, ensure_ascii=False, indent=2)
+
+
+def _parse_iso_date_safe(s: str) -> Optional[date]:
+    try:
+        return date.fromisoformat(s)
+    except (ValueError, TypeError):
+        return None
+
+
+def load_trending_history(path: str) -> TrendingHistory:
+    """读取 trending-history.json;不存在返回空实例。"""
+    p = Path(path)
+    if not p.exists() or p.stat().st_size == 0:
+        return TrendingHistory(path, {})
+    try:
+        with open(p, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        return TrendingHistory(path, data.get("repos", {}))
+    except (json.JSONDecodeError, OSError):
+        print(f"⚠️ trending-history 读取失败,使用空索引: {path}")
+        return TrendingHistory(path, {})
+
+
+_SECTION_ORDER = ("rss", "github", "hackernews", "insights")
+
+
+def assemble_with_sentinels(sections: Dict[str, str]) -> str:
+    """按固定顺序拼装四段 markdown,每段包 sentinel;空段整段省略。"""
+    parts: List[str] = []
+    for key in _SECTION_ORDER:
+        body = (sections.get(key) or "").strip()
+        if not body:
+            continue
+        parts.append(
+            f"<!-- SECTION:{key} BEGIN -->\n{body}\n<!-- SECTION:{key} END -->"
+        )
+    return "\n\n".join(parts)