feat: 军事科技每日资讯推送系统 - Docker部署 + 日志系统 + 数据目录重组

This commit is contained in:
poiuy
2026-07-12 20:01:02 +08:00
commit 54ca4b1b6a
267 changed files with 47047 additions and 0 deletions
View File
+45
View File
@@ -0,0 +1,45 @@
"""手动跑一次 HN 首页 + Algolia enrich,验证选择器与 API"""
import asyncio
import json
import os
import sys
from dotenv import load_dotenv
load_dotenv()
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from src.sections.hackernews.frontpage_scraper import (
fetch_frontpage,
parse_frontpage_html,
)
from src.sections.hackernews.item_enricher import enrich_stories
async def main():
print("📥 抓取 HN 首页...")
html = await fetch_frontpage(timeout=15)
stories = parse_frontpage_html(html)
print(f"📋 解析出 {len(stories)}")
for s in stories[:5]:
print(f" - [{s['points']} pts · {s['comments']} comments] {s['title']} ({s['site']})")
print("\n🔍 enrich 前 1 个外链类故事...")
target = next((s for s in stories if not s["url"].startswith("https://news.ycombinator.com/")), stories[0])
enriched, errors = await enrich_stories(
[target],
top_comments=5,
top_l2_per_l1=3,
comment_max_chars=2000,
comments_total_chars=60000,
link_content_max_chars=1500,
timeout=15,
)
for e in errors:
print(f" ⚠️ {e}")
print(json.dumps(enriched, ensure_ascii=False, indent=2))
if __name__ == "__main__":
asyncio.run(main())
+201
View File
@@ -0,0 +1,201 @@
#!/usr/bin/env python3
"""获取新闻脚本 - 模块化测试第一步:获取RSS并存储"""
import argparse
import asyncio
import os
import sys
from collections import defaultdict
from datetime import datetime, timedelta, timezone
from pathlib import Path
from urllib.parse import urlparse
# 添加项目根目录到路径
sys.path.insert(0, str(Path(__file__).parent.parent))
from dotenv import load_dotenv
load_dotenv()
from src.config import get_timezone, load_config, merge_sources
from src.fetcher import fetch_all_feeds
from src.processor import html_to_markdown
from src.storage import append_entries, get_fetch_file, load_existing_links
def parse_args():
"""解析命令行参数"""
parser = argparse.ArgumentParser(description="RSS新闻获取测试")
parser.add_argument(
"--hours", "-H", type=int, default=1, help="获取过去多少小时的新闻 (默认: 1)"
)
parser.add_argument("--minutes", "-m", type=int, help="获取过去多少分钟的新闻")
parser.add_argument(
"--output-dir",
"-o",
type=str,
default="tests/news-data",
help="输出目录 (默认: tests/news-data)",
)
parser.add_argument(
"--max-per-domain",
type=int,
default=30,
help="同一域名最大保留源数量 (默认: 30)",
)
return parser.parse_args()
def get_cutoff_time(args) -> datetime:
"""根据参数计算截止时间 (返回 UTC 时间)"""
now = datetime.now(timezone.utc)
if args.minutes:
return now - timedelta(minutes=args.minutes)
return now - timedelta(hours=args.hours)
def limit_sources_by_domain(sources: list, max_per_domain: int = 30) -> list:
"""限制同一域名的源数量"""
domain_sources = defaultdict(list)
for source in sources:
url = source.get("xmlUrl", "")
try:
domain = urlparse(url).netloc.lower()
# 移除 www 前缀
if domain.startswith("www."):
domain = domain[4:]
except Exception:
domain = "unknown"
domain_sources[domain].append(source)
limited_sources = []
domain_counts = {}
for domain, src_list in domain_sources.items():
kept = src_list[:max_per_domain]
limited_sources.extend(kept)
domain_counts[domain] = {"total": len(src_list), "kept": len(kept)}
return limited_sources, domain_counts
async def fetch_news():
"""主函数:获取RSS新闻并存储"""
args = parse_args()
tz = get_timezone()
print("=" * 60)
print("📰 RSS新闻获取测试 (Step 1)")
print("=" * 60)
# 1. 加载配置
print("\n📋 加载配置...")
config = load_config()
all_sources = merge_sources(config["sources"])
print(f" OPML 解析完成: {len(all_sources)} 个源")
# 2. 域名限制
print(f"\n🔍 域名限制 (每域名最多 {args.max_per_domain} 个)...")
sources, domain_stats = limit_sources_by_domain(all_sources, args.max_per_domain)
# 打印域名统计
total_domains = len(domain_stats)
limited_domains = sum(
1 for d in domain_stats.values() if d["total"] > args.max_per_domain
)
print(f" 域名总数: {total_domains}")
print(f" 受限域名: {limited_domains}")
print(f" 最终保留: {len(sources)} 个源")
# 显示受限域名详情
for domain, stats in sorted(domain_stats.items(), key=lambda x: -x[1]["total"])[:5]:
if stats["total"] > args.max_per_domain:
print(f" - {domain}: {stats['total']}{stats['kept']}")
# 3. 计算时间窗口
cutoff = get_cutoff_time(args)
print(f"\n⏰ 时间窗口")
print(f" UTC: {cutoff.strftime('%Y-%m-%d %H:%M')}")
print(
f" Local: {(datetime.now(tz) - (datetime.now(timezone.utc) - cutoff)).strftime('%Y-%m-%d %H:%M')}"
)
# 4. 获取RSS数据
print(f"\n📡 开始获取...")
max_workers = config.get("fetch", {}).get("max_workers", 10)
timeout = config.get("fetch", {}).get("timeout", 5)
entries = await fetch_all_feeds(
sources, cutoff, max_workers=max_workers, timeout=timeout
)
# 5. 统计结果
print(f"\n📊 获取统计")
print(f" 读取源数: {len(all_sources)}")
print(f" 保留源数: {len(sources)}")
print(f" 获取条目: {len(entries)}")
if not entries:
print("\n⚠️ 没有获取到新消息")
return 0
# 6. 转换HTML到Markdown
print("\n📝 处理内容...")
for entry in entries:
entry["content"] = html_to_markdown(
entry.get("content", ""), entry.get("link", "")
)
# 7. 保存到文件
print(f"\n💾 保存到文件...")
output_dir = Path(args.output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
# 使用今天的日期作为文件名(JSON格式)
today = datetime.now(tz).date()
fetch_file = output_dir / f"fetch-{today.isoformat()}.json"
# 加载已有链接去重
existing_links = (
load_existing_links(str(fetch_file)) if fetch_file.exists() else set()
)
new_entries = [
e for e in entries if e.get("link") and e["link"] not in existing_links
]
# 添加时间戳并格式化
for entry in entries:
entry["fetched_at"] = datetime.now(tz).isoformat()
if isinstance(entry.get("published"), datetime):
entry["published"] = entry["published"].astimezone(tz).isoformat()
# 使用新的 append_entries 批量保存
meta = {"date": today.isoformat()}
append_entries(str(fetch_file), entries, meta)
print(f" 文件: {fetch_file}")
print(f" 保存: {len(entries)}")
print(f" 新增: {len(new_entries)}")
print(f" 重复: {len(entries) - len(new_entries)}")
print("\n" + "=" * 60)
print("✅ Step 1 完成: RSS获取并存储")
print("=" * 60)
return len(entries)
if __name__ == "__main__":
try:
count = asyncio.run(fetch_news())
sys.exit(0 if count > 0 else 1)
except KeyboardInterrupt:
print("\n\n👋 已取消")
sys.exit(130)
except Exception as e:
print(f"\n❌ 错误: {e}")
import traceback
traceback.print_exc()
sys.exit(1)
+46
View File
@@ -0,0 +1,46 @@
"""手动跑一次 xcancel/nitter 抓取,验证白名单 UA + requests 路径
用法:python tests/fetch_nitter.py
"""
import asyncio
import os
import sys
from datetime import datetime, timedelta, timezone
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from src.fetcher import fetch_all_feeds
FEEDS = [
{
"title": "Alibaba_Qwen",
"xmlUrl": "https://rss.xcancel.com/Alibaba_Qwen/rss",
},
{
"title": "trq212",
"xmlUrl": "https://rss.xcancel.com/trq212/rss",
},
]
async def main():
cutoff = datetime.now(timezone.utc) - timedelta(days=7)
print(f"📥 抓取 {len(FEEDS)} 个 nitter 源(cutoff={cutoff.isoformat()}...")
entries = await fetch_all_feeds(FEEDS, cutoff)
print(f"📋 拿到 {len(entries)}")
for e in entries[:10]:
published = e["published"].isoformat() if e["published"] else "?"
print(f" - [{published}] {e['source']} {e['title']}")
print(f" {e['link']}")
print(f" {e['content']}\n\n")
if not entries:
print("⚠️ 一条都没拿到 —— 检查 UA / TLS 路径是否生效")
sys.exit(1)
if __name__ == "__main__":
asyncio.run(main())
+44
View File
@@ -0,0 +1,44 @@
"""手动跑一次 GH trending 抓取 + deep-dive,验证选择器与 API 接入"""
import asyncio
import json
import os
import sys
from dotenv import load_dotenv
load_dotenv()
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from src.config import load_config
from src.sections.github.repo_enricher import enrich_repos
from src.sections.github.trending_scraper import (
fetch_trending_page,
parse_trending_html,
)
async def main():
config = load_config()
print("📥 抓取 GitHub Trending...")
html = await fetch_trending_page(timeout=15)
repos = parse_trending_html(html)
print(f"📋 解析出 {len(repos)} 个 repo")
for r in repos[:5]:
print(f" - {r['full_name']}{r['stars_today']}/{r['stars_total']} | {r['description'][:80]}")
cfg = config["sections"]["github_trending"]
print(f"\n🔍 enrich 前 {min(3, len(repos))} 个...")
enriched, errors = await enrich_repos(
repos[:3],
token_env=cfg.get("tokenName", "GITHUB_TOKEN"),
readme_max_chars=cfg.get("readme_max_chars", 3000),
timeout=15,
)
for e in errors:
print(f" ⚠️ {e}")
print(json.dumps(enriched, ensure_ascii=False, indent=2))
if __name__ == "__main__":
asyncio.run(main())
@@ -0,0 +1,209 @@
#!/usr/bin/env python3
"""手工触发 LLM 错误告警。
用法:
source ../.venv/bin/activate
python tests/manual_trigger_fetch_loop_llm_error.py --scenario score
python tests/manual_trigger_fetch_loop_llm_error.py --scenario immediate
python tests/manual_trigger_fetch_loop_llm_error.py --scenario digest
python tests/manual_trigger_fetch_loop_llm_error.py --scenario all
场景说明:
- score: 走 fetch_loop -> run_fetch_job -> score_batch,触发评分错误通知
- immediate: 走 run_fetch_job -> generate_immediate_push,触发即时推送生成错误通知
- digest: 走 run_push_job -> compose_digest,触发汇总生成错误通知
- all: 依次触发以上三类真实通知
"""
import argparse
import asyncio
import sys
from pathlib import Path
from unittest.mock import patch
sys.path.insert(0, str(Path(__file__).parent.parent))
from src.config import load_config
from src.main import fetch_loop, run_fetch_job, run_push_job
MOCK_FETCH_ENTRIES = [
{
"title": "Manual Mock News 1",
"link": "https://example.com/mock-1",
"published": "2026-03-08T20:00:00+08:00",
"source": "Manual Mock Source",
"content": "<p>Mock content 1</p>",
"tags": [],
"score": 0,
"summary": "",
},
{
"title": "Manual Mock News 2",
"link": "https://example.com/mock-2",
"published": "2026-03-08T20:05:00+08:00",
"source": "Manual Mock Source",
"content": "<p>Mock content 2</p>",
"tags": [],
"score": 0,
"summary": "",
},
]
MOCK_HOT_SCORED_ENTRIES = [
{
"title": "Manual Hot News",
"link": "https://example.com/hot-1",
"published": "2026-03-08T21:00:00+08:00",
"source": "Manual Mock Source",
"content": "mock content",
"tags": ["AI"],
"score": 95,
"summary": "manual hot summary",
}
]
MOCK_DIGEST_ENTRIES = [
{
"title": "Manual Digest News",
"link": "https://example.com/digest-1",
"published": "2026-03-08T19:00:00+08:00",
"fetched_at": "2026-03-08T21:00:00+08:00",
"source": "Manual Mock Source",
"content": "mock digest content",
"tags": ["AI"],
"score": 88,
"summary": "manual digest summary",
}
]
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="手工触发 LLM 错误告警")
parser.add_argument(
"--scenario",
choices=["score", "immediate", "digest", "all"],
default="score",
help="要触发的错误场景,默认 score",
)
return parser.parse_args()
async def _trigger_score_error(config):
mock_sources = [
{
"title": "Manual Mock Feed",
"xmlUrl": "https://example.com/rss.xml",
"category": "AI",
}
]
tmp_dir = Path("test/news-data")
tmp_dir.mkdir(parents=True, exist_ok=True)
fetch_file = tmp_dir / "manual-fetch-loop-error.json"
async def fake_fetch_all_feeds(*args, **kwargs):
return MOCK_FETCH_ENTRIES
async def fake_sleep(seconds: float):
if seconds <= 0:
return
raise asyncio.CancelledError()
with (
patch("src.main.merge_sources", return_value=mock_sources),
patch("src.main.fetch_all_feeds", side_effect=fake_fetch_all_feeds),
patch("src.main.load_existing_links", return_value=set()),
patch("src.main.get_fetch_file", return_value=str(fetch_file)),
patch(
"src.llm.call_llm",
side_effect=RuntimeError("manual mock llm scoring error"),
),
patch("src.main.asyncio.sleep", side_effect=fake_sleep),
):
await fetch_loop(config)
async def _trigger_immediate_push_error(config):
tmp_dir = Path("test/news-data")
tmp_dir.mkdir(parents=True, exist_ok=True)
fetch_file = tmp_dir / "manual-immediate-push-error.json"
async def fake_fetch_all_feeds(*args, **kwargs):
return [dict(entry) for entry in MOCK_HOT_SCORED_ENTRIES]
with (
patch(
"src.main.merge_sources",
return_value=[
{
"title": "Manual",
"xmlUrl": "https://example.com/rss.xml",
"category": "AI",
}
],
),
patch("src.main.fetch_all_feeds", side_effect=fake_fetch_all_feeds),
patch("src.main.load_existing_links", return_value=set()),
patch("src.main.get_fetch_file", return_value=str(fetch_file)),
patch(
"src.main.score_batch",
return_value=([dict(entry) for entry in MOCK_HOT_SCORED_ENTRIES], []),
),
patch(
"src.main.generate_immediate_push",
return_value=("", "manual mock immediate push error"),
),
):
await run_fetch_job(config)
async def _trigger_digest_error(config):
with (
patch(
"src.main.collect_entries_for_push",
return_value=([dict(entry) for entry in MOCK_DIGEST_ENTRIES], []),
),
patch("src.main.get_last_push_file", return_value=None),
patch(
"src.main.compose_digest",
side_effect=RuntimeError("manual mock compose digest error"),
),
):
await run_push_job(config)
async def _run_selected_scenarios(scenario: str) -> None:
config = load_config()
if scenario in {"score", "all"}:
print("\n🚨 触发 score_batch 错误通知")
await _trigger_score_error(config)
if scenario in {"immediate", "all"}:
print("\n🚨 触发 generate_immediate_push 错误通知")
await _trigger_immediate_push_error(config)
if scenario in {"digest", "all"}:
print("\n🚨 触发 compose_digest 错误通知")
await _trigger_digest_error(config)
def main() -> int:
args = parse_args()
print("🚨 即将触发真实的 LLM 错误提醒")
print(f" - 场景: {args.scenario}")
print(" - 推送: 使用当前 config.json 和环境变量中的真实渠道")
try:
asyncio.run(_run_selected_scenarios(args.scenario))
except KeyboardInterrupt:
print("\n已取消")
return 130
print("✅ 脚本执行完成;如果推送配置正确,你应该已经收到对应错误提醒。")
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,99 @@
#!/usr/bin/env python3
"""预览 LLM 查重上下文 —— 直接打印 load_recent_notify_content / load_recent_push_content
返回的内容,便于调试历史去重时实际喂给 LLM 的素材。
Usage:
# 默认:读取 config.json 的 filter.context_days,打印 notify + push 两段
python tests/preview_recent_context.py
# 指定回溯天数
python tests/preview_recent_context.py --days 5
# 只看一种
python tests/preview_recent_context.py --type notify
python tests/preview_recent_context.py --type push
# 自定义数据目录(例如 tests/news-data 里的样本数据)
python tests/preview_recent_context.py --data-dir tests/news-data
"""
import argparse
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent))
from src.config import load_config
from src.storage import load_recent_notify_content, load_recent_push_content
def _resolve_days(cli_days: int | None) -> int:
if cli_days is not None:
return cli_days
try:
config = load_config()
return int(config.get("filter", {}).get("context_days", 3))
except Exception:
return 3
def _print_section(label: str, content: str) -> None:
sep = "=" * 60
print(f"\n{sep}\n{label}\n{sep}")
if not content:
print("(空)")
return
print(content)
print(
f"\n--- {label} 字符数: {len(content)} | 块数(按 `------` 切分): "
f"{len([b for b in content.split('------') if b.strip()])} ---"
)
def main() -> int:
parser = argparse.ArgumentParser(
description="预览近 N 天 notify / push 文件正文 (剥离 frontmatter 后)"
)
parser.add_argument(
"--days",
type=int,
default=None,
help="回溯天数,默认读取 config.filter.context_days(兜底 3)",
)
parser.add_argument(
"--type",
choices=["notify", "push", "both"],
default="both",
help="预览哪种历史,默认 both",
)
parser.add_argument(
"--data-dir",
default="news-data",
help="数据目录,默认 news-data",
)
parser.add_argument(
"--section",
default="rss",
help="push 文件取哪个 sentinel 段,默认 rss(可选 github / hackernews / insights)",
)
args = parser.parse_args()
days = _resolve_days(args.days)
print(
f"📅 回溯天数: {days} | 📂 数据目录: {args.data_dir} | "
f"类型: {args.type} | push.section: {args.section}"
)
if args.type in ("notify", "both"):
notify_md = load_recent_notify_content(days, args.data_dir)
_print_section("近期即时推送 (notify)", notify_md)
if args.type in ("push", "both"):
push_md = load_recent_push_content(days, args.data_dir, section=args.section)
_print_section(f"近期汇总推送 (push.{args.section})", push_md)
return 0
if __name__ == "__main__":
sys.exit(main())
+140
View File
@@ -0,0 +1,140 @@
"""Probe HN: 抓取首页前 10 条,统计每个 story 评论树的深度/数量/字符数。
用于判断 enrich 策略:是否需要 L2 回复、压缩、过滤短评等。
字符数说明:
- "raw" = Algolia 返回的 text 字段(HTML)
- "md" = html_to_markdown 后的字符数(LLM 实际看到的)
"""
import asyncio
import json
import os
import sys
from typing import Dict, List, Tuple
import aiohttp
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from src.processor import html_to_markdown
from src.sections.hackernews.frontpage_scraper import (
fetch_frontpage,
parse_frontpage_html,
)
ALGOLIA = "https://hn.algolia.com/api/v1/items"
TIMEOUT = 30
async def fetch_item(session: aiohttp.ClientSession, item_id: str) -> Dict:
async with session.get(
f"{ALGOLIA}/{item_id}",
timeout=aiohttp.ClientTimeout(total=TIMEOUT),
) as resp:
if resp.status != 200:
raise RuntimeError(f"{item_id} -> {resp.status}")
return await resp.json()
def walk_tree(node: Dict, depth: int = 0) -> List[Tuple[int, str]]:
"""递归遍历评论树,返回 [(depth, text_html), ...]。depth=0 是 story 自身,1 是顶层评论。"""
result: List[Tuple[int, str]] = []
text = (node or {}).get("text") or ""
if depth > 0 and text:
result.append((depth, text))
for child in (node or {}).get("children") or []:
result.extend(walk_tree(child, depth + 1))
return result
def stats(samples: List[str]) -> Dict:
if not samples:
return {"count": 0, "raw_chars": 0, "md_chars": 0, "avg_md": 0, "max_md": 0}
md_lens = [len(html_to_markdown(s)) for s in samples]
raw_lens = [len(s) for s in samples]
return {
"count": len(samples),
"raw_chars": sum(raw_lens),
"md_chars": sum(md_lens),
"avg_md": round(sum(md_lens) / len(md_lens)),
"max_md": max(md_lens),
}
def by_depth(nodes: List[Tuple[int, str]], d: int) -> List[str]:
return [t for depth, t in nodes if depth == d]
async def probe_one(
session: aiohttp.ClientSession, story: Dict
) -> Dict:
item_id = story["id"]
data = await fetch_item(session, item_id)
flat = walk_tree(data)
l1 = by_depth(flat, 1)
l2 = by_depth(flat, 2)
l3plus = [t for d, t in flat if d >= 3]
all_comments = [t for _, t in flat]
return {
"id": item_id,
"title": story["title"][:60],
"page_comments": story["comments"],
"L1": stats(l1),
"L2": stats(l2),
"L3+": stats(l3plus),
"ALL": stats(all_comments),
}
def print_row(label: str, s: Dict):
print(
f" {label:5s} count={s['count']:4d} md_total={s['md_chars']:7d} "
f"avg={s['avg_md']:5d} max={s['max_md']:6d}"
)
async def main():
print("📥 抓取 HN 首页...")
html = await fetch_frontpage(timeout=15)
stories = parse_frontpage_html(html)[:10]
print(f"📋 取前 10 条 (实际 {len(stories)} 条)\n")
async with aiohttp.ClientSession() as session:
results = await asyncio.gather(
*[probe_one(session, s) for s in stories],
return_exceptions=True,
)
summary = {"L1": [], "L2": [], "L3+": [], "ALL": []}
for r in results:
if isinstance(r, Exception):
print(f"{r}\n")
continue
print(
f"#{r['id']} page_comments={r['page_comments']}\n {r['title']}"
)
print_row("L1", r["L1"])
print_row("L2", r["L2"])
print_row("L3+", r["L3+"])
print_row("ALL", r["ALL"])
print()
for key in summary:
summary[key].append(r[key])
print("=" * 70)
print("📊 10 条 story 汇总(平均/总和)\n")
for key in ["L1", "L2", "L3+", "ALL"]:
rows = summary[key]
total_count = sum(r["count"] for r in rows)
total_md = sum(r["md_chars"] for r in rows)
avg_count_per_story = round(total_count / len(rows), 1) if rows else 0
avg_md_per_story = round(total_md / len(rows)) if rows else 0
print(
f" {key:5s} 总数={total_count:4d} 总md字符={total_md:8d} "
f"平均每story count={avg_count_per_story:5.1f} md_chars={avg_md_per_story:6d}"
)
if __name__ == "__main__":
asyncio.run(main())
+168
View File
@@ -0,0 +1,168 @@
#!/usr/bin/env python3
"""推送功能测试 (Step 2)
测试内容:
1. 读取已获取的新闻数据
2. 测试推送到所有已启用的平台
3. 验证推送格式和内容
使用方法:
python tests/push_news.py # 默认从 fetch 数据读取发送 (--fake)
python tests/push_news.py --fake # 从 fetch 数据读取发送
python tests/push_news.py --real # 从 news-data/push-*.md 最新文件发送
"""
import argparse
import asyncio
import sys
from datetime import date
from pathlib import Path
from typing import Optional
sys.path.insert(0, str(Path(__file__).parent.parent))
from dotenv import load_dotenv
load_dotenv()
from src.config import load_config
from src.push import send_to_platforms
from src.storage import (
convert_fetch_json_to_md,
get_fetch_file,
get_last_push_file,
read_entries,
)
def parse_args():
"""解析命令行参数"""
parser = argparse.ArgumentParser(description="推送功能测试")
parser.add_argument(
"--fake", action="store_true", help="从 fetch 数据读取发送(默认)"
)
parser.add_argument(
"--real", action="store_true", help="从 news-data/push-*.md 最新文件发送"
)
return parser.parse_args()
def load_push_content() -> Optional[str]:
"""从 news-data 获取最新的 push 文件内容(去掉 frontmatter"""
last_file = get_last_push_file(data_dir="tests/news-data")
if not last_file:
print(" ❌ 未找到 push 文件")
return None
print(f" 📂 读取文件: {last_file}")
with open(last_file, "r", encoding="utf-8") as f:
content = f.read()
# 去掉 frontmatter
if content.startswith("---"):
parts = content.split("---", 2)
if len(parts) >= 3:
content = parts[2].strip()
return content
async def test_push(mode: str = "fake"):
"""测试推送功能"""
print("=" * 60)
print("📤 推送功能测试 (Step 2)")
print(f" 模式: {'real' if mode == 'real' else 'fake (fetch数据)'}")
print("=" * 60)
# 加载配置
print("\n📋 加载配置...")
config = load_config()
content: Optional[str] = None
title: Optional[str] = None
if mode == "real":
# 从 push 文件读取
content = load_push_content()
if content:
# 提取标题
first_line = content.split("\n")[0] if content else ""
if first_line.startswith("# "):
title = first_line.replace("# ", "").strip()
print(f" 📝 内容长度: {len(content)} 字符")
print(f" 📝 标题: {title}")
else:
# 从 fetch 数据读取(默认)
print("\n📖 读取新闻数据...")
fetch_file = get_fetch_file(date.today(), data_dir="tests/news-data")
entries = read_entries(fetch_file)
print(f" 文件: {fetch_file}")
print(f" 获取到 {len(entries)} 条新闻")
# 同时生成 Markdown 版本便于阅读
md_file = str(fetch_file).replace(".json", ".md")
convert_fetch_json_to_md(fetch_file, md_file)
print(f" Markdown版本: {md_file}")
if not entries:
print("\n⚠️ 没有新闻数据,请先运行 fetch_news.py")
return
# 构建测试消息
print("\n📝 构建推送消息...")
content = build_test_message(entries[:5])
print(f" 消息长度: {len(content)} 字符")
if not content:
print("\n⚠️ 没有内容可推送")
return
# 推送到所有已启用平台
print("\n📤 推送消息...")
try:
await send_to_platforms(
content, config["push"], "📰 AI Daily 每日精选 | {Test:YYYY-MM-DD}"
)
except Exception as e:
print(f" ❌ 推送失败: {e}")
raise
print("\n" + "=" * 60)
print("✅ Step 2 完成: 推送测试")
print("=" * 60)
def build_test_message(entries: list) -> str:
"""构建测试消息"""
lines = [
"📰 **新闻推送测试**",
"",
f"共获取 {len(entries)} 条新闻:",
"",
]
for i, entry in enumerate(entries, 1):
title = entry.get("title", "无标题")
source = entry.get("source", "未知来源")
link = entry.get("link", "")
published = entry.get("published", "")
content = entry.get("content", "")[:100]
lines.append(f"**{i}. {title}**")
lines.append(f" 📰 来源: {source}")
if published:
lines.append(f" ⏰ 时间: {published}")
if link:
lines.append(f" 🔗 链接: {link}")
if content:
lines.append(f" 📝 内容: {content}...")
lines.append("")
return "\n".join(lines)
if __name__ == "__main__":
args = parse_args()
mode = "real" if args.real else "fake"
asyncio.run(test_push(mode))
+1
View File
@@ -0,0 +1 @@
# pytest tests for daily-news project
+154
View File
@@ -0,0 +1,154 @@
"""pytest fixtures for daily-news project"""
import json
import pytest
import sys
from datetime import datetime, timezone, timedelta
from pathlib import Path
from unittest.mock import MagicMock
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
@pytest.fixture
def temp_dir(tmp_path):
"""临时目录fixture"""
return tmp_path
@pytest.fixture
def sample_config():
"""示例配置"""
return {
"sources": {
"base_opml": "resources/rss.opml",
"add": [],
"block": [],
"block_domains": ["*.substack.com"],
},
"filter": {
"min_score": 60,
"hot_threshold": 90,
"context_days": 2,
"keep_days": 7,
},
"schedule": {
"fetch_interval_minutes": 30,
"push_cron": ["0 8 * * *", "0 17 * * *"],
"timezone_hours": 8,
},
"llm": {
"provider": "openai",
"model": "gpt-4o-mini",
"baseUrl": "https://api.openai.com/v1",
"apiKeyName": "OPENAI_API_KEY",
"max_prompt_chars": 10000,
"max_concurrent_batches": 3,
},
"push": {
"discord": {
"enabled": True,
"webhook_url": "https://discord.com/api/webhooks/test/abc",
},
"feishu": {"enabled": False, "apiKeyName": "FEISHU_WEBHOOK_URL"},
},
}
@pytest.fixture
def sample_opml(temp_dir):
"""示例OPML文件"""
opml_content = """<?xml version="1.0"?>
<opml version="2.0">
<body>
<outline title="Feed1" xmlUrl="http://feed1.com/rss" type="rss" category="tech"/>
<outline title="Feed2" xmlUrl="http://feed2.com/rss" type="rss" category="ai"/>
</body>
</opml>"""
opml_file = temp_dir / "test.opml"
opml_file.write_text(opml_content)
return str(opml_file)
@pytest.fixture
def sample_entry():
"""示例新闻条目"""
return {
"title": "Test Article Title",
"link": "https://example.com/article",
"published": datetime.now(timezone.utc).isoformat(),
"fetched_at": datetime.now(timezone.utc).isoformat(),
"source": "Test Source",
"content": "<p>Test content</p>",
"summary": "Test summary",
"tags": ["AI", "Tech"],
"score": 85,
}
@pytest.fixture
def sample_entries():
"""示例新闻条目列表"""
now = datetime.now(timezone.utc)
return [
{
"title": "Article 1",
"link": "https://example.com/1",
"published": now.isoformat(),
"fetched_at": now.isoformat(),
"source": "Source1",
"content": "Content 1",
"summary": "Summary 1",
"tags": ["AI"],
"score": 85,
},
{
"title": "Article 2",
"link": "https://example.com/2",
"published": (now - timedelta(hours=1)).isoformat(),
"fetched_at": now.isoformat(),
"source": "Source2",
"content": "Content 2",
"summary": "Summary 2",
"tags": ["Tech"],
"score": 70,
},
{
"title": "Article 3",
"link": "https://example.com/3",
"published": (now - timedelta(hours=2)).isoformat(),
"fetched_at": now.isoformat(),
"source": "Source3",
"content": "Content 3",
"summary": "Summary 3",
"tags": ["News"],
"score": 55,
},
]
@pytest.fixture
def sample_fetch_json(temp_dir, sample_entries):
"""示例fetch JSON文件"""
data = {
"meta": {"date": datetime.now().date().isoformat()},
"entries": sample_entries,
}
json_file = temp_dir / "fetch-test.json"
json_file.write_text(json.dumps(data, ensure_ascii=False, indent=2))
return str(json_file)
@pytest.fixture
def mock_httpx_session():
"""Mock httpx/aiohttp session"""
mock_response = MagicMock()
mock_response.status = 200
mock_response.text = "<rss></rss>"
mock_session = MagicMock()
mock_session.__aenter__ = MagicMock(return_value=mock_session)
mock_session.__aexit__ = MagicMock(return_value=None)
mock_session.get = MagicMock(return_value=mock_response)
return mock_session
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+249
View File
@@ -0,0 +1,249 @@
"""配置模块测试"""
import json
import pytest
import sys
from datetime import datetime, timezone, timedelta
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src"))
from config import (
load_config,
parse_opml,
merge_sources,
get_timezone,
)
class TestLoadConfig:
"""测试配置加载"""
def test_load_valid_config(self, temp_dir):
config = {
"sources": {"base_opml": "test.opml", "add": [], "block": []},
"filter": {"min_score": 60},
"schedule": {"fetch_interval_minutes": 30},
"llm": {"provider": "groq", "model": "moonshotai/kimi-k2-instruct"},
"push": {"discord": {"enabled": False}},
}
config_file = temp_dir / "config.json"
config_file.write_text(json.dumps(config))
result = load_config(str(config_file))
assert result["filter"]["min_score"] == 60
assert result["llm"]["provider"] == "groq"
def test_load_missing_file(self):
with pytest.raises(FileNotFoundError):
load_config("nonexistent.json")
def test_load_invalid_json(self, temp_dir):
config_file = temp_dir / "config.json"
config_file.write_text("invalid json")
with pytest.raises(json.JSONDecodeError):
load_config(str(config_file))
def test_load_config_with_sources(self, temp_dir, sample_opml):
config = {
"sources": {
"base_opml": sample_opml,
"add": [
{
"title": "Add1",
"xmlUrl": "http://add1.com/rss",
"category": "test",
}
],
"block": [],
},
"filter": {"min_score": 60},
"schedule": {"fetch_interval_minutes": 30, "timezone_hours": 8},
"llm": {"provider": "test"},
"push": {"discord": {"enabled": False}},
}
config_file = temp_dir / "config.json"
config_file.write_text(json.dumps(config))
result = load_config(str(config_file))
assert len(result["sources"]["add"]) == 1
class TestParseOpml:
"""测试OPML解析"""
def test_parse_valid_opml(self, temp_dir):
opml_content = """<?xml version="1.0"?>
<opml version="2.0">
<body>
<outline title="Feed1" xmlUrl="http://feed1.com/rss" type="rss"/>
</body>
</opml>"""
opml_file = temp_dir / "test.opml"
opml_file.write_text(opml_content)
feeds = parse_opml(str(opml_file))
assert len(feeds) == 1
assert feeds[0]["title"] == "Feed1"
assert feeds[0]["xmlUrl"] == "http://feed1.com/rss"
def test_parse_missing_file(self):
feeds = parse_opml("nonexistent.opml")
assert feeds == []
def test_parse_opml_with_category(self, temp_dir):
opml_content = """<?xml version="1.0"?>
<opml version="2.0">
<body>
<outline title="TechFeed" xmlUrl="http://tech.com/rss" type="rss" category="技术"/>
<outline title="AIFeed" xmlUrl="http://ai.com/rss" type="rss" category="AI"/>
</body>
</opml>"""
opml_file = temp_dir / "test.opml"
opml_file.write_text(opml_content)
feeds = parse_opml(str(opml_file))
assert len(feeds) == 2
assert feeds[0]["category"] == "技术"
assert feeds[1]["category"] == "AI"
def test_parse_opml_empty_body(self, temp_dir):
opml_content = """<?xml version="1.0"?>
<opml version="2.0">
<body>
</body>
</opml>"""
opml_file = temp_dir / "test.opml"
opml_file.write_text(opml_content)
feeds = parse_opml(str(opml_file))
assert feeds == []
class TestMergeSources:
"""测试源合并"""
def test_merge_base_and_add(self, sample_opml):
config = {
"base_opml": sample_opml,
"add": [
{"title": "Feed3", "xmlUrl": "http://feed3.com/rss", "category": "test"}
],
"block": [],
}
sources = merge_sources(config)
assert len(sources) == 3
def test_block_by_xmlUrl(self, sample_opml):
config = {
"base_opml": sample_opml,
"add": [],
"block": [{"title": "Block1", "xmlUrl": "http://feed1.com/rss"}],
}
sources = merge_sources(config)
assert all(s["xmlUrl"] != "http://feed1.com/rss" for s in sources)
assert len(sources) == 1
def test_deduplicate_by_xmlUrl(self, sample_opml):
config = {
"base_opml": sample_opml,
"add": [
{
"title": "Duplicate",
"xmlUrl": "http://feed1.com/rss",
"category": "test",
}
],
"block": [],
}
sources = merge_sources(config)
urls = [s["xmlUrl"] for s in sources]
assert len(urls) == len(set(urls))
assert len(sources) == 2
def test_block_domains_wildcard(self, temp_dir):
opml_content = """<?xml version="1.0"?>
<opml version="2.0">
<body>
<outline title="Substack" xmlUrl="https://tech.substack.com/rss" type="rss"/>
<outline title="Blog" xmlUrl="https://tech.blog/rss" type="rss"/>
</body>
</opml>"""
opml_file = temp_dir / "test.opml"
opml_file.write_text(opml_content)
config = {
"base_opml": str(opml_file),
"add": [],
"block": [],
"block_domains": ["*.substack.com"],
}
sources = merge_sources(config)
assert len(sources) == 1
assert sources[0]["xmlUrl"] == "https://tech.blog/rss"
def test_block_domains_exact(self, temp_dir):
opml_content = """<?xml version="1.0"?>
<opml version="2.0">
<body>
<outline title="YouTube" xmlUrl="https://youtube.com/feed" type="rss"/>
<outline title="Blog" xmlUrl="https://tech.blog/rss" type="rss"/>
</body>
</opml>"""
opml_file = temp_dir / "test.opml"
opml_file.write_text(opml_content)
config = {
"base_opml": str(opml_file),
"add": [],
"block": [],
"block_domains": ["youtube.com"],
}
sources = merge_sources(config)
assert len(sources) == 1
assert sources[0]["xmlUrl"] == "https://tech.blog/rss"
def test_block_domains_subdomain(self, temp_dir):
opml_content = """<?xml version="1.0"?>
<opml version="2.0">
<body>
<outline title="Substack1" xmlUrl="https://substack.com/feed" type="rss"/>
<outline title="Substack2" xmlUrl="https://ai.substack.com/feed" type="rss"/>
<outline title="Blog" xmlUrl="https://tech.blog/rss" type="rss"/>
</body>
</opml>"""
opml_file = temp_dir / "test.opml"
opml_file.write_text(opml_content)
config = {
"base_opml": str(opml_file),
"add": [],
"block": [],
"block_domains": ["*.substack.com"],
}
sources = merge_sources(config)
assert len(sources) == 1
assert sources[0]["xmlUrl"] == "https://tech.blog/rss"
class TestGetTimezone:
"""测试时区获取"""
def test_get_timezone_from_config(self, sample_config):
tz = get_timezone(sample_config)
assert isinstance(tz, timezone)
assert tz.utcoffset(datetime.now()).total_seconds() == 8 * 3600
def test_get_timezone_none_config(self):
tz = get_timezone(None)
assert isinstance(tz, timezone)
def test_get_timezone_no_timezone_hours(self):
config = {"schedule": {}}
tz = get_timezone(config)
assert isinstance(tz, timezone)
def test_get_timezone_custom_hours(self):
config = {"schedule": {"timezone_hours": -5}}
tz = get_timezone(config)
assert tz.utcoffset(datetime.now()).total_seconds() == -5 * 3600
+190
View File
@@ -0,0 +1,190 @@
"""RSS抓取模块测试"""
import pytest
import sys
from datetime import datetime, timezone, timedelta
from pathlib import Path
from unittest.mock import MagicMock, patch, AsyncMock
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src"))
from fetcher import (
parse_entry_time,
fetch_single_feed_async,
fetch_all_feeds,
DEFAULT_FEED_TIMEOUT,
)
class TestParseEntryTime:
"""测试时间解析"""
def test_parse_published_parsed(self):
entry = MagicMock()
entry.published_parsed = (2024, 1, 15, 10, 30, 0, 0, 0, 0)
result = parse_entry_time(entry)
assert result is not None
assert result.year == 2024
assert result.month == 1
assert result.day == 15
assert result.tzinfo == timezone.utc
def test_parse_updated_parsed(self):
entry = MagicMock()
entry.published_parsed = None
entry.updated_parsed = (2024, 1, 15, 10, 30, 0, 0, 0, 0)
result = parse_entry_time(entry)
assert result is not None
assert result.year == 2024
assert result.month == 1
assert result.day == 15
def test_parse_no_time(self):
entry = MagicMock()
entry.published_parsed = None
entry.updated_parsed = None
result = parse_entry_time(entry)
assert result is None
class TestFetchSingleFeedAsync:
"""测试单源抓取"""
@pytest.mark.asyncio
async def test_fetch_success(self, temp_dir):
rss_content = """<?xml version="1.0"?>
<rss version="2.0">
<channel>
<title>Test Feed</title>
<item>
<title>Article 1</title>
<link>https://example.com/1</link>
<pubDate>Mon, 15 Jan 2024 10:00:00 GMT</pubDate>
<description>Test description</description>
</item>
</channel>
</rss>"""
feed_info = {"title": "Test Feed", "xmlUrl": "http://test.com/rss"}
cutoff = datetime(2024, 1, 1, tzinfo=timezone.utc)
mock_response = MagicMock()
mock_response.status = 200
mock_response.text = AsyncMock(return_value=rss_content)
mock_session = MagicMock()
mock_session.__aenter__ = AsyncMock(return_value=mock_session)
mock_session.__aexit__ = AsyncMock(return_value=None)
mock_session.get = MagicMock(
return_value=AsyncMock(
__aenter__=AsyncMock(return_value=mock_response),
__aexit__=AsyncMock(return_value=None),
)
)
with patch("aiohttp.ClientSession", return_value=mock_session):
entries = await fetch_single_feed_async(
feed_info, cutoff, session=mock_session
)
assert len(entries) == 1
assert entries[0]["title"] == "Article 1"
assert entries[0]["link"] == "https://example.com/1"
assert entries[0]["source"] == "Test Feed"
@pytest.mark.asyncio
async def test_fetch_http_error(self):
feed_info = {"title": "Test Feed", "xmlUrl": "http://test.com/rss"}
cutoff = datetime(2024, 1, 1, tzinfo=timezone.utc)
mock_response = MagicMock()
mock_response.status = 404
mock_session = MagicMock()
mock_session.get = MagicMock(
return_value=AsyncMock(
__aenter__=AsyncMock(return_value=mock_response),
__aexit__=AsyncMock(return_value=None),
)
)
entries = await fetch_single_feed_async(feed_info, cutoff, session=mock_session)
assert entries == []
@pytest.mark.asyncio
async def test_fetch_timeout(self):
feed_info = {"title": "Test Feed", "xmlUrl": "http://test.com/rss"}
cutoff = datetime(2024, 1, 1, tzinfo=timezone.utc)
import aiohttp
mock_session = MagicMock()
mock_session.get = MagicMock(side_effect=aiohttp.ServerTimeoutError())
entries = await fetch_single_feed_async(feed_info, cutoff, session=mock_session)
assert entries == []
@pytest.mark.asyncio
async def test_fetch_cutoff_filter(self):
feed_info = {"title": "Test Feed", "xmlUrl": "http://test.com/rss"}
cutoff = datetime(2024, 1, 10, tzinfo=timezone.utc)
import feedparser
with patch(
"fetcher.fetch_single_feed_async", new_callable=AsyncMock
) as mock_fetch:
mock_fetch.return_value = [
{
"title": "New",
"link": "https://example.com/new",
"published": datetime(2024, 1, 15, tzinfo=timezone.utc),
}
]
result = await mock_fetch(feed_info, cutoff)
assert len(result) == 1
class TestFetchAllFeeds:
"""测试并发抓取"""
@pytest.mark.asyncio
async def test_concurrent_limit(self):
feeds = [
{"title": f"Feed{i}", "xmlUrl": f"http://feed{i}.com/rss"}
for i in range(20)
]
cutoff = datetime(2024, 1, 1, tzinfo=timezone.utc)
with patch(
"fetcher.fetch_single_feed_async", new_callable=AsyncMock
) as mock_fetch:
mock_fetch.return_value = []
await fetch_all_feeds(feeds, cutoff, max_workers=5)
assert mock_fetch.call_count == 20
@pytest.mark.asyncio
async def test_empty_feeds(self):
cutoff = datetime(2024, 1, 1, tzinfo=timezone.utc)
entries = await fetch_all_feeds([], cutoff)
assert entries == []
@pytest.mark.asyncio
async def test_default_timeout(self):
feed_info = {"title": "Test", "xmlUrl": "http://test.com"}
cutoff = datetime.now(timezone.utc)
with patch(
"fetcher.fetch_single_feed_async", new_callable=AsyncMock
) as mock_fetch:
mock_fetch.return_value = []
await mock_fetch(feed_info, cutoff, timeout=None)
mock_fetch.assert_called_once()
+330
View File
@@ -0,0 +1,330 @@
"""LLM模块测试"""
import json
import pytest
import sys
from datetime import datetime, timezone, timedelta
from pathlib import Path
from unittest.mock import patch, AsyncMock, MagicMock
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src"))
from llm import (
load_prompt,
_parse_llm_json_response,
_split_entries_for_batch,
_build_batch_prompt,
_merge_scores,
_score_single_batch,
call_llm,
check_llm_available,
generate_immediate_push,
score_batch,
)
class TestLoadPrompt:
"""测试提示词加载"""
def test_load_prompt_basic(self, temp_dir):
prompt_file = temp_dir / "test.txt"
prompt_file.write_text("Hello {name}!")
result = load_prompt(str(prompt_file), name="World")
assert result == "Hello World!"
def test_load_prompt_missing_file(self):
with pytest.raises(FileNotFoundError):
load_prompt("nonexistent.txt")
def test_load_prompt_with_braces(self, temp_dir):
prompt_file = temp_dir / "test.txt"
prompt_file.write_text("Hello {name}, curly braces: { }")
result = load_prompt(str(prompt_file), name="World")
assert result == "Hello World, curly braces: { }"
def test_load_prompt_multiple_vars(self, temp_dir):
prompt_file = temp_dir / "test.txt"
prompt_file.write_text("{greeting} {name}, you have {count} messages")
result = load_prompt(str(prompt_file), greeting="Hi", name="Alice", count=5)
assert result == "Hi Alice, you have 5 messages"
class TestParseLlmJsonResponse:
"""测试LLM响应解析"""
def test_parse_json_array(self):
response = '[{"link": "https://example.com", "score": 80}]'
result = _parse_llm_json_response(response)
assert len(result) == 1
assert result[0]["link"] == "https://example.com"
assert result[0]["score"] == 80
def test_parse_with_markdown_codeblock(self):
response = """```json
[{"link": "https://example.com", "score": 80}]
```"""
result = _parse_llm_json_response(response)
assert len(result) == 1
def test_parse_with_codeblock(self):
response = """```
[{"link": "https://example.com", "score": 80}]
```"""
result = _parse_llm_json_response(response)
assert len(result) == 1
def test_parse_invalid_response(self):
response = "This is not JSON at all"
with pytest.raises(ValueError):
_parse_llm_json_response(response)
class TestSplitEntriesForBatch:
"""测试条目分批"""
def test_split_empty(self):
result = _split_entries_for_batch([])
assert result == []
def test_split_single_batch(self):
entries = [
{
"link": f"https://example.com/{i}",
"title": f"Title{i}",
"content": "x" * 100,
}
for i in range(5)
]
result = _split_entries_for_batch(entries, max_prompt_chars=10000)
assert len(result) == 1
def test_split_multiple_batches(self):
entries = [
{
"link": f"https://example.com/{i}",
"title": f"Title{i}",
"content": "x" * 5000,
}
for i in range(10)
]
result = _split_entries_for_batch(entries, max_prompt_chars=10000)
assert len(result) > 1
class TestBuildBatchPrompt:
"""测试构建批量提示词"""
def test_build_batch_prompt_basic(self):
entries = [
{
"link": "https://example.com/1",
"title": "Title1",
"source": "Source1",
"published": "2024-01-15",
"content": "Content",
}
]
result = _build_batch_prompt(entries)
assert "Title1" in result
assert "https://example.com/1" in result
class TestMergeScores:
"""测试评分合并"""
def test_merge_scores_basic(self):
entries = [
{"link": "https://example.com/1", "title": "Title1"},
{"link": "https://example.com/2", "title": "Title2"},
]
scores = [
{
"link": "https://example.com/1",
"score": 85,
"tags": ["AI"],
"summary": "Summary1",
},
{
"link": "https://example.com/2",
"score": 70,
"tags": ["Tech"],
"summary": "Summary2",
},
]
result = _merge_scores(entries, scores)
assert result[0]["score"] == 85
assert result[0]["tags"] == ["AI"]
assert result[1]["score"] == 70
def test_merge_scores_partial(self):
entries = [
{"link": "https://example.com/1", "title": "Title1", "score": 50},
{"link": "https://example.com/2", "title": "Title2", "score": 60},
]
scores = [{"link": "https://example.com/1", "score": 85}]
result = _merge_scores(entries, scores)
assert result[0]["score"] == 85
assert result[1]["score"] == 60
class TestCallLlm:
"""测试LLM调用"""
@pytest.mark.asyncio
async def test_call_llm_success(self):
config = {
"model": "gpt-4",
"baseUrl": "https://api.openai.com/v1",
"apiKeyName": "OPENAI_API_KEY",
}
with patch("llm.call_llm", new_callable=AsyncMock) as mock_call:
mock_call.return_value = "Test response"
result = await mock_call("Test prompt", config)
assert result == "Test response"
@pytest.mark.asyncio
async def test_call_llm_missing_key(self):
config = {"model": "gpt-4", "apiKeyName": "MISSING_KEY"}
with pytest.raises(ValueError, match="未设置MISSING_KEY"):
await call_llm("Test prompt", config)
class TestLlmHealthCheck:
"""测试LLM可用性检查"""
@pytest.mark.asyncio
async def test_check_llm_available_success(self, sample_config):
with patch("llm.call_llm", new_callable=AsyncMock) as mock_call:
mock_call.return_value = "OK"
result = await check_llm_available(sample_config["llm"])
assert result == "OK"
@pytest.mark.asyncio
async def test_check_llm_available_empty_response(self, sample_config):
with patch("llm.call_llm", new_callable=AsyncMock) as mock_call:
mock_call.return_value = " "
with pytest.raises(RuntimeError, match="返回空响应"):
await check_llm_available(sample_config["llm"])
class TestImmediatePush:
"""测试即时推送生成"""
@pytest.mark.asyncio
async def test_generate_immediate_push_failure_returns_error(
self, sample_entries, sample_config
):
with patch("llm.load_prompt", return_value="prompt"), patch(
"llm.call_llm", new_callable=AsyncMock
) as mock_call:
mock_call.side_effect = RuntimeError("boom")
content, error = await generate_immediate_push(
sample_entries[:1], sample_config["llm"], recent_push_context=""
)
assert content == ""
assert error == "生成即时推送失败: boom"
class TestScoreBatch:
"""测试批量评分"""
@pytest.mark.asyncio
async def test_score_batch_empty(self, sample_config):
result, errors = await score_batch([], sample_config["llm"])
assert result == []
assert errors == []
@pytest.mark.asyncio
async def test_score_batch_single(self, sample_entries, sample_config):
entries = sample_entries[:1]
mock_scores = [
{
"link": entries[0]["link"],
"score": 85,
"tags": ["AI"],
"summary": "Test summary",
}
]
with patch("llm._score_single_batch", new_callable=AsyncMock) as mock_score:
mock_score.return_value = (mock_scores, [])
result, errors = await score_batch(entries, sample_config["llm"])
assert len(result) == 1
assert result[0]["score"] == 85
assert errors == []
@pytest.mark.asyncio
async def test_score_single_batch_failure_returns_empty_results(
self, sample_entries, sample_config
):
with patch("llm.call_llm", new_callable=AsyncMock) as mock_call:
mock_call.side_effect = RuntimeError("boom")
results, errors = await _score_single_batch(
sample_entries[:2], sample_config["llm"]
)
assert results == []
assert errors == ["批次1 评分失败: boom"]
@pytest.mark.asyncio
async def test_score_single_batch_reconcile_partial_results(
self, sample_entries, sample_config
):
entries = sample_entries[:2]
llm_results = [
{
"link": entries[0]["link"],
"score": 91,
"tags": ["AI"],
"summary": "Matched result",
}
]
with patch("llm.call_llm", new_callable=AsyncMock) as mock_call:
mock_call.return_value = json.dumps(llm_results, ensure_ascii=False)
results, errors = await _score_single_batch(entries, sample_config["llm"])
assert len(results) == 1
assert results[0]["score"] == 91
assert len(errors) == 1
assert "评分结果异常" in errors[0]
assert "输入2" in errors[0]
assert "返回1" in errors[0]
assert "匹配1" in errors[0]
@pytest.mark.asyncio
async def test_score_single_batch_keeps_full_results(self, sample_entries, sample_config):
entries = sample_entries[:3]
llm_results = [
{
"link": entry["link"],
"score": 88,
"tags": ["AI"],
"summary": f"Summary for {index}",
}
for index, entry in enumerate(entries, start=1)
]
with patch("llm.call_llm", new_callable=AsyncMock) as mock_call:
mock_call.return_value = json.dumps(llm_results, ensure_ascii=False)
results, errors = await _score_single_batch(entries, sample_config["llm"])
assert len(results) == 3
assert [result["link"] for result in results] == [entry["link"] for entry in entries]
assert errors == []
@@ -0,0 +1,101 @@
"""测试新增 LLM 函数 (summarize_github_trending 等)"""
import sys
from pathlib import Path
from unittest.mock import patch, AsyncMock
import pytest
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src"))
from llm import summarize_github_trending, select_ai_related_hn, summarize_hackernews
@pytest.mark.asyncio
async def test_summarize_github_trending_happy_path(tmp_path):
prompt_path = tmp_path / "section_github.md"
prompt_path.write_text("Repos: {repos_json}\nmax_items={max_items}", encoding="utf-8")
config = {
"model": "x",
"baseUrl": "http://x",
"apiKeyName": "DEEPSEEK_API_KEY",
"prompts": {"section_github": str(prompt_path)},
"sections": {"github_trending": {"max_items": 3}},
}
enriched = [{"full_name": "o/r", "readme_excerpt": "rm"}]
with patch("llm.call_llm", new=AsyncMock(return_value="## md")):
md, err = await summarize_github_trending(enriched, config)
assert md == "## md"
assert err is None
@pytest.mark.asyncio
async def test_summarize_github_trending_llm_failure_returns_error(tmp_path):
prompt_path = tmp_path / "section_github.md"
prompt_path.write_text("x {repos_json} {max_items}", encoding="utf-8")
config = {
"model": "x",
"baseUrl": "http://x",
"apiKeyName": "DEEPSEEK_API_KEY",
"prompts": {"section_github": str(prompt_path)},
"sections": {"github_trending": {"max_items": 3}},
}
with patch("llm.call_llm", new=AsyncMock(side_effect=RuntimeError("boom"))):
md, err = await summarize_github_trending([{"full_name": "o/r"}], config)
assert md == ""
assert "boom" in err
@pytest.mark.asyncio
async def test_select_ai_related_hn_parses_id_array(tmp_path):
prompt_path = tmp_path / "select.md"
prompt_path.write_text("k={k} candidates={candidates_json}", encoding="utf-8")
config = {
"model": "x",
"baseUrl": "http://x",
"apiKeyName": "DEEPSEEK_API_KEY",
"prompts": {"section_hackernews_select": str(prompt_path)},
}
with patch("llm.call_llm", new=AsyncMock(return_value='["111", "222"]')):
ids, err = await select_ai_related_hn(
[{"id": "111"}, {"id": "222"}, {"id": "333"}], k=2, config=config
)
assert ids == ["111", "222"]
assert err is None
@pytest.mark.asyncio
async def test_select_ai_related_hn_empty_array(tmp_path):
prompt_path = tmp_path / "select.md"
prompt_path.write_text("{k}{candidates_json}", encoding="utf-8")
config = {
"model": "x",
"baseUrl": "http://x",
"apiKeyName": "DEEPSEEK_API_KEY",
"prompts": {"section_hackernews_select": str(prompt_path)},
}
with patch("llm.call_llm", new=AsyncMock(return_value="[]")):
ids, err = await select_ai_related_hn([{"id": "1"}], k=1, config=config)
assert ids == []
assert err is None
@pytest.mark.asyncio
async def test_summarize_hackernews_happy(tmp_path):
prompt_path = tmp_path / "hn.md"
prompt_path.write_text("{stories_json}", encoding="utf-8")
config = {
"model": "x",
"baseUrl": "http://x",
"apiKeyName": "DEEPSEEK_API_KEY",
"prompts": {"section_hackernews": str(prompt_path)},
}
with patch("llm.call_llm", new=AsyncMock(return_value="## HN summary")):
md, err = await summarize_hackernews(
[{"id": "1", "title": "t", "link_content": "x", "top_comments": []}], config
)
assert md == "## HN summary"
assert err is None
+285
View File
@@ -0,0 +1,285 @@
"""主程序逻辑测试"""
import json
import pytest
import sys
from datetime import datetime, timezone, timedelta, date
from pathlib import Path
from unittest.mock import patch, MagicMock, AsyncMock
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src"))
from main import (
now_local,
parse_time_to_local,
calculate_push_times,
collect_entries_for_push,
main as run_main,
)
class TestNowLocal:
"""测试获取本地时间"""
def test_now_local_with_config(self, sample_config):
result = now_local(sample_config)
assert isinstance(result, datetime)
assert result.tzinfo is not None
def test_now_local_without_config(self):
result = now_local()
assert isinstance(result, datetime)
class TestParseTimeToLocal:
"""测试时间解析"""
def test_parse_iso_format(self, sample_config):
result = parse_time_to_local("2024-01-15T10:30:00+00:00", sample_config)
assert result is not None
assert result.year == 2024
assert result.month == 1
assert result.day == 15
def test_parse_with_z_suffix(self, sample_config):
result = parse_time_to_local("2024-01-15T10:30:00Z", sample_config)
assert result is not None
assert result.year == 2024
def test_parse_invalid_format(self, sample_config):
result = parse_time_to_local("not-a-date", sample_config)
assert result is None
def test_parse_none(self, sample_config):
result = parse_time_to_local("", sample_config)
assert result is None
class TestCalculatePushTimes:
"""测试推送时间计算"""
def test_calculate_single_cron(self, sample_config):
times = calculate_push_times(["30 8 * * *"], config=sample_config)
assert len(times) == 1
assert times[0].hour == 8
assert times[0].minute == 30
def test_calculate_multiple_crons(self, sample_config):
times = calculate_push_times(["0 8 * * *", "0 17 * * *"], config=sample_config)
assert len(times) == 2
hours = [t.hour for t in times]
assert 8 in hours
assert 17 in hours
def test_calculate_with_offset(self, sample_config):
times = calculate_push_times(["0 8 * * *"], offset_days=1, config=sample_config)
assert len(times) == 1
expected_date = (datetime.now(timezone.utc) + timedelta(days=1)).date()
assert times[0].date() == expected_date
def test_calculate_invalid_cron(self, sample_config):
times = calculate_push_times(["invalid cron"], config=sample_config)
assert times == []
class TestCollectEntriesForPush:
"""测试收集推送条目"""
def test_collect_no_files(self, temp_dir):
to_push, context = collect_entries_for_push(
last_push_time=None, context_days=2, min_score=60, data_dir=str(temp_dir)
)
assert to_push == []
assert context == []
def test_collect_with_low_score(self, temp_dir):
now = datetime.now(timezone.utc)
data = {
"meta": {"date": now.date().isoformat()},
"entries": [
{
"title": "Low Score",
"link": "https://example.com/1",
"score": 30,
"fetched_at": now.isoformat(),
}
],
}
fetch_file = temp_dir / f"fetch-{now.date().isoformat()}.json"
with open(fetch_file, "w") as f:
json.dump(data, f)
to_push, context = collect_entries_for_push(
last_push_time=None, context_days=2, min_score=60, data_dir=str(temp_dir)
)
assert len(to_push) == 0
def test_collect_with_high_score(self, temp_dir):
now = datetime.now(timezone.utc)
data = {
"meta": {"date": now.date().isoformat()},
"entries": [
{
"title": "High Score",
"link": "https://example.com/1",
"score": 85,
"fetched_at": now.isoformat(),
}
],
}
fetch_file = temp_dir / f"fetch-{now.date().isoformat()}.json"
with open(fetch_file, "w") as f:
json.dump(data, f)
to_push, context = collect_entries_for_push(
last_push_time=None, context_days=2, min_score=60, data_dir=str(temp_dir)
)
assert len(to_push) == 1
assert to_push[0]["score"] == 85
def test_collect_with_last_push_time(self, temp_dir):
now = datetime.now(timezone.utc)
last_push = now - timedelta(hours=2)
data = {
"meta": {"date": now.date().isoformat()},
"entries": [
{
"title": "New Entry",
"link": "https://example.com/1",
"score": 80,
"fetched_at": now.isoformat(),
},
{
"title": "Old Entry",
"link": "https://example.com/2",
"score": 80,
"fetched_at": last_push.isoformat(),
},
],
}
fetch_file = temp_dir / f"fetch-{now.date().isoformat()}.json"
with open(fetch_file, "w") as f:
json.dump(data, f)
to_push, context = collect_entries_for_push(
last_push_time=last_push,
context_days=2,
min_score=60,
data_dir=str(temp_dir),
)
assert len(to_push) == 1
assert to_push[0]["title"] == "New Entry"
def test_collect_context_limit(self, temp_dir):
now = datetime.now(timezone.utc)
entries = [
{
"title": f"Entry{i}",
"link": f"https://example.com/{i}",
"score": 50 + i,
"fetched_at": now.isoformat(),
}
for i in range(60)
]
data = {"meta": {"date": now.date().isoformat()}, "entries": entries}
fetch_file = temp_dir / f"fetch-{now.date().isoformat()}.json"
with open(fetch_file, "w") as f:
json.dump(data, f)
to_push, context = collect_entries_for_push(
last_push_time=None, context_days=2, min_score=60, data_dir=str(temp_dir)
)
assert len(context) <= 50
def test_collect_multi_day(self, temp_dir):
from src.config import get_timezone
tz = get_timezone()
today = datetime.now(tz)
yesterday = today - timedelta(days=1)
today_data = {
"meta": {"date": today.date().isoformat()},
"entries": [
{
"title": "Today Entry",
"link": "https://example.com/1",
"score": 80,
"fetched_at": today.isoformat(),
}
],
}
yesterday_data = {
"meta": {"date": yesterday.date().isoformat()},
"entries": [
{
"title": "Yesterday Entry",
"link": "https://example.com/2",
"score": 75,
"fetched_at": yesterday.isoformat(),
}
],
}
(temp_dir / f"fetch-{today.date().isoformat()}.json").write_text(
json.dumps(today_data)
)
(temp_dir / f"fetch-{yesterday.date().isoformat()}.json").write_text(
json.dumps(yesterday_data)
)
to_push, context = collect_entries_for_push(
last_push_time=None, context_days=2, min_score=60, data_dir=str(temp_dir)
)
assert len(to_push) >= 1
class TestMainStartup:
"""测试主程序启动流程"""
@pytest.mark.asyncio
async def test_main_checks_llm_before_starting_loops(self, sample_config):
with patch("main.load_config", return_value=sample_config), patch(
"main.check_llm_available", new_callable=AsyncMock
) as mock_check, patch(
"main.fetch_loop", new_callable=AsyncMock
) as mock_fetch_loop, patch(
"main.push_loop", new_callable=AsyncMock
) as mock_push_loop:
await run_main()
mock_check.assert_awaited_once_with(sample_config["llm"])
mock_fetch_loop.assert_awaited_once_with(sample_config)
mock_push_loop.assert_awaited_once_with(sample_config)
@pytest.mark.asyncio
async def test_main_exits_when_llm_health_check_fails(self, sample_config):
with patch("main.load_config", return_value=sample_config), patch(
"main.check_llm_available", new_callable=AsyncMock
) as mock_check, patch(
"main.fetch_loop", new_callable=AsyncMock
) as mock_fetch_loop, patch(
"main.push_loop", new_callable=AsyncMock
) as mock_push_loop:
mock_check.side_effect = RuntimeError("health failed")
await run_main()
mock_check.assert_awaited_once_with(sample_config["llm"])
mock_fetch_loop.assert_not_called()
mock_push_loop.assert_not_called()
@@ -0,0 +1,123 @@
"""测试早报四模块编排:gather + insights 串行 + sentinel 拼装 + 失败隔离"""
import sys
from pathlib import Path
from unittest.mock import patch, AsyncMock
import pytest
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
from src.main import _run_morning_push
def _insights_meta() -> dict:
return {
"title": "📰 AI Daily 每日精选 | 2026-05-22",
"lead": "lead text",
"highlights": ["a"],
"profile": "morning",
"date": "2026-05-22",
"excerpt": "",
"seotitle": "",
"seodescription": "",
}
@pytest.mark.asyncio
async def test_assembles_all_four_sections(sample_config):
sample_config["filter"]["push_context_days"] = 5
sent = {}
async def fake_send(content, push_cfg, title=None, metadata=None):
sent["content"] = content
sent["metadata"] = metadata
saved = {}
def fake_save(
filepath,
content,
source_count,
total_entries,
profile="default",
metadata=None,
):
saved["profile"] = profile
saved["content"] = content
saved["metadata"] = metadata
with patch(
"src.main.run_rss_section", new=AsyncMock(return_value=("R", None, None))
), patch(
"src.main.run_github_section", new=AsyncMock(return_value=("G", None))
), patch(
"src.main.run_hackernews_section", new=AsyncMock(return_value=("H", None))
), patch(
"src.main.run_insights_section",
new=AsyncMock(return_value=("I", _insights_meta(), None)),
), patch(
"src.main.send_to_platforms", new=AsyncMock(side_effect=fake_send)
), patch(
"src.main.save_push_file", side_effect=fake_save
):
await _run_morning_push(sample_config)
assert "SECTION:rss" in sent["content"]
assert "SECTION:github" in sent["content"]
assert "SECTION:hackernews" in sent["content"]
assert "SECTION:insights" in sent["content"]
assert sent["metadata"]["profile"] == "morning"
assert saved["profile"] == "morning"
assert saved["metadata"]["lead"] == "lead text"
@pytest.mark.asyncio
async def test_rss_failure_raises_to_caller(sample_config):
sample_config["filter"]["push_context_days"] = 5
with patch(
"src.main.run_rss_section",
new=AsyncMock(return_value=("", None, "compose_digest 失败")),
), patch(
"src.main.run_github_section", new=AsyncMock(return_value=("G", None))
), patch(
"src.main.run_hackernews_section", new=AsyncMock(return_value=("H", None))
), patch(
"src.main.notify_llm_errors", new=AsyncMock()
):
with pytest.raises(RuntimeError):
await _run_morning_push(sample_config)
@pytest.mark.asyncio
async def test_section_failure_degrades_to_omission(sample_config):
sample_config["filter"]["push_context_days"] = 5
sent = {}
async def fake_send(content, push_cfg, title=None, metadata=None):
sent["content"] = content
with patch(
"src.main.run_rss_section", new=AsyncMock(return_value=("R", None, None))
), patch(
"src.main.run_github_section", new=AsyncMock(return_value=("", "gh down"))
), patch(
"src.main.run_hackernews_section", new=AsyncMock(return_value=("H", None))
), patch(
"src.main.run_insights_section",
new=AsyncMock(return_value=("I", _insights_meta(), None)),
), patch(
"src.main.notify_llm_errors", new=AsyncMock()
), patch(
"src.main.send_to_platforms", new=AsyncMock(side_effect=fake_send)
), patch(
"src.main.save_push_file"
):
await _run_morning_push(sample_config)
assert "SECTION:rss" in sent["content"]
assert "SECTION:github" not in sent["content"]
assert "SECTION:hackernews" in sent["content"]
@@ -0,0 +1,43 @@
"""测试 run_push_job 的早报/默认路径分发"""
import sys
from pathlib import Path
from unittest.mock import patch, AsyncMock
import pytest
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
from src.main import run_push_job
@pytest.mark.asyncio
async def test_default_path_when_not_morning(sample_config):
sample_config["schedule"]["push_cron"] = ["0 8 * * *", "0 17 * * *"]
sample_config["filter"]["push_context_days"] = 5
with patch("src.main.is_morning_push", return_value=False), patch(
"src.main._run_default_push", new=AsyncMock(return_value=None)
) as default_path, patch(
"src.main._run_morning_push", new=AsyncMock(return_value=None)
) as morning_path:
await run_push_job(sample_config)
default_path.assert_awaited_once()
morning_path.assert_not_awaited()
@pytest.mark.asyncio
async def test_morning_path_when_morning(sample_config):
sample_config["schedule"]["push_cron"] = ["0 8 * * *", "0 17 * * *"]
sample_config["filter"]["push_context_days"] = 5
with patch("src.main.is_morning_push", return_value=True), patch(
"src.main._run_default_push", new=AsyncMock(return_value=None)
) as default_path, patch(
"src.main._run_morning_push", new=AsyncMock(return_value=None)
) as morning_path:
await run_push_job(sample_config)
morning_path.assert_awaited_once()
default_path.assert_not_awaited()
@@ -0,0 +1,60 @@
"""测试早报判定:push_cron 最近最早匹配"""
import sys
from datetime import datetime, timedelta, timezone
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
from src.main import is_morning_push
TZ = timezone(timedelta(hours=8))
def _cfg(push_cron):
return {"schedule": {"push_cron": push_cron, "timezone_hours": 8}}
def test_returns_false_when_push_cron_empty():
assert is_morning_push(datetime(2026, 5, 17, 8, 0, tzinfo=TZ), {"schedule": {}}) is False
assert is_morning_push(datetime(2026, 5, 17, 8, 0, tzinfo=TZ), _cfg([])) is False
def test_single_cron_always_morning():
"""push_cron 只有一项时,任何触发(含手动 / 非整点)都视为早报"""
cfg = _cfg(["0 8 * * *"])
assert is_morning_push(datetime(2026, 5, 17, 8, 0, tzinfo=TZ), cfg) is True
assert is_morning_push(datetime(2026, 5, 17, 12, 30, tzinfo=TZ), cfg) is True
assert is_morning_push(datetime(2026, 5, 17, 23, 59, tzinfo=TZ), cfg) is True
def test_earliest_cron_match_is_morning():
"""触发时刻离最早 cron 最近 → 早报"""
cfg = _cfg(["0 8 * * *", "0 17 * * *"])
assert is_morning_push(datetime(2026, 5, 17, 8, 0, tzinfo=TZ), cfg) is True
assert is_morning_push(datetime(2026, 5, 17, 8, 30, tzinfo=TZ), cfg) is True
assert is_morning_push(datetime(2026, 5, 17, 6, 0, tzinfo=TZ), cfg) is True
def test_later_cron_match_not_morning():
"""触发时刻离非最早 cron 最近 → 默认"""
cfg = _cfg(["0 8 * * *", "0 17 * * *"])
assert is_morning_push(datetime(2026, 5, 17, 17, 0, tzinfo=TZ), cfg) is False
assert is_morning_push(datetime(2026, 5, 17, 16, 30, tzinfo=TZ), cfg) is False
assert is_morning_push(datetime(2026, 5, 17, 22, 0, tzinfo=TZ), cfg) is False
def test_drift_tolerance_via_closest_match():
"""无显式容差,但「最近匹配」自动吸附小幅漂移 (08:01 仍归 08:00)"""
cfg = _cfg(["0 8 * * *", "0 17 * * *"])
assert is_morning_push(datetime(2026, 5, 17, 8, 1, tzinfo=TZ), cfg) is True
assert is_morning_push(datetime(2026, 5, 17, 17, 1, tzinfo=TZ), cfg) is False
def test_three_crons_only_earliest_is_morning():
"""三条 cron 时,只有最早那条对应的触发是早报"""
cfg = _cfg(["0 8 * * *", "0 12 * * *", "0 20 * * *"])
assert is_morning_push(datetime(2026, 5, 17, 8, 0, tzinfo=TZ), cfg) is True
assert is_morning_push(datetime(2026, 5, 17, 12, 0, tzinfo=TZ), cfg) is False
assert is_morning_push(datetime(2026, 5, 17, 20, 0, tzinfo=TZ), cfg) is False
@@ -0,0 +1,87 @@
"""内容处理模块测试"""
import pytest
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src"))
from processor import html_to_markdown
class TestHtmlToMarkdown:
"""测试HTML转Markdown"""
def test_convert_basic_html(self):
html = "<p>Hello World</p>"
result = html_to_markdown(html)
assert "Hello World" in result
def test_convert_with_links(self):
html = '<a href="https://example.com">Click here</a>'
result = html_to_markdown(html)
assert "[Click here](https://example.com)" in result
def test_convert_with_images(self):
html = '<img src="https://example.com/image.png" alt="Image">'
result = html_to_markdown(html)
assert "![Image](https://example.com/image.png)" in result
def test_convert_with_headings(self):
html = "<h1>Title</h1><h2>Subtitle</h2>"
result = html_to_markdown(html)
assert "# Title" in result
assert "## Subtitle" in result
def test_convert_with_lists(self):
html = "<ul><li>Item 1</li><li>Item 2</li></ul>"
result = html_to_markdown(html)
assert "Item 1" in result
assert "Item 2" in result
def test_convert_with_strong_emphasis(self):
html = "<strong>Bold</strong> and <em>italic</em>"
result = html_to_markdown(html)
assert "**Bold**" in result
assert "*italic*" in result
def test_relative_link_conversion(self):
html = '<a href="/article/123">Read more</a>'
result = html_to_markdown(html, base_url="https://example.com/blog")
assert "https://example.com/article/123" in result
def test_relative_image_conversion(self):
html = '<img src="/images/logo.png">'
result = html_to_markdown(html, base_url="https://example.com")
assert "https://example.com/images/logo.png" in result
def test_absolute_link_unchanged(self):
html = '<a href="https://other.com/page">Link</a>'
result = html_to_markdown(html, base_url="https://example.com")
assert "https://other.com/page" in result
def test_remove_xgo_ing_link(self):
html = "<p>Content</p><p>[⚡ Powered by xgo.ing](https://xgo.ing)</p>"
result = html_to_markdown(html)
assert "xgo.ing" not in result
assert "Content" in result
def test_remove_xgo_ing_link_with_slash(self):
html = "<p>Content</p><p>[⚡ Powered by xgo.ing](https://xgo.ing/)</p>"
result = html_to_markdown(html)
assert "xgo.ing" not in result
def test_clean_extra_newlines(self):
html = "<p>Line 1</p>\n\n\n\n<p>Line 2</p>"
result = html_to_markdown(html)
assert "\n\n\n\n" not in result
def test_empty_html(self):
result = html_to_markdown("")
assert result.strip() == ""
def test_html_with_nbsp(self):
html = "<p>Hello&nbsp;World</p>"
result = html_to_markdown(html)
assert "Hello" in result
assert "World" in result
+209
View File
@@ -0,0 +1,209 @@
"""推送模块测试"""
import os
import pytest
import sys
from pathlib import Path
from unittest.mock import patch, AsyncMock, MagicMock
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src"))
from push.discord import DiscordPlatform
from push.feishu import FeishuPlatform
from push import create_platform
class TestDiscordPlatform:
"""测试Discord推送"""
def test_validate_config_valid(self):
config = {
"enabled": True,
"apiKeyName": "DISCORD_WEBHOOK_URL",
}
with patch.dict(
os.environ,
{"DISCORD_WEBHOOK_URL": "https://discord.com/api/webhooks/123456/abcdef"},
):
platform = DiscordPlatform(config)
assert platform.validate_config(config) is True
def test_validate_config_disabled(self):
config = {
"enabled": False,
"apiKeyName": "DISCORD_WEBHOOK_URL",
}
with patch.dict(
os.environ,
{"DISCORD_WEBHOOK_URL": "https://discord.com/api/webhooks/123456/abcdef"},
):
platform = DiscordPlatform(config)
assert platform.validate_config(config) is False
def test_validate_config_missing_webhook(self):
config = {"enabled": True, "apiKeyName": "DISCORD_WEBHOOK_URL"}
with patch.dict(os.environ, {"DISCORD_WEBHOOK_URL": ""}):
platform = DiscordPlatform(config)
assert platform.validate_config(config) is False
def test_validate_config_invalid_url(self):
config = {"enabled": True, "apiKeyName": "DISCORD_WEBHOOK_URL"}
with patch.dict(os.environ, {"DISCORD_WEBHOOK_URL": "not-a-valid-url"}):
platform = DiscordPlatform(config)
assert platform.validate_config(config) is False
def test_validate_config_wrong_domain(self):
config = {"enabled": True, "apiKeyName": "DISCORD_WEBHOOK_URL"}
with patch.dict(
os.environ, {"DISCORD_WEBHOOK_URL": "https://example.com/webhook"}
):
platform = DiscordPlatform(config)
assert platform.validate_config(config) is False
def test_split_content_short(self):
with patch.dict(os.environ, {"DISCORD_WEBHOOK_URL": "https://test.com"}):
config = {"apiKeyName": "DISCORD_WEBHOOK_URL"}
platform = DiscordPlatform(config)
short_content = "Hello"
chunks = platform._split_content(short_content, limit=2000)
assert len(chunks) == 1
assert chunks[0] == "Hello"
def test_split_content_long_message(self):
with patch.dict(os.environ, {"DISCORD_WEBHOOK_URL": "https://test.com"}):
config = {"apiKeyName": "DISCORD_WEBHOOK_URL"}
platform = DiscordPlatform(config)
long_content = "A\n" * 2500
chunks = platform._split_content(long_content, limit=2000)
assert len(chunks) > 1
assert all(len(c) <= 2000 for c in chunks)
def test_split_content_exact_boundary(self):
with patch.dict(os.environ, {"DISCORD_WEBHOOK_URL": "https://test.com"}):
config = {"apiKeyName": "DISCORD_WEBHOOK_URL"}
platform = DiscordPlatform(config)
content = "A" * 2000
chunks = platform._split_content(content, limit=2000)
assert len(chunks) == 1
def test_split_content_unicode(self):
with patch.dict(os.environ, {"DISCORD_WEBHOOK_URL": "https://test.com"}):
config = {"apiKeyName": "DISCORD_WEBHOOK_URL"}
platform = DiscordPlatform(config)
content = "你好" * 500
chunks = platform._split_content(content, limit=100)
assert len(chunks) >= 1
@pytest.mark.asyncio
async def test_send_success(self):
with patch.dict(
os.environ,
{"DISCORD_WEBHOOK_URL": "https://discord.com/api/webhooks/test/abc"},
):
config = {"apiKeyName": "DISCORD_WEBHOOK_URL"}
platform = DiscordPlatform(config)
with patch.object(platform, "send", new_callable=AsyncMock) as mock_send:
mock_send.return_value = True
result = await mock_send("Test message")
assert result is True
@pytest.mark.asyncio
async def test_send_failure(self):
with patch.dict(
os.environ,
{"DISCORD_WEBHOOK_URL": "https://discord.com/api/webhooks/test/abc"},
):
config = {"apiKeyName": "DISCORD_WEBHOOK_URL"}
platform = DiscordPlatform(config)
with patch.object(platform, "send", new_callable=AsyncMock) as mock_send:
mock_send.return_value = False
result = await mock_send("Test message")
assert result is False
class TestFeishuPlatform:
"""测试飞书推送"""
def test_validate_config_valid(self):
config = {
"enabled": True,
"apiKeyName": "FEISHU_WEBHOOK_URL",
}
with patch.dict(os.environ, {"FEISHU_WEBHOOK_URL": "https://open.feishu.cn/open-apis/bot/v2/hook/test"}):
platform = FeishuPlatform(config)
assert platform.validate_config(config) is True
def test_validate_config_disabled(self):
config = {
"enabled": False,
"apiKeyName": "FEISHU_WEBHOOK_URL",
}
with patch.dict(os.environ, {"FEISHU_WEBHOOK_URL": "https://open.feishu.cn/open-apis/bot/v2/hook/test"}):
platform = FeishuPlatform(config)
assert platform.validate_config(config) is False
def test_validate_config_missing_key(self):
config = {"enabled": True, "apiKeyName": "FEISHU_WEBHOOK_URL"}
with patch.dict(os.environ, {"FEISHU_WEBHOOK_URL": ""}):
platform = FeishuPlatform(config)
assert platform.validate_config(config) is False
def test_validate_config_any_non_empty_webhook(self):
config = {"enabled": True, "apiKeyName": "FEISHU_WEBHOOK_URL"}
with patch.dict(os.environ, {"FEISHU_WEBHOOK_URL": "https://open.feishu.cn/open-apis/bot/v2/hook/test"}):
platform = FeishuPlatform(config)
assert platform.validate_config(config) is True
class TestPushFactory:
"""测试平台工厂"""
def test_create_enabled_platform(self):
config = {
"enabled": True,
"apiKeyName": "DISCORD_WEBHOOK_URL",
}
with patch.dict(
os.environ,
{"DISCORD_WEBHOOK_URL": "https://discord.com/api/webhooks/123/abc"},
):
platform = create_platform("discord", config)
assert platform is not None
def test_create_disabled_platform_returns_none(self):
config = {"enabled": False, "apiKeyName": "DISCORD_WEBHOOK_URL"}
platform = create_platform("discord", config)
assert platform is None
def test_create_unknown_platform_raises(self):
with pytest.raises(ValueError):
create_platform("unknown", {})
def test_create_feishu_platform(self):
config = {
"enabled": True,
"apiKeyName": "FEISHU_WEBHOOK_URL",
}
with patch.dict(os.environ, {"FEISHU_WEBHOOK_URL": "https://open.feishu.cn/open-apis/bot/v2/hook/test"}):
platform = create_platform("feishu", config)
assert platform is not None
assert isinstance(platform, FeishuPlatform)
def test_create_discord_platform(self):
config = {
"enabled": True,
"apiKeyName": "DISCORD_WEBHOOK_URL",
}
with patch.dict(
os.environ,
{"DISCORD_WEBHOOK_URL": "https://discord.com/api/webhooks/test/abc"},
):
platform = create_platform("discord", config)
assert platform is not None
assert isinstance(platform, DiscordPlatform)
@@ -0,0 +1,107 @@
"""测试 GitHub REST API enrich 字段映射、archived 过滤、token 鉴权头"""
import base64
import sys
from pathlib import Path
from unittest.mock import patch, AsyncMock, MagicMock
import pytest
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
from src.sections.github.repo_enricher import enrich_repo, _auth_headers
def test_auth_headers_with_token(monkeypatch):
monkeypatch.setenv("GITHUB_TOKEN", "ghp_secret")
headers = _auth_headers(token_env="GITHUB_TOKEN")
assert headers["Authorization"] == "Bearer ghp_secret"
assert headers["Accept"] == "application/vnd.github+json"
def test_auth_headers_without_token(monkeypatch):
monkeypatch.delenv("GITHUB_TOKEN", raising=False)
headers = _auth_headers(token_env="GITHUB_TOKEN")
assert "Authorization" not in headers
assert headers["Accept"] == "application/vnd.github+json"
@pytest.mark.asyncio
async def test_enrich_repo_merges_metadata_and_readme():
readme_body = "# Title\n\nProject description here."
readme_b64 = base64.b64encode(readme_body.encode("utf-8")).decode("ascii")
metadata_payload = {
"description": "real desc",
"topics": ["llm", "rag"],
"license": {"spdx_id": "MIT"},
"pushed_at": "2026-05-16T10:00:00Z",
"archived": False,
}
readme_payload = {"content": readme_b64, "encoding": "base64"}
async def fake_get_json(session, url, **kwargs):
if url.endswith("/readme"):
return readme_payload
return metadata_payload
base = {
"url": "https://github.com/o/r",
"full_name": "o/r",
"description": "from trending",
"language": "Python",
"stars_today": 100,
"stars_total": 5000,
}
with patch(
"src.sections.github.repo_enricher._get_json", new=AsyncMock(side_effect=fake_get_json)
):
enriched = await enrich_repo(
session=MagicMock(), repo=base, token_env="GITHUB_TOKEN", readme_max_chars=200
)
assert enriched["topics"] == ["llm", "rag"]
assert enriched["license"] == "MIT"
assert enriched["pushed_at"] == "2026-05-16T10:00:00Z"
assert "Project description" in enriched["readme_excerpt"]
assert enriched["stars_today"] == 100 # trending 已有字段保留
@pytest.mark.asyncio
async def test_enrich_repo_returns_none_when_archived():
metadata_payload = {"archived": True, "topics": [], "pushed_at": "x"}
async def fake_get_json(session, url, **kwargs):
if url.endswith("/readme"):
return {"content": ""}
return metadata_payload
base = {"url": "https://github.com/o/r", "full_name": "o/r"}
with patch(
"src.sections.github.repo_enricher._get_json", new=AsyncMock(side_effect=fake_get_json)
):
result = await enrich_repo(
session=MagicMock(), repo=base, token_env="GITHUB_TOKEN", readme_max_chars=200
)
assert result is None
@pytest.mark.asyncio
async def test_enrich_repo_truncates_readme():
readme_body = "x" * 5000
readme_b64 = base64.b64encode(readme_body.encode("utf-8")).decode("ascii")
async def fake_get_json(session, url, **kwargs):
if url.endswith("/readme"):
return {"content": readme_b64, "encoding": "base64"}
return {"archived": False, "topics": [], "pushed_at": "p"}
base = {"url": "https://github.com/o/r", "full_name": "o/r"}
with patch(
"src.sections.github.repo_enricher._get_json", new=AsyncMock(side_effect=fake_get_json)
):
enriched = await enrich_repo(
session=MagicMock(), repo=base, token_env="GITHUB_TOKEN", readme_max_chars=100
)
assert len(enriched["readme_excerpt"]) == 100
@@ -0,0 +1,40 @@
"""测试 GitHub trending HTML 解析"""
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
from src.sections.github.trending_scraper import parse_trending_html
def test_parse_trending_html_returns_repo_dicts():
fixture = (
Path(__file__).parent / "fixtures" / "github_trending.html"
).read_text(encoding="utf-8")
repos = parse_trending_html(fixture)
assert len(repos) > 0
first = repos[0]
assert first["url"].startswith("https://github.com/")
assert "/" in first["full_name"]
assert isinstance(first["stars_today"], int)
assert isinstance(first["stars_total"], int)
# description / language 可为空字符串但必须是 str
assert isinstance(first["description"], str)
assert isinstance(first["language"], str)
def test_parse_trending_html_dedupes_by_url():
fixture = (
Path(__file__).parent / "fixtures" / "github_trending.html"
).read_text(encoding="utf-8")
repos = parse_trending_html(fixture)
urls = [r["url"] for r in repos]
assert len(urls) == len(set(urls))
def test_parse_trending_html_empty_input():
assert parse_trending_html("") == []
assert parse_trending_html("<html><body>no repos</body></html>") == []
@@ -0,0 +1,145 @@
"""测试 GitHub 板块编排:抓取 → history 过滤 → enrich → LLM 总结"""
import sys
from pathlib import Path
from unittest.mock import patch, AsyncMock
import pytest
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
from src.sections.github.section import run_github_section
# Stub summarize_github_trending until Task 11 provides it
import src.llm as _llm
if not hasattr(_llm, "summarize_github_trending"):
async def _stub(*a, **k):
return "", None
_llm.summarize_github_trending = _stub
def _cfg(history_file: str, max_deep_dive: int = 10) -> dict:
return {
"filter": {"keep_days": 7},
"sections": {
"github_trending": {
"enabled": True,
"max_items": 3,
"max_deep_dive": max_deep_dive,
"readme_max_chars": 3000,
"history_file": history_file,
"request_timeout": 10,
"tokenName": "GITHUB_TOKEN",
}
},
"llm": {
"model": "x",
"baseUrl": "http://x",
"apiKeyName": "DEEPSEEK_API_KEY",
"prompts": {"section_github": "prompts/section_github.md"},
},
}
@pytest.mark.asyncio
async def test_disabled_returns_empty(tmp_path):
cfg = _cfg(str(tmp_path / "h.json"))
cfg["sections"]["github_trending"]["enabled"] = False
md, err = await run_github_section(cfg, now=None)
assert md == ""
assert err is None
@pytest.mark.asyncio
async def test_no_candidates_after_history_returns_empty(tmp_path):
history_path = tmp_path / "h.json"
# 预置 history,使得今日 scrape 出来的 repo 都已存在
history_path.write_text(
'{"repos": {"https://github.com/a/b": "2026-05-16"}, "updated_at": "x"}',
encoding="utf-8",
)
cfg = _cfg(str(history_path))
with patch(
"src.sections.github.section.fetch_trending_page", new=AsyncMock(return_value="<html>")
), patch(
"src.sections.github.section.parse_trending_html",
return_value=[{"url": "https://github.com/a/b", "full_name": "a/b"}],
):
md, err = await run_github_section(cfg, now=None)
assert md == ""
assert err is None
@pytest.mark.asyncio
async def test_happy_path_enriches_and_summarizes(tmp_path):
history_path = tmp_path / "h.json"
cfg = _cfg(str(history_path), max_deep_dive=10)
repos = [
{
"url": "https://github.com/o1/r1",
"full_name": "o1/r1",
"description": "d1",
"language": "Python",
"stars_today": 100,
"stars_total": 1000,
}
]
enriched = [{**repos[0], "topics": ["llm"], "license": "MIT", "pushed_at": "p", "readme_excerpt": "rm"}]
with patch(
"src.sections.github.section.fetch_trending_page", new=AsyncMock(return_value="<html>")
), patch(
"src.sections.github.section.parse_trending_html", return_value=repos
), patch(
"src.sections.github.section.enrich_repos",
new=AsyncMock(return_value=(enriched, [])),
), patch(
"src.llm.summarize_github_trending",
new=AsyncMock(return_value=("## GH section md", None)),
):
md, err = await run_github_section(cfg, now=None)
assert md == "## GH section md"
assert err is None
import json as _j
saved = _j.loads(history_path.read_text(encoding="utf-8"))
assert "https://github.com/o1/r1" in saved["repos"]
@pytest.mark.asyncio
async def test_truncates_candidates_to_max_deep_dive(tmp_path):
cfg = _cfg(str(tmp_path / "h.json"), max_deep_dive=2)
repos = [
{"url": f"https://github.com/o/r{i}", "full_name": f"o/r{i}"} for i in range(5)
]
captured = {}
async def fake_enrich(candidates, **kwargs):
captured["count"] = len(candidates)
return [], []
with patch(
"src.sections.github.section.fetch_trending_page", new=AsyncMock(return_value="<html>")
), patch(
"src.sections.github.section.parse_trending_html", return_value=repos
), patch(
"src.sections.github.section.enrich_repos", new=AsyncMock(side_effect=fake_enrich)
):
await run_github_section(cfg, now=None)
assert captured["count"] == 2
@pytest.mark.asyncio
async def test_scrape_failure_returns_error(tmp_path):
cfg = _cfg(str(tmp_path / "h.json"))
with patch(
"src.sections.github.section.fetch_trending_page",
new=AsyncMock(side_effect=RuntimeError("HTTP 500")),
):
md, err = await run_github_section(cfg, now=None)
assert md == ""
assert "HTTP 500" in err
@@ -0,0 +1,246 @@
"""测试 HN enrich(Algolia 评论树 + 外链正文)"""
import sys
from pathlib import Path
from unittest.mock import patch, AsyncMock, MagicMock
import pytest
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
from src.sections.hackernews.item_enricher import enrich_story
def _kwargs(**overrides):
base = dict(
top_comments=3,
top_l2_per_l1=2,
comment_max_chars=500,
comments_total_chars=60000,
link_content_max_chars=3000,
algolia_base="https://hn.algolia.com/api/v1",
timeout=10,
)
base.update(overrides)
return base
@pytest.mark.asyncio
async def test_enrich_external_link_story_returns_tree():
story = {
"id": "111",
"title": "T",
"url": "https://example.com/post",
"site": "example.com",
"points": 100,
"comments": 5,
"comments_url": "https://news.ycombinator.com/item?id=111",
}
algolia_payload = {
"text": None,
"children": [
{
"text": "<p>comment one</p>",
"children": [
{"text": "<p>reply 1a</p>"},
{"text": "<p>reply 1b</p>"},
{"text": "<p>reply 1c (should be dropped)</p>"},
],
},
{"text": "<p>comment two</p>", "children": []},
{"text": "<p>comment three</p>"},
{"text": "<p>comment four (over top_comments cap)</p>"},
],
}
async def fake_algolia(session, item_id, **kw):
return algolia_payload
async def fake_external(session, url, **kw):
return "link body"
with patch(
"src.sections.hackernews.item_enricher._fetch_algolia_item",
new=AsyncMock(side_effect=fake_algolia),
), patch(
"src.sections.hackernews.item_enricher._fetch_external_markdown",
new=AsyncMock(side_effect=fake_external),
):
enriched = await enrich_story(
session=MagicMock(), story=story, **_kwargs()
)
tree = enriched["top_comments"]
assert len(tree) == 3
assert "comment one" in tree[0]["l1"]
assert len(tree[0]["replies"]) == 2
assert "reply 1a" in tree[0]["replies"][0]
assert "reply 1b" in tree[0]["replies"][1]
assert tree[1]["replies"] == []
assert tree[2]["replies"] == []
assert "link body" in enriched["link_content"]
@pytest.mark.asyncio
async def test_enrich_show_hn_uses_root_text_no_external_fetch():
story = {
"id": "222",
"title": "Show HN: T",
"url": "https://news.ycombinator.com/item?id=222",
"site": "",
"points": 200,
"comments": 10,
"comments_url": "https://news.ycombinator.com/item?id=222",
}
algolia_payload = {
"text": "<p>post body text</p>",
"children": [{"text": "<p>c1</p>"}],
}
link_calls = []
async def fake_algolia(session, item_id, **kw):
return algolia_payload
async def fake_external(session, url, **kw):
link_calls.append(url)
return "should not be called"
with patch(
"src.sections.hackernews.item_enricher._fetch_algolia_item",
new=AsyncMock(side_effect=fake_algolia),
), patch(
"src.sections.hackernews.item_enricher._fetch_external_markdown",
new=AsyncMock(side_effect=fake_external),
):
enriched = await enrich_story(
session=MagicMock(), story=story, **_kwargs()
)
assert link_calls == []
assert "post body text" in enriched["link_content"]
assert enriched["top_comments"][0]["l1"].startswith("c1") or "c1" in enriched["top_comments"][0]["l1"]
@pytest.mark.asyncio
async def test_enrich_truncates_comments_and_link():
story = {
"id": "333",
"title": "T",
"url": "https://example.com/a",
"site": "example.com",
"points": 100,
"comments": 2,
"comments_url": "x",
}
long_comment = "<p>" + ("y" * 2000) + "</p>"
long_reply = "<p>" + ("z" * 2000) + "</p>"
long_link = "z" * 5000
async def fake_algolia(session, item_id, **kw):
return {
"text": None,
"children": [
{"text": long_comment, "children": [{"text": long_reply}]}
],
}
async def fake_external(session, url, **kw):
return long_link
with patch(
"src.sections.hackernews.item_enricher._fetch_algolia_item",
new=AsyncMock(side_effect=fake_algolia),
), patch(
"src.sections.hackernews.item_enricher._fetch_external_markdown",
new=AsyncMock(side_effect=fake_external),
):
enriched = await enrich_story(
session=MagicMock(),
story=story,
**_kwargs(comment_max_chars=100, link_content_max_chars=200),
)
assert len(enriched["top_comments"][0]["l1"]) <= 100
assert len(enriched["top_comments"][0]["replies"][0]) <= 100
assert len(enriched["link_content"]) <= 200
@pytest.mark.asyncio
async def test_enrich_failure_returns_partial():
story = {
"id": "444",
"title": "T",
"url": "https://example.com/x",
"site": "example.com",
"points": 100,
"comments": 2,
"comments_url": "x",
}
async def fake_algolia(session, item_id, **kw):
raise RuntimeError("algolia down")
async def fake_external(session, url, **kw):
return "ok"
with patch(
"src.sections.hackernews.item_enricher._fetch_algolia_item",
new=AsyncMock(side_effect=fake_algolia),
), patch(
"src.sections.hackernews.item_enricher._fetch_external_markdown",
new=AsyncMock(side_effect=fake_external),
):
enriched = await enrich_story(
session=MagicMock(), story=story, **_kwargs()
)
assert enriched["top_comments"] == []
assert "ok" in enriched["link_content"]
@pytest.mark.asyncio
async def test_enrich_total_budget_stops_early():
"""累计字符达 comments_total_chars 立即停止,后续 L1 / L2 都不再加入"""
story = {
"id": "555",
"title": "T",
"url": "https://example.com/q",
"site": "example.com",
"points": 100,
"comments": 5,
"comments_url": "x",
}
big = "<p>" + ("a" * 500) + "</p>" # markdown 约 500 chars
async def fake_algolia(session, item_id, **kw):
return {
"text": None,
"children": [{"text": big, "children": [{"text": big}, {"text": big}]} for _ in range(10)],
}
async def fake_external(session, url, **kw):
return "x"
with patch(
"src.sections.hackernews.item_enricher._fetch_algolia_item",
new=AsyncMock(side_effect=fake_algolia),
), patch(
"src.sections.hackernews.item_enricher._fetch_external_markdown",
new=AsyncMock(side_effect=fake_external),
):
enriched = await enrich_story(
session=MagicMock(),
story=story,
**_kwargs(
top_comments=10,
top_l2_per_l1=2,
comment_max_chars=500,
comments_total_chars=1500,
),
)
tree = enriched["top_comments"]
total = sum(len(n["l1"]) + sum(len(r) for r in n["replies"]) for n in tree)
# 累计应该在 1500 附近停下(允许多收一条到 ~2000),不应该收全 10*3=30 条
assert total <= 2000
assert len(tree) < 10
@@ -0,0 +1,59 @@
"""测试 HN 首页 HTML 解析"""
import json
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
from src.sections.hackernews.frontpage_scraper import parse_frontpage_html
def test_parse_frontpage_returns_stories():
fixture = (Path(__file__).parent / "fixtures" / "hn_frontpage.html").read_text(
encoding="utf-8"
)
stories = parse_frontpage_html(fixture)
assert len(stories) >= 25
s = stories[0]
print(json.dumps(stories[:5], indent=4, ensure_ascii=False))
assert s["id"]
assert s["title"]
assert s["url"]
assert isinstance(s["points"], int)
assert isinstance(s["comments"], int)
assert s["comments_url"].startswith("https://news.ycombinator.com/item?id=")
def test_parse_frontpage_detects_show_hn_internal_url():
html = """
<table>
<tr class="athing" id="111">
<td class="title">
<span class="titleline">
<a href="item?id=111">Ask HN: what's new?</a>
</span>
</td>
</tr>
<tr>
<td class="subtext">
<span class="subline">
<span class="score">50 points</span>
by <a href="user?id=alice">alice</a>
<span class="age"><a href="item?id=111">2 hours ago</a></span>
| <a href="item?id=111">5&nbsp;comments</a>
</span>
</td>
</tr>
</table>
"""
stories = parse_frontpage_html(html)
assert len(stories) == 1
s = stories[0]
assert s["id"] == "111"
assert s["url"].startswith("https://news.ycombinator.com/item?id=")
assert s["site"] == ""
assert s["points"] == 50
assert s["comments"] == 5
@@ -0,0 +1,111 @@
"""测试 HN 板块编排:scrape → select → enrich → LLM"""
import sys
from pathlib import Path
from unittest.mock import patch, AsyncMock
import pytest
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
# Stub HN LLM functions until Task 16 lands them
import src.llm as _llm
if not hasattr(_llm, "select_ai_related_hn"):
async def _stub_select(*a, **k):
return [], None
_llm.select_ai_related_hn = _stub_select
if not hasattr(_llm, "summarize_hackernews"):
async def _stub_summarize(*a, **k):
return "", None
_llm.summarize_hackernews = _stub_summarize
from src.sections.hackernews.section import run_hackernews_section
def _cfg() -> dict:
return {
"filter": {"keep_days": 7},
"sections": {
"hackernews": {
"enabled": True,
"select_k": 1,
"top_comments": 20,
"comment_max_chars": 500,
"link_content_max_chars": 3000,
"request_timeout": 10,
"algolia_base": "https://hn.algolia.com/api/v1",
}
},
"llm": {
"model": "x",
"baseUrl": "http://x",
"apiKeyName": "DEEPSEEK_API_KEY",
"prompts": {
"section_hackernews_select": "prompts/section_hackernews_select.md",
"section_hackernews": "prompts/section_hackernews.md",
},
},
}
@pytest.mark.asyncio
async def test_disabled_returns_empty():
cfg = _cfg()
cfg["sections"]["hackernews"]["enabled"] = False
md, err = await run_hackernews_section(cfg, now=None)
assert md == ""
assert err is None
@pytest.mark.asyncio
async def test_select_empty_returns_silent():
cfg = _cfg()
with patch(
"src.sections.hackernews.section.fetch_frontpage", new=AsyncMock(return_value="<html>")
), patch(
"src.sections.hackernews.section.parse_frontpage_html",
return_value=[{"id": "1", "title": "x"}],
), patch(
"src.llm.select_ai_related_hn",
new=AsyncMock(return_value=([], None)),
):
md, err = await run_hackernews_section(cfg, now=None)
assert md == ""
assert err is None
@pytest.mark.asyncio
async def test_happy_path():
cfg = _cfg()
front = [{"id": "1", "title": "AI thing", "url": "https://e.com/a", "site": "e.com", "points": 100, "comments": 5, "comments_url": "x"}]
enriched = [{**front[0], "link_content": "body", "top_comments": ["c1"]}]
with patch(
"src.sections.hackernews.section.fetch_frontpage", new=AsyncMock(return_value="<html>")
), patch(
"src.sections.hackernews.section.parse_frontpage_html", return_value=front
), patch(
"src.llm.select_ai_related_hn",
new=AsyncMock(return_value=(["1"], None)),
), patch(
"src.sections.hackernews.section.enrich_stories",
new=AsyncMock(return_value=(enriched, [])),
), patch(
"src.llm.summarize_hackernews",
new=AsyncMock(return_value=("## HN md", None)),
):
md, err = await run_hackernews_section(cfg, now=None)
assert md == "## HN md"
assert err is None
@pytest.mark.asyncio
async def test_scrape_failure_returns_error():
cfg = _cfg()
with patch(
"src.sections.hackernews.section.fetch_frontpage",
new=AsyncMock(side_effect=RuntimeError("net")),
):
md, err = await run_hackernews_section(cfg, now=None)
assert md == ""
assert "net" in err
@@ -0,0 +1,66 @@
"""测试 insights 模块"""
import sys
from pathlib import Path
from unittest.mock import patch, AsyncMock
import pytest
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
# Stub Task 17's LLM function until it lands
import src.llm as _llm
if not hasattr(_llm, "generate_trend_insights"):
async def _stub(*a, **k):
return "", None
_llm.generate_trend_insights = _stub
from src.sections.insights.section import run_insights_section
def _cfg() -> dict:
return {
"filter": {"push_context_days": 5},
"sections": {"insights": {"enabled": True}},
"llm": {
"model": "x",
"baseUrl": "http://x",
"apiKeyName": "DEEPSEEK_API_KEY",
"prompts": {"insights": "prompts/insights.md"},
},
}
@pytest.mark.asyncio
async def test_disabled_returns_empty():
cfg = _cfg()
cfg["sections"]["insights"]["enabled"] = False
md, meta, err = await run_insights_section("rss", "gh", "hn", cfg, now=None)
assert md == ""
assert meta is None
assert err is None
@pytest.mark.asyncio
async def test_marks_empty_sections_for_llm():
cfg = _cfg()
captured = {}
async def fake_gen(sections, config):
captured["sections"] = sections
return "insights md", None
with patch(
"src.llm.generate_trend_insights",
new=AsyncMock(side_effect=fake_gen),
):
md, meta, err = await run_insights_section("", "gh md", "", cfg, now=None)
assert md == "insights md"
assert err is None
# metadata 由 parse_insights_with_metadata 注入默认标题/profile
assert meta["profile"] == "morning"
assert "📰 AI Daily 每日精选" in meta["title"]
assert captured["sections"]["rss"] == "(本次无内容)"
assert captured["sections"]["github"] == "gh md"
assert captured["sections"]["hackernews"] == "(本次无内容)"
@@ -0,0 +1,76 @@
"""src.main.collect_entries_for_push patched at source (lazy import in section)"""
import sys
from pathlib import Path
from unittest.mock import patch, AsyncMock
import pytest
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
from src.sections.rss.section import run_rss_section
@pytest.mark.asyncio
async def test_returns_markdown_when_entries_present(sample_config, tmp_path):
digest_raw = (
"---\n"
'title: "🌙 AI Daily 晚报 | 测试"\n'
'lead: "今日测试导读"\n'
"highlights:\n - 重点1\n"
"---\n\n"
"### 1️⃣ digest body"
)
with patch(
"src.main.collect_entries_for_push",
return_value=([{"link": "x", "title": "t", "score": 80}], []),
), patch(
"src.sections.rss.section.compose_digest",
new=AsyncMock(return_value=digest_raw),
), patch(
"src.sections.rss.section.load_recent_push_content", return_value=""
), patch(
"src.sections.rss.section.get_last_push_file", return_value=None
):
md, meta, err = await run_rss_section(sample_config, now=None)
assert md == "### 1️⃣ digest body"
assert err is None
assert meta["title"] == "🌙 AI Daily 晚报 | 测试"
assert meta["lead"] == "今日测试导读"
assert meta["highlights"] == ["重点1"]
assert meta["profile"] == "default"
@pytest.mark.asyncio
async def test_returns_empty_when_no_entries(sample_config):
with patch(
"src.main.collect_entries_for_push", return_value=([], [])
), patch(
"src.sections.rss.section.get_last_push_file", return_value=None
):
md, meta, err = await run_rss_section(sample_config, now=None)
assert md == ""
assert meta is None
assert err is None
@pytest.mark.asyncio
async def test_returns_error_on_compose_failure(sample_config):
with patch(
"src.main.collect_entries_for_push",
return_value=([{"link": "x"}], []),
), patch(
"src.sections.rss.section.compose_digest",
new=AsyncMock(side_effect=RuntimeError("LLM down")),
), patch(
"src.sections.rss.section.load_recent_push_content", return_value=""
), patch(
"src.sections.rss.section.get_last_push_file", return_value=None
):
md, meta, err = await run_rss_section(sample_config, now=None)
assert md == ""
assert meta is None
assert "LLM down" in err
+257
View File
@@ -0,0 +1,257 @@
"""存储模块测试"""
import json
import pytest
import sys
from datetime import datetime, date, timedelta, timezone
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src"))
from storage import (
get_fetch_file,
get_push_file,
get_last_push_file,
extract_push_time,
read_entries,
read_fetch_data,
save_fetch_file,
append_entries,
format_entry,
json_to_md,
save_push_file,
load_existing_links,
cleanup_old_files,
)
class TestGetFetchFile:
"""测试获取fetch文件路径"""
def test_get_fetch_file_default(self):
result = get_fetch_file()
assert "fetch-" in result
assert result.endswith(".json")
def test_get_fetch_file_specific_date(self):
result = get_fetch_file(date(2024, 1, 15))
assert "fetch-2024-01-15.json" in result
class TestGetPushFile:
"""测试获取push文件路径"""
def test_get_push_file_default(self):
result = get_push_file()
assert "push-" in result
assert result.endswith(".md")
def test_get_push_file_specific_time(self):
dt = datetime(2024, 1, 15, 8, 30, 0)
result = get_push_file(dt)
assert "push-2024-01-15-08-30-00.md" in result
class TestExtractPushTime:
"""测试从文件名提取时间"""
def test_extract_valid_time(self):
result = extract_push_time("news-data/push-2024-01-15-08-30-00.md")
assert result is not None
assert result.year == 2024
assert result.month == 1
assert result.day == 15
def test_extract_invalid_filename(self):
result = extract_push_time("invalid.md")
assert result is None
class TestGetLastPushFile:
"""测试获取最新push文件"""
def test_get_last_push_file_empty(self, temp_dir):
result = get_last_push_file(str(temp_dir))
assert result is None
def test_get_last_push_file_exists(self, temp_dir):
(temp_dir / "push-2024-01-14-10-00-00.md").touch()
(temp_dir / "push-2024-01-15-10-00-00.md").touch()
result = get_last_push_file(str(temp_dir))
assert "2024-01-15" in result
class TestReadWriteEntries:
"""测试读写条目"""
def test_read_entries(self, sample_fetch_json):
entries = read_entries(sample_fetch_json)
assert len(entries) == 3
assert entries[0]["title"] == "Article 1"
def test_read_entries_missing_file(self):
entries = read_entries("nonexistent.json")
assert entries == []
def test_read_fetch_data(self, sample_fetch_json):
data = read_fetch_data(sample_fetch_json)
assert "meta" in data
assert "entries" in data
assert len(data["entries"]) == 3
class TestSaveFetchFile:
"""测试保存fetch文件"""
def test_save_fetch_file(self, temp_dir):
filepath = str(temp_dir / "test.json")
meta = {"date": "2024-01-15"}
entries = [{"title": "Test", "link": "https://example.com", "score": 80}]
save_fetch_file(filepath, meta, entries)
data = read_fetch_data(filepath)
assert data["meta"]["date"] == "2024-01-15"
assert len(data["entries"]) == 1
class TestAppendEntries:
"""测试追加条目"""
def test_append_new_entries(self, temp_dir):
filepath = str(temp_dir / "test.json")
meta = {"date": "2024-01-15"}
entries1 = [{"title": "Entry1", "link": "https://example.com/1", "score": 80}]
count1 = append_entries(filepath, entries1, meta)
assert count1 == 1
entries2 = [{"title": "Entry2", "link": "https://example.com/2", "score": 70}]
count2 = append_entries(filepath, entries2, meta)
assert count2 == 1
def test_append_duplicate_entries(self, temp_dir):
filepath = str(temp_dir / "test.json")
meta = {"date": "2024-01-15"}
entries = [{"title": "Entry1", "link": "https://example.com/1", "score": 80}]
append_entries(filepath, entries, meta)
all_entries = read_entries(filepath)
assert len(all_entries) == 1
count = append_entries(filepath, entries, meta)
all_entries_after = read_entries(filepath)
assert len(all_entries_after) == 1
def test_append_to_existing_file(self, temp_dir):
filepath = str(temp_dir / "test.json")
data = {
"meta": {"date": "2024-01-15"},
"entries": [{"title": "Old", "link": "https://old.com", "score": 60}],
}
with open(filepath, "w") as f:
json.dump(data, f)
new_entries = [{"title": "New", "link": "https://new.com", "score": 70}]
count = append_entries(filepath, new_entries)
entries = read_entries(filepath)
assert len(entries) == 2
class TestFormatEntry:
"""测试格式化条目"""
def test_format_entry_basic(self, sample_entry):
result = format_entry(sample_entry)
assert "## Test Article Title" in result
assert "source: Test Source" in result
assert "score: 85" in result
def test_format_entry_with_tags(self, sample_entry):
result = format_entry(sample_entry)
assert "AI" in result
assert "Tech" in result
class TestJsonToMd:
"""测试JSON转Markdown"""
def test_json_to_md_basic(self, sample_fetch_json):
data = read_fetch_data(sample_fetch_json)
result = json_to_md(data)
assert "Article 1" in result
assert "Article 2" in result
assert "Article 3" in result
def test_json_to_md_empty(self):
data = {"meta": {}, "entries": []}
result = json_to_md(data)
assert result == ""
class TestSavePushFile:
"""测试保存推送文件"""
def test_save_push_file(self, temp_dir):
filepath = str(temp_dir / "push-test.md")
content = "# Test Push\n\nContent here"
save_push_file(filepath, content, 5, 10)
with open(filepath, "r") as f:
content = f.read()
assert "pushDate:" in content
assert "sourceCount: 5" in content
assert "totalEntries: 10" in content
assert "# Test Push" in content
class TestLoadExistingLinks:
"""测试加载已有链接"""
def test_load_existing_links_json(self, sample_fetch_json):
links = load_existing_links(sample_fetch_json)
assert len(links) == 3
assert "https://example.com/1" in links
def test_load_existing_links_missing(self):
links = load_existing_links("nonexistent.json")
assert links == set()
def test_load_existing_links_empty_string(self):
links = load_existing_links("")
assert links == set()
class TestCleanupOldFiles:
"""测试清理旧文件"""
def test_cleanup_old_files(self, temp_dir):
old_date = (datetime.now() - timedelta(days=10)).date()
new_date = (datetime.now() - timedelta(days=1)).date()
(temp_dir / f"fetch-{old_date}.json").touch()
(temp_dir / f"fetch-{new_date}.json").touch()
cleanup_old_files(days=7, data_dir=str(temp_dir))
assert not (temp_dir / f"fetch-{old_date}.json").exists()
assert (temp_dir / f"fetch-{new_date}.json").exists()
def test_cleanup_push_files(self, temp_dir):
old_time = datetime.now() - timedelta(days=10)
new_time = datetime.now() - timedelta(days=1)
(temp_dir / f"push-{old_time.strftime('%Y-%m-%d-%H-%M-%S')}.md").touch()
(temp_dir / f"push-{new_time.strftime('%Y-%m-%d-%H-%M-%S')}.md").touch()
cleanup_old_files(days=7, data_dir=str(temp_dir))
files = list(temp_dir.glob("push-*.md"))
assert len(files) == 1
@@ -0,0 +1,163 @@
"""测试新增的 sentinel 切片与 section-aware 读取"""
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src"))
from storage import extract_section
class TestExtractSection:
def test_extract_section_with_sentinel(self):
md = (
"intro\n"
"<!-- SECTION:rss BEGIN -->\n"
"RSS body\n"
"<!-- SECTION:rss END -->\n"
"\n"
"<!-- SECTION:github BEGIN -->\n"
"GH body\n"
"<!-- SECTION:github END -->\n"
)
assert extract_section(md, "rss").strip() == "RSS body"
assert extract_section(md, "github").strip() == "GH body"
assert extract_section(md, "hackernews") == ""
def test_extract_section_legacy_file_rss(self):
legacy = "# AI Daily\n### 1️⃣ foo\n### 2️⃣ bar\n"
assert extract_section(legacy, "rss") == legacy
def test_extract_section_legacy_file_non_rss(self):
legacy = "# AI Daily\n### 1️⃣ foo\n"
assert extract_section(legacy, "github") == ""
assert extract_section(legacy, "hackernews") == ""
assert extract_section(legacy, "insights") == ""
def test_extract_section_missing_end_marker(self):
broken = "<!-- SECTION:rss BEGIN -->\ncontent only\n"
assert extract_section(broken, "rss") == ""
from datetime import date, datetime, timedelta
from storage import save_push_file
from storage import TrendingHistory, load_trending_history
class TestTrendingHistory:
def test_load_missing_file(self, tmp_path):
path = tmp_path / "trending.json"
h = load_trending_history(str(path))
assert h.repos == {}
def test_touch_then_save_then_reload(self, tmp_path):
path = tmp_path / "trending.json"
h = load_trending_history(str(path))
today = date(2026, 5, 17)
h.touch("https://github.com/a/b", today)
h.touch("https://github.com/c/d", today)
h.save()
h2 = load_trending_history(str(path))
assert h2.repos == {
"https://github.com/a/b": "2026-05-17",
"https://github.com/c/d": "2026-05-17",
}
def test_contains_returns_membership(self, tmp_path):
h = load_trending_history(str(tmp_path / "x.json"))
h.touch("https://github.com/a/b", date(2026, 5, 17))
assert "https://github.com/a/b" in h
assert "https://github.com/x/y" not in h
def test_cleanup_removes_expired_entries(self, tmp_path):
path = tmp_path / "trending.json"
path.write_text(
'{"repos": {'
'"https://github.com/old/repo": "2026-05-01", '
'"https://github.com/new/repo": "2026-05-15"'
'}, "updated_at": "2026-05-15T00:00:00+08:00"}',
encoding="utf-8",
)
h = load_trending_history(str(path))
h.cleanup(today=date(2026, 5, 17), keep_days=7)
assert "https://github.com/old/repo" not in h
assert "https://github.com/new/repo" in h
def test_cleanup_keeps_today_inclusive(self, tmp_path):
h = load_trending_history(str(tmp_path / "x.json"))
h.touch("https://github.com/a/b", date(2026, 5, 10))
# 2026-05-10 + 7 days = 2026-05-17 (last_seen 2026-05-10 仍在 keep 区间)
h.cleanup(today=date(2026, 5, 17), keep_days=7)
assert "https://github.com/a/b" in h
# 再过 1 天就出区间
h.cleanup(today=date(2026, 5, 18), keep_days=7)
assert "https://github.com/a/b" not in h
class TestSavePushFileProfile:
def test_default_profile_when_not_specified(self, tmp_path):
f = tmp_path / "push-x.md"
save_push_file(str(f), "body content", source_count=1, total_entries=1)
text = f.read_text(encoding="utf-8")
assert 'profile: "default"' in text
assert "body content" in text
def test_morning_profile(self, tmp_path):
f = tmp_path / "push-x.md"
save_push_file(
str(f), "body", source_count=2, total_entries=3, profile="morning"
)
text = f.read_text(encoding="utf-8")
assert 'profile: "morning"' in text
from storage import cleanup_old_files
import json as _j
class TestCleanupOldFilesTrendingHistory:
def test_prunes_trending_history_entries_not_file(self, tmp_path):
path = tmp_path / "trending-history.json"
old_date = (datetime.now().date() - timedelta(days=30)).isoformat()
fresh_date = datetime.now().date().isoformat()
path.write_text(
'{"repos": {'
f'"https://github.com/a/b": "{old_date}", '
f'"https://github.com/c/d": "{fresh_date}"'
'}, "updated_at": "..."}',
encoding="utf-8",
)
cleanup_old_files(days=7, data_dir=str(tmp_path))
# 文件应保留
assert path.exists()
# 过期条目应被剪枝
data = _j.loads(path.read_text(encoding="utf-8"))
assert "https://github.com/a/b" not in data["repos"]
assert "https://github.com/c/d" in data["repos"]
from storage import assemble_with_sentinels
class TestAssembleWithSentinels:
def test_assembles_all_sections_in_order(self):
out = assemble_with_sentinels(
{"rss": "R", "github": "G", "hackernews": "H", "insights": "I"}
)
assert out.index("SECTION:rss") < out.index("SECTION:github")
assert out.index("SECTION:github") < out.index("SECTION:hackernews")
assert out.index("SECTION:hackernews") < out.index("SECTION:insights")
assert "<!-- SECTION:rss BEGIN -->\nR\n<!-- SECTION:rss END -->" in out
def test_omits_empty_sections(self):
out = assemble_with_sentinels({"rss": "R", "github": "", "hackernews": "H", "insights": ""})
assert "SECTION:github" not in out
assert "SECTION:insights" not in out
assert "SECTION:rss" in out
assert "SECTION:hackernews" in out
def test_returns_empty_when_all_empty(self):
assert assemble_with_sentinels({"rss": "", "github": "", "hackernews": "", "insights": ""}) == ""
+101
View File
@@ -0,0 +1,101 @@
"""时区处理测试"""
import pytest
import sys
from datetime import datetime, timezone, timedelta
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src"))
from config import get_timezone
class TestTimezoneBasics:
"""测试时区基础功能"""
def test_get_timezone_positive_hours(self):
config = {"schedule": {"timezone_hours": 8}}
tz = get_timezone(config)
offset = tz.utcoffset(datetime.now())
assert offset.total_seconds() == 8 * 3600
def test_get_timezone_negative_hours(self):
config = {"schedule": {"timezone_hours": -5}}
tz = get_timezone(config)
offset = tz.utcoffset(datetime.now())
assert offset.total_seconds() == -5 * 3600
def test_get_timezone_zero_hours(self):
config = {"schedule": {"timezone_hours": 0}}
tz = get_timezone(config)
offset = tz.utcoffset(datetime.now())
assert offset.total_seconds() == 0
def test_get_timezone_missing_schedule(self):
config = {}
tz = get_timezone(config)
assert tz is not None
def test_get_timezone_none_config(self):
tz = get_timezone(None)
assert tz is not None
class TestTimezoneConversions:
"""测试时区转换"""
def test_utc_to_local(self):
config = {"schedule": {"timezone_hours": 8}}
tz = get_timezone(config)
utc_time = datetime(2024, 1, 15, 10, 0, 0, tzinfo=timezone.utc)
local_time = utc_time.astimezone(tz)
assert local_time.hour == 18
def test_cross_day_conversion(self):
config = {"schedule": {"timezone_hours": 8}}
tz = get_timezone(config)
utc_time = datetime(2024, 1, 15, 20, 0, 0, tzinfo=timezone.utc)
local_time = utc_time.astimezone(tz)
assert local_time.day == 16
def test_negative_timezone(self):
config = {"schedule": {"timezone_hours": -5}}
tz = get_timezone(config)
utc_time = datetime(2024, 1, 15, 10, 0, 0, tzinfo=timezone.utc)
local_time = utc_time.astimezone(tz)
assert local_time.hour == 5
class TestTimezoneAwareDatetime:
"""测试带时区的datetime操作"""
def test_now_in_config_timezone(self):
config = {"schedule": {"timezone_hours": 8}}
tz = get_timezone(config)
now_local = datetime.now(tz)
assert now_local.tzinfo == tz
def test_timezone_aware_comparison(self):
config = {"schedule": {"timezone_hours": 8}}
tz = get_timezone(config)
dt1 = datetime(2024, 1, 15, 10, 0, 0, tzinfo=timezone.utc)
dt2 = datetime(2024, 1, 15, 18, 0, 0, tzinfo=tz)
assert dt1 == dt2
def test_naive_to_aware(self):
config = {"schedule": {"timezone_hours": 8}}
tz = get_timezone(config)
naive = datetime(2024, 1, 15, 10, 0, 0)
aware = naive.replace(tzinfo=tz)
assert aware.tzinfo == tz
+142
View File
@@ -0,0 +1,142 @@
#!/usr/bin/env python3
"""重新推送即时消息(notify 文件)
Usage:
python tests/resend_notify.py news-data/notify-2026-05-21.md
python tests/resend_notify.py news-data/notify-2026-05-21.md --index 0 # 推送第一个块
"""
import argparse
import asyncio
import sys
from pathlib import Path
from dotenv import load_dotenv
load_dotenv()
sys.path.insert(0, str(Path(__file__).parent.parent))
from src.config import load_config
from src.markdown_utils import parse_frontmatter
from src.push import send_to_platforms
def parse_notify_file(filepath: str):
"""解析 notify 文件,返回所有推送块列表
Returns:
List[Dict]: [{"metadata": {...}, "content": "..."}, ...]
"""
with open(filepath, "r", encoding="utf-8") as f:
content = f.read()
blocks = []
for block in content.split("------"):
block = block.strip()
if not block:
continue
metadata, body = parse_frontmatter(block)
if not metadata:
continue
blocks.append({"metadata": metadata, "content": body})
return blocks
async def resend_notify(filepath: str, config: dict, index: int = -1):
"""重新推送 notify 文件中的即时消息
Args:
filepath: notify 文件路径
config: 配置字典
index: 推送第几个块(-1 表示最新的一个)
Returns:
bool: 推送是否成功
"""
if not Path(filepath).exists():
print(f"❌ 文件不存在: {filepath}")
return False
blocks = parse_notify_file(filepath)
if not blocks:
print("❌ 文件中没有有效的推送块")
return False
print(f"\n📋 文件中共有 {len(blocks)} 个推送块")
# 选择要推送的块
if index == -1:
block = blocks[-1]
print(f" 使用最新的一个(第 {len(blocks)} 个)")
elif 0 <= index < len(blocks):
block = blocks[index]
print(f" 使用第 {index + 1}")
else:
print(f"❌ 索引超出范围: {index} (有效范围: 0-{len(blocks)-1})")
return False
metadata = block["metadata"]
content = block["content"]
# 拼接推送标题,与 main.py 的即时推送保持一致
raw_title = metadata.get("title", "")
title = "🚨 AI Daily 快讯 | " + raw_title if raw_title else "🚨 AI Daily 快讯"
print(f"\n📤 准备推送:")
print(f" 标题: {title}")
print(f" 时间: {metadata.get('pushTime', 'N/A')}")
# 推送
try:
await send_to_platforms(content, config["push"], title=title, metadata=metadata)
print("\n✅ 推送成功!")
return True
except Exception as e:
print(f"\n❌ 推送失败: {e}")
import traceback
traceback.print_exc()
return False
async def main():
parser = argparse.ArgumentParser(description="重新推送即时消息(notify 文件)")
parser.add_argument("filepath", help="notify 文件路径")
parser.add_argument("--index", type=int, default=-1, help="推送第几个块(-1=最新,默认)")
args = parser.parse_args()
print("=" * 60)
print("🚨 即时消息推送工具")
print("=" * 60)
# 加载配置
try:
config = load_config()
print("✅ 配置加载成功")
except Exception as e:
print(f"❌ 加载配置失败: {e}")
return 1
# 显示推送平台配置状态
print("\n📋 推送平台配置:")
import os
for platform_name, platform_conf in config.get("push", {}).items():
enabled = platform_conf.get("enabled", False)
api_key_name = platform_conf.get("apiKeyName", "")
has_key = bool(os.environ.get(api_key_name, ""))
status = "" if (enabled and has_key) else "⚠️"
print(f" {status} {platform_name}: enabled={enabled}, has_key={has_key}")
# 推送
success = await resend_notify(args.filepath, config, args.index)
return 0 if success else 1
if __name__ == "__main__":
try:
sys.exit(asyncio.run(main()))
except KeyboardInterrupt:
print("\n\n👋 已取消")
sys.exit(130)
+108
View File
@@ -0,0 +1,108 @@
#!/usr/bin/env python3
"""重新推送已保存的 push 文件
Usage:
python tests/resend_push.py news-data/push-2026-05-21-08-00-00.md
"""
import argparse
import asyncio
import sys
from pathlib import Path
from dotenv import load_dotenv
load_dotenv()
sys.path.insert(0, str(Path(__file__).parent.parent))
from src.config import load_config
from src.markdown_utils import parse_frontmatter
from src.push import send_to_platforms
async def resend_push_file(filepath: str, config: dict):
"""读取 push 文件,解析 metadata 并重新推送
Args:
filepath: push 文件路径
config: 配置字典
Returns:
bool: 推送是否成功
"""
if not Path(filepath).exists():
print(f"❌ 文件不存在: {filepath}")
return False
# 读取文件内容
with open(filepath, "r", encoding="utf-8") as f:
content = f.read()
# 解析 frontmatter(复用 markdown_utils
metadata, body = parse_frontmatter(content)
if not metadata:
print("❌ 文件格式错误:无法解析 YAML frontmatter")
return False
# 拼接推送标题,与 main.py 的 push 任务保持一致
raw_title = metadata.get("title", "")
title = "📰 AI Daily 每日精选 | " + raw_title if raw_title else "📰 AI Daily 每日精选"
print(f"\n📤 准备推送文件: {Path(filepath).name}")
print(f" 标题: {title}")
print(f" Profile: {metadata.get('profile', 'N/A')}")
print(f" 日期: {metadata.get('date') or metadata.get('pushDate', 'N/A')}")
# 推送到所有平台
try:
await send_to_platforms(body, config["push"], title=title, metadata=metadata)
print("\n✅ 推送成功!")
return True
except Exception as e:
print(f"\n❌ 推送失败: {e}")
import traceback
traceback.print_exc()
return False
async def main():
parser = argparse.ArgumentParser(description="重新推送已保存的 push 文件")
parser.add_argument("filepath", help="push 文件路径")
args = parser.parse_args()
print("=" * 60)
print("📤 重新推送工具")
print("=" * 60)
# 加载配置
try:
config = load_config()
print("✅ 配置加载成功")
except Exception as e:
print(f"❌ 加载配置失败: {e}")
return 1
# 显示推送平台配置状态
print("\n📋 推送平台配置:")
import os
for platform_name, platform_conf in config.get("push", {}).items():
enabled = platform_conf.get("enabled", False)
api_key_name = platform_conf.get("apiKeyName", "")
has_key = bool(os.environ.get(api_key_name, ""))
status = "" if (enabled and has_key) else "⚠️"
print(f" {status} {platform_name}: enabled={enabled}, has_key={has_key} ({api_key_name})")
# 推送文件
success = await resend_push_file(args.filepath, config)
return 0 if success else 1
if __name__ == "__main__":
try:
sys.exit(asyncio.run(main()))
except KeyboardInterrupt:
print("\n\n👋 已取消")
sys.exit(130)
+383
View File
@@ -0,0 +1,383 @@
#!/usr/bin/env python3
"""测试LLM评分和推送功能 - 独立运行脚本
Usage:
# 先激活虚拟环境
source ../.venv/bin/activate
# 测试评分
python tests/run_llm_test.py --score
# 测试即时推送
python tests/run_llm_test.py --immediate-push --push
"""
import argparse
import asyncio
import json
import os
import sys
from datetime import datetime, timedelta, timezone
from pathlib import Path
# 检查是否在虚拟环境中
if not hasattr(sys, "real_prefix") and not (
hasattr(sys, "base_prefix") and sys.base_prefix != sys.prefix
):
print("⚠️ 建议先激活虚拟环境: source .venv/bin/activate")
print("")
# 加载 .env 文件
from dotenv import load_dotenv
load_dotenv()
# 添加项目根目录到路径
sys.path.insert(0, str(Path(__file__).parent.parent))
from src.config import get_timezone, load_config
from src.llm import compose_digest, generate_immediate_push, score_batch
from src.push import send_to_platforms
from src.storage import read_fetch_data, save_fetch_file
def parse_args():
"""解析命令行参数"""
parser = argparse.ArgumentParser(description="测试LLM评分和推送")
parser.add_argument(
"--input",
"-i",
type=str,
default="tests/news-data/fetch-{date}.json",
help="输入文件路径,支持{date}占位符 (默认: tests/news-data/fetch-{date}.json)",
)
parser.add_argument(
"--date",
"-d",
type=str,
default=datetime.now(get_timezone()).strftime("%Y-%m-%d"),
help="日期,格式YYYY-MM-DD (默认: 今天)",
)
parser.add_argument(
"--limit", "-l", type=int, default=0, help="测试的消息数量 (默认: 0表示全部)"
)
# 测试模式选择
parser.add_argument("--score", action="store_true", help="测试评分")
parser.add_argument("--immediate-push", action="store_true", help="测试即时推送")
parser.add_argument("--digest", action="store_true", help="测试汇总推送")
parser.add_argument("--push", action="store_true", help="推送到Discord")
parser.add_argument("--all", action="store_true", help="运行所有测试")
return parser.parse_args()
def should_run(args, mode: str) -> bool:
"""判断是否运行某个模式"""
# 如果没有任何特定模式指定,默认运行评分
if args.all:
return True
# 检查是否指定了任何模式
any_mode = args.score or args.immediate_push or args.digest
if mode == "score":
return args.score or not any_mode # 默认运行评分
elif mode == "immediate_push":
return args.immediate_push
elif mode == "digest":
return args.digest
return False
async def run_llm_test():
"""主函数"""
args = parse_args()
print("=" * 60)
print("🤖 LLM测试脚本")
print("=" * 60)
# 构建输入文件路径
input_path = args.input.format(date=args.date)
print(f"\n📂 输入文件: {input_path}")
# 读取数据
if not Path(input_path).exists():
print(f"❌ 文件不存在: {input_path}")
print("\n💡 提示: 先运行 fetch_news.py 获取新闻数据")
print(" python tests/fetch_news.py --hours 1")
return False
print(input_path)
data = read_fetch_data(input_path)
entries = data.get("entries", [])
meta = data.get("meta", {})
print(f" ✓ 共 {len(entries)}")
if not entries:
print("❌ 没有条目可测试")
return False
# 限制测试数量 (0表示全部)
if args.limit > 0:
test_entries = entries[: args.limit]
print(f" 测试前 {len(test_entries)}")
else:
test_entries = entries
print(f" 测试全部 {len(test_entries)}")
# 显示待评分条目
print(f"\n📄 测试条目:")
for i, e in enumerate(test_entries[:5], 1):
print(f" [{i}] {e.get('title', 'N/A')[:45]}...")
print(f" 来源: {e.get('source', 'N/A')}")
# 加载配置
print("\n⚙️ 加载配置...")
config = load_config()
llm_config = config["llm"]
print(f" ✓ 提供商: {llm_config.get('provider', 'openai')}")
print(f" ✓ 模型: {llm_config.get('model', 'N/A')}")
print(f" ✓ BaseURL: {llm_config.get('baseUrl', 'N/A')}")
# 检查API key
api_key_name = llm_config.get("apiKeyName", "OPENAI_API_KEY")
api_key = os.environ.get(api_key_name)
if not api_key:
print(f"\n❌ 未设置环境变量: {api_key_name}")
return False
print(f" ✓ API Key: {api_key[:10]}...")
# 检查是否启用推送
push_enabled = args.push and config.get("push")
if push_enabled:
print("\n🔌 推送已启用 (将推送到所有已配置的平台)")
# ========== 测试评分 ==========
if should_run(args, "score"):
print("\n" + "-" * 60)
print("🎯 测试: 评分 (score_batch)")
print("-" * 60)
try:
scored, score_errors = await score_batch(test_entries, llm_config)
if score_errors:
print("\n⚠️ 评分存在异常:")
for error in score_errors:
print(f" - {error}")
print("\n✅ 评分完成!")
# 显示评分结果
print("\n📊 评分结果:")
for i, e in enumerate(scored[:5], 1):
print(f"\n [{i}] {e['title'][:40]}...")
print(f" 评分: {e.get('score', 'N/A')}/100")
print(f" 标签: {e.get('tags', [])}")
print(f" 摘要: {e.get('summary', 'N/A')[:60]}...")
# 保存评分结果到JSON文件
print(f"\n💾 保存评分结果到: {input_path}")
# 构建link到评分的映射
score_map = {e.get("link"): e for e in scored if e.get("link")}
# 更新所有entries的评分
all_entries = data.get("entries", [])
for i, entry in enumerate(all_entries):
link = entry.get("link")
if link in score_map:
all_entries[i] = score_map[link]
save_fetch_file(input_path, meta, all_entries)
print(f" ✅ 已保存 {len(scored)} 条评分结果")
# 更新test_entries为评分后的数据
test_entries = scored
except Exception as e:
print(f"\n❌ 评分失败: {e}")
import traceback
traceback.print_exc()
return False
# ========== 测试即时推送 ==========
if should_run(args, "immediate_push"):
print("\n" + "-" * 60)
print("🔥 测试: 即时推送 (generate_immediate_push)")
print("-" * 60)
# 筛选高分条目 (>=80分)用于推送
hot_entries = [e for e in test_entries if e.get("score", 0) >= 90]
if not hot_entries:
hot_entries = test_entries[-3:-1] # 如果没有高分,取前2条
print(f"\n使用 {len(hot_entries)} 条高分消息生成推送...")
# 加载近期推送上下文用于测试
context_days = config.get("filter", {}).get("context_days", 3)
from src.llm import parse_immediate_push_with_metadata
from src.storage import (
get_notify_file,
load_recent_notify_content,
load_recent_push_content,
save_notify_file,
)
recent_notify = load_recent_notify_content(context_days)
recent_push = load_recent_push_content(context_days)
recent_context = (
f"=== 近期即时推送 ===\n{recent_notify}\n\n"
f"=== 近期汇总推送 ===\n{recent_push}"
)
try:
# 传入上下文参数
push_content, immediate_push_error = await generate_immediate_push(
hot_entries, llm_config, recent_push_context=recent_context
)
timestamp = datetime.now(get_timezone()).strftime("%Y-%m-%d")
content_without_title, metadata = parse_immediate_push_with_metadata(
push_content, f"🚨 AI Daily 快讯 | {timestamp}"
)
metadata["pushTime"] = datetime.now(get_timezone()).isoformat()
push_content = content_without_title
if immediate_push_error:
print(f"\n⚠️ 即时推送生成异常: {immediate_push_error}")
push_content = ""
print(f"\n✅ 推送内容生成完成!")
print(f"\n📤 推送内容预览:")
print("-" * 40)
print(
push_content[:500] + "..." if len(push_content) > 500 else push_content
)
print("-" * 40)
# 检查是否有实际内容需要推送
no_content_marker = config.get("filter", {}).get(
"no_content_marker", "[NO_NEW_CONTENT]"
)
if no_content_marker in push_content:
print(f"\nℹ️ 无新内容需要推送 (LLM判定为重复内容)")
else:
# 推送到所有启用的平台
if push_enabled:
print("\n📤 推送消息...")
await send_to_platforms(
push_content,
config["push"],
title="🚨 AI Daily 快讯 | " + metadata["title"],
metadata=metadata,
)
print(" ✅ 推送成功!")
# 保存到 notify 文件
notify_file = get_notify_file()
save_notify_file(notify_file, push_content, metadata)
print(f"\n💾 已保存即时推送到 {notify_file}")
except Exception as e:
print(f"\n❌ 即时推送生成失败: {e}")
import traceback
traceback.print_exc()
# ========== 测试汇总推送 ==========
if should_run(args, "digest"):
print("\n" + "-" * 60)
print("📰 测试: 汇总推送 (compose_digest)")
print("-" * 60)
# 构建上下文(从 fetch 文件读取的历史数据)
context = test_entries[:10] # 使用前10条作为模拟上下文
print(f"\n使用 {len(test_entries)} 条消息生成汇总...")
# 加载近期推送上下文
push_context_days = config.get(
"filter",
).get("push_context_days", 5)
from src.storage import get_push_file, load_recent_push_content, save_push_file
recent_push_context_str = load_recent_push_content(push_context_days)
try:
raw_digest = await compose_digest(
test_entries,
context,
llm_config,
recent_push_context=recent_push_context_str,
)
from src.llm import parse_digest_with_metadata
date_str = datetime.now(get_timezone()).strftime("%Y-%m-%d")
digest_content, metadata = parse_digest_with_metadata(raw_digest, date_str)
metadata["pushTime"] = datetime.now(get_timezone()).isoformat()
print(f"\n✅ 汇总内容生成完成!")
print(f" 标题: {metadata['title']}")
print(f" 导读: {metadata.get('lead', '')[:60]}")
print(f" 重点: {metadata.get('highlights', [])}")
print(f"\n📰 汇总内容预览:")
print("-" * 40)
print(
digest_content[:500] + "..."
if len(digest_content) > 500
else digest_content
)
print("-" * 40)
# 推送到所有启用的平台
if push_enabled:
print("\n📤 推送消息...")
await send_to_platforms(
digest_content,
config["push"],
title="📰 AI Daily 每日精选 | " + metadata["title"],
metadata=metadata,
)
print(" ✅ 推送成功!")
# 保存到 push 文件
push_file = get_push_file()
save_push_file(
push_file,
digest_content,
len(test_entries),
len(test_entries),
profile="default",
metadata=metadata,
)
print(f"\n💾 已保存汇总到 {push_file}")
except Exception as e:
print(f"\n❌ 汇总推送生成失败: {e}")
import traceback
traceback.print_exc()
print("\n" + "=" * 60)
print("✅ LLM测试完成!")
print("=" * 60)
return True
if __name__ == "__main__":
try:
success = asyncio.run(run_llm_test())
sys.exit(0 if success else 1)
except KeyboardInterrupt:
print("\n\n👋 已取消")
sys.exit(130)
except Exception as e:
print(f"\n❌ 错误: {e}")
import traceback
traceback.print_exc()
sys.exit(1)
+30
View File
@@ -0,0 +1,30 @@
"""模拟一次完整早报推送(强制 is_morning=True,但不发送到推送渠道)"""
import asyncio
import os
import sys
from unittest.mock import AsyncMock, patch
from dotenv import load_dotenv
load_dotenv()
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from src.config import load_config
from src.main import _run_morning_push
async def main():
config = load_config()
async def fake_send(content, push_cfg):
print("\n" + "=" * 60)
print("📤 假推送内容(实际不会发送)")
print("=" * 60)
print(content)
with patch("src.main.send_to_platforms", new=AsyncMock(side_effect=fake_send)):
await _run_morning_push(config)
if __name__ == "__main__":
asyncio.run(main())
@@ -0,0 +1,138 @@
#!/usr/bin/env python3
"""测试 cleanup_old_files 函数
功能:
1. 在 tests/cleanup_test_data 文件夹内创建不同日期的测试文件
2. 运行 cleanup_old_files
3. 显示清理结果
"""
import os
import sys
from datetime import date, datetime, timedelta
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent))
from src.storage import cleanup_old_files
def create_test_files():
"""创建测试文件"""
test_dir = Path("tests/cleanup_test_data")
test_dir.mkdir(parents=True, exist_ok=True)
today = date.today()
created_files = []
# 创建不同日期的文件
file_specs = [
# (文件名模式, 日期偏移, 文件类型)
("fetch", -10, "json"), # 10天前 - 应该删除
("fetch", -8, "json"), # 8天前 - 应该删除
("fetch", -7, "json"), # 7天前 - 应该删除
("fetch", -6, "json"), # 6天前 - 应该保留
("fetch", -3, "json"), # 3天前 - 应该保留
("fetch", -1, "json"), # 1天前 - 应该保留
("fetch", 0, "json"), # 今天 - 应该保留
("push", -10, "md"), # 10天前 - 应该删除
("push", -7, "md"), # 7天前 - 应该删除
("push", -5, "md"), # 5天前 - 应该保留
("push", -2, "md"), # 2天前 - 应该保留
("push", 0, "md"), # 今天 - 应该保留
("notify", -9, "md"), # 9天前 - 应该删除
("notify", -6, "md"), # 6天前 - 应该保留
("notify", -1, "md"), # 1天前 - 应该保留
("notify", 0, "md"), # 今天 - 应该保留
]
print(f"\n📅 今天是: {today}")
print(f" cutoff: {today - timedelta(days=7)} (7天前)")
print(
f" 将删除 < {today - timedelta(days=6)} (< {(today - timedelta(days=6)).strftime('%m-%d')}) 的文件"
)
print(
f" 保留 >= {today - timedelta(days=6)} (>= {(today - timedelta(days=6)).strftime('%m-%d')}) 的文件"
)
print(f"\n📂 创建测试文件到: {test_dir}")
print("-" * 50)
for prefix, offset, ext in file_specs:
file_date = today + timedelta(days=offset)
if prefix == "push":
# push 文件带时间戳
filename = f"push-{file_date.isoformat()}-08-00-00.{ext}"
else:
filename = f"{prefix}-{file_date.isoformat()}.{ext}"
filepath = test_dir / filename
# 创建文件并写入内容
with open(filepath, "w") as f:
f.write(f"测试文件 - {filename}")
status = "🗑️ 将删除" if offset <= -7 else "✅ 将保留"
print(f" {status}: {filename}")
created_files.append(filename)
print("-" * 50)
print(f"✅ 创建了 {len(created_files)} 个测试文件")
return test_dir
def list_files_after_cleanup(test_dir: Path):
"""显示清理后的文件"""
print(f"\n📂 清理后的文件列表:")
print("-" * 50)
files = sorted(test_dir.glob("*"))
if not files:
print(" (空目录)")
else:
for f in files:
print(f"{f.name}")
print("-" * 50)
print(f"{len(files)} 个文件")
def main():
"""主函数"""
print("=" * 60)
print("🧪 cleanup_old_files 测试")
print("=" * 60)
# 1. 创建测试文件
test_dir = create_test_files()
# 2. 列出清理前的文件
print(f"\n📂 清理前的文件列表:")
print("-" * 50)
for f in sorted(test_dir.glob("*")):
print(f" {f.name}")
print("-" * 50)
print(f"{len(list(test_dir.glob('*')))} 个文件")
# 3. 运行清理
print("\n🚀 运行 cleanup_old_files...")
cleanup_old_files(days=7, data_dir=str(test_dir))
# 4. 列出清理后的文件
list_files_after_cleanup(test_dir)
print("\n" + "=" * 60)
print("🎉 测试完成!")
print("=" * 60)
# 自动清理测试目录
import shutil
shutil.rmtree(test_dir)
print(f"✅ 已删除测试目录: {test_dir}")
if __name__ == "__main__":
main()
+213
View File
@@ -0,0 +1,213 @@
#!/usr/bin/env python3
"""测试 fetch_lookback_minutes 功能脚本
测试内容:
1. cutoff 时间计算是否正确
2. load_existing_links 阈值逻辑
3. 跨天边界的去重逻辑
"""
import os
import sys
from datetime import datetime, timedelta, timezone
from pathlib import Path
from unittest.mock import patch
sys.path.insert(0, str(Path(__file__).parent.parent))
from dotenv import load_dotenv
load_dotenv()
from src.config import get_timezone, load_config
from src.storage import get_fetch_file, load_existing_links, read_entries
def test_cutoff_calculation():
"""测试 cutoff 时间计算"""
print("\n" + "=" * 60)
print("📊 Test 1: cutoff 时间计算")
print("=" * 60)
config = load_config()
interval = config["schedule"]["fetch_interval_minutes"]
lookback = config["schedule"].get("fetch_lookback_minutes", 120)
# 确保 lookback >= interval
lookback = max(lookback, interval)
threshold = lookback + interval
print(f"\n配置值:")
print(f" fetch_interval_minutes: {interval}")
print(
f" fetch_lookback_minutes: {config['schedule'].get('fetch_lookback_minutes', 120)}"
)
print(f" 修正后的 lookback: {lookback}")
print(f" threshold (lookback + interval): {threshold}")
# 模拟计算 cutoff
now_utc = datetime.now(timezone.utc)
cutoff = now_utc - timedelta(minutes=lookback)
print(f"\n计算结果:")
print(f" 当前 UTC 时间: {now_utc.strftime('%Y-%m-%d %H:%M:%S')}")
print(f" cutoff 时间: {cutoff.strftime('%Y-%m-%d %H:%M:%S')}")
print(f" 过去 {lookback} 分钟")
assert lookback >= interval, f"lookback ({lookback}) 应该 >= interval ({interval})"
print("\n✅ Test 1 通过: cutoff 计算正确")
def test_threshold_logic():
"""测试 load_existing_links 阈值逻辑"""
print("\n" + "=" * 60)
print("📊 Test 2: load_existing_links 阈值逻辑")
print("=" * 60)
config = load_config()
interval = config["schedule"]["fetch_interval_minutes"]
lookback = config["schedule"].get("fetch_lookback_minutes", 120)
lookback = max(lookback, interval)
threshold = lookback + interval
tz = get_timezone(config)
# 测试不同时间点
test_cases = [
("02:00", threshold, True, "凌晨2点应该需要昨天"),
("02:29", threshold, True, "02:29 应该需要昨天"),
("02:30", threshold, False, "02:30 开始只需要当天"),
("12:00", threshold, False, "中午12点只需要当天"),
("23:59", threshold, False, "23:59 只需要当天"),
]
print(f"\n阈值: {threshold} 分钟 (即 {threshold // 60}小时 {threshold % 60}分钟)")
print(f"\n测试结果:")
all_passed = True
for time_str, thresh, expected_need_yesterday, desc in test_cases:
hour, minute = map(int, time_str.split(":"))
current_minutes = hour * 60 + minute
need_yesterday = current_minutes < thresh
status = "" if need_yesterday == expected_need_yesterday else ""
print(
f" {status} {time_str}: 需要昨天={need_yesterday} (预期: {expected_need_yesterday}) - {desc}"
)
if need_yesterday != expected_need_yesterday:
all_passed = False
if all_passed:
print("\n✅ Test 2 通过: 阈值逻辑正确")
else:
print("\n❌ Test 2 失败: 阈值逻辑有问题")
def test_load_existing_links_files():
"""测试实际加载文件功能"""
print("\n" + "=" * 60)
print("📊 Test 3: 实际加载文件测试")
print("=" * 60)
config = load_config()
interval = config["schedule"]["fetch_interval_minutes"]
lookback = config["schedule"].get("fetch_lookback_minutes", 120)
lookback = max(lookback, interval)
threshold = lookback + interval
tz = get_timezone(config)
now = datetime.now(tz)
current_minutes = now.hour * 60 + now.minute
need_yesterday = current_minutes < threshold
print(f"\n当前时间: {now.strftime('%Y-%m-%d %H:%M:%S')}")
print(f"当前分钟数: {current_minutes}")
print(f"阈值: {threshold}")
print(f"需要加载昨天: {need_yesterday}")
# 测试当天文件
today_file = get_fetch_file()
print(f"\n当天文件: {today_file}")
print(f"文件存在: {Path(today_file).exists()}")
# 测试 load_existing_links 函数
existing = load_existing_links(today_file, threshold)
print(f"加载到的链接数: {len(existing)}")
if need_yesterday:
yesterday = (now - timedelta(days=1)).date()
yesterday_file = get_fetch_file(yesterday)
print(f"\n昨天文件: {yesterday_file}")
print(f"文件存在: {Path(yesterday_file).exists()}")
if Path(yesterday_file).exists():
yesterday_entries = read_entries(yesterday_file)
print(f"昨天文件条目数: {len(yesterday_entries)}")
print("\n✅ Test 3 完成: 文件加载功能正常")
def test_mock_time():
"""模拟不同时间测试阈值逻辑"""
print("\n" + "=" * 60)
print("📊 Test 4: 模拟时间测试")
print("=" * 60)
config = load_config()
interval = config["schedule"]["fetch_interval_minutes"]
lookback = config["schedule"].get("fetch_lookback_minutes", 120)
lookback = max(lookback, interval)
threshold = lookback + interval
print(f"\n配置: interval={interval}, lookback={lookback}, threshold={threshold}")
test_times = [
(0, 0), # 00:00
(2, 20), # 02:20
(2, 30), # 02:30
(3, 0), # 03:00
(8, 0), # 08:00
(12, 0), # 12:00
(23, 59), # 23:59
]
print("\n模拟时间测试:")
for hour, minute in test_times:
current_minutes = hour * 60 + minute
need_yesterday = current_minutes < threshold
time_str = f"{hour:02d}:{minute:02d}"
status = "🔴 需要昨天" if need_yesterday else "🟢 只需当天"
print(f" {time_str} ({current_minutes:4d}分钟): {status}")
print("\n✅ Test 4 完成")
def main():
"""主函数"""
print("\n" + "=" * 60)
print("🧪 fetch_lookback_minutes 功能测试")
print("=" * 60)
try:
test_cutoff_calculation()
test_threshold_logic()
test_load_existing_links_files()
test_mock_time()
print("\n" + "=" * 60)
print("🎉 所有测试完成!")
print("=" * 60)
return 0
except Exception as e:
print(f"\n❌ 测试失败: {e}")
import traceback
traceback.print_exc()
return 1
if __name__ == "__main__":
sys.exit(main())
@@ -0,0 +1,93 @@
"""测试 insights section - 从早报文件解析三个板块并生成洞察"""
import asyncio
import re
import sys
from datetime import datetime
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent))
from dotenv import load_dotenv
from src.config import get_timezone, load_config
from src.sections.insights.section import run_insights_section
load_dotenv()
def parse_sections(content: str) -> dict:
"""从早报内容中解析出各个板块"""
sections = {}
pattern = r"<!-- SECTION:(\w+) BEGIN -->\n(.*?)\n<!-- SECTION:\1 END -->"
for match in re.finditer(pattern, content, re.DOTALL):
section_name = match.group(1)
section_content = match.group(2)
sections[section_name] = section_content
return sections
async def main():
if len(sys.argv) < 2:
print("用法: python test_insights_section.py <早报文件路径>")
print(
"示例: python test_insights_section.py news-data/push-2026-05-25-08-00-00.md"
)
sys.exit(1)
filepath = sys.argv[1]
# 读取早报文件
with open(filepath, "r", encoding="utf-8") as f:
content = f.read()
# 解析板块
sections = parse_sections(content)
print(f"📄 解析文件: {filepath}")
print(f"📋 找到板块: {list(sections.keys())}")
print()
rss_md = sections.get("rss", "")
gh_md = sections.get("github", "")
hn_md = sections.get("hackernews", "")
if not rss_md and not gh_md and not hn_md:
print("❌ 未找到任何板块内容")
sys.exit(1)
print(f"RSS 板块: {len(rss_md)} 字符")
print(f"GitHub 板块: {len(gh_md)} 字符")
print(f"HackerNews 板块: {len(hn_md)} 字符")
print()
# 加载配置
config = load_config()
now = datetime.now(get_timezone(config))
# 调用 insights section
print("🤖 生成洞察中...")
insights_md, metadata, error = await run_insights_section(
rss_md, gh_md, hn_md, config, now
)
if error:
print(f"❌ 错误: {error}")
sys.exit(1)
print("\n" + "=" * 60)
print("📊 Insights 板块结果:")
print("=" * 60)
print(insights_md)
print("=" * 60)
if metadata:
print("\n📋 Metadata:")
for key, value in metadata.items():
print(f" {key}: {value}")
if __name__ == "__main__":
asyncio.run(main())
+113
View File
@@ -0,0 +1,113 @@
#!/usr/bin/env python3
"""测试 push_loop 时间逻辑 - 直接调用 main.py"""
import asyncio
import sys
from datetime import datetime, timedelta
from pathlib import Path
from unittest.mock import patch
sys.path.insert(0, str(Path(__file__).parent.parent))
from src.config import get_timezone, load_config
# 记录调用时间
call_times = []
async def mock_run_push_job(config):
"""模拟 push job"""
now = datetime.now(get_timezone(config))
call_times.append(now)
print(f"\n{'='*40}")
print(f"📤 Mock Push Job 被调用 | {now.strftime('%Y-%m-%d %H:%M:%S')}")
print(f"{'='*40}\n")
async def main():
print("🧪 测试 push_loop 时间逻辑")
print("="*50)
# 加载配置
config = load_config()
tz = get_timezone(config)
# 动态设置 cron:约 30 秒后 和 60 秒后触发
# 标准 cron 是5字段(分 时 日 月 周),通过计算实现秒级等待
now = datetime.now(tz)
# 计算等待时间:
# 第一次:约 30 秒后(下一分钟,等待 = 60 - 当前秒数)
# 第二次:约 60 秒后(下两分钟,等待 = 120 - 当前秒数)
sec_to_wait_1 = 60 - now.second # 到下一分钟的剩余秒数
sec_to_wait_2 = sec_to_wait_1 + 60 # 再加一分钟
min_1 = (now.minute + 1) % 60
min_2 = (now.minute + 2) % 60
cron1 = f"{min_1} {now.hour} * * *"
cron2 = f"{min_2} {now.hour} * * *"
config['schedule']['push_cron'] = [cron1, cron2]
print(f"\n当前时间: {now.strftime('%H:%M:%S')}")
print(f"测试配置:")
print(f" - 第1次推送: {cron1} (约 {sec_to_wait_1}s 后)")
print(f" - 第2次推送: {cron2} (约 {sec_to_wait_2}s 后)")
print()
# 使用 patch mock run_push_job,设置超时 3 分钟
from src import main as main_module
test_task = None
push_task = None
async def run_test():
with patch.object(main_module, 'run_push_job', mock_run_push_job):
await main_module.push_loop(config)
async def timeout_guard():
await asyncio.sleep(180) # 3分钟超时
print("\n⏱️ 测试超时")
if push_task:
push_task.cancel()
try:
# 同时运行 push_loop 和超时守卫
push_task = asyncio.create_task(run_test())
timeout_task = asyncio.create_task(timeout_guard())
# 等待 push_task 完成或超时
while push_task and not push_task.done():
if len(call_times) >= 2:
push_task.cancel()
break
await asyncio.sleep(0.1)
timeout_task.cancel()
except asyncio.CancelledError:
pass
# 验证结果
print("\n" + "="*50)
print("📊 测试结果验证")
print("="*50)
if len(call_times) >= 2:
print(f"✅ 成功调用 {len(call_times)}")
for i, t in enumerate(call_times, 1):
print(f"{i}次: {t.strftime('%H:%M:%S')}")
interval = (call_times[1] - call_times[0]).total_seconds()
print(f"\n实际间隔: {interval:.1f}")
if 55 <= interval <= 65:
print("✅ 间隔正确 (约60秒)")
else:
print(f"⚠️ 间隔异常 (期望 ~60秒)")
else:
print(f"❌ 只调用了 {len(call_times)} 次,期望 2 次")
if __name__ == "__main__":
try:
asyncio.run(main())
except KeyboardInterrupt:
print("\n\n👋 测试已取消")
sys.exit(130)