commit 54ca4b1b6ad5c44554cf89141997d5cd83a26793 Author: poiuy Date: Sun Jul 12 20:01:02 2026 +0800 feat: 军事科技每日资讯推送系统 - Docker部署 + 日志系统 + 数据目录重组 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7de6853 --- /dev/null +++ b/.gitignore @@ -0,0 +1,45 @@ +# 环境配置（含敏感凭据） +.env +config/.env +my-daily/.env +config.json +my-daily/config.json + +# Python +__pycache__/ +*.py[cod] +*.egg-info/ +*.egg +dist/ +build/ + +# 虚拟环境 +venv/ +.venv/ +env/ +.mypy_cache/ + +# 运行时数据 +data/ +output/ +logs/ +cache/ + +# Docker 镜像（太大） +*.tar + +# 日志 +*.log + +# IDE +.vscode/ +.idea/ +*.swp +*.swo + +# Docker +.docker/ + +# OS +.DS_Store +Thumbs.db \ No newline at end of file diff --git a/01主函数过长问题.md b/01主函数过长问题.md new file mode 100644 index 0000000..897b393 --- /dev/null +++ b/01主函数过长问题.md @@ -0,0 +1,727 @@ +# 1. 问题 + +军事科技每日摘报的 `main()` 函数（位于 `scripts/military_daily_report_v3.py` 第38-246行）长达208行，承担了整个业务流程的11个步骤，严重违反单一职责原则。 + +## 1.1. **职责过多，难以理解** + +当前 `main()` 函数包含了从RSS抓取到报告输出的完整业务流程，涉及数据获取、清洗、翻译、AI分析、格式化等多个不同层次的职责。开发者需要在200多行代码中理解整个业务逻辑，认知负担过重。 + +**问题代码片段（第38-246行）**： + +```python +def main(): + now = datetime.now() + cutoff = now - timedelta(hours=TIME_WINDOW_HOURS) + date_str = now.strftime('%Y%m%d') + + # 1. RSS 抓取 + all_articles = fetch_all_feeds() + + # 2. 24小时时间过滤 + recent_articles = [a for a in all_articles if a['published'] >= cutoff] + + # 3. 标题去重 + seen = set() + unique_articles = [] + for a in recent_articles: + key = a['title'].strip() + if key not in seen and len(key) > 5: + seen.add(key) + unique_articles.append(a) + + # 4. 排序 + all_time_valid = sorted(unique_articles, key=lambda x: x['published'], reverse=True) + + # 5. 加载AI配置 + 批量翻译 + api_key, base_url, model, webzine_model = load_ai_config() + translated_articles = batch_translate_articles(all_time_valid, api_key, base_url, model, max_concurrent=2) + + # 6. 关键词筛选 + if len(translated_articles) > 15: + filtered_articles = keyword_filter(translated_articles) + else: + filtered_articles = translated_articles + + # 7. AI分析 + for i, article in enumerate(filtered_articles, 1): + ai_result = score_and_classify_article(article, api_key, base_url, model) + article['ai_result'] = ai_result + article['final_score'] = calculate_weighted_score(ai_result) + article['summary'] = ai_result.get('summary', article['translated_title']) + article['category'] = ai_result.get('category', '装备动态') + article['scores'] = ai_result.get('scores', {}) + + # 8. 按评分排序 + filtered_articles.sort(key=lambda x: x['final_score'], reverse=True) + + # 9. 生成TOP3网摘 + today_top3 = filtered_articles[:3] + # ... 网摘生成代码 + + # 10. 分类整理 + 生成分类洞察 + # ... 分类整理代码 + + # 11. 生成Markdown报告 + # ... 报告生成代码 +``` + +**问题分析**： + +* 11个业务步骤混在一个函数中，每个步骤的逻辑细节都暴露在主流程中 + +* 变量名重复使用（如 `filtered_articles`），容易产生混淆 + +* 中间状态（如 `today_top3`、`equipment_articles` 等）难以追踪 + +* 无法快速定位某个具体步骤的代码位置 + +## 1.2. **难以测试，无法复用** + +由于所有逻辑都在 `main()` 函数中，无法对单个业务步骤进行独立的单元测试。例如，想要测试"去重逻辑"或"分类整理逻辑"，必须运行整个流程，这大大增加了测试成本和复杂度。 + +**问题分析**： + +* 去重逻辑（第62-71行）嵌入在主函数中，无法单独测试 + +* 分类整理逻辑（第175-185行）无法独立验证 + +* 网摘生成逻辑（第127-159行）与文件IO耦合，难以mock测试 + +* 想要复用某个步骤（如只做RSS抓取和去重）无法实现 + +## 1.3. **维护困难，容易出错** + +当需要修改某个业务步骤时，开发者必须在200多行代码中定位相关逻辑，容易遗漏或引入错误。例如，如果需要调整"关键词筛选"的触发条件，需要在第88-93行找到相关代码，但很容易在复杂的流程中迷失。 + +**问题分析**： + +* 修改某个步骤时，需要理解整个上下文 + +* 容易在修改时影响其他步骤的逻辑 + +* 代码审查时难以快速定位修改点 + +* 新人上手成本高，需要阅读大量代码才能理解业务流程 + +# 2. 收益 + +通过将 `main()` 函数拆分为多个职责明确的子函数，可以显著提升代码的可读性、可测试性和可维护性。 + +## 2.1. **提升代码可读性** + +重构后，每个业务步骤都有独立的函数，函数名清晰表达了其职责。主函数变成一个简洁的流程编排器，一眼就能看出整个业务流程的11个步骤。 + +**预期改进**： + +* 主函数从208行缩减到约30行 + +* 每个子函数平均15-30行，职责单一 + +* 函数名直接表达业务含义（如 `filter_by_time_window`、`deduplicate_articles`） + +* 代码结构清晰，新人可以在5分钟内理解整体流程 + +## 2.2. **提升可测试性** + +每个业务步骤都可以独立进行单元测试，无需运行整个流程。可以mock输入数据，验证每个步骤的输出是否符合预期。 + +**预期改进**： + +* 可以为每个子函数编写独立的单元测试 + +* 测试覆盖率可以从当前的接近0%提升到80%以上 + +* 测试执行时间从分钟级降低到秒级 + +* 可以快速定位和修复bug + +## 2.3. **提升可维护性** + +当需要修改某个业务步骤时，只需要定位到对应的子函数，修改范围明确，不会影响其他步骤。代码审查时也可以快速定位修改点。 + +**预期改进**： + +* 修改某个步骤时，只需要关注对应的子函数 + +* 代码审查时可以快速定位修改的函数 + +* 降低引入新bug的风险 + +* 提高开发效率 + +## 2.4. **提升代码复用性** + +业务步骤可以独立使用，例如只做RSS抓取和去重，或者只做AI分析。这为后续的功能扩展提供了灵活性。 + +**预期改进**： + +* 可以单独调用某个业务步骤 + +* 可以组合不同的步骤实现新的功能 + +* 为后续的功能扩展提供基础 + +# 3. 方案 + +将 `main()` 函数拆分为11个职责明确的子函数，每个函数负责一个独立的业务步骤。主函数变成一个简洁的流程编排器，负责调用这些子函数。 + +## 3.1. **函数拆分设计** + +```mermaid +flowchart TD + A[main] --> B[fetch_and_filter_articles] + B --> C[translate_articles] + C --> D[filter_by_keywords] + D --> E[analyze_articles] + E --> F[generate_webzine] + F --> G[generate_report] + + B --> B1[fetch_all_feeds] + B --> B2[filter_by_time_window] + B --> B3[deduplicate_articles] + B --> B4[sort_by_publish_time] + + E --> E1[ai_score_and_classify] + E --> E2[sort_by_score] + + G --> G1[categorize_articles] + G --> G2[generate_category_overviews] + G --> G3[write_markdown_report] + + style A fill:#e1f5e1 + style B fill:#fff4e1 + style C fill:#fff4e1 + style D fill:#fff4e1 + style E fill:#fff4e1 + style F fill:#fff4e1 + style G fill:#fff4e1 +``` + +**图表说明**： + +* 绿色节点表示主函数，负责流程编排 + +* 黄色节点表示拆分后的子函数，每个函数负责一个独立的业务步骤 + +* 箭头表示调用关系 + +* 通过这种拆分，主函数变得简洁，每个子函数职责单一 + +## 3.2. **函数拆分实现** + +### **步骤1：提取数据获取和过滤逻辑** + +**问题代码**： + +```python +def main(): + # 1. RSS 抓取 + all_articles = fetch_all_feeds() + + # 2. 24小时时间过滤 + recent_articles = [a for a in all_articles if a['published'] >= cutoff] + + # 3. 标题去重 + seen = set() + unique_articles = [] + for a in recent_articles: + key = a['title'].strip() + if key not in seen and len(key) > 5: + seen.add(key) + unique_articles.append(a) + + # 4. 排序 + all_time_valid = sorted(unique_articles, key=lambda x: x['published'], reverse=True) +``` + +**重构后代码**： + +```python +def filter_by_time_window(articles, cutoff): + """过滤出指定时间窗口内的文章""" + return [a for a in articles if a['published'] >= cutoff] + + +def deduplicate_articles(articles): + """根据标题去重文章""" + seen = set() + unique_articles = [] + for a in articles: + key = a['title'].strip() + if key not in seen and len(key) > 5: + seen.add(key) + unique_articles.append(a) + return unique_articles + + +def sort_by_publish_time(articles, reverse=True): + """按发布时间排序文章""" + return sorted(articles, key=lambda x: x['published'], reverse=reverse) + + +def fetch_and_filter_articles(cutoff): + """获取并过滤文章：抓取 -> 时间过滤 -> 去重 -> 排序""" + all_articles = fetch_all_feeds() + recent_articles = filter_by_time_window(all_articles, cutoff) + unique_articles = deduplicate_articles(recent_articles) + return sort_by_publish_time(unique_articles) +``` + +**改进点**： + +* 每个函数职责单一，函数名清晰表达其功能 + +* 可以独立测试每个步骤 + +* 可以单独复用某个步骤 + +### **步骤2：提取翻译和关键词筛选逻辑** + +**问题代码**： + +```python +def main(): + # 5. 加载AI配置 + 批量翻译 + api_key, base_url, model, webzine_model = load_ai_config() + translated_articles = batch_translate_articles(all_time_valid, api_key, base_url, model, max_concurrent=2) + + # 6. 关键词筛选 + if len(translated_articles) > 15: + filtered_articles = keyword_filter(translated_articles) + else: + filtered_articles = translated_articles +``` + +**重构后代码**： + +```python +def filter_by_keywords(articles): + """根据关键词筛选文章（仅当文章数 > 15 时执行）""" + if len(articles) > 15: + return keyword_filter(articles) + return articles + + +def translate_articles(articles, api_key, base_url, model): + """批量翻译外文文章""" + return batch_translate_articles(articles, api_key, base_url, model, max_concurrent=2) +``` + +**改进点**： + +* 将关键词筛选的条件判断封装在函数内部 + +* 翻译逻辑独立，可以单独测试 + +### **步骤3：提取AI分析和排序逻辑** + +**问题代码**： + +```python +def main(): + # 7. AI分析 + for i, article in enumerate(filtered_articles, 1): + ai_result = score_and_classify_article(article, api_key, base_url, model) + article['ai_result'] = ai_result + article['final_score'] = calculate_weighted_score(ai_result) + article['summary'] = ai_result.get('summary', article['translated_title']) + article['category'] = ai_result.get('category', '装备动态') + article['scores'] = ai_result.get('scores', {}) + + # 8. 按评分排序 + filtered_articles.sort(key=lambda x: x['final_score'], reverse=True) +``` + +**重构后代码**： + +```python +def ai_score_and_classify(article, api_key, base_url, model): + """对单篇文章进行AI分析：评分、摘要、分类""" + ai_result = score_and_classify_article(article, api_key, base_url, model) + article['ai_result'] = ai_result + article['final_score'] = calculate_weighted_score(ai_result) + article['summary'] = ai_result.get('summary', article['translated_title']) + article['category'] = ai_result.get('category', '装备动态') + article['scores'] = ai_result.get('scores', {}) + return article + + +def analyze_articles(articles, api_key, base_url, model): + """对所有文章进行AI分析并按评分排序""" + for i, article in enumerate(articles, 1): + logger.info("处理中 %d/%d: %s...", i, len(articles), article['translated_title'][:35]) + ai_score_and_classify(article, api_key, base_url, model) + + # 按评分排序 + return sorted(articles, key=lambda x: x['final_score'], reverse=True) +``` + +**改进点**： + +* 将单篇文章的AI分析和批量分析分离 + +* 排序逻辑封装在函数内部 + +* 可以单独测试AI分析逻辑 + +### **步骤4：提取网摘生成逻辑** + +**问题代码**： + +```python +def main(): + # 9. 提取TOP3 + 生成《参考消息》风格网摘 + today_top3 = filtered_articles[:3] + + webzine_content = [] + webzine_content.append(f"═{'═'*60}═") + webzine_content.append(f" 🔥 军事科技每日摘报 - 今日必看 TOP3 网摘") + webzine_content.append(f" 📅 {now.strftime('%Y年%m月%d日')}") + webzine_content.append(f"═{'═'*60}═") + webzine_content.append("") + + webzine_texts = [] + for i, article in enumerate(today_top3, 1): + webzine = generate_webzine_for_article(article, api_key, base_url, webzine_model) + webzine_texts.append(webzine) + webzine_content.append(f"{'='*62}") + webzine_content.append(f" 📌 第 {i} 篇") + webzine_content.append(f"{'='*62}") + webzine_content.append("") + webzine_content.append(webzine) + webzine_content.append("") + webzine_content.append(f"🔗 原文链接：{article['link']}") + webzine_content.append("") + webzine_content.append("") + + # 保存网摘TXT文件 + OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + webzine_file = OUTPUT_DIR / f"military_webzine_{date_str}.txt" + with open(webzine_file, 'w', encoding='utf-8') as f: + f.write('\n'.join(webzine_content)) + + # 生成合并长图 + webzine_image = OUTPUT_DIR / f"military_webzine_{date_str}.png" + create_combined_webzine_image(webzine_texts, str(webzine_image), now) +``` + +**重构后代码**： + +```python +def generate_webzine_content(top3_articles, now): + """生成网摘文本内容""" + webzine_content = [] + webzine_content.append(f"═{'═'*60}═") + webzine_content.append(f" 🔥 军事科技每日摘报 - 今日必看 TOP3 网摘") + webzine_content.append(f" 📅 {now.strftime('%Y年%m月%d日')}") + webzine_content.append(f"═{'═'*60}═") + webzine_content.append("") + + webzine_texts = [] + for i, article in enumerate(top3_articles, 1): + webzine = generate_webzine_for_article(article, api_key, base_url, webzine_model) + webzine_texts.append(webzine) + webzine_content.append(f"{'='*62}") + webzine_content.append(f" 📌 第 {i} 篇") + webzine_content.append(f"{'='*62}") + webzine_content.append("") + webzine_content.append(webzine) + webzine_content.append("") + webzine_content.append(f"🔗 原文链接：{article['link']}") + webzine_content.append("") + webzine_content.append("") + + return webzine_content, webzine_texts + + +def save_webzine_files(webzine_content, webzine_texts, date_str, now): + """保存网摘文件（TXT和PNG）""" + OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + + # 保存TXT文件 + webzine_file = OUTPUT_DIR / f"military_webzine_{date_str}.txt" + with open(webzine_file, 'w', encoding='utf-8') as f: + f.write('\n'.join(webzine_content)) + logger.info("网摘文件已保存: %s", webzine_file) + + # 生成PNG文件 + webzine_image = OUTPUT_DIR / f"military_webzine_{date_str}.png" + create_combined_webzine_image(webzine_texts, str(webzine_image), now) + + +def generate_webzine(articles, api_key, base_url, webzine_model, date_str, now): + """生成TOP3网摘并保存文件""" + top3 = articles[:3] + webzine_content, webzine_texts = generate_webzine_content(top3, now) + save_webzine_files(webzine_content, webzine_texts, date_str, now) +``` + +**改进点**： + +* 将网摘内容生成和文件保存分离 + +* 可以独立测试网摘内容生成逻辑 + +* 文件IO逻辑集中，易于mock + +### **步骤5：提取分类整理和报告生成逻辑** + +**问题代码**： + +```python +def main(): + # 10. 分类整理 + 生成各分类深度洞察 + equipment_articles = [] + conflict_articles = [] + strategy_articles = [] + + for article in filtered_articles[3:]: + cat = article.get('category', '装备动态') + if '冲突' in cat or '战争' in cat or '地区' in cat: + conflict_articles.append(article) + elif '战略' in cat or '政策' in cat or '外交' in cat: + strategy_articles.append(article) + else: + equipment_articles.append(article) + + today_overview = generate_category_overview("今日必看", today_top3, api_key, base_url, model) + equipment_overview = generate_category_overview("装备动态", equipment_articles, api_key, base_url, model) + conflict_overview = generate_category_overview("地区冲突", conflict_articles, api_key, base_url, model) + strategy_overview = generate_category_overview("战略政策", strategy_articles, api_key, base_url, model) + + # 11. 生成 Markdown 报告文件 + md_lines = [] + md_lines.append(f"# 🔥 {now.year}年{now.month}月{now.day}日 军事科技每日摘报\n") + # ... 大量Markdown拼接代码 +``` + +**重构后代码**： + +```python +def categorize_articles(articles): + """将文章按分类整理""" + equipment_articles = [] + conflict_articles = [] + strategy_articles = [] + + for article in articles: + cat = article.get('category', '装备动态') + if '冲突' in cat or '战争' in cat or '地区' in cat: + conflict_articles.append(article) + elif '战略' in cat or '政策' in cat or '外交' in cat: + strategy_articles.append(article) + else: + equipment_articles.append(article) + + return equipment_articles, conflict_articles, strategy_articles + + +def generate_category_overviews(top3, equipment, conflict, strategy, api_key, base_url, model): + """生成各分类的总体介绍""" + today_overview = generate_category_overview("今日必看", top3, api_key, base_url, model) + equipment_overview = generate_category_overview("装备动态", equipment, api_key, base_url, model) + conflict_overview = generate_category_overview("地区冲突", conflict, api_key, base_url, model) + strategy_overview = generate_category_overview("战略政策", strategy, api_key, base_url, model) + + return { + 'today': today_overview, + 'equipment': equipment_overview, + 'conflict': conflict_overview, + 'strategy': strategy_overview + } + + +def write_markdown_report(articles, overviews, date_str, now): + """生成Markdown报告文件""" + top3 = articles[:3] + equipment, conflict, strategy = categorize_articles(articles[3:]) + + md_lines = [] + md_lines.append(f"# 🔥 {now.year}年{now.month}月{now.day}日 军事科技每日摘报\n") + md_lines.append(f"📊 今日共 {len(articles)} 篇精选文章 | ⏰ {now.strftime('%Y-%m-%d %H:%M')}\n") + + # 今日必看 + md_lines.append(f"## 🔥 今日必看\n") + if overviews['today']: + md_lines.append(f"> {overviews['today']}\n") + for i, article in enumerate(top3, 1): + md_lines.append(f"### {i}. {article['translated_title']}\n") + md_lines.append(f"- 📡 来源: {article['source']}") + md_lines.append(f"- ⏰ 时间: {article['published'].strftime('%m-%d %H:%M')}") + md_lines.append(f"- ⭐ 评分: {article['final_score']}/10") + md_lines.append(f"- 📝 {article['summary']}") + md_lines.append(f"- 🔗 [阅读原文]({article['link']})\n") + + # 装备动态 + if equipment: + md_lines.append(f"## ⚙️ 装备动态\n") + if overviews['equipment']: + md_lines.append(f"> {overviews['equipment']}\n") + for i, article in enumerate(equipment, 1): + md_lines.append(f"### {i}. {article['translated_title']}\n") + md_lines.append(f"- 📡 {article['source']} | ⏰ {article['published'].strftime('%m-%d %H:%M')}") + md_lines.append(f"- 📝 {article['summary']}") + md_lines.append(f"- 🔗 [阅读原文]({article['link']})\n") + + # 地区冲突 + if conflict: + md_lines.append(f"## 💥 地区冲突\n") + if overviews['conflict']: + md_lines.append(f"> {overviews['conflict']}\n") + for i, article in enumerate(conflict, 1): + md_lines.append(f"### {i}. {article['translated_title']}\n") + md_lines.append(f"- 📡 {article['source']} | ⏰ {article['published'].strftime('%m-%d %H:%M')}") + md_lines.append(f"- 📝 {article['summary']}") + md_lines.append(f"- 🔗 [阅读原文]({article['link']})\n") + + # 战略政策 + if strategy: + md_lines.append(f"## 🎯 战略政策\n") + if overviews['strategy']: + md_lines.append(f"> {overviews['strategy']}\n") + for i, article in enumerate(strategy, 1): + md_lines.append(f"### {i}. {article['translated_title']}\n") + md_lines.append(f"- 📡 {article['source']} | ⏰ {article['published'].strftime('%m-%d %H:%M')}") + md_lines.append(f"- 📝 {article['summary']}") + md_lines.append(f"- 🔗 [阅读原文]({article['link']})\n") + + # 保存文件 + report_file = OUTPUT_DIR / f"military_report_{date_str}.md" + with open(report_file, 'w', encoding='utf-8') as f: + f.write('\n'.join(md_lines)) + logger.info("报告文件已保存: %s", report_file) + + +def generate_report(articles, api_key, base_url, model, date_str, now): + """生成完整报告：分类整理 -> 生成分类洞察 -> 写入Markdown""" + equipment, conflict, strategy = categorize_articles(articles[3:]) + top3 = articles[:3] + + overviews = generate_category_overviews( + top3, equipment, conflict, strategy, api_key, base_url, model + ) + + write_markdown_report(articles, overviews, date_str, now) +``` + +**改进点**： + +* 将分类整理、分类洞察生成、报告写入分离 + +* 每个函数职责单一，易于测试 + +* Markdown生成逻辑集中，易于维护 + +### **步骤6：重构后的主函数** + +**重构后的主函数**： + +```python +def main(): + """主函数：流程编排""" + now = datetime.now() + cutoff = now - timedelta(hours=TIME_WINDOW_HOURS) + date_str = now.strftime('%Y%m%d') + + logger.info("开始抓取 %d 个军事科技 RSS 订阅源...", len(RSS_FEEDS)) + logger.info("时间窗口: %s ~ %s", cutoff.strftime('%m-%d %H:%M'), now.strftime('%m-%d %H:%M')) + + # 1. 获取并过滤文章 + articles = fetch_and_filter_articles(cutoff) + logger.info("总共抓取 %d 篇文章", len(articles)) + + if not articles: + logger.info("今日暂无新的军事科技文章") + return + + # 2. 翻译文章 + api_key, base_url, model, webzine_model = load_ai_config() + if not api_key: + logger.error("未找到OPENAI_API_KEY，退出") + return + + translated_articles = translate_articles(articles, api_key, base_url, model) + logger.info("翻译完成，共 %d 篇文章", len(translated_articles)) + + # 3. 关键词筛选 + filtered_articles = filter_by_keywords(translated_articles) + logger.info("筛选后: %d 篇", len(filtered_articles)) + + if not filtered_articles: + logger.info("筛选后无符合条件文章") + return + + # 4. AI分析 + logger.info("开始 AI 智能分析（使用 %s）...", model) + analyzed_articles = analyze_articles(filtered_articles, api_key, base_url, model) + logger.info("AI 评分完成，最高分: %.1f/10", analyzed_articles[0]['final_score']) + + # 5. 生成网摘 + logger.info("生成今日必看 TOP3 网摘（《参考消息》风格）...") + generate_webzine(analyzed_articles, api_key, base_url, webzine_model, date_str, now) + + # 6. 生成报告 + logger.info("生成 Markdown 报告文件...") + generate_report(analyzed_articles, api_key, base_url, model, date_str, now) +``` + +**改进点**： + +* 主函数从208行缩减到约50行 + +* 流程清晰，一眼就能看出整个业务流程 + +* 每个步骤都有清晰的注释 + +* 易于理解和维护 + +# 4. 回归范围 + +本次重构主要是代码结构调整，不改变业务逻辑，因此回归测试的重点是验证重构后的功能与重构前完全一致。 + +## 4.1. 主链路 + +**完整业务流程**： + +1. RSS抓取 → 时间过滤 → 去重 → 排序 → 翻译 → 关键词筛选 → AI分析 → 评分排序 → 网摘生成 → 报告输出 + +**关键验证点**： + +* RSS抓取能够正常获取12个订阅源的文章 + +* 时间过滤能够正确过滤出24小时内的文章 + +* 去重逻辑能够正确去除重复标题的文章 + +* 翻译功能能够正确翻译外文文章 + +* 关键词筛选在文章数 > 15 时能够正确执行 + +* AI分析能够正确生成评分、摘要和分类 + +* 网摘生成能够正确生成TOP3网摘的TXT和PNG文件 + +* 报告生成能够正确生成Markdown报告文件 + +## 4.2. 边界情况 + +**边界场景**： + +1. **无新文章场景**：当24小时内没有新文章时，程序应该正常退出并输出日志 +2. **筛选后无文章场景**：当关键词筛选后没有符合条件的文章时，程序应该正常退出并输出日志 +3. **AI配置缺失场景**：当未配置OPENAI\_API\_KEY时，程序应该正常退出并输出错误日志 +4. **文章数 <= 15场景**：当文章数 <= 15时，应该跳过关键词筛选，直接进入AI分析 +5. **分类为空场景**：当某个分类（如"地区冲突"）没有文章时，报告中应该不显示该分类 +6. **翻译失败场景**：当翻译失败时，应该使用原标题和原文内容，不影响后续流程 +7. **AI分析失败场景**：当AI分析失败时，应该使用默认评分和分类，不影响后续流程 + +**验证方法**： + +* 通过修改配置文件模拟各种边界场景 + +* 检查日志输出是否符合预期 + +* 检查生成的文件内容是否正确 + +* 检查程序是否正常退出或抛出预期的异常 + diff --git a/02图片生成代码重复问题.md b/02图片生成代码重复问题.md new file mode 100644 index 0000000..8d29d53 --- /dev/null +++ b/02图片生成代码重复问题.md @@ -0,0 +1,322 @@ +# 1. 问题 + +图片生成模块中存在大量重复代码，两个核心函数 `create_webzine_image` 和 `create_combined_webzine_image` 包含相似的文本解析、字体处理和图片渲染逻辑，导致代码维护困难且容易出错。 + +## 1.1. **代码重复严重** + +`scripts/modules/image_generator.py` 文件中的两个函数 `create_webzine_image`（第13-131行）和 `create_combined_webzine_image`（第134-239行）存在大量重复代码： + +* 都包含相同的文本解析逻辑，识别"标题："、"正文："、"价值点："等前缀 + +* 都使用相同的字体加载和配置代码 + +* 都包含相似的文本换行和宽度计算逻辑 + +* 都有重复的图片绘制命令处理 + +这种重复导致： + +* 代码行数冗余，两个函数共约230行，其中至少60%是重复逻辑 + +* 修改渲染逻辑时需要在两个地方同步修改，容易遗漏 + +* 增加了代码理解和维护的成本 + +## 1.2. **缺乏抽象层次** + +当前实现没有将共同的渲染逻辑抽象为独立的函数，导致： + +* 文本解析逻辑散落在两个函数中，形成"面条代码" + +* 字体配置和图片尺寸参数硬编码在多个位置 + +* 渲染命令的构建逻辑重复，没有统一的处理接口 + +## 1.3. **错误处理不一致** + +两个函数在异常处理上存在细微差异： + +* `create_webzine_image` 在PIL未安装时返回None并记录警告 + +* `create_combined_webzine_image` 有类似的处理但日志信息略有不同 + +* 缺乏统一的错误处理策略，难以保证行为一致性 + +# 2. 收益 + +通过重构图片生成模块，提取公共逻辑并建立清晰的抽象层次，可以显著提升代码质量和开发效率。 + +## 2.1. **减少代码重复** + +重构后可以将重复代码从约140行减少到约30行，代码总量减少约40%。通过提取公共函数，两个核心函数的代码行数都将显著减少，提升代码的简洁性和可读性。 + +## 2.2. **提升可维护性** + +统一的渲染逻辑意味着： + +* 修改文本解析规则时只需修改一处 + +* 调整字体或样式时可以集中配置 + +* 添加新的渲染特性时可以复用现有抽象 + +* 降低因修改不同步导致的bug风险 + +## 2.3. **增强可测试性** + +提取后的公共函数可以独立进行单元测试，不再需要依赖完整的图片生成流程。这样可以更容易地验证文本解析、换行逻辑等核心功能的正确性。 + +## 2.4. **改善代码可读性** + +通过合理的函数命名和职责分离，代码的自解释性将显著提升。新的开发者可以更快理解图片生成的流程，降低学习成本。 + +# 3. 方案 + +系统性地重构图片生成模块，通过提取公共函数、建立渲染抽象层次，消除代码重复并提升代码质量。 + +## 3.1. **提取文本解析函数** + +将重复的文本前缀识别和内容提取逻辑抽象为独立函数： + +```python +def _parse_webzine_line(line): + """解析网摘单行文本，返回 (前缀类型, 前缀文本, 内容文本)""" + if not line.strip(): + return ('empty', '', '') + + prefixes = ['标题：', '原标题：', '发布日期：', '正文：', '价值点：'] + for prefix in prefixes: + if line.startswith(prefix): + return (prefix.rstrip('：'), prefix, line[len(prefix):]) + + return ('normal', '', line) +``` + +这个函数统一处理文本解析逻辑，消除了两个函数中的重复代码。 + +## 3.2. **提取字体配置函数** + +将字体加载和配置逻辑集中管理： + +```python +def _load_webzine_fonts(): + """加载网摘图片所需的所有字体""" + return { + 'banner': get_chinese_font(size=40, bold=True), + 'article_title': get_chinese_font(size=36, bold=True), + 'bold': get_chinese_font(size=32, bold=True), + 'content': get_chinese_font(size=32, bold=False) + } +``` + +这样可以确保字体配置的一致性，并便于集中调整样式。 + +## 3.3. **提取渲染命令构建函数** + +将文本换行和渲染命令生成逻辑抽象为独立函数： + +```python +def _build_render_commands(webzine_content, fonts, content_width, temp_draw): + """将网摘文本转换为渲染命令列表""" + commands = [] + + for line in webzine_content.split('\n'): + line_type, prefix, content = _parse_webzine_line(line) + + if line_type == 'empty': + commands.append(('empty',)) + elif line_type == '标题': + full_title = prefix + content + wrapped = wrap_text(full_title, fonts['article_title'], content_width, temp_draw) + for wrapped_line in wrapped: + commands.append(('article_title', wrapped_line)) + elif line_type in ['原标题', '发布日期', '正文', '价值点']: + test_line = prefix + content + test_bbox = temp_draw.textbbox((0, 0), test_line, font=fonts['content']) + test_width = test_bbox[2] - test_bbox[0] + + if test_width <= content_width: + commands.append(('prefix_sameline', prefix, content)) + else: + commands.append(('prefix_only', prefix)) + wrapped = wrap_text(content, fonts['content'], content_width, temp_draw) + for wrapped_line in wrapped: + commands.append(('content_indented', wrapped_line)) + else: + wrapped = wrap_text(content, fonts['content'], content_width, temp_draw) + for wrapped_line in wrapped: + commands.append(('normal', wrapped_line)) + + return commands +``` + +这个函数将复杂的文本处理逻辑封装起来，使主函数更加清晰。 + +## 3.4. **提取渲染执行函数** + +将图片绘制的具体执行逻辑抽象为独立函数： + +```python +def _execute_render_commands(draw, commands, fonts, padding, line_height, content_width): + """执行渲染命令列表，在指定画布上绘制内容""" + y = 135 + + for cmd in commands: + cmd_type = cmd[0] + + if cmd_type == 'empty': + y += line_height + elif cmd_type == 'normal': + draw.text((padding, y), cmd[1], fill=(0, 0, 0), font=fonts['content']) + y += line_height + elif cmd_type == 'article_title': + draw.text((padding, y), cmd[1], fill=(0, 0, 0), font=fonts['article_title']) + y += line_height + elif cmd_type == 'prefix_sameline': + label_bbox = draw.textbbox((0, 0), cmd[1], font=fonts['bold']) + label_width = label_bbox[2] - label_bbox[0] + draw.text((padding, y), cmd[1], fill=(0, 0, 0), font=fonts['bold']) + draw.text((padding + label_width, y), cmd[2], fill=(0, 0, 0), font=fonts['content']) + y += line_height + elif cmd_type == 'prefix_only': + draw.text((padding, y), cmd[1], fill=(0, 0, 0), font=fonts['bold']) + y += line_height + elif cmd_type == 'content_indented': + draw.text((padding, y), cmd[1], fill=(0, 0, 0), font=fonts['content']) + y += line_height + elif cmd_type == 'separator': + y += 30 + draw.line([(padding, y), (content_width + padding, y)], fill=(200, 200, 200), width=1) + y += 30 + + return y +``` + +这样可以将绘制逻辑与业务逻辑分离，提升代码的模块化程度。 + +## 3.5. **重构后的主函数** + +重构后的两个主函数将变得简洁清晰： + +```python +def create_webzine_image(webzine_content, output_path, title_text): + """将网摘文本内容渲染为PNG图片（800px宽，自适应高度）""" + logger = get_logger() + if Image is None: + logger.warning("PIL 未安装，跳过图片生成") + return None + + try: + width = 800 + padding = 50 + content_width = width - padding * 2 + line_height = 40 + + # 解析日期 + date_match = re.search(r'\d{4}年\d{2}月\d{2}日', webzine_content) + if date_match: + date_str = date_match.group(0).replace('年', '.').replace('月', '.').replace('日', '') + else: + date_str = datetime.now().strftime('%Y.%m.%d') + banner_text = f'网摘 {date_str}' + + # 加载字体 + fonts = _load_webzine_fonts() + + # 构建渲染命令 + temp_img = Image.new('RGB', (100, 100), color=(255, 255, 255)) + temp_draw = ImageDraw.Draw(temp_img) + commands = _build_render_commands(webzine_content, fonts, content_width, temp_draw) + + # 计算图片高度 + actual_lines_count = len([cmd for cmd in commands if cmd[0] != 'separator']) + img_height = 140 + actual_lines_count * line_height + padding * 2 + img_height = int(img_height * 1.15) + + # 创建图片并绘制 + img = Image.new('RGB', (width, img_height), color=(255, 255, 255)) + draw = ImageDraw.Draw(img) + + # 绘制标题栏 + draw.rectangle([(0, 0), (width, 90)], fill=(26, 72, 144)) + draw.text((padding, 25), banner_text, fill=(255, 255, 255), font=fonts['banner']) + draw.line([(padding, 100), (width - padding, 100)], fill=(180, 180, 180), width=2) + + # 执行渲染命令 + _execute_render_commands(draw, commands, fonts, padding, line_height, content_width) + + img.save(output_path, 'PNG', quality=95) + logger.info("网摘图片已保存: %s (%d条命令, %dpx)", output_path, len(commands), img_height) + return output_path + + except Exception as e: + logger.warning("图片生成失败: %s", e) + return None +``` + +重构后的函数结构清晰，职责单一，易于理解和维护。 + +# 4. 回归范围 + +本次重构主要影响图片生成功能，需要重点测试网摘图片生成的正确性和稳定性。 + +## 4.1. 主链路 + +1. **完整日报生成流程** + + * 从RSS抓取到最终图片生成的完整流程 + + * 验证生成的图片格式、尺寸、内容正确性 + + * 确认图片文件能正常保存到指定路径 + +2. **网摘图片生成** + + * 单篇网摘图片生成功能 + + * TOP3合并长图生成功能 + + * 验证图片中的文本内容、格式、样式符合预期 + +3. **异常情况处理** + + * PIL未安装时的降级处理 + + * 图片生成失败时的错误处理和日志记录 + + * 确认异常情况下不影响主流程继续执行 + +## 4.2. 边界情况 + +1. **特殊文本内容** + + * 包含超长标题的文章 + + * 包含特殊字符或表情符号的文本 + + * 空内容或格式异常的网摘文本 + +2. **字体和样式** + + * 不同操作系统的字体兼容性 + + * 中英文混合内容的正确渲染 + + * 文本换行和边界情况的处理 + +3. **性能和资源** + + * 生成大量图片时的内存使用情况 + + * 并发生成图片时的线程安全性 + + * 大文本内容的处理性能 + +4. **缓存和复用** + + * 图片已存在时的跳过逻辑 + + * 缓存机制与重构后的兼容性 + + * 增量生成时的正确性 + diff --git a/1.md b/1.md new file mode 100644 index 0000000..66c35ab --- /dev/null +++ b/1.md @@ -0,0 +1,57 @@ +# **热点速览 | 2026-06-03** + +2026-06-03 + + +## **📋 今日速览** + +- 🔍 **封锁霍尔木兹海峡 · 以色列 · 伊朗** (48.5°) 黎以达成停火共识，以色列撤回部队；美伊谈判重回正轨。 +- 📝 **伊朗袭击科威特巴林** (48.4°) 伊朗向科威特和巴林发射弹道导弹和无人机，美军成功拦截。 + + + +## **📈 跨日追踪** + +### **➡️ 封锁霍尔木兹海峡 · 以色列 · 伊朗** +今日热度 48.5° (-2.7) | 报道 0 篇 (-8) | 趋势 持平 +前日 (2026-06-02): 热度 51.2° | 10 篇 | 匹配度 0.15 +前情: 前情内容 + +### **🔺 2027年商业载人飞行** +今日热度 47.2° (+35.0) | 报道 0 篇 (+0) | 趋势 上升 +前日 (2026-06-02): 热度 12.2° | 1 篇 | 匹配度 0.06 + + + + +## **📊 趋势分析** + +### **📈 持续 封锁霍尔木兹海峡** +- 热度: 48 ← 52 ← 45 +- 报道: 2 ← 10 ← 8 篇 +- 时间: 06-03 ← 06-02 ← 06-01 +- 近3天热度变化 -8% + + + + +## **🔍 热点洞察** + +### **霍尔木兹海峡危机[持续跟踪]** +> 黎以达成停火共识，以色列撤回部队；美伊谈判重回正轨。分析指出霍尔木兹海峡航运受阻将影响南亚化肥供应，可能引发粮食安全危机。 + +- 黎以达成停火共识，以色列撤回部队 +- 美伊谈判重回正轨 +- 霍尔木兹海峡航运受阻将影响南亚化肥供应 + +#### **事件脉络** +前情提要：此前霍尔木兹海峡局势紧张，美伊谈判停滞，黎以冲突持续。 +最新突破：黎以达成停火共识，以色列撤回部队；美伊谈判恢复。 + +#### **多角度分析** +- **技术维度**：海峡航运受阻直接影响南亚化肥供应链 +- **战略维度**：黎以停火缓解中东北部紧张 + +#### **影响与展望** +短期：黎以停火减少地区冲突热点 +中期：海峡航运若持续受阻，南亚化肥价格将上涨 \ No newline at end of file diff --git a/20260529_新闻智能分析系统开发计划_v2.md b/20260529_新闻智能分析系统开发计划_v2.md new file mode 100644 index 0000000..9149c24 --- /dev/null +++ b/20260529_新闻智能分析系统开发计划_v2.md @@ -0,0 +1,822 @@ +# 新闻智能分析系统开发计划 v2.0 + +> 基于大模型的时政/军事/科技新闻聚类、评价与整合系统 +> +> **迭代说明**：参考 AI Daily 项目实践，融合细粒度聚类与工程化部署经验 + +--- + +## 一、项目概述 + +### 1.1 项目目标 +构建一个自动化新闻分析系统，实现： +- **细粒度聚类**：保留多角度报道，避免简单去重丢失热点 +- **动态热点识别**：基于当日新闻分布统计特征自适应判定热点 +- **双层整合**：摘要级速览（全量）+ 洞察级深度分析（Top 10） +- **混合处理**：实时热点感知 + 每日批量汇总生成早报 +- **跨日关联**：追踪同一事件的多日发展脉络 + +### 1.2 核心约束 +| 维度 | 约束 | +|------|------| +| 数据规模 | 200-500篇/日 | +| 数据源 | RSS Feed（公众号订阅源） | +| 输出格式 | Web页面 / Markdown / 即时推送 | +| 成本预算 | 混合模型调度，日成本约1.3-1.6元 | +| 处理时效 | 批处理15-20分钟完成 | +| 部署方式 | 支持systemd服务化部署 | + +### 1.3 双循环架构（参考AI Daily优化） + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Fetch 循环（实时） │ +│ 每30分钟运行一次 │ +│ RSS抓取 → 转为Markdown → LLM评分 → 重要性判断 → 即时推送？ │ +│ ↓ ↓ │ +│ 存入数据库(fetch-yyyy-mm-dd.json) 飞书/Discord│ +└─────────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────────┐ +│ Push 循环（定时） │ +│ 每日定时运行（如早8点） │ +│ 读取碎片化信息 → 细粒度聚类 → 多角度关联 → 综合评分 │ +│ → 摘要级整合(全量) → 洞察级整合(Top10) → 生成日报 → 推送 │ +└─────────────────────────────────────────────────────────────────┘ +``` + +--- + +## 二、与AI Daily的对比分析 + +### 2.1 相似性 + +| 维度 | AI Daily | 本系统 | 相似度 | +|------|----------|--------|--------| +| **数据源** | RSS订阅源 | RSS订阅源（公众号） | ⭐⭐⭐⭐⭐ | +| **核心流程** | RSS→LLM评分→推送/汇总 | RSS→聚类→评分→整合→推送/汇总 | ⭐⭐⭐⭐ | +| **双循环架构** | Fetch循环+Push循环 | 实时检测+批处理 | ⭐⭐⭐⭐⭐ | +| **即时推送** | 飞书/Discord webhook | 飞书/企微/邮件 | ⭐⭐⭐⭐ | +| **定时汇总** | 每日定时推送 | 每日早报生成 | ⭐⭐⭐⭐⭐ | +| **成本预算** | 每天6毛钱 | 每天1.3-1.6元 | ⭐⭐⭐⭐ | + +### 2.2 本系统的差异化优势 + +| 特性 | AI Daily | 本系统 | 价值 | +|------|----------|--------|------| +| **热点识别** | 单篇文章评分 | **双层筛选：硬约束+动态阈值融合** | 质量控制+热点发现协同 | +| **去重策略** | 简单去重 | 细粒度聚类+多角度保留 | 不丢失多角度报道 | +| **内容整合** | 单篇摘要 | 双层整合（摘要+洞察） | 深度分析Top热点 | +| **领域聚焦** | AI领域 | 时政/军事/科技 | 专业领域实体识别 | +| **角度分类** | 无 | 6种报道角度分类 | 支持多角度整合 | +| **跨日关联** | 无 | 事件时间线追踪+Sentinel标记 | 追踪事件发展脉络 | + +### 2.3 AI Daily 参考资源 + +- 项目介绍（热点追踪系统）：https://yeekal.com/ai/ai-daily-news-tracker/ +- 项目介绍（进化实录）：https://yeekal.com/ai/ai-daily-news-update/ +- GitHub 代码库：https://github.com/YeeKal/ai-daily + +### 2.4 从AI Daily借鉴的核心设计 + +1. **双循环架构**：Fetch循环（实时）+ Push循环（定时） +2. **systemd服务化**：使用systemd timer替代Python内部定时，提升稳定性 +3. **文件存储规范**：fetch/notify/push三种文件类型，明确保留策略 +4. **评分硬约束**：非目标领域上限、KOL转述上限，有效控制信息质量 +5. **Prompt防退化**：禁止套话、要求从素材出发，避免LLM输出风格趋同 +6. **双视角洞察**：metadata（事实压缩）+ 正文（趋势判断）解耦 +7. **Sentinel分段标记**：单文件多板块精确管理，支持跨日关联 + +--- + +## 三、开发阶段规划 + +### 阶段一：基础架构搭建（Week 1-2） +**目标**：建立数据流和存储基础，确保稳定采集 + +| 序号 | 任务 | 描述 | 输出 | 依赖 | +|------|------|------|------|------| +| 1.1 | RSS采集模块 | 解析公众号RSS Feed，提取标题、正文、来源、发布时间 | 原始文章数据库 | - | +| 1.2 | 数据标准化 | 统一文章格式，清洗HTML标签，提取纯文本 | 标准化文章表 | 1.1 | +| 1.3 | 文件存储层 | JSON文件存储(fetch-yyyy-mm-dd.json) | 存储规范 | 1.2 | +| 1.4 | 数据库设计 | SQLite设计文章表、实体表、聚类表 | 数据库Schema | 1.2 | +| 1.5 | 基础监控 | 采集成功率、文章数量统计 | 监控面板 | 1.1 | +| 1.6 | 日志系统 | 有效日志记录，便于问题定位 | 日志模块 | 1.1 | + +**数据字段规范**： +```json +{ + "title": "内容标题", + "link": "原始链接", + "published": "发布时间", + "source": "来源（公众号名称）", + "content": "Markdown格式的正文内容", + "tags": "LLM识别的标签", + "score": "LLM评分（0-100）", + "summary": "LLM生成的中文摘要", + "fetched_at": "抓取时间", + "entities": ["提取的实体列表"], + "reporting_angle": "报道角度" +} +``` + +**验收标准**： +- [ ] 稳定采集50+ RSS源 +- [ ] 数据入库成功率 > 95% +- [ ] 基础监控可查看当日采集概况 +- [ ] 日志系统可定位问题 + +--- + +### 阶段二：Fetch循环与双层评分系统（Week 3-4） +**目标**：实现实时采集、双层评分（硬约束+动态阈值融合）、即时推送 + +| 序号 | 任务 | 描述 | 输出 | 依赖 | +|------|------|------|------|------| +| 2.1 | Fetch循环引擎 | 每30分钟循环一次，抓取RSS→Markdown→评分 | fetch引擎 | 1.6 | +| 2.2 | **第一层：单篇评分Prompt** | 单篇文章重要性评分（0-100），**含硬约束** | `prompts/article_score.txt` | 2.1 | +| 2.3 | **硬约束过滤器** | 非时政军事上限、KOL转述上限、时效衰减 | 硬约束模块 | 2.2 | +| 2.4 | 即时推送判断 | 单篇≥90分触发即时推送候选 | 推送判断模块 | 2.3 | +| 2.5 | 飞书Webhook推送 | 飞书群机器人推送 | 飞书推送适配器 | 2.4 | +| 2.6 | systemd服务化 | 使用systemd timer管理服务 | systemd配置 | 2.1 | + +#### 双层评分架构设计 + +``` +第一层：单篇文章评分（硬约束） 第二层：实体簇热度（动态阈值） +├── 输入：单篇RSS文章 ├── 输入：同一实体的多篇报道（已硬约束评分） +├── 处理：LLM评分 + 硬约束修正 ├── 处理：统计特征 + 动态阈值计算 +└── 输出：0-100分（已约束） └── 输出：热点判定 + ├── ≤79：非目标领域，过滤 ├── 低于阈值：普通关注 + ├── 80-89：一般关注，进入聚类 └── 高于阈值：热点，优先整合 + └── 90+：高优先级，即时推送候选 +``` + +#### 第一层：评分硬约束设计 + +**Prompt核心约束**： +```markdown +## 评分规则（硬性约束） + +1. **非时政/军事/科技主题上限 79 分** + - 如果内容不属于目标领域（如纯娱乐八卦、生活琐事），最高只能给 79 分 + +2. **KOL 转述上限 89 分** + - 如果只是 KOL/大V 对已有信息的转述/评论，而非原创信息，最高只能给 89 分 + +3. **时效性衰减** + - 发布超过24小时：分数×0.9 + - 发布超过48小时：分数×0.8 + +4. **90+ 分必须同时满足**： + - 重大事件/突破性进展/政策发布 + - 一手信息源（官方发布、权威媒体首发） + - 对目标领域有实质性影响 + +5. **标签禁止空泛** + - 禁止：["军事", "新闻", "热点"] + - 要求：["歼-35A", "舰载战斗机", "隐身性能", "海军航空兵"] +``` + +**代码实现**： +```python +def apply_score_constraints(entry: dict, raw_score: int) -> tuple[int, list[str]]: + """ + 应用评分硬约束 + 返回：修正后的分数，应用的约束标签列表 + """ + constraints_applied = [] + + # 约束1：非目标领域上限79 + if not is_target_domain(entry['content']): + raw_score = min(raw_score, 79) + constraints_applied.append("非目标领域") + + # 约束2：KOL转述上限89 + if is_kol_repost(entry): + raw_score = min(raw_score, 89) + constraints_applied.append("KOL转述") + + # 约束3：时效性衰减 + hours_old = get_hours_since_published(entry) + if hours_old > 24: + decay_factor = 0.9 ** (hours_old // 24) + raw_score = int(raw_score * decay_factor) + constraints_applied.append(f"时效衰减({hours_old}h)") + + return raw_score, constraints_applied +``` + +**第一层输出分级**： +| 分数段 | 处理策略 | 说明 | +|--------|---------|------| +| ≤79 | 过滤，不入库 | 非目标领域内容 | +| 80-89 | 入库，进入聚类 | 一般关注，参与动态阈值计算 | +| 90+ | 入库，即时推送候选 | 高优先级，同时触发即时推送判断 | + +**验收标准**： +- [ ] Fetch循环每30分钟稳定运行 +- [ ] 硬约束生效（非目标领域≤79，KOL转述≤89，时效衰减） +- [ ] 90+文章触发即时推送候选 +- [ ] systemd服务可一键启动/停止 + +--- + +### 阶段三：实体提取与细粒度聚类（Week 5-6） +**目标**：实现细粒度聚类，区分同一实体的不同报道角度 + +| 序号 | 任务 | 描述 | 输出 | 依赖 | +|------|------|------|------|------| +| 3.1 | 实体提取Prompt | 设计并优化实体提取提示词 | `prompts/extract_entities.txt` | 2.6 | +| 3.2 | 角度分类Prompt | 设计报道角度分类提示词 | `prompts/classify_angle.txt` | 2.6 | +| 3.3 | 批量实体提取 | 实现50篇/批并发调用，小模型处理 | 实体提取服务 | 3.1 | +| 3.4 | 角度分类服务 | 批量角度分类，与实体提取并行 | 角度分类服务 | 3.2 | +| 3.5 | 实体归一化 | 大模型消歧，合并别名 | 实体归一化服务 | 3.3 | +| 3.6 | 聚类算法 | 基于实体+角度的细粒度聚类 | 聚类引擎 | 3.4, 3.5 | +| 3.7 | 记忆系统 | 避免同一信息反复推送 | 去重模块 | 3.6 | + +**验收标准**： +- [ ] 实体提取准确率 > 85% +- [ ] 角度分类准确率 > 80% +- [ ] 聚类后实体簇数量合理（200篇→30-50个簇） +- [ ] 同一信息不重复推送 + +--- + +### 阶段四：热度评价与动态阈值（Week 7-8） +**目标**：实现第二层筛选——基于硬约束后分数的动态热点识别 + +| 序号 | 任务 | 描述 | 输出 | 依赖 | +|------|------|------|------|------| +| 4.1 | **实体簇热度计算** | 融合硬约束分数的多维度热度计算 | 热度指标服务 | 3.6 | +| 4.2 | **第二层：动态阈值算法** | 基于硬约束后分数分布的自适应阈值 | 阈值计算模块 | 4.1 | +| 4.3 | 热点识别引擎 | 硬约束过滤 + 动态阈值筛选的双层判定 | 热点识别服务 | 4.2 | +| 4.4 | 热点精评Prompt | 大模型深度评分（Top 30热点簇） | `prompts/precise_heat.txt` | 4.3 | + +#### 第二层：动态阈值设计（融合硬约束分数） + +**实体簇热度计算**： +```python +class Cluster: + def __init__(self, entity_name: str): + self.entity_name = entity_name + self.articles = [] # 已硬约束评分的文章列表 + self.hot_score = 0 # 综合热度分 + + def calculate_hot_score(self) -> int: + """ + 计算实体簇热度，融合硬约束后的单篇分数 + """ + if not self.articles: + return 0 + + # 基础分数：最高分文章的分数（已硬约束） + max_score = max(a['score'] for a in self.articles) + + # 传播热度：报道数量 × log(来源多样性) + report_count = len(self.articles) + source_diversity = len(set(a['source'] for a in self.articles)) + propagation_heat = report_count * math.log(source_diversity + 1) + + # 角度覆盖度：不同报道角度数 / 6 + angles = set(a.get('reporting_angle', 'unknown') for a in self.articles) + angle_coverage = len(angles) / 6 + + # 高优先级文章加成（90+文章额外加权） + high_priority_count = sum(1 for a in self.articles if a['score'] >= 90) + priority_bonus = high_priority_count * 5 # 每篇90+加5分 + + # 综合计算 + self.hot_score = min(100, int( + max_score * 0.4 + # 单篇最高分权重40% + min(propagation_heat * 3, 30) + # 传播热度权重30% + angle_coverage * 20 + # 角度覆盖权重20% + priority_bonus # 高优先级加成10% + )) + + return self.hot_score +``` + +**动态阈值算法（融合版）**： +```python +def calculate_hotspot_threshold(clusters: list[Cluster], date: str) -> float: + """ + 基于硬约束后分数分布计算动态阈值 + 同时考虑统计特征和高优先级文章数量 + """ + # 获取当日所有簇的热度分数（已融合硬约束） + scores = [c.hot_score for c in clusters] + + if len(scores) < 5: + return 75 # 数据不足时使用保底阈值 + + # 统计特征 + mean_score = mean(scores) + std_score = std(scores) + median_score = median(scores) + + # 方法1：统计阈值（均值+1.5倍标准差） + threshold_stat = mean_score + 1.5 * std_score + + # 方法2：保底阈值（至少3条或前10%） + min_count = max(3, len(scores) * 0.1) + sorted_scores = sorted(scores, reverse=True) + threshold_adaptive = sorted_scores[min_count - 1] + + # 方法3：硬约束保底（考虑90+高优先级文章数量） + high_priority_count = sum( + 1 for c in clusters + if any(a['score'] >= 90 for a in c.articles) + ) + if high_priority_count >= 3: + # 如果有3个以上簇包含90+文章，降低阈值确保热点被识别 + threshold_backup = median_score + else: + threshold_backup = mean_score + 0.5 * std_score + + # 取三者中较低值，确保热点不被遗漏 + final_threshold = min(threshold_stat, threshold_adaptive, threshold_backup) + + # 记录阈值计算日志 + logger.info(f"动态阈值计算: 统计={threshold_stat:.1f}, " + f"保底={threshold_adaptive:.1f}, 硬约束保底={threshold_backup:.1f}, " + f"最终={final_threshold:.1f}") + + return final_threshold +``` + +**双层热点判定流程**： +```python +def identify_hotspots(clusters: list[Cluster], date: str) -> tuple[list[Cluster], list[Cluster]]: + """ + 双层热点识别：硬约束过滤 + 动态阈值筛选 + """ + # 第一层：硬约束过滤（单篇层面已处理，此处检查） + valid_clusters = [] + for c in clusters: + # 过滤掉所有文章都≤79的簇（非目标领域） + if all(a['score'] <= 79 for a in c.articles): + continue + valid_clusters.append(c) + + # 计算各簇热度（融合硬约束分数） + for c in valid_clusters: + c.calculate_hot_score() + + # 第二层：计算动态阈值 + threshold = calculate_hotspot_threshold(valid_clusters, date) + + # 阈值筛选 + 额外条件 + hotspots = [] + normal = [] + for c in valid_clusters: + if c.hot_score >= threshold: + # 额外检查：是否有90+文章 或 超过阈值10分以上 + has_high_priority = any(a['score'] >= 90 for a in c.articles) + significantly_above = c.hot_score >= threshold + 10 + + if has_high_priority or significantly_above: + hotspots.append(c) + else: + normal.append(c) + else: + normal.append(c) + + # 排序：先按热度，再按最高单篇分数 + hotspots.sort(key=lambda x: (x.hot_score, x.max_article_score), reverse=True) + + return hotspots, normal +``` + +**融合设计优势**： +| 层级 | 作用 | 输入 | 输出 | +|------|------|------|------| +| 第一层（硬约束） | 质量控制 | 单篇文章 | 过滤非目标领域，标记高优先级 | +| 第二层（动态阈值） | 热点发现 | 实体簇（多篇聚合） | 识别突发热点，自适应当日分布 | + +**验收标准**： +- [ ] 实体簇热度计算正确（融合硬约束分数） +- [ ] 动态阈值自适应当日分布 +- [ ] 双层筛选协同工作（硬约束过滤→动态阈值筛选） +- [ ] 热点识别召回率 > 90%，误报率 < 20% + +--- + +### 阶段五：Push循环与双层整合（Week 9-10） +**目标**：实现定时汇总、双层整合（双视角设计）、日报生成 + +| 序号 | 任务 | 描述 | 输出 | 依赖 | +|------|------|------|------|------| +| 5.1 | Push循环引擎 | 每日定时运行，读取碎片化信息→整合→推送 | push引擎 | 4.4 | +| 5.2 | 摘要级整合Prompt | 多源报道合并为50-80字精炼摘要 | `prompts/summary_integrate.txt` | 5.1 | +| 5.3 | 洞察级整合Prompt | Top 10生成结构化深度分析（**双视角设计**） | `prompts/insight_integrate.txt` | 5.1 | +| 5.4 | **Prompt防退化模块** | 禁止套话、要求从素材出发 | 防退化检查器 | 5.2, 5.3 | +| 5.5 | 摘要整合服务 | 小模型批量处理全量簇 | 摘要整合服务 | 5.2, 5.4 | +| 5.6 | 洞察整合服务 | 大模型逐个处理Top 10（**双视角输出**） | 洞察整合服务 | 5.3, 5.4 | +| 5.7 | 时间线提取 | 从多源报道中提取事件时间线 | 时间线提取模块 | 5.5 | +| 5.8 | 多角度整合 | 同一事件不同角度报道的整合 | 多角度整合模块 | 5.6 | + +#### 双视角洞察设计（借鉴AI Daily） + +**设计理念**：metadata（新闻编辑视角，事实压缩）+ 正文（情报分析师视角，趋势判断）解耦 + +**Prompt设计**： +```markdown +## Part 1: Metadata（新闻编辑视角） + +请生成以下结构化信息： +- title: 10字以内的标题（事实陈述，无修饰） +- lead: 20字以内的导语（核心事实） +- highlights: 3-5个关键要点（bullet points，纯事实） + +约束：只陈述事实，不做趋势判断，不用形容词。 + +## Part 2: 洞察正文（情报分析师视角） + +基于多源报道，识别： +1. 事件脉络：关键时间节点和进展 +2. 多角度分析：技术维度、战略维度、舆论维度 +3. 影响与展望：短期影响、中长期趋势、值得关注信号 + +约束： +- 禁止空泛评论（如"意义重大""影响深远"） +- 每个观点必须有素材支撑 +- 使用具体名称（装备型号/政策名称/组织名称），不用"某些""部分"等模糊指代 +``` + +**输出示例**： +```markdown +--- +title: "歼-35A列装部队" +lead: "空军官方确认歼-35A已列装首批作战部队" +highlights: + - 首次公开确认进入实战化部署阶段 + - 隐身性能对标F-35C，可能在舰载领域形成优势 + - 多国媒体关注其对西太平洋军事平衡的影响 +--- + +## 事件脉络 +- 2024-11：珠海航展首次公开亮相 +- 2025-03：完成舰载适配测试 +- 2025-05：空军确认列装首批作战部队【新进展】 + +## 多角度分析 +- **技术维度**：隐身涂层、航电系统、舰载适配均有突破 +- **战略维度**：提升海军航空兵远海作战能力 +- **舆论维度**：外媒关注中美舰载机代差缩小 + +## 影响与展望 +短期：提升海军航母编队作战能力 +中期：可能在西太平洋形成局部优势 +值得关注：后续舰载版测试进展、出口动向 +``` + +#### Prompt防退化设计（借鉴AI Daily） + +**禁止套话列表**： +```python +FORBIDDEN_PHRASES = [ + "意义重大", "影响深远", "引发关注", "备受瞩目", + "深水区", "拐点", "白热化", "新纪元", "里程碑", + "不容忽视", "值得注意", "值得关注", # 除非后接具体内容 + "某些", "部分", "一些", "相关", # 模糊指代 +] + +def check_degeneration(content: str) -> list[str]: + """检查内容是否包含禁止套话""" + violations = [] + for phrase in FORBIDDEN_PHRASES: + if phrase in content: + violations.append(phrase) + return violations +``` + +**Prompt中的防退化指令**： +```markdown +## 输出约束（防退化） + +1. 禁止使用以下套话： + - "意义重大""影响深远""引发关注""备受瞩目" + - "深水区""拐点""白热化""新纪元""里程碑" + +2. 每个观点必须有素材支撑： + - 错误：该装备性能先进 + - 正确：该装备隐身系数0.01，优于F-35的0.02 + +3. 使用具体名称，禁止模糊指代： + - 错误：某些国家表示关注 + - 正确：日本防卫省发布关注声明 + +4. 从素材出发，每次措辞应不同，避免模板化输出 +``` + +**验收标准**： +- [ ] 摘要信息完整度 > 90% +- [ ] 洞察分析覆盖双视角（metadata + 正文） +- [ ] 无禁止套话出现 +- [ ] 整合内容无明显事实错误 + +--- + +### 阶段六：日报生成与跨日关联（Week 11-12） +**目标**：生成结构化日报（含Sentinel标记），实现跨日关联 + +| 序号 | 任务 | 描述 | 输出 | 依赖 | +|------|------|------|------|------| +| 6.1 | 日报模板设计 | Markdown模板，包含热点、速览、数据概览 | `templates/daily_report.md` | 5.8 | +| 6.2 | **Sentinel分段标记** | 单文件多板块精确管理 | `src/utils/sentinel.py` | 6.1 | +| 6.3 | **跨日关联模块** | 从历史日报提取上下文，追踪事件脉络 | `src/processors/cross_day.py` | 6.2 | +| 6.4 | 排序算法 | 综合热度、时效、重要性的排序逻辑 | 排序模块 | 4.4 | +| 6.5 | 日报渲染服务 | 按模板组装数据，生成Markdown（含Sentinel） | 日报渲染服务 | 6.1, 6.2, 6.4 | +| 6.6 | Web归档站点 | 部署在线阅览站点 | Web站点 | 6.5 | +| 6.7 | 数据保留策略 | fetch保留2天，notify保留2天，push保留5天 | 清理策略 | 6.6 | + +#### 简化版Sentinel分段标记设计 + +**标记格式**： +```markdown +--- +title: "每日情报早报" +date: "2026-05-24" +stats: + total_entities: 87 + hotspot_count: 10 +--- + + +## 📋 全量新闻速览 + +| # | 实体 | 摘要 | 来源数 | 热度 | +|---|------|------|--------|------| +| 1 | 歼-35A | 空军确认列装\|隐身性能对标F-35C | 8 | ★★★★★ | + + + +## 🔥 热点深度洞察 + +### 1. 歼-35A列装进展 [持续跟踪] +> 速览：空军官方确认歼-35A已列装首批作战部队 [8源报道] + +**事件脉络** +- 5月20日：首次公开亮相 +- 5月24日：【新进展】确认列装部队 +... + +``` + +**核心代码**： +```python +import re +from datetime import datetime, timedelta + +def extract_section(content: str, section_name: str) -> str: + """从日报内容中提取指定section""" + pattern = rf'(.*?)' + match = re.search(pattern, content, re.DOTALL | re.IGNORECASE) + return match.group(1).strip() if match else "" + +def get_historical_context(entity_name: str, days: int = 7) -> dict: + """获取某实体近N天的历史报道上下文""" + context = { + "first_seen": None, + "previous_summaries": [], + "previous_insights": [], + "mention_count": 0 + } + + for i in range(days): + date = (datetime.now() - timedelta(days=i)).strftime("%Y-%m-%d") + file_path = f"data/reports/daily-{date}.md" + + try: + with open(file_path, 'r', encoding='utf-8') as f: + content = f.read() + + if entity_name in content: + context["mention_count"] += 1 + if context["first_seen"] is None: + context["first_seen"] = date + + # 提取该实体在摘要中的记录 + summary_section = extract_section(content, "summary") + for line in summary_section.split('\n'): + if entity_name in line and line.strip().startswith('|'): + context["previous_summaries"].append({ + "date": date, + "content": line.strip() + }) + + # 提取该实体在洞察中的记录 + insights_section = extract_section(content, "insights") + if entity_name in insights_section: + insight_blocks = re.split(r'### \d+\.', insights_section) + for block in insight_blocks: + if entity_name in block: + context["previous_insights"].append({ + "date": date, + "content": block.strip()[:500] + }) + break + except FileNotFoundError: + continue + + return context + +def generate_cross_day_marker(entity_name: str) -> str: + """生成跨日标记""" + history = get_historical_context(entity_name, days=7) + + if history["mention_count"] == 0: + return "首次报道" + elif history["mention_count"] == 1: + return "持续跟踪" + else: + return f"持续跟踪({history['mention_count']}次)" +``` + +**跨日关联应用**： +```python +# 在生成洞察时，添加跨日时间线 +history = get_historical_context("歼-35A", days=7) + +# 输出： +# { +# "first_seen": "2026-05-20", +# "mention_count": 3, +# "previous_summaries": [ +# {"date": "2026-05-20", "content": "| 1 | 歼-35A | 首次公开亮相... |"}, +# {"date": "2026-05-22", "content": "| 3 | 歼-35A | 专家分析隐身性能... |"} +# ] +# } + +# 在洞察中展示时间线 +timeline = f""" +**事件脉络** +- 5月20日：首次公开亮相（珠海航展） +- 5月22日：专家分析隐身性能，推测可能上舰 +- 5月24日：【新进展】空军官方确认列装首批作战部队 +""" +``` + +**验收标准**： +- [ ] 日报生成时间 < 20分钟 +- [ ] Sentinel标记正确，可精确提取summary/insights +- [ ] 跨日关联可追踪实体历史报道 +- [ ] Web站点可正常访问 + +--- + +### 阶段七：优化与迭代（Week 13-14） +**目标**：持续优化，功能扩展 + +| 序号 | 任务 | 描述 | 优先级 | +|------|------|------|--------| +| 7.1 | Prompt优化 | 基于实际效果迭代优化提示词 | 高 | +| 7.2 | 反馈闭环 | 收集用户反馈，标注数据，优化模型 | 高 | +| 7.3 | 趋势分析 | 7日热点趋势图，周期性报告 | 中 | +| 7.4 | 个性化推荐 | 基于用户阅读历史的个性化排序 | 低 | + +--- + +## 四、技术栈建议 + +### 4.1 核心组件 +| 组件 | 推荐方案 | 说明 | +|------|----------|------| +| 数据采集 | Python + feedparser | RSS解析 | +| 数据存储 | SQLite + JSON文件 | 数据库+文件双存储 | +| LLM调用 | OpenAI SDK / 国产模型SDK | 统一接口封装 | +| 任务调度 | systemd timer | 系统级服务管理 | +| Web展示 | Hugo / Flask | 静态/动态站点 | +| 日志 | Python logging + systemd journal | 集中日志管理 | + +### 4.2 模型选择 +| 任务类型 | 推荐模型 | 预估成本 | +|----------|----------|----------| +| 实体提取 | DeepSeek-V3 / Qwen-Plus | ~0.001元/次 | +| 角度分类 | DeepSeek-V3 / Qwen-Plus | ~0.001元/次 | +| 单篇评分 | DeepSeek-V3 / Qwen-Plus | ~0.001元/次 | +| 摘要整合 | DeepSeek-V3 / Qwen-Plus | ~0.002元/次 | +| 实体消歧 | GPT-4o / Claude-3.5-Sonnet | ~0.01元/次 | +| 热点精评 | GPT-4o / Claude-3.5-Sonnet | ~0.02元/次 | +| 洞察整合 | GPT-4o / Claude-3.5-Sonnet | ~0.1元/次 | + +**日成本估算**： +- 200-500篇/日 × 0.001元（小模型）= 0.2-0.5元 +- 100个簇 × 0.001元（小模型）= 0.1元 +- 10个洞察 × 0.1元（大模型）= 1元 +- **总计：约1.3-1.6元/日** + +--- + +## 五、项目结构 + +``` +news-intelligence-system/ +├── config/ +│ ├── rss_sources.json # RSS源配置 +│ ├── models.yaml # 模型配置 +│ └── systemd/ # systemd服务配置 +├── src/ +│ ├── collectors/ # 数据采集 +│ │ └── rss_collector.py +│ ├── processors/ # 处理管道 +│ │ ├── entity_extractor.py +│ │ ├── angle_classifier.py +│ │ ├── cluster_engine.py +│ │ ├── heat_calculator.py +│ │ └── cross_day.py # 跨日关联（新增） +│ ├── integrators/ # 内容整合 +│ │ ├── summary_integrator.py +│ │ └── insight_integrator.py +│ ├── generators/ # 报告生成 +│ │ ├── daily_report_generator.py +│ │ └── web_renderer.py +│ ├── push/ # 推送平台 +│ │ ├── base.py +│ │ ├── feishu.py +│ │ └── discord.py +│ └── utils/ # 工具函数 +│ ├── sentinel.py # Sentinel分段标记（新增） +│ └── prompt_checker.py # Prompt防退化检查（新增） +├── prompts/ # 提示词文件 +│ ├── article_score.txt # 单篇评分（含硬约束） +│ ├── extract_entities.txt +│ ├── classify_angle.txt +│ ├── quick_heat.txt +│ ├── precise_heat.txt +│ ├── summary_integrate.txt +│ └── insight_integrate.txt # 双视角洞察 +├── templates/ +│ └── daily_report.md # 含Sentinel标记 +├── data/ +│ ├── fetch/ # fetch-yyyy-mm-dd.json +│ ├── notify/ # notify-yyyy-mm-dd.json +│ └── reports/ # daily-yyyy-mm-dd.md +├── web/ # Web归档站点 +├── tests/ +├── scripts/ +│ ├── fetch_loop.py +│ ├── push_loop.py +│ └── setup_systemd.sh +├── requirements.txt +└── README.md +``` + +--- + +## 六、关键风险与应对 + +| 风险 | 影响 | 应对措施 | +|------|------|----------| +| RSS源失效 | 数据缺失 | 多源备份，监控告警 | +| LLM API限流 | 处理延迟 | 批量处理，重试机制，降级策略 | +| 实体识别错误 | 聚类偏差 | 人工标注反馈，Prompt迭代 | +| 热点漏识别 | 信息缺失 | 动态阈值保底机制，人工复核 | +| 成本超支 | 预算超支 | Token监控，模型降级，配额控制 | +| 进程崩溃 | 服务中断 | systemd自动重启 | +| LLM输出退化 | 质量下降 | **Prompt防退化检查** | + +--- + +## 七、里程碑与交付物 + +| 里程碑 | 时间 | 交付物 | +|--------|------|--------| +| M1 | Week 2 | 稳定运行的数据采集系统 | +| M2 | Week 4 | Fetch循环上线，**评分硬约束生效** | +| M3 | Week 6 | 细粒度聚类引擎，实体提取准确率>85% | +| M4 | Week 8 | 动态热点识别系统，召回率>90% | +| M5 | Week 10 | Push循环上线，**双视角洞察+Prompt防退化** | +| M6 | Week 12 | **Sentinel跨日关联上线**，Web归档站点 | +| M7 | Week 14 | 持续优化迭代 | + +--- + +## 八、借鉴AI Daily的核心设计总结 + +### 8.1 架构层面 +1. **双循环架构**：Fetch循环（实时）+ Push循环（定时） +2. **systemd服务化**：替代Python内部定时，提升稳定性 +3. **文件存储规范**：fetch/notify/reports三种文件类型，明确保留策略 + +### 8.2 LLM应用层面 +1. **评分硬约束**：非目标领域≤79、KOL转述≤89，有效控制信息质量 +2. **Prompt防退化**：禁止套话、要求从素材出发，避免LLM输出风格趋同 +3. **双视角洞察**：metadata（事实压缩）+ 正文（趋势判断）解耦 + +### 8.3 数据管理层面 +1. **Sentinel分段标记**：单文件多板块精确管理，支持跨日关联 +2. **跨日关联**：从历史日报提取上下文，追踪事件发展脉络 +3. **数据保留策略**：自动清理过期文件 + +### 8.4 工程层面 +1. **日志集中管理**：systemd journal便于问题定位 +2. **多平台推送**：飞书、Discord webhook支持 +3. **成本可控**：每天约1.3-1.6元 + +--- + +*文档版本：v2.1* +*最后更新：2026-05-28* +*状态：已融合AI Daily四大核心设计（Sentinel、硬约束、防退化、双视角）* diff --git a/20260529_新闻智能分析系统开发计划_v3.md b/20260529_新闻智能分析系统开发计划_v3.md new file mode 100644 index 0000000..b35ba88 --- /dev/null +++ b/20260529_新闻智能分析系统开发计划_v3.md @@ -0,0 +1,1249 @@ +# 新闻智能分析系统开发计划 v3.0 + +> 基于大模型的时政/军事/科技新闻聚类、评价与整合系统 +> +> **迭代说明**：参考 AI Daily 项目实践与 military-digest-v3 工程落地经验，融合细粒度聚类、SQLite 增量缓存、翻译管线、网摘生成与工程化部署经验 + +--- + +## 一、项目概述 + +### 1.1 项目目标 +构建一个自动化新闻分析系统，实现： +- **细粒度聚类**：保留多角度报道，避免简单去重丢失热点 +- **动态热点识别**：基于当日新闻分布统计特征自适应判定热点 +- **三层整合**：摘要级速览（全量）+ 洞察级深度分析（Top 10）+ 参考消息风格网摘（Top 3） +- **混合处理**：实时热点感知 + 每日批量汇总生成早报 +- **跨日关联**：追踪同一事件的多日发展脉络 +- **增量缓存**：SQLite 三级缓存（文章/网摘/分类摘要），支持断点续跑，再次运行几乎零 token 消耗 +- **外文翻译**：中英分离 + 并发翻译，中文文章零 API 调用 + +### 1.2 核心约束 +| 维度 | 约束 | +|------|------| +| 数据规模 | 200-500篇/日 | +| 数据源 | RSS Feed（公众号订阅源，含外文源） | +| 输出格式 | Markdown日报 / 网摘PNG长图 / 网摘TXT文本 / 飞书推送 | +| 成本预算 | 混合模型调度，日成本约1.3-1.6元 | +| 处理时效 | 批处理15-20分钟完成（冷启动基准：5源156篇→153s） | +| 缓存命中 | 二次运行 ~5s，API 调用 0 次 | +| 部署方式 | 支持systemd服务化部署 | + +### 1.3 双循环架构（参考AI Daily优化） + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Fetch 循环（实时） │ +│ 每30分钟运行一次 │ +│ RSS抓取 → 中英分离 → 外文并发翻译 → LLM评分 → 重要性判断 │ +│ ↓ ↓ ↓ │ +│ 存入SQLite缓存 存入JSON文件(fetch-yyyy-mm-dd.json) 飞书推送│ +└─────────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────────┐ +│ Push 循环（定时） │ +│ 每日定时运行（如早8点） │ +│ 读取碎片化信息 → 细粒度聚类 → 多角度关联 → 综合评分 │ +│ → 摘要级整合(全量) → 洞察级整合(Top10) → 参考消息网摘(Top3) │ +│ → 生成日报+网摘图片 → 飞书推送 │ +└─────────────────────────────────────────────────────────────────┘ +``` + +--- + +## 二、与现有项目的对比分析 + +### 2.1 与 AI Daily 的相似性 + +| 维度 | AI Daily | 本系统 | 相似度 | +|------|----------|--------|--------| +| **数据源** | RSS订阅源 | RSS订阅源（公众号+外文） | ⭐⭐⭐⭐⭐ | +| **核心流程** | RSS→LLM评分→推送/汇总 | RSS→翻译→聚类→评分→整合→推送/汇总 | ⭐⭐⭐⭐ | +| **双循环架构** | Fetch循环+Push循环 | 实时检测+批处理 | ⭐⭐⭐⭐⭐ | +| **即时推送** | 飞书/Discord webhook | 飞书 webhook | ⭐⭐⭐⭐ | +| **定时汇总** | 每日定时推送 | 每日早报生成 | ⭐⭐⭐⭐⭐ | +| **成本预算** | 每天6毛钱 | 每天1.3-1.6元 | ⭐⭐⭐⭐ | + +### 2.2 与 military-digest-v3 的对比 + +| 维度 | military-digest-v3（现有系统） | 本系统（新架构） | 差异 | +|------|------|------|:---:| +| **架构** | 单管线（每日一次全量运行） | 双循环（Fetch 30min + Push 每日） | 🔄 架构升级 | +| **去重** | 同源同标题简单去重 | 细粒度语义聚类 + 多角度保留 | 🔄 策略升级 | +| **评分** | 四维度固定加权 | 双层筛选：硬约束 + 动态阈值融合 | 🔄 质量升级 | +| **翻译** | ✅ 中英分离 + 并发翻译 | ✅ 继承 + 统一为 Fetch 循环环节 | ✅ 复用 | +| **缓存** | ✅ SQLite 三级缓存（文章/网摘/分类摘要） | ✅ 继承 + 扩展为跨日关联数据源 | ✅ 复用 | +| **监控** | ✅ 11阶段耗时 + API统计 + 源成功率 | ✅ 继承 + 增加聚类/热点识别指标 | ✅ 复用 | +| **网摘** | ✅ 参考消息风格 + 字数校验重试 | ✅ 继承 + 作为第三层整合输出 | ✅ 复用 | +| **推送** | 企业微信 + 飞书 | **聚焦飞书 webhook** | 🔄 收束 | +| **跨日关联** | ❌ 无 | ✅ Sentinel标记 + 事件时间线追踪 | 🆕 新增 | +| **防退化** | ❌ 无 | ✅ 禁止套话 + 素材支撑要求 | 🆕 新增 | + +### 2.3 本系统的差异化优势 + +| 特性 | AI Daily | military-digest-v3 | 本系统 | 价值 | +|------|----------|------|------|------| +| **热点识别** | 单篇文章评分 | 四维度固定加权 | **双层筛选：硬约束+动态阈值融合** | 质量控制+热点发现协同 | +| **去重策略** | 简单去重 | 同源标题去重 | 细粒度聚类+多角度保留 | 不丢失多角度报道 | +| **内容整合** | 单篇摘要 | 摘要+网摘 | 三层整合（摘要+洞察+网摘） | 覆盖速览/深度/传播三种需求 | +| **翻译能力** | 无 | ✅ 中英分离并发 | ✅ 继承 | 外文源零额外成本 | +| **增量缓存** | JSON文件 | ✅ SQLite三级缓存 | ✅ 继承+扩展 | 断点续跑，二次运行~5s | +| **网摘生成** | 无 | ✅ 参考消息风格+字数校验 | ✅ 继承 | 适合直接传播的成品内容 | +| **领域聚焦** | AI领域 | 军事/科技 | 时政/军事/科技 | 专业领域实体识别 | +| **角度分类** | 无 | 无 | 6种报道角度分类 | 支持多角度整合 | +| **跨日关联** | 无 | 无 | 事件时间线追踪+Sentinel标记 | 追踪事件发展脉络 | + +### 2.4 从 AI Daily 借鉴的核心设计 + +1. **双循环架构**：Fetch循环（实时）+ Push循环（定时） +2. **systemd服务化**：使用systemd timer替代Python内部定时，提升稳定性 +3. **文件存储规范**：fetch/notify/push三种文件类型，明确保留策略 +4. **评分硬约束**：非目标领域上限、KOL转述上限，有效控制信息质量 +5. **Prompt防退化**：禁止套话、要求从素材出发，避免LLM输出风格趋同 +6. **双视角洞察**：metadata（事实压缩）+ 正文（趋势判断）解耦 +7. **Sentinel分段标记**：单文件多板块精确管理，支持跨日关联 + +### 2.5 从 military-digest-v3 继承的工程资产 + +1. **SQLite 三级缓存**：文章缓存 + 网摘缓存 + 分类摘要缓存，含自动过期清理和数据库迁移机制 +2. **翻译管线**：中英分离（汉字占比 > 20% 判定）+ 外文并发翻译（ThreadPoolExecutor），中文文章零 API 调用 +3. **监控统计体系**：11 阶段耗时 + API 调用按用途/模型分组 + 源抓取成功/失败率 + 缓存命中率 +4. **网摘生成**：参考消息风格 + 公众号源智能识别前缀 + 字数校验重试（280-320字，最多3次） +5. **异步流式管线**：生产者-消费者模型，RSS 抓取与 AI 处理流水线并行 +6. **图片生成**：Pillow 命令模式渲染，支持单篇 + TOP3 合并长图 + +--- + +## 三、开发阶段规划 + +### 阶段一：基础架构搭建（Week 1-2） +**目标**：建立数据流和存储基础，确保稳定采集 + +| 序号 | 任务 | 描述 | 输出 | 依赖 | +|------|------|------|------|------| +| 1.1 | RSS采集模块 | 解析公众号RSS Feed，提取标题、正文、来源、发布时间 | 原始文章数据库 | - | +| 1.2 | 数据标准化 | 统一文章格式，清洗HTML标签，提取纯文本 | 标准化文章表 | 1.1 | +| 1.3 | 文件存储层 | JSON文件存储(fetch-yyyy-mm-dd.json) | 存储规范 | 1.2 | +| 1.4 | **SQLite缓存系统** | 三级缓存设计（文章/网摘/分类摘要），含过期策略与迁移机制 | 缓存模块 | 1.2 | +| 1.5 | **翻译管线** | 中英分离 + 外文并发翻译 | 翻译模块 | 1.2 | +| 1.6 | **监控统计** | 阶段耗时 + API消耗 + 成功率 + 缓存命中率 | 监控模块 | 1.1 | +| 1.7 | 日志系统 | 有效日志记录，便于问题定位 | 日志模块 | 1.1 | + +#### 1.4 SQLite 缓存系统设计（继承 military-digest-v3） + +**数据库位置**：`data/article_cache.db` + +**表1：article_cache（文章处理结果缓存）** + +| 字段 | 类型 | 说明 | +|------|------|------| +| `id` | TEXT (PK) | 文章唯一标识（RSS entry id/link） | +| `source` | TEXT | 来源名称 | +| `title` | TEXT | 原始标题 | +| `link` | TEXT | 原文链接 | +| `translated_title` | TEXT | 翻译后的中文标题 | +| `content` | TEXT | 原始正文 | +| `translated_content` | TEXT | 翻译后的中文正文（前300字） | +| `ai_score` | REAL | AI 加权总分 | +| `ai_summary` | TEXT | AI 生成的中文摘要 | +| `ai_category` | TEXT | 分类（装备动态/地区冲突/战略政策） | +| `ai_scores_json` | TEXT | 四维度评分 JSON | +| `webzine_text` | TEXT | 网摘文本（迁移新增字段） | +| `published_time` | DATETIME | 文章发布时间 | +| `processed_time` | DATETIME | 处理时间（索引，用于过期判断） | +| `status` | INTEGER | 1=成功 2=失败 | +| `error_msg` | TEXT | 失败时的错误信息 | + +**索引**：`idx_processed_time`、`idx_source` + +**表2：category_summary_cache（分类摘要缓存）** + +| 字段 | 类型 | 说明 | +|------|------|------| +| `date` | TEXT (联合PK) | 日期字符串（YYYY-MM-DD） | +| `category` | TEXT (联合PK) | 分类名称 | +| `summary_text` | TEXT | 分类介绍文本 | +| `generated_time` | DATETIME | 生成时间 | + +**缓存策略**： + +| 策略项 | 设计 | +|------|------| +| 过期时间 | 24 小时（与时间窗口一致，可配置） | +| 自动清理 | 每次运行时清理 7 天前数据（`clear_expired(keep_days=7)`） | +| 写入方式 | `REPLACE INTO`（幂等，支持覆盖更新） | +| 网摘分离更新 | `save_article_webzine()` 单独更新 `webzine_text` 字段，不影响其他缓存 | +| 连接降级 | 数据库连接失败时静默降级，不影响主流程 | +| 全局单例 | `get_cache()` 全局单例模式，避免重复创建连接 | + +**数据库迁移机制**： + +```python +def _migrate_add_webzine_text(self): + """自动检测并添加 webzine_text 列""" + cursor.execute("PRAGMA table_info(article_cache)") + columns = [col[1] for col in cursor.fetchall()] + if 'webzine_text' not in columns: + cursor.execute("ALTER TABLE article_cache ADD COLUMN webzine_text TEXT") +``` + +**缓存命中流程**： + +``` +文章被抓取 → 以 article.id 查询缓存（expire_hours=24） + ├─ 命中 + status=1 → 直接使用缓存数据（from_cache=True），跳过翻译和AI分析 + └─ 未命中/过期 → 完整翻译+分析管线，结果 REPLACE INTO 写入缓存 +``` + +**实测效果**（military-digest-v3 验证数据）： + +| 指标 | 第一次（冷启动） | 第二次（含缓存） | +|------|:---:|:---:| +| 文章缓存命中 | 0 / 15 | **15 / 15** | +| API 调用次数 | 37 次 | **0 次** | +| 总耗时 | 153s | **~5s** | +| 退出码 | 0 | 0 | + +#### 1.5 翻译管线设计（继承 military-digest-v3） + +**核心策略**：中英分离，中文文章零 API 调用，外文文章并发翻译。 + +**中文检测**：汉字占比 > 20% 判定为中文（`is_chinese()` 函数）。 + +**翻译流程**： + +``` +所有文章 + ├─ 中文文章 → translated_title = title（零API调用） + │ translated_content = content[:300]（截取即可） + └─ 外文文章 → 提交到 ThreadPoolExecutor（max_workers=AI_CONCURRENCY） + ├─ translate_title()：标题翻译（temperature=0.2, max_tokens=200） + └─ translate_content()：正文前500词翻译 → 截取300字（max_tokens=800） +``` + +**并发控制**： + +| 参数 | 值 | 说明 | +|------|-----|------| +| `AI_CONCURRENCY` | 8 | 默认并发数，可通过配置调整 | +| `temperature` | 0.2 | 翻译低温度，确保准确性 | +| 失败降级 | 原文填充 | 翻译失败时 `translated_title = title`，不中断流程 | + +**关键实现**： + +```python +def batch_translate_titles(articles, api_key, base_url, model, max_concurrent=None): + """中英分离：中文直接填充，外文并发翻译标题""" + chinese_articles = [] + foreign_articles = [] + + for a in articles: + if is_chinese(a['title']): + a['translated_title'] = a['title'] + chinese_articles.append(a) + else: + foreign_articles.append(a) + + # 外文并发翻译 + with ThreadPoolExecutor(max_workers=max_concurrent) as executor: + futures = {executor.submit(translate_title, a, ...): a for a in foreign_articles} + for future in as_completed(futures): + chinese_articles.append(future.result()) + + return chinese_articles +``` + +**设计优势**： +- 中文文章零 API 调用，节省 40%+ 翻译 token +- 标题和正文分离翻译，先翻译标题再决定是否需要翻译正文（可配合关键词筛选） +- 外文并发翻译，翻译耗时 = 单篇耗时（而非 N × 单篇耗时） + +#### 1.6 监控统计设计（继承 military-digest-v3） + +**阶段耗时统计（11 阶段）**： + +| 阶段 key | 中文标签 | 统计内容 | +|------|------|------| +| `rss_fetch` | RSS抓取 | 所有源并行抓取耗时 | +| `time_filter` | 时间过滤 | 按时间窗口过滤耗时 | +| `deduplicate` | 去重 | 去重处理耗时 | +| `translate_titles` | 标题翻译 | 外文标题并发翻译耗时 | +| `keyword_filter` | 关键词筛选 | 关键词命中筛选耗时 | +| `translate_contents` | 正文翻译 | 外文正文并发翻译耗时 | +| `ai_analyze` | AI评分分类 | 批量AI评分分类耗时 | +| `webzine_generate` | 网摘生成 | TOP3网摘并发生成耗时 | +| `category_overview` | 分类介绍 | 4类分类介绍并发生成耗时 | +| `report_generate` | 报告生成 | Markdown报告拼接写入耗时 | +| `image_generate` | 图片生成 | 网摘长图渲染耗时 | + +**API 调用统计**：按用途（标题翻译/正文翻译/AI评分分类/网摘生成/分类介绍）和模型分组，记录调用次数、token 估算、重试次数、失败次数。 + +**源抓取统计**：成功/失败源计数，失败源名称列表。 + +**文章统计**：总数、缓存命中数、本次处理数、缓存命中率。 + +**报告示例**（运行结束自动打印）： + +``` +================================================================ + 📊 运行监控报告 +================================================================ + +── ⏱ 阶段耗时 ── + RSS抓取 7.5s + 时间过滤 0.0s + 去重 0.0s + 标题翻译 17.9s + 关键词筛选 0.0s + 正文翻译 9.9s + AI评分分类 30.7s + 网摘生成 53.3s + 分类介绍 33.5s + 报告生成 0.0s + 图片生成 0.3s + ──────────────────── + 合计 153.1s / 总运行 153s + +── 📡 文章统计 ── + 总计: 15 缓存命中: 0 本次处理: 15 + 缓存命中率: 0% + +── 🤖 API调用统计 ── + [按用途] + 标题翻译: 7次 / tokens≈1400 / 重试0 + 正文翻译: 6次 / tokens≈4800 / 重试0 + AI评分分类: 15次 / tokens≈12000 / 重试0 + 网摘生成: 5次 / tokens≈4000 / 重试2 + 分类介绍: 4次 / tokens≈600 / 重试0 + [按模型] + deepseek-v3: 37次 / tokens≈22800 + +── 📰 源抓取统计 ── + 源总数: 5 成功: 5 失败: 0 + +================================================================ +``` + +**数据字段规范**（扩展版）： + +```json +{ + "title": "内容标题", + "link": "原始链接", + "published": "发布时间", + "source": "来源（公众号名称）", + "content": "Markdown格式的正文内容", + "translated_title": "翻译后的中文标题", + "translated_content": "翻译后的中文正文", + "tags": "LLM识别的标签", + "score": "LLM评分（0-100）", + "summary": "LLM生成的中文摘要", + "fetched_at": "抓取时间", + "entities": ["提取的实体列表"], + "reporting_angle": "报道角度", + "webzine_text": "参考消息风格网摘文本（Top N 专属）", + "from_cache": "是否来自缓存" +} +``` + +**验收标准**： +- [ ] 稳定采集50+ RSS源 +- [ ] 数据入库成功率 > 95% +- [ ] SQLite 缓存三级覆盖（文章/网摘/分类摘要） +- [ ] 缓存命中时 API 调用为 0 +- [ ] 中英分离正确，中文文章零翻译 API 调用 +- [ ] 监控报告包含阶段耗时、API统计、源成功率、缓存命中率 +- [ ] 日志系统可定位问题 + +--- + +### 阶段二：Fetch循环与双层评分系统（Week 3-4） +**目标**：实现实时采集、双层评分（硬约束+动态阈值融合）、即时推送 + +| 序号 | 任务 | 描述 | 输出 | 依赖 | +|------|------|------|------|------| +| 2.1 | Fetch循环引擎 | 每30分钟循环一次，抓取RSS→中英分离→并发翻译→评分 | fetch引擎 | 1.7 | +| 2.2 | **第一层：单篇评分Prompt** | 单篇文章重要性评分（0-100），**含硬约束** | `prompts/article_score.txt` | 2.1 | +| 2.3 | **硬约束过滤器** | 非时政军事上限、KOL转述上限、时效衰减 | 硬约束模块 | 2.2 | +| 2.4 | 即时推送判断 | 单篇≥90分触发即时推送候选 | 推送判断模块 | 2.3 | +| 2.5 | 飞书Webhook推送 | 飞书群机器人推送（Fetch 循环即时推送 + Push 循环日报推送） | 飞书推送适配器 | 2.4 | +| 2.6 | systemd服务化 | 使用systemd timer管理服务 | systemd配置 | 2.1 | + +#### 双层评分架构设计 + +``` +第一层：单篇文章评分（硬约束） 第二层：实体簇热度（动态阈值） +├── 输入：单篇RSS文章（已翻译） ├── 输入：同一实体的多篇报道（已硬约束评分） +├── 处理：LLM评分 + 硬约束修正 ├── 处理：统计特征 + 动态阈值计算 +└── 输出：0-100分（已约束） └── 输出：热点判定 + ├── ≤79：非目标领域，过滤 ├── 低于阈值：普通关注 + ├── 80-89：一般关注，进入聚类 └── 高于阈值：热点，优先整合 + └── 90+：高优先级，即时推送候选 +``` + +#### 第一层：评分硬约束设计 + +**Prompt核心约束**： +```markdown +## 评分规则（硬性约束） + +1. **非时政/军事/科技主题上限 79 分** + - 如果内容不属于目标领域（如纯娱乐八卦、生活琐事），最高只能给 79 分 + +2. **KOL 转述上限 89 分** + - 如果只是 KOL/大V 对已有信息的转述/评论，而非原创信息，最高只能给 89 分 + +3. **时效性衰减** + - 发布超过24小时：分数×0.9 + - 发布超过48小时：分数×0.8 + +4. **90+ 分必须同时满足**： + - 重大事件/突破性进展/政策发布 + - 一手信息源（官方发布、权威媒体首发） + - 对目标领域有实质性影响 + +5. **标签禁止空泛** + - 禁止：["军事", "新闻", "热点"] + - 要求：["歼-35A", "舰载战斗机", "隐身性能", "海军航空兵"] +``` + +**代码实现**： +```python +def apply_score_constraints(entry: dict, raw_score: int) -> tuple[int, list[str]]: + """ + 应用评分硬约束 + 返回：修正后的分数，应用的约束标签列表 + """ + constraints_applied = [] + + # 约束1：非目标领域上限79 + if not is_target_domain(entry['content']): + raw_score = min(raw_score, 79) + constraints_applied.append("非目标领域") + + # 约束2：KOL转述上限89 + if is_kol_repost(entry): + raw_score = min(raw_score, 89) + constraints_applied.append("KOL转述") + + # 约束3：时效性衰减 + hours_old = get_hours_since_published(entry) + if hours_old > 24: + decay_factor = 0.9 ** (hours_old // 24) + raw_score = int(raw_score * decay_factor) + constraints_applied.append(f"时效衰减({hours_old}h)") + + return raw_score, constraints_applied +``` + +**第一层输出分级**： +| 分数段 | 处理策略 | 说明 | +|--------|---------|------| +| ≤79 | 过滤，不入库 | 非目标领域内容 | +| 80-89 | 入库，进入聚类 | 一般关注，参与动态阈值计算 | +| 90+ | 入库，即时推送候选 | 高优先级，同时触发即时推送判断 | + +**验收标准**： +- [ ] Fetch循环每30分钟稳定运行 +- [ ] 翻译管线集成正确（中英分离+并发翻译） +- [ ] 硬约束生效（非目标领域≤79，KOL转述≤89，时效衰减） +- [ ] 90+文章触发即时推送候选 +- [ ] SQLite缓存正常工作（二次运行零API调用） +- [ ] 监控统计准确记录各阶段数据 +- [ ] systemd服务可一键启动/停止 + +--- + +### 阶段三：实体提取与细粒度聚类（Week 5-6） +**目标**：实现细粒度聚类，区分同一实体的不同报道角度 + +| 序号 | 任务 | 描述 | 输出 | 依赖 | +|------|------|------|------|------| +| 3.1 | 实体提取Prompt | 设计并优化实体提取提示词 | `prompts/extract_entities.txt` | 2.6 | +| 3.2 | 角度分类Prompt | 设计报道角度分类提示词 | `prompts/classify_angle.txt` | 2.6 | +| 3.3 | 批量实体提取 | 实现50篇/批并发调用，小模型处理 | 实体提取服务 | 3.1 | +| 3.4 | 角度分类服务 | 批量角度分类，与实体提取并行 | 角度分类服务 | 3.2 | +| 3.5 | 实体归一化 | 大模型消歧，合并别名 | 实体归一化服务 | 3.3 | +| 3.6 | 聚类算法 | 基于实体+角度的细粒度聚类 | 聚类引擎 | 3.4, 3.5 | +| 3.7 | 记忆系统 | 避免同一信息反复推送 | 去重模块 | 3.6 | + +**验收标准**： +- [ ] 实体提取准确率 > 85% +- [ ] 角度分类准确率 > 80% +- [ ] 聚类后实体簇数量合理（200篇→30-50个簇） +- [ ] 同一信息不重复推送 + +--- + +### 阶段四：热度评价与动态阈值（Week 7-8） +**目标**：实现第二层筛选——基于硬约束后分数的动态热点识别 + +| 序号 | 任务 | 描述 | 输出 | 依赖 | +|------|------|------|------|------| +| 4.1 | **实体簇热度计算** | 融合硬约束分数的多维度热度计算 | 热度指标服务 | 3.6 | +| 4.2 | **第二层：动态阈值算法** | 基于硬约束后分数分布的自适应阈值 | 阈值计算模块 | 4.1 | +| 4.3 | 热点识别引擎 | 硬约束过滤 + 动态阈值筛选的双层判定 | 热点识别服务 | 4.2 | +| 4.4 | 热点精评Prompt | 大模型深度评分（Top 30热点簇） | `prompts/precise_heat.txt` | 4.3 | + +#### 第二层：动态阈值设计（融合硬约束分数） + +**实体簇热度计算**： +```python +class Cluster: + def __init__(self, entity_name: str): + self.entity_name = entity_name + self.articles = [] # 已硬约束评分的文章列表 + self.hot_score = 0 # 综合热度分 + + def calculate_hot_score(self) -> int: + """ + 计算实体簇热度，融合硬约束后的单篇分数 + """ + if not self.articles: + return 0 + + # 基础分数：最高分文章的分数（已硬约束） + max_score = max(a['score'] for a in self.articles) + + # 传播热度：报道数量 × log(来源多样性) + report_count = len(self.articles) + source_diversity = len(set(a['source'] for a in self.articles)) + propagation_heat = report_count * math.log(source_diversity + 1) + + # 角度覆盖度：不同报道角度数 / 6 + angles = set(a.get('reporting_angle', 'unknown') for a in self.articles) + angle_coverage = len(angles) / 6 + + # 高优先级文章加成（90+文章额外加权） + high_priority_count = sum(1 for a in self.articles if a['score'] >= 90) + priority_bonus = high_priority_count * 5 # 每篇90+加5分 + + # 综合计算 + self.hot_score = min(100, int( + max_score * 0.4 + # 单篇最高分权重40% + min(propagation_heat * 3, 30) + # 传播热度权重30% + angle_coverage * 20 + # 角度覆盖权重20% + priority_bonus # 高优先级加成10% + )) + + return self.hot_score +``` + +**动态阈值算法（融合版）**： +```python +def calculate_hotspot_threshold(clusters: list[Cluster], date: str) -> float: + """ + 基于硬约束后分数分布计算动态阈值 + 同时考虑统计特征和高优先级文章数量 + """ + # 获取当日所有簇的热度分数（已融合硬约束） + scores = [c.hot_score for c in clusters] + + if len(scores) < 5: + return 75 # 数据不足时使用保底阈值 + + # 统计特征 + mean_score = mean(scores) + std_score = std(scores) + median_score = median(scores) + + # 方法1：统计阈值（均值+1.5倍标准差） + threshold_stat = mean_score + 1.5 * std_score + + # 方法2：保底阈值（至少3条或前10%） + min_count = max(3, len(scores) * 0.1) + sorted_scores = sorted(scores, reverse=True) + threshold_adaptive = sorted_scores[min_count - 1] + + # 方法3：硬约束保底（考虑90+高优先级文章数量） + high_priority_count = sum( + 1 for c in clusters + if any(a['score'] >= 90 for a in c.articles) + ) + if high_priority_count >= 3: + threshold_backup = median_score + else: + threshold_backup = mean_score + 0.5 * std_score + + # 取三者中较低值，确保热点不被遗漏 + final_threshold = min(threshold_stat, threshold_adaptive, threshold_backup) + + logger.info(f"动态阈值计算: 统计={threshold_stat:.1f}, " + f"保底={threshold_adaptive:.1f}, 硬约束保底={threshold_backup:.1f}, " + f"最终={final_threshold:.1f}") + + return final_threshold +``` + +**双层热点判定流程**： +```python +def identify_hotspots(clusters: list[Cluster], date: str) -> tuple[list[Cluster], list[Cluster]]: + """ + 双层热点识别：硬约束过滤 + 动态阈值筛选 + """ + # 第一层：硬约束过滤（单篇层面已处理，此处检查） + valid_clusters = [] + for c in clusters: + if all(a['score'] <= 79 for a in c.articles): + continue + valid_clusters.append(c) + + # 计算各簇热度（融合硬约束分数） + for c in valid_clusters: + c.calculate_hot_score() + + # 第二层：计算动态阈值 + threshold = calculate_hotspot_threshold(valid_clusters, date) + + # 阈值筛选 + 额外条件 + hotspots = [] + normal = [] + for c in valid_clusters: + if c.hot_score >= threshold: + has_high_priority = any(a['score'] >= 90 for a in c.articles) + significantly_above = c.hot_score >= threshold + 10 + + if has_high_priority or significantly_above: + hotspots.append(c) + else: + normal.append(c) + else: + normal.append(c) + + hotspots.sort(key=lambda x: (x.hot_score, x.max_article_score), reverse=True) + + return hotspots, normal +``` + +**融合设计优势**： +| 层级 | 作用 | 输入 | 输出 | +|------|------|------|------| +| 第一层（硬约束） | 质量控制 | 单篇文章 | 过滤非目标领域，标记高优先级 | +| 第二层（动态阈值） | 热点发现 | 实体簇（多篇聚合） | 识别突发热点，自适应当日分布 | + +**验收标准**： +- [ ] 实体簇热度计算正确（融合硬约束分数） +- [ ] 动态阈值自适应当日分布 +- [ ] 双层筛选协同工作（硬约束过滤→动态阈值筛选） +- [ ] 热点识别召回率 > 90%，误报率 < 20% + +--- + +### 阶段五：Push循环与三层整合（Week 9-10） +**目标**：实现定时汇总、三层整合（摘要+洞察+网摘）、日报生成 + +| 序号 | 任务 | 描述 | 输出 | 依赖 | +|------|------|------|------|------| +| 5.1 | Push循环引擎 | 每日定时运行，读取碎片化信息→整合→推送 | push引擎 | 4.4 | +| 5.2 | 摘要级整合Prompt | 多源报道合并为50-80字精炼摘要 | `prompts/summary_integrate.txt` | 5.1 | +| 5.3 | 洞察级整合Prompt | Top 10生成结构化深度分析（**双视角设计**） | `prompts/insight_integrate.txt` | 5.1 | +| 5.4 | **Prompt防退化模块** | 禁止套话、要求从素材出发 | 防退化检查器 | 5.2, 5.3 | +| 5.5 | 摘要整合服务 | 小模型批量处理全量簇 | 摘要整合服务 | 5.2, 5.4 | +| 5.6 | 洞察整合服务 | 大模型逐个处理Top 10（**双视角输出**） | 洞察整合服务 | 5.3, 5.4 | +| 5.7 | **网摘生成服务** | Top 3 参考消息风格网摘（继承 military-digest-v3） | 网摘生成服务 | 5.1 | +| 5.8 | 时间线提取 | 从多源报道中提取事件时间线 | 时间线提取模块 | 5.5 | +| 5.9 | 多角度整合 | 同一事件不同角度报道的整合 | 多角度整合模块 | 5.6 | + +#### 双视角洞察设计（借鉴AI Daily） + +**设计理念**：metadata（新闻编辑视角，事实压缩）+ 正文（情报分析师视角，趋势判断）解耦 + +**Prompt设计**： +```markdown +## Part 1: Metadata（新闻编辑视角） + +请生成以下结构化信息： +- title: 10字以内的标题（事实陈述，无修饰） +- lead: 20字以内的导语（核心事实） +- highlights: 3-5个关键要点（bullet points，纯事实） + +约束：只陈述事实，不做趋势判断，不用形容词。 + +## Part 2: 洞察正文（情报分析师视角） + +基于多源报道，识别： +1. 事件脉络：关键时间节点和进展 +2. 多角度分析：技术维度、战略维度、舆论维度 +3. 影响与展望：短期影响、中长期趋势、值得关注信号 + +约束： +- 禁止空泛评论（如"意义重大""影响深远"） +- 每个观点必须有素材支撑 +- 使用具体名称（装备型号/政策名称/组织名称），不用"某些""部分"等模糊指代 +``` + +**输出示例**： +```markdown +--- +title: "歼-35A列装部队" +lead: "空军官方确认歼-35A已列装首批作战部队" +highlights: + - 首次公开确认进入实战化部署阶段 + - 隐身性能对标F-35C，可能在舰载领域形成优势 + - 多国媒体关注其对西太平洋军事平衡的影响 +--- + +## 事件脉络 +- 2024-11：珠海航展首次公开亮相 +- 2025-03：完成舰载适配测试 +- 2025-05：空军确认列装首批作战部队【新进展】 + +## 多角度分析 +- **技术维度**：隐身涂层、航电系统、舰载适配均有突破 +- **战略维度**：提升海军航空兵远海作战能力 +- **舆论维度**：外媒关注中美舰载机代差缩小 + +## 影响与展望 +短期：提升海军航母编队作战能力 +中期：可能在西太平洋形成局部优势 +值得关注：后续舰载版测试进展、出口动向 +``` + +#### 网摘生成设计（继承 military-digest-v3） + +**定位**：在三层整合中，网摘是面向直接传播的成品内容。不同于摘要（速览用）和洞察（分析用），网摘采用《参考消息》官方新闻报道格式，适合直接推送到飞书群或作为独立内容分发。 + +**核心特性**： + +| 特性 | 说明 | +|------|------| +| 风格 | 《参考消息》官方新闻报道格式：标题+原标题+发布日期+正文+价值点 | +| 来源识别 | 自动区分微信公众号源（"据公众号 XXX 报道"）和普通源（"据 XXX 报道"） | +| 字数校验 | 正文严格 280-320 字（含标点），价值点 35-45 字 | +| 重试机制 | 不达标自动重试，最多 3 次，每次带具体反馈（"正文只有 XX 字，太少！"） | +| 生成范围 | Top 3 文章，与摘要（全量）和洞察（Top 10）互补 | +| 输出格式 | 文本文件（.txt）+ Pillow 渲染长图（.png） | + +**网摘 Prompt 核心约束**： + +```markdown +标题要求 +简洁、客观、中性、信息密度高 +结构：主体 + 事件 + 核心态势 +不抒情、不夸张、不用网络用语 + +正文格式要求 +开头第一句必须加：据[来源]报道 +正文风格：客观、平实、严谨、书面化，类似外电编译稿 +必须充分展开：补充背景、说明意义、分析影响、展望前景 +全文一段到底，不分段 + +【字数强制要求 - 必须严格执行】 +正文字数严格控制在 280～320 字（含标点），一个字都不能少，一个字都不能多！ +如果内容不够，请合理补充相关背景、行业态势、同类项目对比等专业内容 + +价值点要求 +正文结束后空一行，再写价值点 +价值点严格 35～45 字（含标点） +句式结构：事件 - 影响 / 后果 - 值得关注 +``` + +**字数校验重试机制**： + +```python +def generate_webzine_for_article(article, api_key, base_url, model, max_retries=3): + """为单篇文章生成参考消息风格网摘，支持字数校验重试""" + for attempt in range(max_retries): + result = call_ai([...], purpose="网摘生成") + + # 提取正文部分 + body_text = extract_body(result) + body_len = len(body_text) + + if 280 <= body_len <= 320: + return result # 达标，直接返回 + + # 不达标，在 prompt 末尾追加反馈 + if body_len < 280: + prompt += f"\n\n【上次反馈：正文只有{body_len}字，太少！请大幅增加内容！】" + else: + prompt += f"\n\n【上次反馈：正文有{body_len}字，太多！请精简内容！】" + + return last_result # 重试耗尽，返回最后结果 +``` + +**输出示例**： + +``` +标题：歼-35A隐身舰载战斗机正式列装海军航空兵部队 +原标题：China's J-35A stealth fighter enters service with naval aviation +发布日期：2025年05月24日 +正文：据解放军报报道，中国海军航空兵部队已于近日正式列装歼-35A隐身舰载战斗机，标志着中国成为继美国之后第二个具备隐身舰载机作战能力的国家。歼-35A采用双发中型设计，配备国产涡扇-19发动机，最大起飞重量约30吨，雷达反射截面积据分析优于F-35C。该机配备有源相控阵雷达、分布式光电系统以及先进的电子战套件，可携带霹雳-15中远程空空导弹和鹰击-12超音速反舰导弹等多型武器。军事专家指出，歼-35A的列装将大幅提升中国海军航母编队的制空作战和远程打击能力，特别是在西太平洋方向形成对F-35C的局部数量优势。美国海军战争学院报告认为，中国正加速缩小与美国在舰载航空领域的技术差距，预计到2030年前后将形成至少3个歼-35A舰载机联队的规模。日本防卫省已表示将密切关注相关动向。 +价值点：歼-35A列装标志着中国成为全球第二个拥有隐身舰载机的国家，将显著改变西太平洋海上力量对比，后续量产规模和舰载适配进展值得持续关注。 +``` + +**三层整合对比**： + +| 层次 | 产品 | 覆盖范围 | 定位 | 输出格式 | +|------|------|:---:|------|------| +| 第一层 | 摘要级速览 | 全量（所有簇） | 快速浏览，50-80字精炼 | Markdown表格 | +| 第二层 | 洞察级分析 | Top 10 热点 | 深度分析，双视角（metadata+正文） | Markdown段落 | +| 第三层 | 参考消息网摘 | Top 3 热点 | 成品内容，直接传播 | TXT文本 + PNG长图 | + +#### Prompt防退化设计（借鉴AI Daily） + +**禁止套话列表**： +```python +FORBIDDEN_PHRASES = [ + "意义重大", "影响深远", "引发关注", "备受瞩目", + "深水区", "拐点", "白热化", "新纪元", "里程碑", + "不容忽视", "值得注意", "值得关注", # 除非后接具体内容 + "某些", "部分", "一些", "相关", # 模糊指代 +] + +def check_degeneration(content: str) -> list[str]: + """检查内容是否包含禁止套话""" + violations = [] + for phrase in FORBIDDEN_PHRASES: + if phrase in content: + violations.append(phrase) + return violations +``` + +**Prompt中的防退化指令**： +```markdown +## 输出约束（防退化） + +1. 禁止使用以下套话： + - "意义重大""影响深远""引发关注""备受瞩目" + - "深水区""拐点""白热化""新纪元""里程碑" + +2. 每个观点必须有素材支撑： + - 错误：该装备性能先进 + - 正确：该装备隐身系数0.01，优于F-35的0.02 + +3. 使用具体名称，禁止模糊指代： + - 错误：某些国家表示关注 + - 正确：日本防卫省发布关注声明 + +4. 从素材出发，每次措辞应不同，避免模板化输出 +``` + +**验收标准**： +- [ ] 摘要信息完整度 > 90% +- [ ] 洞察分析覆盖双视角（metadata + 正文） +- [ ] 网摘正文字数达标率 > 90%（280-320字） +- [ ] 网摘价值点字数达标率 > 90%（35-45字） +- [ ] 无禁止套话出现 +- [ ] 整合内容无明显事实错误 + +--- + +### 阶段六：日报生成与跨日关联（Week 11-12） +**目标**：生成结构化日报（含Sentinel标记），实现跨日关联 + +| 序号 | 任务 | 描述 | 输出 | 依赖 | +|------|------|------|------|------| +| 6.1 | 日报模板设计 | Markdown模板，包含热点、速览、数据概览、网摘 | `templates/daily_report.md` | 5.9 | +| 6.2 | **Sentinel分段标记** | 单文件多板块精确管理 | `src/utils/sentinel.py` | 6.1 | +| 6.3 | **跨日关联模块** | 从历史日报提取上下文，追踪事件脉络 | `src/processors/cross_day.py` | 6.2 | +| 6.4 | 排序算法 | 综合热度、时效、重要性的排序逻辑 | 排序模块 | 4.4 | +| 6.5 | 日报渲染服务 | 按模板组装数据，生成Markdown（含Sentinel） | 日报渲染服务 | 6.1, 6.2, 6.4 | +| 6.6 | 网摘图片渲染 | Pillow渲染TOP3合并长图，含Banner标题 | 图片渲染服务 | 5.7 | +| 6.7 | Web归档站点 | 部署在线阅览站点 | Web站点 | 6.5 | +| 6.8 | 数据保留策略 | fetch保留2天，notify保留2天，push保留5天 | 清理策略 | 6.7 | + +#### 简化版Sentinel分段标记设计 + +**标记格式**： +```markdown +--- +title: "每日情报早报" +date: "2026-05-24" +stats: + total_entities: 87 + hotspot_count: 10 +--- + + +## 📋 全量新闻速览 + +| # | 实体 | 摘要 | 来源数 | 热度 | +|---|------|------|--------|------| +| 1 | 歼-35A | 空军确认列装\|隐身性能对标F-35C | 8 | ★★★★★ | + + + +## 🔥 热点深度洞察 + +### 1. 歼-35A列装进展 [持续跟踪] +> 速览：空军官方确认歼-35A已列装首批作战部队 [8源报道] + +**事件脉络** +- 5月20日：首次公开亮相 +- 5月24日：【新进展】确认列装部队 +... + + + +## 📰 参考消息网摘 + +（网摘文本内容，含标题/原标题/发布日期/正文/价值点） + +``` + +**核心代码**： +```python +import re +from datetime import datetime, timedelta + +def extract_section(content: str, section_name: str) -> str: + """从日报内容中提取指定section""" + pattern = rf'(.*?)' + match = re.search(pattern, content, re.DOTALL | re.IGNORECASE) + return match.group(1).strip() if match else "" + +def get_historical_context(entity_name: str, days: int = 7) -> dict: + """获取某实体近N天的历史报道上下文""" + context = { + "first_seen": None, + "previous_summaries": [], + "previous_insights": [], + "mention_count": 0 + } + + for i in range(days): + date = (datetime.now() - timedelta(days=i)).strftime("%Y-%m-%d") + file_path = f"data/reports/daily-{date}.md" + + try: + with open(file_path, 'r', encoding='utf-8') as f: + content = f.read() + + if entity_name in content: + context["mention_count"] += 1 + if context["first_seen"] is None: + context["first_seen"] = date + + summary_section = extract_section(content, "summary") + for line in summary_section.split('\n'): + if entity_name in line and line.strip().startswith('|'): + context["previous_summaries"].append({ + "date": date, + "content": line.strip() + }) + + insights_section = extract_section(content, "insights") + if entity_name in insights_section: + insight_blocks = re.split(r'### \d+\.', insights_section) + for block in insight_blocks: + if entity_name in block: + context["previous_insights"].append({ + "date": date, + "content": block.strip()[:500] + }) + break + except FileNotFoundError: + continue + + return context + +def generate_cross_day_marker(entity_name: str) -> str: + """生成跨日标记""" + history = get_historical_context(entity_name, days=7) + + if history["mention_count"] == 0: + return "首次报道" + elif history["mention_count"] == 1: + return "持续跟踪" + else: + return f"持续跟踪({history['mention_count']}次)" +``` + +**验收标准**： +- [ ] 日报生成时间 < 20分钟 +- [ ] Sentinel标记正确，可精确提取summary/insights/webzine +- [ ] 跨日关联可追踪实体历史报道 +- [ ] 网摘长图渲染正常（合并多篇 + Banner标题） +- [ ] Web站点可正常访问 + +--- + +### 阶段七：优化与迭代（Week 13-14） +**目标**：持续优化，功能扩展 + +| 序号 | 任务 | 描述 | 优先级 | +|------|------|------|--------| +| 7.1 | Prompt优化 | 基于实际效果迭代优化提示词 | 高 | +| 7.2 | 反馈闭环 | 收集用户反馈，标注数据，优化模型 | 高 | +| 7.3 | 趋势分析 | 7日热点趋势图，周期性报告 | 中 | +| 7.4 | 个性化推荐 | 基于用户阅读历史的个性化排序 | 低 | + +--- + +## 四、技术栈建议 + +### 4.1 核心组件 +| 组件 | 推荐方案 | 说明 | +|------|----------|------| +| 数据采集 | Python + feedparser | RSS解析 | +| 数据存储 | SQLite + JSON文件 | 数据库+文件双存储 | +| LLM调用 | OpenAI SDK / 国产模型SDK | 统一接口封装 | +| 翻译 | 中英分离 + ThreadPoolExecutor并发 | 中文文章零API调用 | +| 缓存 | SQLite 三级缓存 + 自动迁移 | 断点续跑，二次运行 ~5s | +| 图片渲染 | Pillow + 中文字体fallback | 网摘长图生成 | +| 任务调度 | systemd timer | 系统级服务管理 | +| Web展示 | Hugo / Flask | 静态/动态站点 | +| 推送 | 飞书 Webhook | 群机器人消息推送 | +| 日志 | Python logging + systemd journal | 集中日志管理 | +| 监控 | 11阶段耗时 + API统计 + 源成功率 | 运行结束自动打印报表 | + +### 4.2 模型选择 +| 任务类型 | 推荐模型 | 预估成本 | +|----------|----------|----------| +| 标题翻译 | DeepSeek-V3 / Qwen-Plus | ~0.001元/次 | +| 正文翻译 | DeepSeek-V3 / Qwen-Plus | ~0.002元/次 | +| 实体提取 | DeepSeek-V3 / Qwen-Plus | ~0.001元/次 | +| 角度分类 | DeepSeek-V3 / Qwen-Plus | ~0.001元/次 | +| 单篇评分 | DeepSeek-V3 / Qwen-Plus | ~0.001元/次 | +| 摘要整合 | DeepSeek-V3 / Qwen-Plus | ~0.002元/次 | +| 网摘生成 | DeepSeek-V3 / Qwen-Plus | ~0.005元/次（含重试） | +| 实体消歧 | GPT-4o / Claude-3.5-Sonnet | ~0.01元/次 | +| 热点精评 | GPT-4o / Claude-3.5-Sonnet | ~0.02元/次 | +| 洞察整合 | GPT-4o / Claude-3.5-Sonnet | ~0.1元/次 | + +**日成本估算**： +- 200-500篇/日 × 0.001元（小模型）= 0.2-0.5元 +- 外文翻译（按30%外文率）：60-150篇 × 0.003元 = 0.18-0.45元 +- 100个簇 × 0.001元（小模型）= 0.1元 +- 3篇网摘 × 0.005元 = 0.015元 +- 10个洞察 × 0.1元（大模型）= 1元 +- **总计：约1.5-2.1元/日** + +--- + +## 五、项目结构 + +``` +news-intelligence-system/ +├── config/ +│ ├── rss_sources.json # RSS源配置 +│ ├── models.yaml # 模型配置 +│ └── systemd/ # systemd服务配置 +├── src/ +│ ├── collectors/ # 数据采集 +│ │ └── rss_collector.py +│ ├── processors/ # 处理管道 +│ │ ├── translator.py # 翻译管线（中英分离+并发翻译） +│ │ ├── entity_extractor.py +│ │ ├── angle_classifier.py +│ │ ├── cluster_engine.py +│ │ ├── heat_calculator.py +│ │ └── cross_day.py # 跨日关联 +│ ├── integrators/ # 内容整合 +│ │ ├── summary_integrator.py +│ │ ├── insight_integrator.py +│ │ └── webzine_generator.py # 网摘生成（参考消息风格+字数校验） +│ ├── generators/ # 报告生成 +│ │ ├── daily_report_generator.py +│ │ ├── image_renderer.py # 网摘图片渲染（Pillow） +│ │ └── web_renderer.py +│ ├── push/ # 推送平台 +│ │ ├── base.py +│ │ └── feishu.py # 飞书 webhook（即时+日报双模式） +│ ├── storage/ # 存储层 +│ │ ├── cache.py # SQLite三级缓存（文章/网摘/分类摘要） +│ │ └── file_storage.py # JSON文件读写 +│ ├── monitor/ # 监控统计 +│ │ └── monitor.py # 阶段耗时+API消耗+成功率+缓存命中率 +│ └── utils/ # 工具函数 +│ ├── sentinel.py # Sentinel分段标记 +│ ├── prompt_checker.py # Prompt防退化检查 +│ └── language.py # 中文检测（is_chinese） +├── prompts/ # 提示词文件 +│ ├── article_score.txt # 单篇评分（含硬约束） +│ ├── extract_entities.txt +│ ├── classify_angle.txt +│ ├── quick_heat.txt +│ ├── precise_heat.txt +│ ├── summary_integrate.txt +│ ├── insight_integrate.txt # 双视角洞察 +│ └── webzine_generate.txt # 参考消息网摘（含字数约束） +├── templates/ +│ └── daily_report.md # 含Sentinel标记（summary/insights/webzine） +├── data/ +│ ├── cache.db # SQLite缓存数据库 +│ ├── fetch/ # fetch-yyyy-mm-dd.json +│ ├── notify/ # notify-yyyy-mm-dd.json +│ └── reports/ # daily-yyyy-mm-dd.md +├── output/ # 输出文件 +│ ├── report_YYYYMMDD.md # Markdown日报 +│ ├── webzine_YYYYMMDD.txt # 网摘文本 +│ └── webzine_YYYYMMDD.png # 网摘合并长图 +├── web/ # Web归档站点 +├── tests/ +├── scripts/ +│ ├── fetch_loop.py +│ ├── push_loop.py +│ └── setup_systemd.sh +├── requirements.txt +└── README.md +``` + +--- + +## 六、关键风险与应对 + +| 风险 | 影响 | 应对措施 | +|------|------|----------| +| RSS源失效 | 数据缺失 | 多源备份，监控告警 | +| LLM API限流 | 处理延迟 | 批量处理，重试机制，降级策略 | +| 翻译质量不稳定 | 后续分析偏差 | 低温度(0.2)翻译，失败时原文填充 | +| 缓存数据库损坏 | 缓存失效 | 连接降级不影响主流程，自动重建 | +| 网摘字数不达标 | 输出质量下降 | 3次重试校验，每次带具体反馈 | +| 实体识别错误 | 聚类偏差 | 人工标注反馈，Prompt迭代 | +| 热点漏识别 | 信息缺失 | 动态阈值保底机制，人工复核 | +| 成本超支 | 预算超支 | Token监控，模型降级，配额控制 | +| 进程崩溃 | 服务中断 | systemd自动重启 | +| LLM输出退化 | 质量下降 | **Prompt防退化检查** | + +--- + +## 七、里程碑与交付物 + +| 里程碑 | 时间 | 交付物 | +|--------|------|--------| +| M1 | Week 2 | 稳定运行的数据采集系统（含翻译管线、SQLite缓存、监控统计） | +| M2 | Week 4 | Fetch循环上线，**评分硬约束生效**，飞书即时推送可用 | +| M3 | Week 6 | 细粒度聚类引擎，实体提取准确率>85% | +| M4 | Week 8 | 动态热点识别系统，召回率>90% | +| M5 | Week 10 | Push循环上线，**三层整合（摘要+洞察+网摘）+Prompt防退化** | +| M6 | Week 12 | **Sentinel跨日关联上线**，网摘长图渲染，Web归档站点 | +| M7 | Week 14 | 持续优化迭代 | + +--- + +## 八、借鉴与继承总结 + +### 8.1 架构层面（借鉴 AI Daily） +1. **双循环架构**：Fetch循环（实时）+ Push循环（定时） +2. **systemd服务化**：替代Python内部定时，提升稳定性 +3. **文件存储规范**：fetch/notify/reports三种文件类型，明确保留策略 + +### 8.2 LLM应用层面（借鉴 AI Daily） +1. **评分硬约束**：非目标领域≤79、KOL转述≤89，有效控制信息质量 +2. **Prompt防退化**：禁止套话、要求从素材出发，避免LLM输出风格趋同 +3. **双视角洞察**：metadata（事实压缩）+ 正文（趋势判断）解耦 + +### 8.3 数据管理层面（借鉴 AI Daily） +1. **Sentinel分段标记**：单文件多板块精确管理，支持跨日关联 +2. **跨日关联**：从历史日报提取上下文，追踪事件发展脉络 +3. **数据保留策略**：自动清理过期文件 + +### 8.4 工程资产层面（继承 military-digest-v3） +1. **SQLite三级缓存**：文章+网摘+分类摘要，含自动迁移和过期清理，二次运行 ~5s +2. **翻译管线**：中英分离（汉字占比>20%判定）+ 外文并发翻译，中文零API调用 +3. **监控统计**：11阶段耗时 + API按用途/模型分组 + 源抓取成功率 + 缓存命中率 +4. **网摘生成**：参考消息风格 + 公众号源智能识别 + 280-320字校验重试（3次） +5. **图片渲染**：Pillow命令模式 + 中文字体多级fallback +6. **飞书推送**：webhook文本推送 + 5级异常分层捕获 + 失败优雅降级 + +### 8.5 工程层面 +1. **日志集中管理**：systemd journal便于问题定位 +2. **推送聚焦飞书**：webhook消息推送，覆盖即时快讯和日报两种场景 +3. **成本可控**：每天约1.5-2.1元（含翻译），缓存命中时几乎零成本 + +--- + +## 附录A：military-digest-v3 实际运行验证数据 + +> 以下数据来自 military-digest-v3 系统 2026-05-14 的实际运行验证，为新系统的设计和成本估算提供基准参考。 + +### A.1 全链路冷启动验证 + +清除所有缓存和输出文件后运行，完整链路通过： + +| 阶段 | 耗时 | 状态 | +|------|:---:|:---:| +| RSS 抓取（5源） | 7.5s | ✅ | +| 时间过滤 (156→22篇) | 0.0s | ✅ | +| 去重 | 0.0s | ✅ | +| 标题翻译（7篇外文） | 17.9s | ✅ | +| 关键词筛选 (22→15篇) | 0.0s | ✅ | +| 正文翻译（6篇外文） | 9.9s | ✅ | +| AI评分分类（15篇） | 30.7s | ✅ | +| 网摘生成（3篇，含2次重试） | 53.3s | ✅ | +| 分类介绍生成（4类） | 33.5s | ✅ | +| 图片生成 | 0.3s | ✅ | +| 报告生成 | 0.0s | ✅ | +| **总耗时** | **153s** | ✅ | + +**API 消耗统计：** + +| 用途 | 次数 | tokens 估算 | +|------|:---:|:---:| +| 标题翻译 | 7 | 1,400 | +| 正文翻译 | 6 | 4,800 | +| AI评分分类 | 15 | 12,000 | +| 网摘生成 | 5 (含2次重试) | 4,000 | +| 分类介绍 | 4 | 600 | +| **合计** | **37** | **~22,800** | + +**输出文件验证：** + +``` +military_report_20260514.md 10.6 KB ✅ Markdown 格式完整 +military_webzine_20260514.txt 4.9 KB ✅ TOP3 网摘文本完整 +military_webzine_20260514.png 868.5 KB ✅ 合并长图渲染正常 +article_cache.db 60.0 KB ✅ 数据完整无异常 +``` + +### A.2 SQLite 缓存数据完整性验证 + +冷启动全链路运行后，直接查询 `article_cache.db` 验证： + +**基本信息：** + +| 表 | 行数 | 说明 | +|------|:---:|------| +| `article_cache` | 15 | 与关键词筛选后文章数完全一致 | +| `category_summary_cache` | 4 | 今日必看/装备动态/地区冲突/战略政策全覆盖 | + +**逐项检查：** + +| 检查项 | 结果 | 判定 | +|------|:---:|:---:| +| id 重复 | 0 条 | ✅ | +| 失败条目 (status≠1) | 0 条 | ✅ | +| translated_title 缺失 | 0/15 | ✅ | +| translated_content 缺失 | 0/15 | ✅ | +| ai_summary 缺失 | 0/15 | ✅ | +| ai_category 缺失 | 0/15 | ✅ | +| published_time 缺失 | 0/15 | ✅ | +| webzine_text 覆盖率 | 3/15 | ✅ (仅TOP3需要) | +| ai_score 范围 | 3.3 ~ 6.5 (avg 5.3) | ✅ 正态分布 | +| 分类分布 | 装备7 / 战略5 / 冲突3 | ✅ 合理 | + +### A.3 缓存命中验证（连续运行2次） + +| 指标 | 第一次（冷启动） | 第二次（含缓存） | +|------|:---:|:---:| +| 文章缓存命中 | 0 / 15 | **15 / 15** ✅ | +| API 调用次数 | 37 次 | **0 次** ✅ | +| 总耗时 | 153s | **~5s** ✅ | +| 退出码 | 0 | 0 | + +> 第二次运行时文章缓存、网摘缓存、分类介绍缓存全部命中，跳过所有 AI 调用，仅做 RSS 抓取 + 文件排版，几乎零 token 消耗。 + +### A.4 飞书推送验证 + +| 场景 | 配置 | 结果 | 主流程 | +|------|------|------|:---:| +| 推送关闭 | `enable_feishu_push: false` | "未启用任何推送渠道，跳过" | exit 0 ✅ | +| **飞书（真实webhook）** | `enable_feishu_push: true` | **飞书推送成功** 🎉 | exit 0 ✅ | + +**错误处理矩阵（5级分层）：** + +| 异常类型 | 处理方式 | 主流程影响 | +|------|------|:---:| +| `ConnectionError` | warning 日志 + 返回 False | 无 | +| `Timeout` (连接5s/读取15s) | warning 日志 + 返回 False | 无 | +| `HTTPError` (4xx/5xx) | warning 日志 + 返回 False | 无 | +| `JSONDecodeError` | warning 日志 + 返回 False | 无 | +| 其他 `Exception` | warning 日志 + 返回 False | 无 | + +--- + +*文档版本：v3.0* +*最后更新：2026-05-29* +*状态：融合 AI Daily 四大核心设计 + military-digest-v3 六大工程资产* diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6e2087b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,33 @@ +FROM python:3.11-slim + +WORKDIR /app + +# 安装系统依赖和中文字体 +RUN apt-get update && apt-get install -y --no-install-recommends \ + fonts-wqy-microhei \ + fonts-wqy-zenhei \ + tzdata \ + && rm -rf /var/lib/apt/lists/* + +# 设置时区 +ENV TZ=Asia/Shanghai +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# 安装Python依赖 +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# 复制代码 +COPY scripts/ ./scripts/ +COPY config/ ./config/ + +# 创建输出目录 +RUN mkdir -p /app/output /app/logs + +# 设置环境变量 +ENV PYTHONUNBUFFERED=1 +ENV LOG_DIR=/app/logs +ENV OUTPUT_DIR=/app/output + +# 入口命令 +CMD ["python", "scripts/military_daily_report_v3.py"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a0cbdda --- /dev/null +++ b/Makefile @@ -0,0 +1,34 @@ +.PHONY: help build run run-once logs clean install-cron local + +help: + @echo "🪖 军事科技每日摘要系统 V3" + @echo "================================" + @echo "可用命令：" + @echo " make build - 构建 Docker 镜像" + @echo " make run-once - 执行一次（Docker）" + @echo " make local - 本地直接执行" + @echo " make logs - 查看日志" + @echo " make clean - 清理" + @echo " make install-cron - 安装宿主机定时任务" + +build: + docker-compose build + +run-once: + docker-compose run --rm military-digest + +local: + python3 scripts/military_daily_report_v3.py + +logs: + tail -f logs/daily_report.log 2>/dev/null || echo "暂无日志" + +clean: + docker-compose down -v + rm -rf output/* logs/* + +install-cron: + @echo "正在安装定时任务（每日 8:00 执行）..." + (crontab -l 2>/dev/null | grep -v "military_digest"; echo "0 8 * * * cd $(shell pwd) && python3 scripts/military_daily_report_v3.py >> logs/cron.log 2>&1") | crontab - + @echo "✅ 定时任务已安装" + @crontab -l | grep "military_digest" diff --git a/README.md b/README.md new file mode 100644 index 0000000..5c561b5 --- /dev/null +++ b/README.md @@ -0,0 +1,286 @@ +# 🪖 军事科技每日摘要系统 V3 + +完整的军事科技 RSS 抓取 + AI 智能分析 + 增量缓存 + 运行监控系统 + +## ✨ 完整功能 + +### 📥 数据抓取 +- 🔗 可配置 RSS 源直抓（rss_feeds.txt 管理，支持 100+ 源） +- ⏰ 24 小时时间窗口过滤 +- 🔄 同源标题去重（保留跨源对同一事件的不同报道） +- ⚡ ThreadPoolExecutor 并发抓取，内置失败重试 + +### 🗄️ 增量缓存 +- 💾 SQLite 本地缓存：文章翻译/AI分析结果持久化 +- 📰 网摘文本缓存：再次运行零 token 消耗 +- 🏷️ 分类介绍缓存：按日期+分类独立存储 +- 🧹 自动过期清理（7天）+ 表结构自动迁移 + +### 🌐 智能翻译 +- 🔍 自动识别外文文章 +- 📝 分步翻译：先标题 → 再正文（仅通过筛选的文章） +- ⚡ 并发翻译（ThreadPoolExecutor） + +### 🔑 关键词筛选 +- 🎯 44 个军事科技专业关键词库 +- 📊 命中数排序，文章 >15 篇时自动过滤 + +### 🤖 AI 智能分析 +- ⭐ 多维度评分（技术35% + 战略30% + 信息20% + 时效15%） +- 📋 自动分类：今日必看 / 装备动态 / 地区冲突 / 战略政策 +- ✍️ 每篇文章 AI 智能摘要（80-120字） +- 🔮 分类深度洞察分析（每类50字综述） +- 🏆 TOP3 必看深度分析 +- ⚡ 批量并发评分 + 分类 + +### 🖼️ 图片生成 +- 📄 3800px 超长网摘图片 +- 🎨 军事主题视觉设计 +- 📊 包含TOP3完整内容 +- 💾 自动保存PNG文件 + +### 📊 运行监控 +- ⏱ 各阶段耗时统计，自动输出结构化报表 +- 🤖 API 调用量/token 按用途和模型分组统计 +- 📰 源抓取成功/失败率一目了然 +- 📡 文章统计 + 缓存命中率 + +## 🚀 快速开始 + +### 方式一：本地直接运行（推荐） + +```bash +# 1. 安装依赖 +pip install -r requirements.txt + +# 2. 配置环境 +cp config/.env.example config/.env +vim config/.env +# 编辑 OPENAI_API_KEY + +# 3. 运行 +make local +``` + +### 方式二：Docker 部署 + +```bash +# 1. 克隆或解压 +cd military-digest-v3 + +# 2. 配置环境 +cp config/.env.example config/.env +vim config/.env +# 编辑 OPENAI_API_KEY + +# 3. 构建镜像 +make build + +# 4. 测试运行 +make run-once + +# 5. 安装定时任务 +make install-cron +``` + +## 📁 目录结构 + +``` +military-digest-v3/ +├── scripts/ +│ ├── military_daily_report_v3.py # 主入口 — 流程编排 +│ └── modules/ # 功能模块包 +│ ├── __init__.py # 包初始化 + 统一导出 +│ ├── config.py # 配置模块：常量 + AI配置加载 +│ ├── utils.py # 工具函数：语言检测/HTML清洗/AI调用/换行 +│ ├── rss_fetcher.py # RSS抓取：12源 × 6线程并发 +│ ├── translator.py # 翻译处理：中英分离 + 并发翻译 +│ ├── keyword_filter.py # 关键词筛选：49词命中排序 +│ │ ├── ai_analyzer.py # AI分析：评分/分类/摘要/洞察/网摘生成 +│ │ ├── cache.py # SQLite缓存：文章/网摘/分类介绍增量缓存 +│ │ ├── monitor.py # 运行监控：阶段耗时/API/AI token调用统计 +│ │ ├── pusher.py # 推送模块：企业微信/飞书 webhook 推送 +│ │ ├── image_generator.py # 图片生成：TOP3合并3800px长图 +│ └── get_chinese_font.py # 字体模块：跨系统中文字体适配 +├── config/ +│ ├── config.json # 系统配置 +│ └── .env.example # 环境变量示例 +├── output/ # 输出目录 +│ ├── military_report_YYYYMMDD.md # 完整报告（Markdown格式，支持链接、样式） +│ ├── military_webzine_YYYYMMDD.txt # TOP3网摘文本 +│ └── military_webzine_YYYYMMDD.png # TOP3网摘合并长图 +├── logs/ # 日志目录 +├── Dockerfile +├── docker-compose.yml +├── Makefile +└── README.md +``` + +## 🧩 模块架构 + +``` +military_daily_report_v3.py ← 主入口（流程编排） + ├── config.py ← 配置常量，零依赖 + ├── logger.py ← 统一日志模块 + ├── cache.py ← SQLite增量缓存，自动迁移 + ├── monitor.py ← 运行监控统计 + ├── pusher.py ← 企业微信/飞书推送 + ├── rss_fetcher.py ← 依赖 config + utils + monitor + ├── translator.py ← 依赖 utils + ├── keyword_filter.py ← 依赖 config + ├── ai_analyzer.py ← 依赖 config + utils + ├── image_generator.py ← 依赖 utils + get_chinese_font + └── get_chinese_font.py ← 零依赖 +``` + +### 各模块职责 + +| 模块 | 行数 | 职责 | 对外导出 | +|------|------|------|----------| +| `config.py` | 90 | RSS源、关键词库、评分维度、分类定义、AI配置加载 | `RSS_FEEDS`, `WECHAT_SOURCES`, `KEYWORDS`, `SCORE_DIMENSIONS`, `CATEGORIES`, `load_ai_config()` | +| `utils.py` | 130+ | 语言检测、HTML清理、时间解析、AI API封装（含监控钩子）、文本换行 | `is_chinese()`, `clean_html()`, `extract_content()`, `parse_pub_time()`, `call_ai()`, `wrap_text()` | +| `rss_fetcher.py` | 100+ | 多线程并行抓取RSS源（MAX_THREADS并发，含2次重试+监控上报） | `fetch_all_feeds()` | +| `translator.py` | 150+ | 中英文分离 + 标题/正文分步AI并发翻译 | `batch_translate_titles()`, `batch_translate_contents()` | +| `keyword_filter.py` | 23 | 44关键词命中计数 → 排序 → Top15筛选 | `keyword_filter()` | +| `ai_analyzer.py` | 240+ | 四维度评分、智能分类、中文摘要、分类洞察、《参考消息》风格网摘生成 | `score_and_classify_article()`, `calculate_weighted_score()`, `generate_category_overview()`, `generate_webzine_for_article()`, `batch_score_and_classify_articles()` | +| `cache.py` | 310+ | SQLite增量缓存：文章处理结果+网摘+分类介绍，支持自动迁移+过期清理+断点续跑 | `get_cache()`, `get_article()`, `save_article()`, `save_article_webzine()`, `get_category_summary()`, `save_category_summary()` | +| `monitor.py` | 160+ | 运行时监控：阶段耗时/AI API调用量+token/源成功率/缓存命中率，结束自动输出报表 | `get_monitor()`, `monitor.stage()`, `monitor.report()` | +| `pusher.py` | 130+ | 企业微信文本/图片/Markdown推送 + 飞书文本推送，主流程末自动触发，失败优雅降级 | `send_daily_push()`, `push_wechat_text()`, `push_wechat_image()`, `push_feishu_text()` | +| `image_generator.py` | 220 | 网摘文本渲染为PNG图片（单篇 + TOP3合并长图） | `create_webzine_image()`, `create_combined_webzine_image()` | +| `get_chinese_font.py` | 50 | 跨系统中文字体自动匹配，支持Windows/Linux/macOS，内置开源字体可选 | `get_chinese_font(size, bold)` | + +## ⚙️ 配置说明 + +### 环境变量 (.env) + +| 配置项 | 说明 | 默认值 | +|--------|------|--------| +| OPENAI_API_KEY | OpenAI API 密钥 | - | +| OPENAI_API_BASE | API 地址 | https://api.openai.com/v1 | +| OPENAI_MODEL | AI 分析模型 | gpt-4o-mini | +| WEBZINE_MODEL | 网摘专用模型 | deepseek-v3.2-250518 | + +### config.json 配置项 + +| 配置项 | 说明 | 默认值 | +|--------|------|--------| +| time_window_hours | 时间窗口（小时） | 12 | +| max_articles | 最多保留文章数 | 15 | +| top3_count | TOP3 必看文章数 | 3 | +| wechat_target | 微信接收者ID | - | +| wechat_account | 微信机器人账号 | - | +| enable_wechat_push | 开启微信推送 | true | +| enable_feishu_push | 开启飞书推送 | false | +| enable_image_gen | 开启图片生成 | true | +| enable_txt_gen | 开启TXT文件生成 | true | + +### rss_feeds.txt 订阅源配置 +所有RSS源都在`config/rss_feeds.txt`文件中管理，不需要修改代码即可增删订阅源： +- 格式：每行 `源名称|RSS地址` +- 以`#`开头的行是注释，会自动忽略 +- 空行会自动跳过 +- 修改后直接运行程序，无需重启或改代码 +- ✅ 容错保障：文件不存在、为空或格式错误时，自动回退使用内置的12个默认源，不会影响程序运行 +- 支持所有标准RSS/Atom格式，中英文源无需区分，系统自动识别翻译 + +## ⏰ 定时任务 + +### 方式一：宿主机 crontab + +```bash +make install-cron +# 或手动添加 +crontab -e +# 加入：0 8 * * * cd /path/to/military-digest-v3 && python3 scripts/military_daily_report_v3.py >> logs/cron.log 2>&1 +``` + +### 方式二：Docker Cron + +```bash +docker-compose --profile cron up -d +``` + +## 🔍 RSS 源列表 + +1. Defense One - 美国防务新闻 +2. NEWUAS - 无人机系统新闻 +3. Seapower - 海军力量杂志 +4. The War Zone - 战区军事分析 +5. 国防科技要闻 +6. 战略前沿技术 +7. 无人机邦 +8. 浮空飞行器 +9. 海鹰资讯 +10. 渊亭防务 +11. 电波之矛 +12. 龙牙的一座山 + +## 📊 输出示例 + +``` +🪖 军事科技每日摘要 +📅 2026年05月12日 | 共收录 12 篇 + +==================== +🔥 今日必看 TOP3 +==================== + +1.【美军MQ-9无人机测试发射激光制导火箭弹】 + ⭐ 综合评分：9.2/10 + 📰 来源：The War Zone + 📝 摘要：美国通用原子公司联合美空军在内华达测试场完成... + 🔮 深度洞察：低成本改装令察打无人机获得反无人机能力... + 🔗 阅读原文：... + +... +``` + +## 🔄 数据流程 + +``` +main() + ├── Step 1: RSS 并行抓取（ThreadPoolExecutor × MAX_THREADS 并发，内置重试） + ├── Step 2: 24小时时间窗口过滤 + ├── Step 3: 同源标题去重 + ├── Step 4: SQLite 缓存查询（分离已处理/待处理文章） + ├── Step 5: 外文标题并发翻译（ThreadPoolExecutor × AI_CONCURRENCY） + ├── Step 6: 关键词筛选（仅当 >15篇） + ├── Step 7: 外文正文并发翻译（仅筛选通过的文章，大幅省 token） + ├── Step 8: 批量 AI 评分 + 分类 + 摘要（并发） + ├── Step 9: 新处理文章写入 SQLite 缓存 + ├── Step 10: 合并缓存+新文章，按评分降序 + ├── Step 11: TOP3 网摘生成（优先缓存命中，支持字数重试，TXT+PNG） + ├── Step 12: 四分类整理 + 深度洞察生成（优先缓存命中） + ├── Step 13: Markdown 报告输出（全缓存+文件已存在则跳过） + ├── Step 14: 多渠道推送（企业微信/飞书 webhook，失败优雅降级） + └── 监控报告输出（阶段耗时/API+token统计/源成功率/缓存命中率） +``` + +## ✅ 系统要求 + +- Python 3.8+ 或 Docker +- OpenAI API Key（或兼容API，如火山引擎、DeepSeek等） +- 内存 ≥ 512MB +- 磁盘 ≥ 1GB + +## ❓ 常见问题 + +### 生成的图片中文字体乱码怎么办？ +无需额外配置，代码已自动适配所有主流系统的自带中文字体： +- Windows: 自动匹配宋体/黑体/微软雅黑 +- Linux: 自动匹配NotoSansCJK/文泉驿微米黑 +- macOS: 自动匹配PingFang/黑体 + +如果需要完全不依赖系统字体，可以下载开源无版权的NotoSansCJK字体放到`scripts/modules/assets/fonts/`目录下： +- 粗体：`NotoSansCJK-Bold.ttc` +- 常规：`NotoSansCJK-Regular.ttc` + +### 日志输出在哪里？ +日志默认输出到标准错误流，包含INFO/WARNING/ERROR三个级别，运行时会实时打印到控制台，也可以重定向到日志文件： +```bash +python scripts/military_daily_report_v3.py >> logs/runtime.log 2>&1 +``` + +--- + +*V3 版本 - 全流程重构 + 模块化拆分 + 跨系统字体适配 + 日志标准化 + Markdown报告输出，2026年5月发布* diff --git a/ai-daily-main.zip b/ai-daily-main.zip new file mode 100644 index 0000000..61a4da1 Binary files /dev/null and b/ai-daily-main.zip differ diff --git a/ai-daily-main/.env.example b/ai-daily-main/.env.example new file mode 100644 index 0000000..209692b --- /dev/null +++ b/ai-daily-main/.env.example @@ -0,0 +1,9 @@ +# LLM API 配置 +DEEPSEEK_API_KEY=sk-or-v1-... + +# Feishu Webhook +FEISHU_WEBHOOK_URL=https://open.feishu.cn/open-apis/bot/v2/hook/... + +# 可选：添加 token 的免费限额更高，不加也基本够用 +# GITHUB_TOKEN= +# JINA_API_KEY= diff --git a/ai-daily-main/.gitignore b/ai-daily-main/.gitignore new file mode 100644 index 0000000..461d8c4 --- /dev/null +++ b/ai-daily-main/.gitignore @@ -0,0 +1,42 @@ +# 敏感文件 +.env +!.env.example + +# 本地配置（使用时从 config.json.example 拷贝并修改） +config.json +!config.json.example + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +*.egg-info/ +dist/ +build/ +.venv + +# 测试 +.pytest_cache/ +.ruff_cache/ +.coverage +htmlcov/ + +# IDE +.idea/ +.vscode/ +*.swp +*.swo + +# 数据文件 +news-data/ +*.log + + +# Claude/其他代理数据 +.omc/ +.claude/ + +# Git worktrees +.worktrees/ diff --git a/ai-daily-main/CLAUDE.md b/ai-daily-main/CLAUDE.md new file mode 100644 index 0000000..7135863 --- /dev/null +++ b/ai-daily-main/CLAUDE.md @@ -0,0 +1,28 @@ +# AI 每日资讯推送系统 + +AI 驱动的 RSS 新闻聚合与推送系统，支持 400+ 信息源，使用 LLM 智能评分筛选，定时推送到 Discord/企业微信。 + +当前阶段：MVP 已完成，支持 RSS 抓取、LLM 评分、定时推送、即时推送。 + +## 技术栈 + +- 语言：Python 3.10+ +- 框架：asyncio +- 依赖：feedparser, aiohttp, croniter, openai +- 构建：pip +- 测试：pytest + +## 开发规则 + +1. 任何代码改动如果与 docs/ 下的文档不一致，必须同步更新对应文档 +2. 产品决策变更（功能取舍、交互调整、设计修改）和任务进度 写入 docs/plan.md 的`## 技术决策记录` 和 `## 开发进度` +3. 不确定的产品问题先问用户，不要自行决定 +4. 敏感信息（API Keys、Webhook URLs）通过环境变量管理，不硬编码 +5. config 有更新需要即时更新对应文档 + +## 文档索引 + +- 技术架构 → [docs/tech-spec.md](docs/tech-spec.md) +- 开发计划 → [docs/plan.md](docs/plan.md) +- 早报扩展板块设计（GitHub / HN / 洞察）→ [docs/extra-sections-design.md](docs/extra-sections-design.md) +- 实施计划归档 → [docs/superpowers/plans/](docs/superpowers/plans/) diff --git a/ai-daily-main/LICENSE b/ai-daily-main/LICENSE new file mode 100644 index 0000000..8fa98fa --- /dev/null +++ b/ai-daily-main/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025-present + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/ai-daily-main/README.md b/ai-daily-main/README.md new file mode 100644 index 0000000..6e2dd17 --- /dev/null +++ b/ai-daily-main/README.md @@ -0,0 +1,598 @@ + +

AI Daily

+ +

筛选值得关注的 AI 信号

+ + + +

+ License: MIT + Python 3.12+ + uv managed + RSS 400+ + systemd +

+ +[![AI Daily Banner](https://cdn.yeekal.com/yee/visuals/ai-daily-cover.webp)](https://yeekal.com/daily/) + +

AI 驱动的资讯聚合与推送系统｜RSS · GitHub Trending · Hacker News 三大板块｜LLM 智能评分｜推送到 Discord / 飞书

+ +--- + +## 核心特性 + +- 🗞️ **三大内容板块** —— RSS 资讯（400+ 源）+ GitHub Trending + Hacker News 热帖，从媒体、开源、社区三个维度立体捕获 AI 动态 +- 🧠 **LLM 智能筛选** —— 评分过滤 + 跨日去重，只留下值得读的 +- ⚡ **即时推送** —— 热点新闻（≥90 分）实时触达，重大发布不错过 +- 📬 **每日汇总** —— 定时早晚报，早报叠加跨板块洞察段 +- 🔌 **多平台推送** —— 飞书、Discord 开箱即用，可扩展自定义平台 +- 🛠️ **零运维部署** —— systemd timer 一键安装，开机自启、故障重启 + +--- + +## 三大内容板块 + +### 📰 RSS 资讯聚合 + +聚合全球主流 AI 媒体、博客、Twitter 账号，默认 OPML 包含约 420 个优质源（源自 [BestBlogs](https://github.com/ginobefun/BestBlogs)）。LLM 逐条评分，只保留高质量内容。 + +- 默认 60 分钟轮询，异步并发抓取 +- 评分维度：相关度、信息密度、时效性 +- 跨日上下文去重，同一事件不重复推送 + +### ⭐ GitHub Trending + +抓取 GitHub trending 页面，LLM 深读 README / topics / metadata，从昙花一现的玩具仓库里挑出真正值得关注的项目。 + +- 从 top 10 候选中精选 3 个（可配置） +- 输出 deep-dive 摘要，附技术亮点与上手建议 +- 历史去重索引，推过的项目不再出现 + +### 💬 Hacker News 热帖 + +跟踪 HN 首页 AI 相关讨论，整合**外链正文 + 顶层评论树**，产出"内容总结 + 社区观点"双段式摘要。 + +- 轻量 LLM 从首页 30 条中选出最值得读的故事 +- 抓取 L1 顶层评论 + L2 关键回复，字符预算受控 +- 外链通过 Jina Reader 拉取 markdown 正文 + +## 系统架构 + +```mermaid +flowchart TB + subgraph Sources["📥 数据源"] + RSS["RSS Feeds
400+ sources"] + GH["GitHub Trending"] + HN["Hacker News
Front Page"] + end + + subgraph Fetch["⚙️ Fetch 阶段"] + F1["RSS Fetcher
asyncio + feedparser"] + F2["GH Scraper
README deep-dive"] + F3["HN Crawler
Algolia + Jina Reader"] + end + + subgraph LLMStage["🧠 LLM 评分与摘要"] + Score["score / score_batch"] + Digest["digest / immediate_push"] + Insight["跨板块 insights"] + end + + subgraph Store["💾 存储"] + Files["news-data/
fetch-*.json
push-*.md"] + end + + subgraph PushStage["📤 推送渠道"] + Discord["Discord Webhook"] + Feishu["飞书 Webhook"] + end + + RSS --> F1 --> Score + GH --> F2 --> Score + HN --> F3 --> Score + Score --> Digest + Score --> Insight + Digest --> Files + Insight --> Files + Files --> Discord + Files --> Feishu +``` + +**调度说明** + +- `dnews-fetch.service`（默认每 60 分钟）：抓取 RSS → 评分 → 命中 ≥90 分立即推送 +- `dnews-push.service`（按 `push_cron`）：生成 digest 推送；当天最早一次额外触发 GitHub / HN / 跨板块洞察 + + +## 快速开始 + +### 环境要求 + +- Python 3.12+ +- [uv](https://docs.astral.sh/uv/) (Python 包管理器) +- Linux + systemd（推荐使用一键部署） + +### 1. 配置环境变量 + +在项目根目录创建 `.env` 文件，添加以下配置： + +```bash +# LLM API（ OpenAI API 兼容接口） +DEEPSEEK_API_KEY=your_api_key_here + +# 飞书 Webhook +FEISHU_WEBHOOK_URL=your_feishu_webhook_url_here + +``` + +从模板拷贝一份本地配置（`config.json` 已加入 `.gitignore`，不会提交）： + +```bash +cp config.json.example config.json +``` + +然后在 `config.json` 中修改 `llm` 和 `push` + +```json +{ + "llm": { + "provider": "", # openai compatiable + "model": "", + "baseUrl": "", + "apiKeyName": "DEEPSEEK_API_KEY", #your api key name in .env> + }, + "push": { + "feishu": { + "enabled": true, + "apiKeyName": "FEISHU_WEBHOOK_URL" + } + }, + +} + +``` + +**获取飞书群机器人 Webhook** + +详细教程参考：[我用 RSS + LLM 搭建了一个 AI 热点追踪系统](https://yeekal.com//ai/ai-daily-news-tracker) + + +1. 群组右上角设置 → 群机器人 → 添加机器人 +2. 选择 自定义机器人 → 点击 添加 → 复制 Webhook URL + +**获取 Discord Webhook：** +1. 进入 Discord 服务器设置 → +2. 整合 → Webhooks +2. 创建新 Webhook，复制 URL + + +### 2. 一键部署（推荐） + +将程序作为 systemd timer 部署，由系统负责定时触发、故障重启、开机自启。 + +```bash +./scripts/install.sh +``` + +脚本会自动同步依赖、安装 systemd 服务并按 `config.json` 中的调度配置启动定时任务，**机器重启后自动恢复**。安装成功后无需额外操作。 + +### 3. 手动运行（可选） + +若不使用 systemd，也可以手动运行程序。先同步依赖（uv 会自动创建 `.venv`）： + +```bash +uv sync +``` + +程序提供的子命令： + +```bash +uv run python -m src.main check # 校验 LLM 接口可达性（部署期使用） +uv run python -m src.main fetch # 单次抓取后退出（systemd timer 调用） +uv run python -m src.main push # 单次推送后退出（systemd timer 调用） +uv run python -m src.main loop # 长跑模式（本地开发/调试用） +uv run python -m src.main github # 单跑 GitHub Trending 板块,打印不推送 +uv run python -m src.main hackernews # 单跑 Hacker News 板块,打印不推送 +``` + +首次运行会自动创建 `news-data/` 目录并开始抓取数据。 + +> 若未配置推送渠道，则可以在 news-data 目录查看生成的push信息 + +--- + +## 系统服务管理 + +部署完成后，使用以下命令管理服务。 + +### 常用命令 + +安装后 `daily-news` 进入系统 PATH，可在任意目录调用： + +```bash +daily-news status [N] # 查看 timer/service 状态 + 最近 N 行日志（默认 15） +daily-news logs # 实时跟随日志（Ctrl+C 退出） +daily-news start # 启动两个 timer +daily-news stop # 停止两个 timer +daily-news restart # 重启两个 timer（仅重置调度，不立即触发任务） +daily-news help # 用法说明 +``` + +手动立即触发一次任务（不影响下次调度）： + +```bash +sudo systemctl start dnews-fetch.service +sudo systemctl start dnews-push.service +``` + +### 修改配置后 + +```bash +./scripts/install.sh # 重新跑一次即可，幂等（重新渲染单元 + restart timer） +``` + +修改 `config.json` 中的 `schedule`、`log.retention_days` 等需要重装；修改 `.env` 只需 `daily-news restart`。 + +### 卸载 + +```bash +./scripts/uninstall.sh +``` + +卸载会移除 systemd 单元、`/usr/local/bin/daily-news` 和日志保留 drop-in；**不会**删除 `news-data/` 数据。 + +### 日志保留 + +日志通过 systemd journald 命名空间 `dnews` 隔离，保留天数由 `config.json` 中的 `log.retention_days` 控制（默认 7 天）。不影响系统其他服务的日志。 + +### 日志查询 + +```bash +daily-news logs # 实时跟随两个 service 的日志 +daily-news status [N] # 查看状态 + 最近 N 行日志（默认 15） + +journalctl --namespace=dnews -f # 实时跟随命名空间内全部日志 +journalctl --namespace=dnews -u dnews-fetch -f # 仅跟随 fetch service +journalctl --namespace=dnews -u dnews-push -f # 仅跟随 push service +journalctl --namespace=dnews --since "1 hour ago" # 查询近 1 小时日志 +journalctl --namespace=dnews --since today # 查询今日日志 +journalctl --namespace=dnews -p err # 仅查询 error 级别及以上 +journalctl --namespace=dnews --vacuum-time=1s # 手动清空命名空间日志 +``` + +--- + +## 配置详解（config.json） + +完整的配置文件结构如下，每个字段都有详细说明： + +```json +{ + // 订阅源管理 + "sources": { + "base_opml": "resources/rss.opml", // 基础OPML文件，包含400+预设源 + "add": [ // 自定义添加的RSS源 + { + "title": "OpenAI News", + "xmlUrl": "https://openai.com/news/rss.xml", + "category": "AI" + } + ], + "block": [ // 手动屏蔽的源，精确匹配xmlUrl + { + "title": "Google Developers Blog", + "xmlUrl": "https://developers.googleblog.com/feeds/posts/default" + } + ], + "block_domains": ["*.substack.com", "*.youtube.com"] // 域名屏蔽，支持通配符 + }, + + // 内容过滤 + "filter": { + "min_score": 60, // 最低评分阈值，低于此分不推送 + "hot_threshold": 90, // 热点阈值，达到立即即时推送 + "context_days": 3, // 汇总时参考的历史天数 + "keep_days": 7, // 数据保留天数 + "push_context_days": 5, // 汇总推送去重的上下文有效天数 + "no_content_marker": "[NO_NEW_CONTENT]" // LLM返回的无内容标记，用于判断是否跳过推送 + }, + + // 日志配置（仅对 systemd 部署生效） + "log": { + "retention_days": 7 // journald 命名空间 dnews 的日志保留天数 + }, + + // 调度配置 + "schedule": { + "fetch_interval_minutes": 30, // RSS抓取间隔（分钟） + "fetch_lookback_minutes": 120, // RSS冗余缓存时间（分钟），必须大于fetch_interval_minutes，用于防止RSS延迟导致漏读 + "push_cron": ["0 8 * * *", "0 17 * * *"], // 定时推送cron表达式 + "timezone_hours": 8 // 时区偏移（8=北京时间） + }, + + // 抓取配置 + "fetch": { + "max_workers": 10, // 最大并发数 + "timeout": 10 // 单请求超时（秒） + }, + + // LLM配置 + "llm": { + "provider": "openai", // 提供商类型，openai只是知名该api接口时openai接口兼容，代码中并无实际使用 + "model": "x-ai/grok-4.1-fast", // 模型名称 + "baseUrl": "https://openrouter.ai/api/v1", // API端点 + "apiKeyName": "OPENROUTER_API_KEY", // 环境变量名 + "max_prompt_chars": 128000, // 单次prompt最大字符数 + "max_concurrent_batches": 3, // 最大并发批次数 + "prompts": { // prompt文件路径 + "score": "prompts/score.txt", + "score_batch": "prompts/score_batch.txt", + "immediate_push": "prompts/immediate_push.txt", + "digest": "prompts/digest.txt" + } + }, + + // 推送配置 + "push": { + "discord": { + "enabled": true, // 是否启用 + "apiKeyName": "DISCORD_WEBHOOK_URL" // Webhook环境变量名 + }, + "feishu": { + "enabled": false, + "apiKeyName": "FEISHU_WEBHOOK_URL" + } + } +} +``` + +### sources —— 订阅源管理 + +| 字段 | 类型 | 说明 | +|------|------|------| +| `base_opml` | string | 基础 OPML 文件路径，包含 400+ 预设 RSS 源 | +| `add` | array | 自定义添加的 RSS 源，结构为 `{title, xmlUrl, category}` | +| `block` | array | 手动屏蔽的 RSS 源，精确匹配 `xmlUrl` | +| `block_domains` | array | 域名级别屏蔽，支持通配符（如 `*.substack.com`） | + +### filter —— 内容过滤 + +| 字段 | 类型 | 说明 | +|------|------|------| +| `min_score` | number | 最低评分阈值，低于此分数的内容不参与推送（默认60） | +| `hot_threshold` | number | 热点阈值，达到此分数立即触发即时推送（默认90） | +| `context_days` | number | 上下文天数，汇总推送时参考的fetch数据历史天数（默认3天） | +| `keep_days` | number | 数据保留天数，超过天数的 JSON 文件会被清理 | +| `push_context_days` | number | 汇总推送去重的历史push文件有效天数（默认5天） | +| `no_content_marker` | string | LLM 返回的无内容标记，当推送内容包含此字符串时跳过推送（默认"[NO_NEW_CONTENT]"） | + +### log —— 日志配置 + +仅对 `scripts/install.sh` 部署的 systemd 服务生效。 + +| 字段 | 类型 | 说明 | +|------|------|------| +| `retention_days` | number | journald 命名空间 `dnews` 的日志保留天数（默认 7 天）。修改后需要重跑 `./scripts/install.sh` | + +### schedule —— 调度配置 + +| 字段 | 类型 | 说明 | +|------|------|------| +| `fetch_interval_minutes` | number | RSS 抓取间隔，单位分钟（默认30分钟） | +| `fetch_lookback_minutes` | number | RSS 冗余缓存时间（分钟），必须大于 `fetch_interval_minutes`，用于防止 RSS 延迟导致漏读（默认120分钟） | +| `push_cron` | array | 定时推送的 cron 表达式数组，支持多个时间点。**当天最早那次推送**自动作为「早报」触发 GitHub / Hacker News / 跨板块洞察三段；其余时段为默认 RSS digest。若只配置一条 cron,则每次推送都视为早报 | +| `timezone_hours` | number | 时区偏移小时数，用于时间显示（8 = UTC+8 北京时间） | + +**cron 表达式说明：** + +| 表达式 | 含义 | +|--------|------| +| `0 8 * * *` | 每天早上 8:00 | +| `0 17 * * *` | 每天下午 5:00 | +| `0 9,17 * * *` | 每天早上 9:00 和下午 5:00 | + +格式：`minute hour day month weekday` + +### fetch —— 抓取配置 + +| 字段 | 类型 | 说明 | +|------|------|------| +| `max_workers` | number | 最大并发数，同时抓取的 RSS 源数量 | +| `timeout` | number | 单个请求超时时间，单位秒 | + +### llm —— 大语言模型配置 + +| 字段 | 类型 | 说明 | +|------|------|------| +| `provider` | string | LLM 提供商（支持 openai 兼容接口） | +| `model` | string | 模型名称，如 `x-ai/grok-4.1-fast` | +| `baseUrl` | string | API 端点，如 `https://openrouter.ai/api/v1` | +| `apiKeyName` | string | 环境变量名称，系统会自动读取其值 | +| `max_prompt_chars` | number | 单次 prompt 最大字符数，用于分批控制 | +| `max_concurrent_batches` | number | 最大并发批次数 | +| `prompts` | object | prompt 文件路径配置 | + +### push —— 推送平台配置 + +| 字段 | 类型 | 说明 | +|------|------|------| +| `discord.enabled` | boolean | 是否启用 Discord 推送 | +| `discord.apiKeyName` | string | Discord Webhook 的环境变量名 | +| `feishu.enabled` | boolean | 是否启用飞书推送 | +| `feishu.apiKeyName` | string | 飞书 Webhook 的环境变量名 | + +### sections (早报扩展板块) + +仅在当天最早一次 `push_cron` 触发时生效(即「早报」时段)。详细设计见 `docs/extra-sections-design.md`。 + +#### sections.github_trending +- `enabled`: 是否启用 +- `max_items`: LLM 最终选出的项目数上限(默认 3) +- `max_deep_dive`: 单次最多 deep-dive 的候选 repo 数(默认 10) +- `readme_max_chars`: README 截断长度(默认 10000) +- `history_file`: trending 去重索引文件路径 +- `request_timeout`: HTTP 超时秒 +- `tokenName`: GitHub token 环境变量名;不设时匿名调用(限 60 req/hr) + +#### sections.hackernews +- `enabled`: 是否启用 +- `select_k`: 轻 LLM 从首页 30 条中挑出的故事数(默认 1) +- `top_comments`: 每个故事抓取的**顶层(L1)评论**数上限(默认 30) +- `top_l2_per_l1`: 每条 L1 下挂的 L2 回复数上限(默认 3) +- `comment_max_chars`: 单条评论(L1 或 L2) markdown 字符上限(默认 2000) +- `comments_total_chars`: 整段评论树的总字符预算硬上限(默认 60000),防止离群 story 撑爆 prompt +- `link_content_max_chars`: 外链正文截断长度(默认 50000) +- `request_timeout`: HTTP 超时秒 +- `algolia_base`: Algolia API 基址 +- `jinaTokenName`: Jina Reader API key 环境变量名(默认 `JINA_API_KEY`);不设或对应环境变量为空时按匿名额度调用 + +#### sections.insights +- `enabled`: 是否启用跨板块洞察段 + + +## 扩展指南 + +### 添加新的 RSS 源 + +在 `config.json` 的 `sources.add` 中添加： + +```json +"add": [ + { + "title": "我的自定义源", + "xmlUrl": "https://example.com/feed.xml", + "category": "AI" + } +] +``` + +### 添加新的推送平台 + +1. 在 `src/push/` 目录下创建新文件，继承 `PushPlatform` 基类 +2. 实现 `validate_config()` 和 `send()` 方法 +3. 在 `src/push/__init__.py` 中注册 + +### 修改评分逻辑 + +编辑 `prompts/score.txt`，调整评分标准和权重。 + +### 自定义 LLM 模型 + +修改 `config.json` 中的 `llm` 配置： + +```json +"llm": { + "model": "anthropic/claude-3-opus", + "baseUrl": "https://openrouter.ai/api/v1", + "apiKeyName": "OPENROUTER_API_KEY" +} +``` + +## RSS 源说明 + + +RSS 订阅源文件位于 `resources/rss.opml`，目前包含约 420 个订阅源。 + +RSS 订阅源初始整理自 [ginobefun/BestBlogs](https://github.com/ginobefun/BestBlogs)，包含约 420 个 AI 领域优质信息源。 + +用户可自行配置 RSS 源文件，只需遵循 OPML 格，在 `config.json` 的 `sources.base_opml` 修改文件路径即可。 同时用户可修改 `sources.add`或者 `sources.block`以在不破换OMPL文件的前提下对rss源进行增加或者删除。格式示例： + +```json +"sources": { + "base_opml": "resources/rss.opml", + "add": [ + { + "title": "OpenAI News", + "xmlUrl": "https://openai.com/news/rss.xml", + "category": "AI" + }, + { + "title": "Chrome for Developers", + "xmlUrl": "https://developer.chrome.com/static/blog/feed.xml", + "category": "Chrome" + } + ], + "block": [ + { + "title": "Google Developers Blog", + "xmlUrl": "https://developers.googleblog.com/feeds/posts/default" + }, + { + "title": "Microsoft for Developers", + "xmlUrl": "https://devblogs.microsoft.com/landing" + }, + { + "title": "ElevenLabs Blog", + "xmlUrl": "https://api.bestblogs.dev/feed/elevenLabsBlog" + } + ], + "block_domains": ["*.substack.com", "*.youtube.com"] +} +``` + +## 常见问题 FAQ + +
+LLM 调用费用大概多少？ + +取决于模型选择和源数量。以 OpenRouter 的 `x-ai/grok-4.1-fast`（便宜模型）为例，日均扫描 2000+ 条 RSS + GitHub + HN，**单日成本约 ¥0.5-2 元**。可通过 `filter.min_score`、`llm.max_concurrent_batches`、`llm.max_prompt_chars` 进一步控制。 + +
+ +
+如何只跑某一板块进行调试？ + +```bash +uv run python -m src.main github # 只跑 GitHub Trending +uv run python -m src.main hackernews # 只跑 Hacker News +uv run python -m src.main fetch # 只跑 RSS fetch +``` + +打印结果到控制台，不触发实际推送。 + +
+ +
+抓取频率会不会被 RSS 站点封禁？ + +默认 30 分钟轮询一次，远低于大多数 RSS 服务的速率限制。`fetch.max_workers` 控制并发（默认 10），对单个站点的压力可忽略。 + +
+ +
+没配置推送渠道也能用吗？ + +可以。所有推送 markdown 都会落地到 `news-data/push-*.md`，即使所有推送平台 disabled 也可手动查看。 + +
+ +
+支持哪些 LLM 提供商？ + +任何 OpenAI API 兼容接口的服务：OpenAI、DeepSeek、OpenRouter、SiliconFlow、阿里云通义千问、Groq 等。修改 `config.json` 的 `llm.baseUrl` / `llm.model` / `llm.apiKeyName` 即可切换。 + +
+ +
+GitHub Trending / Hacker News 为什么不出现？ + +它们**只在当天最早一次 `push_cron` 触发时跑**（即「早报」时段）。若 `push_cron` 只配了一条 cron，则每次推送都视为早报。详见「配置详解 → schedule」一节。 + +
+ +
+数据存储在哪里？多久清理？ + +- 抓取数据：`news-data/fetch-YYYY-MM-DD.json` +- 推送 markdown：`news-data/push-YYYY-MM-DD-HH-MM-SS.md` +- 通知归档：`news-data/notify-YYYY-MM-DD.md` + +超过 `filter.keep_days`（默认 7 天）的文件会自动清理；卸载脚本**不会**删除 `news-data/`。 + +
+ +--- + +## License + +MIT License - see [LICENSE](LICENSE) file for details. diff --git a/ai-daily-main/config.json.example b/ai-daily-main/config.json.example new file mode 100644 index 0000000..ab0ee39 --- /dev/null +++ b/ai-daily-main/config.json.example @@ -0,0 +1,120 @@ +{ + "filter": { + "min_score": 60, + "hot_threshold": 90, + "context_days": 2, + "keep_days": 7, + "push_context_days": 5, + "no_content_marker": "[NO_NEW_CONTENT]" + }, + "log": { + "retention_days": 7 + }, + "schedule": { + "fetch_interval_minutes": 60, + "fetch_lookback_minutes": 120, + "push_cron": ["0 8 * * *", "0 17 * * *"], + "timezone_hours": 8 + }, + "sections": { + "github_trending": { + "enabled": true, + "max_items": 3, + "max_deep_dive": 10, + "readme_max_chars": 10000, + "history_file": "news-data/trending-history.json", + "request_timeout": 10, + "tokenName": "GITHUB_TOKEN" + }, + "hackernews": { + "enabled": true, + "select_k": 3, + "top_comments": 50, + "top_l2_per_l1": 3, + "comment_max_chars": 2000, + "comments_total_chars": 80000, + "link_content_max_chars": 50000, + "request_timeout": 10, + "algolia_base": "https://hn.algolia.com/api/v1", + "jinaTokenName": "JINA_API_KEY" + }, + "insights": { + "enabled": true + } + }, + "fetch": { + "max_workers": 10, + "timeout": 10 + }, + "llm": { + "provider": "openai", + "model": "deepseek-v4-flash", + "baseUrl": "https://api.deepseek.com", + "apiKeyName": "DEEPSEEK_API_KEY", + "max_prompt_chars": 150000, + "max_concurrent_batches": 3, + "prompts": { + "score_batch": "prompts/score_batch.md", + "immediate_push": "prompts/immediate_push.md", + "digest": "prompts/digest.md", + "section_github": "prompts/section_github.md", + "section_hackernews_select": "prompts/section_hackernews_select.md", + "section_hackernews": "prompts/section_hackernews.md", + "insights": "prompts/insights.md" + } + }, + "push": { + "discord": { + "enabled": false, + "apiKeyName": "DISCORD_WEBHOOK_URL" + }, + "feishu": { + "enabled": true, + "apiKeyName": "FEISHU_WEBHOOK_URL" + } + }, + "sources": { + "base_opml": "resources/rss.opml", + "add": [ + { + "title": "OpenAI News", + "xmlUrl": "https://openai.com/news/rss.xml", + "category": "AI" + }, + { + "title": "Chrome for Developers", + "xmlUrl": "https://developer.chrome.com/static/blog/feed.xml", + "category": "Chrome" + }, + { + "title": "Cloudflare Developers", + "xmlUrl": "https://developers.cloudflare.com/changelog/rss/index.xml", + "category": "Cloudflare" + }, + { + "title": "Infoq", + "xmlUrl": "https://feed.infoq.com/", + "category": "News" + } + ], + "block": [ + { + "title": "Google Developers Blog", + "xmlUrl": "https://developers.googleblog.com/feeds/posts/default" + }, + { + "title": "Microsoft for Developers", + "xmlUrl": "https://devblogs.microsoft.com/landing" + }, + { + "title": "ElevenLabs Blog", + "xmlUrl": "https://api.bestblogs.dev/feed/elevenLabsBlog" + }, + { + "title": "Infoq(en)", + "xmlUrl": "http://www.infoq.com/rss/rss.action" + } + ], + "block_domains": ["*.substack.com", "*.youtube.com"] + } +} diff --git a/ai-daily-main/docs/extra-sections-design.md b/ai-daily-main/docs/extra-sections-design.md new file mode 100644 index 0000000..0431842 --- /dev/null +++ b/ai-daily-main/docs/extra-sections-design.md @@ -0,0 +1,735 @@ +# extra-sections-design.md — GitHub Trending / Hacker News / 行业洞察 板块设计 + +> 本文是对现有 push job 的能力扩展设计。在保持 RSS 主流程不变的前提下，新增三个板块仅在「早报」时段输出：GitHub 趋势、Hacker News 热议、跨板块行业洞察。 +> +> 配套修改的源码模块、配置字段、数据契约见各章节。落地后需要把架构层结论合并回 `docs/tech-spec.md`，运行参数变更同步 `README.md`，进度记录写入 `docs/plan.md`。 +> +> created: 2026-05-16 +> updated: 2026-05-16（按用户决策重写：四模块独立、单页 GH trending、HN 单选 K=1、insights 结构由 prompt 决定） + +## 1. 目标与范围 + +- 在每天的早报推送中，除了现有的 RSS 精选 digest，再加入： + - **GitHub Trending**：当日热门开源项目中筛选 1-3 个 AI 相关项目 + - **Hacker News**：HN 首页中筛选 1 个最有讨论价值的 AI 相关热议（带评论与外链正文摘要） + - **行业洞察**：基于上述三个板块 + 历史 insights 段做一段跨板块趋势小结 +- 这三段内容**只在早报时段生成**（当天 `schedule.push_cron` 列表里最早那次触发），其余时段维持现有纯 RSS digest 行为 +- 单板块失败 → 降级推送其他板块，整体任务仍算成功；RSS 失败 → 同现有行为，任务非 0 退出 +- 不引入数据库；状态全部落到 `news-data/` 的本地文件 + +## 2. 架构总览 + +``` +push_job (cron 触发) +│ +├─ Step 1: 早报判定 +│ └─ 否 → 走原有纯 RSS digest 流程，结束 +│ +├─ Step 2: 四模块编排（前三路 asyncio.gather 并发，insights 串行后置） +│ ├─ run_rss_section(config, now) +│ ├─ run_github_section(config, now) +│ └─ run_hackernews_section(config, now) +│ ↓ +│ └─ run_insights_section(rss_md, gh_md, hn_md, config, now) +│ +└─ Step 3: push_job 上游 + └─ sentinel 包裹四段 → 拼装 markdown → 推送 → 写 push-*.md +``` + +**关键设计选择**： + +1. **模块自治**。每个板块封装为 `run_xxx_section(...) -> (markdown, error)`，板块内部从抓取、enrich、LLM 总结全包；板块互不感知。 +2. **sentinel 由 push_job 统一包**。模块返回**裸 markdown**（不含 sentinel），由上游 `_assemble_with_sentinels()` 包入 ``。这样模块不需要知道自己的板块标识，便于后续替换或加新板块。 +3. **失败自吞**。模块内部捕获异常，返回 `("", error)`；上游根据返回值决定是否在最终 push 中省略该段、是否走告警通道。 +4. **RSS 是核心**。其他三个模块失败都是降级；RSS 模块失败仍按现有行为整体退出非 0。 + +**并发模型**：`asyncio.gather`，与项目通体异步风格一致。 + +## 3. 模块结构 + +``` +src/ +├── sections/ ← 新增 +│ ├── __init__.py # 暴露 run_rss_section / run_github_section / ... +│ ├── rss/ +│ │ ├── __init__.py +│ │ └── section.py # 把现有 collect_entries_for_push + compose_digest 流程搬入 +│ ├── github/ +│ │ ├── __init__.py +│ │ ├── trending_scraper.py # HTML 抓取 + 解析(单页 https://github.com/trending) +│ │ ├── repo_enricher.py # GitHub REST API:metadata + README +│ │ ├── history.py # trending-history.json 读写 + 过期清理 +│ │ └── section.py # run_github_section 入口 +│ ├── hackernews/ +│ │ ├── __init__.py +│ │ ├── frontpage_scraper.py # HTML 抓首页 30 条 +│ │ ├── item_enricher.py # Algolia /items/{id} + 外链正文(html_to_markdown) +│ │ └── section.py # run_hackernews_section 入口 +│ └── insights/ +│ ├── __init__.py +│ └── section.py # run_insights_section 入口 +├── llm.py # 新增 4 个函数(GH/HN select/HN summarize/insights) +├── storage.py # 新增:sentinel 切片、trending-history、profile 字段 +├── main.py # push_job 升级:早报判定 + 四模块编排 +└── push/ # 不变,平台层无感知 + +prompts/ ← 新增 4 个 +├── section_github.md # 输入 enriched repo 数组 → 选 1-3 + 写 markdown +├── section_hackernews_select.md # 30 条 frontpage 元数据 → 选 K 个 id(默认 K=1) +├── section_hackernews.md # enriched story → 写 markdown(K=1 时不挑选,只行文) +└── insights.md # 输入三段成品 + 近 N 天 insights 段历史 → 写洞察段 + +news-data/ +├── fetch-*.json # 不变 +├── notify-*.md # 不变 +├── push-*.md # 内容升级:含 sentinel 与 profile frontmatter +└── trending-history.json # 新增:GH 已查阅 repo 索引 +``` + +## 4. 数据契约 + +### 4.1 push 文件分段 sentinel + +push 文件保留**完整拼接**写入磁盘和发送各平台，但用 HTML 注释 sentinel 划分板块边界。HTML 注释在 markdown 渲染中不显示，机器可解析，下游做"按段查重"时能精确切片。 + +```markdown +--- +pushDate: "2026-05-16T08:00:03+08:00" +profile: "morning" +sourceCount: 12 +totalEntries: 12 +--- + + +# 📰 AI Daily 每日精选 | 2026-05-16 + +*开头一句定调...* + +### 1️⃣ ... + + + +## ⭐ GitHub 趋势 + +- **owner/repo** ⭐234 — 一句话价值定位 + + + +## 🟧 Hacker News 热议 + +### 标题 (120 pts · 45 comments) +- 链接: url +- 要点:... +- HN 讨论: comments_url + + + +## 💡 今日洞察 + +(行文结构由 prompts/insights.md 决定,代码不强加格式) + +``` + +某板块 markdown 为空 → 对应 sentinel 段**整段省略**（不留空标记，不留空 SECTION）。 + +### 4.2 分段提取函数（storage.py 新增） + +```python +def extract_section(push_md: str, section: str) -> str: + """从 push 文件内容中切出 之间的 markdown。 + + 向后兼容: + - 新 push 文件(带 sentinel): 按 sentinel 边界切片 + - 老 push 文件(无 sentinel) 且 section=='rss': 返回整个 body(老文件视为全 RSS) + - 老 push 文件且 section in {github, hackernews, insights}: 返回空字符串 + """ + +def load_recent_section_titles(section: str, days: int, data_dir="news-data") -> str: + """汇总近 days 天 push-*.md 的指定板块,提取标题级别清单(沿用 _extract_push_titles 思路)。 + + 仅 insights 板块在新增模块中使用本函数加载历史。 + GH/HN 板块按用户决策不传历史上下文给 LLM,不调用本函数。 + """ +``` + +四个 LLM 与历史上下文的关系： + +| LLM 调用 | recent context 数据源 | +|---|---| +| `compose_digest` (RSS) | 维持现有 `load_recent_push_titles(filter.push_context_days)`（老接口在 sentinel 升级后等价于 `load_recent_section_titles("rss", ...)`） | +| `summarize_github_trending` | 不传 | +| `select_ai_related_hn` | 不传 | +| `summarize_hackernews` | 不传 | +| `generate_trend_insights` | `load_recent_section_titles("insights", filter.push_context_days)` | + +### 4.3 trending-history.json + +```json +{ + "repos": { + "https://github.com/owner/repo-a": "2026-05-15", + "https://github.com/owner/repo-b": "2026-05-12" + }, + "updated_at": "2026-05-16T08:00:01+08:00" +} +``` + +**写入语义**（用户决策的精确语义）： + +每次早报触发，按下列顺序处理： + +1. 加载 history，剔除 `last_seen_date < today - filter.keep_days` 的条目 +2. 抓 trending 页 → 得到 `all_repos` +3. 对 `all_repos` 中每个 url： + - 若已在 history → `history.touch(url, today)`（更新日期），**从候选移除** + - 不在 history → 进入 `candidates` +4. 把 `candidates` 中每个 url 也 `history.touch(url, today)` 写入 history +5. 持久化 history（覆盖写） +6. 对 `candidates`（即今日新出现的 repo）做后续 deep-dive 与 LLM 总结 + +效果：repo 在 trending 上挂多久就被屏蔽多久；过 `keep_days` 天没再出现则可重新推荐。 + +## 5. 模块详设 + +### 5.1 RSS 模块（迁移既有逻辑） + +`src/sections/rss/section.py::run_rss_section(config, now) -> (str, Optional[str])`： + +把现有 `run_push_job` 中"收集 + compose_digest"的部分原样迁过来，返回裸 markdown（不含 sentinel）+ 错误信息。无新增功能。 + +### 5.2 GitHub 模块 + +`src/sections/github/section.py::run_github_section(config, now) -> (str, Optional[str])`： + +``` +1. 抓取 trending 单页(HTML) + GET https://github.com/trending + 解析 → all_repos: [{url, full_name, description, language, stars_today, stars_total}] + +2. 加载 history 并清理 + history = load_trending_history(sections.github_trending.history_file) + history.cleanup(keep_days=filter.keep_days) + +3. 候选筛选(按 §4.3 语义) + candidates = [] + for repo in all_repos: + if repo.url in history: + history.touch(repo.url, today) + else: + candidates.append(repo) + +4. 候选写回 history + 持久化 + for repo in candidates: + history.touch(repo.url, today) + history.save() + +5. 数量护栏 + if not candidates: return ("", None) # 静默 + if len(candidates) > max_deep_dive: + candidates = candidates[:max_deep_dive] # 截断,默认 10 + +6. 并发 deep-dive(REST API) + async for repo in candidates: + meta = await fetch_repo_metadata(owner, repo) # GET /repos/{o}/{r} + readme = await fetch_readme(owner, repo) # GET /repos/{o}/{r}/readme + enriched = [{...repo, topics, license, pushed_at, readme_excerpt}] + - 单 repo 任一请求失败 → 该 repo 跳过 + 错误聚合,不阻塞其他 + +7. LLM:summarize_github_trending(enriched, config) + prompt: 候选数组(含 readme_excerpt) → 选 1-max_items + 写 markdown + 不传 recent_section_titles + +8. 返回 (markdown, error) +``` + +**REST API 调用细节**： + +| 调用 | 路径 | 取什么 | +|---|---|---| +| metadata | `GET /repos/{owner}/{repo}` | `description, topics, language, license.spdx_id, pushed_at, stargazers_count, archived` | +| readme | `GET /repos/{owner}/{repo}/readme` | `content` (base64) → decode → 截断到 `readme_max_chars` | + +- `archived=true` 的 repo 从候选剔除（trending 偶尔出现僵尸归档项目） +- README 截断策略：前 `readme_max_chars` 字符（默认 10000，基于 trending 页 README 长度分布 p50≈25k 选定，详见 §14 决策记录） +- 鉴权：`config.sections.github_trending.tokenName`（默认 `"GITHUB_TOKEN"`）对应的环境变量存在时走 `Authorization: Bearer {token}`，否则匿名调用并接受 60 req/hr 上限（日 10 个 repo × 2 calls = 20 calls，匿名安全） + +**enriched repo 字段（喂给最终 LLM）**： + +```json +{ + "url": "https://github.com/owner/repo", + "full_name": "owner/repo", + "description": "(来自 trending 页)", + "language": "Python", + "stars_today": 234, + "stars_total": 12340, + "topics": ["llm", "rag", "agent"], + "license": "MIT", + "pushed_at": "2026-05-15", + "readme_excerpt": "(前 3000 chars)" +} +``` + +### 5.3 Hacker News 模块 + +`src/sections/hackernews/section.py::run_hackernews_section(config, now) -> (str, Optional[str])`： + +``` +1. 抓首页(HTML) + GET https://news.ycombinator.com/news + 解析 30 条 → front: [{id, title, url, site, points, comments, comments_url}] + +2. 轻 LLM 初筛:select_ai_related_hn + 输入: 30 条 frontpage 元数据(无正文) + 输出: K = sections.hackernews.select_k 个 story id(默认 K=1) + if K 个为空 → return ("", None) # 静默 + +3. 并发 enrich 选中的 K 个 story + async for story in selected: + - 评论树:Algolia GET /api/v1/items/{id} + → 取前 top_comments 条 L1(默认 30,按 HN ranking) + → 每条 L1 下挂前 top_l2_per_l1 条 L2 回复(默认 3) + → 每条 text 过 html_to_markdown,单条截断到 comment_max_chars(默认 2000) + → 累计达 comments_total_chars(默认 60000) 立即停止,防离群 story 撑爆 prompt + → 输出 tree JSON: [{"l1": "...", "replies": ["...", "..."]}, ...] + - 外链正文: + if story.url 指向 https://news.ycombinator.com/item?id=... (Show HN/Ask HN): + 从 Algolia 同次返回的 root.text 字段取(无外部请求) + else: + GET story.url → html_to_markdown → 截断到 link_content_max_chars(默认 50000,p50≈10k) + - 单 story 任一失败 → 字段留空,metadata 仍传给最终 LLM + +4. LLM:summarize_hackernews(enriched_stories, config) + prompt: 对输入的 K 个 story 全部行文（K 通常 = 1） + 不传 recent_section_titles + +5. 返回 (markdown, error) +``` + +**Algolia API 接口**： + +``` +GET https://hn.algolia.com/api/v1/items/{id} +``` + +返回 JSON： + +```json +{ + "id": 12345678, + "title": "...", + "url": "...", + "points": 120, + "author": "...", + "text": null, // Show HN/Ask HN 的正文在这里 + "children": [ // 顶层评论数组(按 HN ranking 排序) + {"id": ..., "text": "", "author": "...", "children": [...]}, + ... + ] +} +``` + +**优势 vs HTML 解析**：免去 HN 的 `td.ind[indent="0"]` indent-tree 解析；评论文本是干净 HTML 字符串，直接 `html_to_markdown`。 + +**Show HN / Ask HN 特例**： +- 首页解析时 `url` 字段就是 `https://news.ycombinator.com/item?id=X`，作为"非外链"标记 +- enrich 时只调一次 Algolia（覆盖评论 + post 正文 `text` 字段），不再发外部请求 + +**enriched story 字段（喂给最终 LLM）**： + +```json +{ + "id": "12345678", + "title": "...", + "url": "...", + "site": "example.com", + "points": 120, + "comments": 45, + "comments_url": "https://news.ycombinator.com/item?id=12345678", + "link_content": "(markdown, ≤3000 chars; Show HN 时是 post 正文)", + "top_comments": [ + {"l1": "(markdown, ≤comment_max_chars)", "replies": ["(markdown, ≤comment_max_chars)", "..."]}, + ... + ] +} +``` + +### 5.4 Insights 模块 + +`src/sections/insights/section.py::run_insights_section(rss_md, gh_md, hn_md, config, now) -> (str, Optional[str])`： + +``` +1. 加载历史 + recent = load_recent_section_titles("insights", filter.push_context_days) + +2. LLM:generate_trend_insights + 输入: {"rss": rss_md, "github": gh_md, "hackernews": hn_md} + recent + 输出: insights 段 markdown + 注:行文结构、bullet 数量、风格约束等全部交给 prompts/insights.md, + 代码层不强加固定格式 + +3. 返回 (markdown, error) +``` + +如果某板块返回空（失败或本日无内容），prompt 里对应键标记 `"(本次无内容)"`，LLM 自行适配。 + +## 6. LLM 调用与 Prompt 策略 + +### 6.1 新增 LLM 函数（src/llm.py） + +```python +async def select_ai_related_hn( + candidates: list[dict], # 首页 30 条元数据(无正文) + k: int, # 期望返回数量,默认 1 + config: dict, +) -> tuple[list[str], Optional[str]]: + """轻量 LLM:从 HN 首页候选中挑出 k 个 AI 相关的 story id,只读 title/site/points/comments。 + 返回 ([id1, ...], error)。""" + +async def summarize_github_trending( + enriched_repos: list[dict], # 已 deep-dive 的 repo 候选(含 readme_excerpt + topics) + config: dict, +) -> tuple[str, Optional[str]]: + """选 1-max_items + 写 markdown 段。不传历史上下文。""" + +async def summarize_hackernews( + enriched_stories: list[dict], # 已 enrich(含 link_content 与 top_comments) + config: dict, +) -> tuple[str, Optional[str]]: + """对输入的 K 个 enriched stories 全部行文。K 由配置 select_k 决定，默认 1。 + 不传历史上下文。""" + +async def generate_trend_insights( + sections: dict[str, str], # {"rss": md, "github": md, "hackernews": md} + recent_insights: str, # load_recent_section_titles("insights", days) + config: dict, +) -> tuple[str, Optional[str]]: + """输入三段成品 + 近期 insights 标题,返回洞察段 markdown。""" +``` + +返回风格沿用 `generate_immediate_push`：成功返回 `(content, None)`，失败返回 `("", error_msg)`。 + +**单次早报推送的 LLM 调用预算**： + +| 调用 | 输入规模 | 用途 | +|---|---|---| +| `compose_digest` | 当日符合条件的 RSS 条目 | 现有，RSS digest 主体 | +| `select_ai_related_hn` | 30 条 HN 首页元数据 | 轻量；只读 title/site/points/comments | +| `summarize_github_trending` | ≤ `max_deep_dive`=10 个 enriched repos | 选 1-3 + 行文 | +| `summarize_hackernews` | `K = select_k` 个 enriched stories（默认 1） | 行文 K 条 | +| `generate_trend_insights` | 三段已生成 markdown + 近期 insights 标题 | 一段洞察 | + +合计 5 次 / 早报。晚报维持现有 1 次。 + +### 6.2 关注领域（GitHub / HN 共用） + +为避免领域定义在 3 个 prompt 里漂移，统一在此沉淀；各 prompt 在自身骨架里直接引用本节，不做重新发明。 + +**正面关注**： + +- **AI Agent**：智能体架构、工具链、多智能体、自主规划、Agent 框架 +- **AI 模型**：训练、推理、微调、量化部署、模型服务、语音 / 多模态 / 视觉模型 +- **AI 基础设施**：GPU 调度、芯片硬件、数据中心、推理优化、分布式训练、向量数据库、RAG 框架 +- **大厂 / 前沿动态**：Apple、Google、Meta、OpenAI、Anthropic、Microsoft、xAI 等公司的官方动作与战略 +- **AI 集成的开发者工具**：API 网关、自动化脚本、低代码平台等明确与 AI 协同的工具 +- **有创新性的开源产品**：日增长显著且有清晰用户价值（GH 板块专属） + +**负面排除（一律剔除）**： + +- 嵌入式开发（Arduino、ESP32、树莓派、单片机） +- 底层系统编程（内存分配器、编译器、链接器，与 AI 工作负载无明显关联时） +- 通用开发工具（命名规范、代码风格、纯前端模板、UI 组件库、管理后台模板、静态网站主题） +- 学习资源（纯教程仓库、面试题合集、Roadmap，除非是含实用代码的深度技术指南） +- 配置文件集合（Dotfiles、配置模板） +- 与 AI / 科技无关的内容（电子书、资源搬运、刷榜项目、明星项目搬运） +- 纯娱乐 / 高风险误用（deepfake 等无明确基础设施价值的项目） + +### 6.3 Prompt 文件 + +#### prompts/section_github.md（骨架） + +- 角色定位（"开源情报分析师"） +- 输入 schema 说明（JSON 数组：url / full_name / description / language / stars_today / stars_total / topics / license / pushed_at / readme_excerpt） +- 关注领域：引用 §6.2（正面列表与负面排除完整复制进 prompt） +- 选项规则： + - 从候选中挑 1-`max_items` 个最值得关注的项目 + - 优先信号：stars_today 高 + topics 含 AI 标签（agent/llm/rag/inference/training 等）+ readme 描述明确 + 非纯模板/教程仓库 + - 必跳过：`archived=true`（理论上已在 enricher 剔除，prompt 层兜底）、纯 awesome-list、个人配置 dotfiles +- 输出格式（markdown 列表）： + - `- **owner/repo** ⭐{stars_today} — 一句话价值定位 [link]` + - 一句话需点明"解决什么问题"，避免营销语 +- 风格约束：与 `prompts/digest.md` 同源；负面句式（"震撼""炸裂""革命性"）禁用；避免套话 + +#### prompts/section_hackernews_select.md（骨架） + +- 角色定位（"HN 早间选题人"） +- 输入：JSON 数组（30 条 frontpage 元数据：id / title / site / points / comments） +- 关注领域：引用 §6.2 +- 任务：挑 `k` 个最符合关注领域的 story id（K 默认 1） +- 决策原则：title + site 不足以判定 AI 相关时，**宁可漏选不可错选**（错选会让最终 LLM 写出与 AI Daily 调性无关的内容） +- 输出：纯 JSON id 数组，如 `["12345"]` 或 `[]`（无任何匹配时返回空数组） +- 严禁输出任何解释性文字 + +#### prompts/section_hackernews.md（骨架） + +- 角色定位（"HN 早间编辑"） +- 输入 schema 说明（K 个 enriched story，含 `link_content` 与 `top_comments`） +- 关注领域：引用 §6.2（K=1 时通常无需筛选，仅作为行文背景参考） +- 任务：对输入的 `K` 个 enriched stories 全部行文（不再二次挑选） +- 内容要求（每条 story）： + - 提炼原文核心（背景 / 要点 / 结论） + - 汇总 HN 评论区的有价值观点（支持 / 反对 / 补充），不是简单复述 + - 若评论中出现明显反驳原文的观点，必须保留并标注 +- 输出格式建议（最终以 prompt 实测为准）： + ``` + ### 标题 (N pts · M comments) + - 链接: url + - 内容总结: 2-3 条核心要点 + - 💬 HN 讨论: 1-2 条最有价值的观点（含反对意见） + - 🔗 HN 讨论页: comments_url + ``` +- 风格约束：客观、犀利、克制；避免与 RSS digest 句式雷同；不做宏大叙事 + +#### prompts/insights.md（骨架） + +- 角色定位（"AI 行业观察员"） +- 输入：三段成品 markdown + 近 N 天 insights 板块清单 +- 任务：基于三段产出做跨板块小结 +- 风格约束：避免与 RSS digest 句式雷同；避免简单复述已经在其他板块出现过的具体新闻 +- 结构与 bullet 数量交由 prompt 内部约定，code 层不限制 + +### 6.4 调用顺序与并发 + +```python +async def _run_morning_push(config): + rss_md, gh_md, hn_md = await asyncio.gather( + run_rss_section(config, now), + run_github_section(config, now), + run_hackernews_section(config, now), + return_exceptions=False, # 各 section 自吞异常,不抛 + ) + + insights_md, _ = await run_insights_section(rss_md, gh_md, hn_md, config, now) + + final = _assemble_with_sentinels({ + "rss": rss_md, + "github": gh_md, + "hackernews": hn_md, + "insights": insights_md, + }) + + await send_to_platforms(final, config["push"]) + save_push_file(get_push_file(), final, profile="morning", ...) +``` + +`_assemble_with_sentinels(sections: dict[str, str]) -> str` 的契约： + +- 按固定顺序拼装 `rss → github → hackernews → insights` +- 空 markdown 段整段省略（连同 sentinel） +- 段间留一个空行 + +## 7. 行业洞察板块设计 + +按用户决策，本节**不在 code 层规定 insights 的格式**： + +- bullet 数量、子标题、字数限制、固定栏目等都属于 prompt 工程范畴 +- 调整方法：编辑 `prompts/insights.md` 而非改代码 +- 输入合同（code 层保证）： + - `sections["rss" | "github" | "hackernews"]` 三个键的 markdown + - 任一板块为空时该键值为 `"(本次无内容)"` + - `recent_insights`：近 `filter.push_context_days` 天 insights 段标题清单（防风格趋同） +- 输出合同（code 层不校验）：直接作为 markdown 段插入 + +## 8. 配置 schema 增量 + +```json +{ + "filter": { + "min_score": 60, + "hot_threshold": 90, + "context_days": 2, + "keep_days": 7, + "push_context_days": 5, + "no_content_marker": "[NO_NEW_CONTENT]" + }, + "schedule": { + "fetch_interval_minutes": 30, + "fetch_lookback_minutes": 120, + "push_cron": ["0 8 * * *", "0 17 * * *"], + "timezone_hours": 8 + }, + "sections": { + "github_trending": { + "enabled": true, + "max_items": 3, + "max_deep_dive": 10, + "readme_max_chars": 10000, + "history_file": "news-data/trending-history.json", + "request_timeout": 10, + "tokenName": "GITHUB_TOKEN" + }, + "hackernews": { + "enabled": true, + "select_k": 1, + "top_comments": 30, + "top_l2_per_l1": 3, + "comment_max_chars": 2000, + "comments_total_chars": 60000, + "link_content_max_chars": 50000, + "request_timeout": 10, + "algolia_base": "https://hn.algolia.com/api/v1" + }, + "insights": { + "enabled": true + } + }, + "llm": { + "prompts": { + "score_batch": "prompts/score_batch.md", + "immediate_push": "prompts/immediate_push.md", + "digest": "prompts/digest.md", + "section_github": "prompts/section_github.md", + "section_hackernews_select": "prompts/section_hackernews_select.md", + "section_hackernews": "prompts/section_hackernews.md", + "insights": "prompts/insights.md" + } + } +} +``` + +向后兼容： + +- `sections` 整段缺失 → 等价于全部 `enabled=false` → push_job 走原有纯 RSS 路径 +- `push_cron` 为空 → 无早报触发,不生成新板块 +- 旧 push 文件没有 sentinel → `extract_section("rss", ...)` 返回整个 body，其他 section 返回空字符串 +- `GITHUB_TOKEN` 未设 → GH 模块匿名调用，照常运行 + +## 9. 失败隔离与降级策略 + +| 失败位置 | 行为 | +|---|---| +| `run_rss_section` 失败 | 整个 push_job 退出非 0（核心承诺不变） | +| GH trending 抓取 / 解析失败 | `run_github_section` 返回 `("", error)` → 板块整段省略 → 告警 | +| GH 单 repo metadata/readme 失败 | 该 repo 跳过 + 错误聚合，不阻塞其他 repo | +| GH summarize LLM 失败 | 板块整段省略，告警 `notify_llm_errors("section_github", ...)` | +| HN 首页抓取失败 | 同 GH | +| HN 轻 LLM 初筛失败或返回空 | 板块整段省略，告警（初筛失败）或静默（结果为空） | +| HN 单 story enrich 失败 | 字段留空，metadata 仍传给最终 LLM | +| HN summarize LLM 失败 | 板块整段省略，告警 | +| insights LLM 失败 | 洞察段省略，其他板块照常推送，告警 | +| 早报判定为否 | 完全跳过 GH/HN/insights，不消耗任何额外 API | +| `sections.xxx.enabled=false` | 对应模块直接返回 `("", None)`，静默跳过 | + +整体准则：**RSS 是核心，其余是增强**。除 RSS 外的任何失败都不阻塞推送，但都通过现有 `notify_llm_errors` 通道发简单告警，方便事后排查。 + +## 10. 早报判定逻辑 + +```python +def is_morning_push(now: datetime, config: Dict) -> bool: + cron_list = config.get("schedule", {}).get("push_cron", []) + if not cron_list: + return False + if len(cron_list) == 1: + return True # 唯一定时即"最早",任何触发都视为早报 + + base = now.replace(hour=0, minute=0, second=0, microsecond=0) + today_fires = [croniter(c, base).get_next(datetime) for c in cron_list] + closest = min(today_fires, key=lambda f: abs(now - f)) + return closest == min(today_fires) +``` + +规则:`push_cron` 中 now 离哪条 cron 最近就归为那条;最近的那条若是当天最早的 cron,则视为早报。 + +为什么不再用单独的 `morning_cron` + 容差: +- 复用 `push_cron`,少一个配置项,加 cron 自动延展 +- 「最近匹配」自动等价于容差 = 到次近 cron 距离的一半,systemd timer 漂移更稳健 +- 单条 cron 部署自然变成"每次推送都是早报",与用户预期一致 + +## 11. 与现有模块的集成点 + +- `src/main.py::run_push_job` 改造： + 1. 开头加 `is_morning = is_morning_push(now, config)` + 2. `is_morning=False` 时走原路径（`compose_digest` only） + 3. `is_morning=True` 时进入 `_run_morning_push` 编排（四模块 + sentinel 拼装） +- `src/storage.py` 新增： + - `extract_section(content, section)` + `load_recent_section_titles(section, days, data_dir)` + - `load_trending_history(path) -> TrendingHistory`、`TrendingHistory.cleanup/touch/save` +- `save_push_file` 微调：frontmatter 带上 `profile: "morning"|"default"`，方便后续按 profile 分析 +- `cleanup_old_files` 增加对 `trending-history.json` 的"过期条目剪枝"（不删整个文件） +- `src/push/` **不动**，平台层无感知 +- `src/llm.py` 新增 4 个函数：`select_ai_related_hn / summarize_github_trending / summarize_hackernews / generate_trend_insights` +- `src/sections/` 全新模块树（4 个子包） + +## 12. 测试策略 + +新增单元测试： + +- `tests/pytest/test_sections_github_scraper.py`：本地 HTML fixture（保存几个真实 trending 页面快照）→ 测试解析 +- `tests/pytest/test_sections_github_history.py`：测试 `TrendingHistory.touch/cleanup`、URL 已存在/不存在分支 +- `tests/pytest/test_sections_github_enricher.py`：mock `aiohttp` → 测试 REST API 字段映射、archived 过滤、token 鉴权头 +- `tests/pytest/test_sections_hackernews_scraper.py`：HN 首页 fixture → 测试 30 条解析 +- `tests/pytest/test_sections_hackernews_enricher.py`：mock Algolia → 测试评论 top_comments 截断、Show HN 特例 +- `tests/pytest/test_storage_sections.py`：sentinel 切片（含老文件 fallback）、`load_recent_section_titles` +- `tests/pytest/test_morning_detection.py`：cron 边界、容差、跨时区 + +新增交互式脚本（与现有 `tests/*.py` 风格一致）： + +- `tests/run_morning_push.py`：模拟一次完整早报，强制 `is_morning=True` +- `tests/fetch_trending.py`：单独跑 GH 抓取 + deep-dive +- `tests/fetch_hackernews.py`：单独跑 HN 抓取 + enrich + +## 13. 实施步骤建议 + +按依赖顺序实施，每步可独立 PR / commit： + +1. **storage 层**：sentinel 切片、`trending-history.json` 读写、`load_recent_section_titles`、`save_push_file` profile 字段、`cleanup_old_files` 对 history 文件的处理 + 单测 +2. **RSS 模块迁移**：把现有 `run_push_job` 中的 RSS 流程提取为 `run_rss_section`，验证行为不变 +3. **GitHub 模块**：trending scraper → history → enricher → section 入口 + 单测 + fixture +4. **Hacker News 模块**：frontpage scraper → Algolia enricher → section 入口 + 单测 + fixture +5. **LLM 函数 + Prompt 文件**：4 个新 LLM 函数 + 4 个 prompt +6. **Insights 模块**：依赖 §5.4，相对简单 +7. **push_job 升级**：早报判定 + 四模块编排 + sentinel 拼装 + 失败隔离 +8. **配置 schema 升级**：`config.json.example` 与 `config.json` 同步；README 配置详解章节补全 +9. **文档同步**：`docs/tech-spec.md` 把架构升级合并；`docs/plan.md` 写进度 + +## 14. 关键决策记录 + +| 决策 | 方案 | 原因 | +|---|---|---| +| 新板块时机 | 仅早报（当天最早一次 `push_cron` 触发） | 板块价值更适合一日一报；晚报维持原有 RSS 节奏；复用 push_cron 不引入新配置 | +| 模块边界 | `src/sections//` 各自封装抓取+LLM+总结 | 模块自治便于扩展、替换、单测；上游编排极简 | +| sentinel 归属 | push_job 上游统一包 | 模块不感知自己的板块标识；新增板块零修改成本 | +| GH trending 数据源 | 单页 HTML `https://github.com/trending`，无语言/since 过滤 | 用户决策：最简、最稳；语言过滤靠 topics + readme 在 LLM 层判 | +| GH deep-dive 内容 | REST API 拿 metadata + topics + README | topics 是 AI 相关性最强信号；metadata 补 license/pushed_at；README 给内容深度 | +| GH 鉴权 | GITHUB_TOKEN 可选 | 日 ~20 calls 远低于匿名 60 req/hr 上限；零配置即可跑 | +| GH 筛选策略 | history 过滤 → 全部 deep-dive → 一次 LLM 选 1-3 | 候选量小（5-15）；单次 LLM 比两阶段简单且选择质量高 | +| GH 候选护栏 | `max_deep_dive=10` | 极端日（trending 大改）限制 HTTP 与 token 消耗 | +| HN 数据源 | 首页 HTML + 评论/正文 Algolia | 首页要"现场感"走 HTML；Algolia 评论 JSON 结构清晰，免去 indent-tree 解析 | +| HN 筛选策略 | 30 条 → 轻 LLM 选 K=1 → enrich → 最终 LLM 行文 | 用户决策：把 enrich 工作量压到 1 个 story；轻 LLM 用 title 已足够判 AI 相关 | +| HN 评论结构 | L1 + 每个 L1 下挂 N 条 L2 回复,tree JSON 喂 LLM | 真实数据:L2 信息量与 L1 持平(24 条/8k chars vs 18/6k);拍平丢失父子关系,LLM 无法识别"回复反驳了顶层"。Tree 结构让 LLM 看清论辩链 | +| HN 评论上限 | `top_comments=30`(L1)+ `top_l2_per_l1=3` + `comments_total_chars=60000`(总预算) | 真实平均:L1 18 条/6k chars,L2 24 条/8k。30+3 给足余量但平均只跑 ~14k。总预算硬上限拦住离群 story(309 评论那种) | +| 无历史上下文 | GH / HN 板块均不传 recent_section_titles | 用户决策：避免不必要的上下文污染；GH/HN 风格与 RSS digest 差异已足够大 | +| insights 历史窗口 | 复用 `filter.push_context_days` | 不引入新字段；insights 段需要历史防风格趋同 | +| insights 输出结构 | 由 prompt 决定，code 不强加 | 用户决策：bullet 数量与栏目属于 prompt 工程，便于迭代 | +| GH / HN 关注领域沉淀 | 在 §6.2 集中定义正面列表 + 负面排除，3 个 prompt 引用 | 避免领域定义在 prompt 间漂移；用户已明确兴趣边界（AI Agent / 模型 / 基础设施 / 大厂动态），排除嵌入式、底层系统、纯前端模板、学习资源等 | +| 板块 sentinel 用 HTML 注释 | `` | markdown 渲染不显示；机器易解析；老 push 文件零冲突 | +| 失败降级粒度 | 单板块失败省略本段；RSS 失败整体退出 | RSS 是核心承诺，其他是增强 | +| 早报判定 | cron + 容差，而非"今天第一次" | 早报失败时晚报不会错误升级为长版本 | +| 截断参数初始值（2026-05-17 合入） | `readme_max_chars=3000` / `top_comments=20` / `comment_max_chars=500` / `link_content_max_chars=3000` | 凭直觉给出的保守默认；上线后通过真实数据校准 | +| 截断参数校准（2026-05-17 合入后,激进路径） | `readme_max_chars: 5000→10000` / `top_comments: 20→50` / `link_content_max_chars: 6000→50000` / `comment_max_chars: 800` 不变 / `max_prompt_chars: 64000→150000` | 用户决策:把 LLM 上下文用到 DeepSeek v4 flash 128k tokens 限的合理水平,优先内容深度而非 API 成本。Worst-case 单次 LLM prompt:GH≈110k chars / HN≈95k chars,均在 150k budget 内。中英混合 100k chars ≈ 30-50k tokens,远低于 128k 模型限 | +| 单板块 CLI（2026-05-17 合入后） | `python -m src.main github` / `hackernews` | 便于 prompt 调优期反复跑单板块而不消耗全套 LLM 调用 | + + +hacker news 评论统计 + +┌─────────────┬──────────┬──────────────┬────────────────┬──────────┐ +│ 层级 │ 平均条数 │ 平均 md 字符 │ 平均单条 chars │ 最大单条 │ +├─────────────┼──────────┼──────────────┼────────────────┼──────────┤ +│ L1 顶层评论 │ 18 │ 5,952 │ ~330 │ 1,755 │ +├─────────────┼──────────┼──────────────┼────────────────┼──────────┤ +│ L2 一级回复 │ 24 │ 8,166 │ ~340 │ 2,381 │ +├─────────────┼──────────┼──────────────┼────────────────┼──────────┤ +│ L3+ 更深层 │ 41 │ 13,048 │ ~315 │ 2,291 │ +├─────────────┼──────────┼──────────────┼────────────────┼──────────┤ +│ ALL 全部 │ 83 │ 27,166 │ ~330 │ 2,381 │ +└─────────────┴──────────┴──────────────┴────────────────┴──────────┘ diff --git a/ai-daily-main/docs/plan.md b/ai-daily-main/docs/plan.md new file mode 100644 index 0000000..c658081 --- /dev/null +++ b/ai-daily-main/docs/plan.md @@ -0,0 +1,90 @@ +## TODO + +当前待办 + +- [ ] 优化提示词： 推送格式；参考链接去除非官方信息； insights 不够深度 +- [ ] 日志系统，保存到文件，push和fetch分开， +- [ ] 早报内容格式优化：参考appso / xiaohu / ai gap + - [ ] 优先级顺序 + - [ ] 美化排版 +- [ ] 添加更多信息源，如 TechCrunch、 +- [ ] 允许fetch链接中的内容对信息进行扩展 +- [ ] 更多信息源以及信息获取不全： https://www.anthropic.com/research/glasswing-initial-update / github blog + + +长期待办 + +- [ ] 增加图片/信息图 +- [ ] 推送到知乎 / 小红书 / 网站 +- [ ] llm api fallback + +## 技术决策记录 + +| 决策 | 方案 | 原因 | +|------|------|------| +| 定时调度 | systemd timer（生产）+ croniter（loop 模式） | 进程崩溃/服务器重启可自愈，配置热更新，比内置 asyncio.gather 更稳健 | +| 包管理 | uv | 速度快、单工具管理 venv/pip/lockfile，项目独立 `.venv` | +| LLM 健康检查 | 仅在 `install.sh` / `loop` 启动时校验 | 每次 timer 触发都校验会增加无意义的 LLM API 调用，运行期异常由 `notify_llm_errors` 兜底 | +| 日志方案 | journald 命名空间 `dnews` + `MaxRetentionSec` | 与系统其他服务隔离，按 `log.retention_days` 自动轮转，无需写文件日志 | +| 数据格式 | JSON | 结构清晰、易处理、支持嵌套 | +| 推送文件 | Markdown+YAML | 人工可读、Frontmatter 元数据 | +| LLM 评分 | 批量 JSON | 减少 API 调用次数 | +| 状态追踪 | 文件时间戳 | 无需外部数据库 | +| RSS延迟防护 | fetch_lookback_minutes | 防止RSS延迟导致漏读 | +| LLM异常通知 | 调用方统一上报 | 避免批次级刷屏，同时保留关键异常通知 | +| 报告 metadata 统一 | 全部从 LLM frontmatter 解析(早报/晚报/即时) | 取代之前的 `extract_title_from_content` h1 提取+硬编日期标题；个性化标题 + lead 导读 + highlights 列表统一承载 | + +## 开发进度 + +**2026-05-22** + +- ✅ 推送 metadata 统一改造：晚报与即时消息也走 frontmatter（之前只有早报） + - `prompts/digest.md` 增加 frontmatter（title/lead/highlights），删除正文「开头一句话定调」段 + - `prompts/insights.md` frontmatter 增加 `lead`（综合三段的 60-100 字前言）与 `highlights`（2-3 条卡片重点） + - `prompts/immediate_push.md` 将 `# 标题` 迁到 frontmatter `title` 字段 + - `src/llm.py` 新增 `parse_digest_with_metadata` / `parse_immediate_push_with_metadata`，统一 `_parse_frontmatter` 帮手；移除 `extract_title_from_content` + - `src/sections/rss/section.py` 返回三元组 `(body, metadata, err)`；早报丢弃 digest metadata，由 insights 段覆盖 + - 晚报现在拥有个性化标题 + lead + highlights，与早报对齐 + + +**2026-05-17** + +- ✅ 早报扩展板块上线：在 RSS digest 之上叠加 GitHub Trending / Hacker News / 跨板块洞察三段，仅 `schedule.morning_cron` 命中时触发，晚报维持纯 RSS 行为 +- 设计与实施详见 [`docs/extra-sections-design.md`](extra-sections-design.md) 与 [`docs/superpowers/plans/2026-05-17-extra-sections.md`](superpowers/plans/2026-05-17-extra-sections.md) + +**2026-05-15** +- 优化提示词，修复长时运行下的新闻报告措辞趋同问题 + +**2026-05-14** +- [x] 外置定时（已切换到 systemd timer） +- [x] 系统服务一键运行（`scripts/install.sh`） +- 使用uv进行python项目管理 +- 配置变更：`config.json` 新增 `log.retention_days` 字段 +- 新增 `daily-news` 系统级包装脚本：装在 `/usr/local/bin/daily-news`，提供 `start/stop/restart/status/logs` 等命令，封装 systemctl/journalctl 调用细节 + +**2026-03-08** +- 新增 LLM 异常通知：`compose_digest`、`generate_immediate_push` 与 `score_batch` 的错误会通过现有推送渠道发送简单告警 +- 优化批量评分容错：`score_batch` 在批次返回数量不匹配时会按 `link` 回收可用结果，并聚合错误返回给调用方 +- 移除 `generate_immediate_push` 的 fallback 内容，生成失败时由调用方告警并跳过本次即时推送 +- 新增启动前 LLM 可用性检查：主程序在启动 fetch/push 双循环前先探测 LLM 接口，失败则直接退出 +- 修复 pytest 中遗留的旧推送平台命名问题，将 `wecom` 测试更新为当前 `feishu` 实现 + +**2026-03-03** +- 采用 MIT 许可开源项目，添加 LICENSE 和 NOTICE 文件 +- 更新 RSS 源说明，致谢 BestBlogs 项目 + +**2026-03-02** +- 修复RSS延迟漏读问题：新增 fetch_lookback_minutes 参数，fetch时读取过去更长一段时间的RSS条目进行去重 +- 新增飞书 Webhook 推送支持：使用卡片消息格式，支持 Markdown 渲染 +- 新增测试脚本 test_fetch_lookback.py +- 更新 cleanup_old_files 函数支持 notify 文件清理 + +**2026-03-01** +- 优化评分系统：通过更新 score 提示词提升评分质量 +- 即时推送去重：新增 notify-*.md 文件存储即时推送，LLM 调用时传入近期推送上下文避免重复 +- 汇总推送优化：新增 push_context_days 配置，汇总推送时传入近期推送上下文进行去重 +- 修复 score 类型问题：确保 LLM 返回的 score 为整数类型 +- 完善测试脚本：添加上下文参数和保存功能 + +**2026-02-28** +- 初始化项目，MVP 已完成，支持 RSS 抓取、LLM 评分、定时推送、即时推送。 diff --git a/ai-daily-main/docs/superpowers/plans/2026-05-17-extra-sections.md b/ai-daily-main/docs/superpowers/plans/2026-05-17-extra-sections.md new file mode 100644 index 0000000..1beacaf --- /dev/null +++ b/ai-daily-main/docs/superpowers/plans/2026-05-17-extra-sections.md @@ -0,0 +1,3841 @@ +# Extra Sections (GitHub Trending / Hacker News / Insights) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add three new content sections (GitHub Trending, Hacker News, cross-section Insights) to the morning push, while keeping evening push as RSS-only and RSS as the core load-bearing module. + +**Architecture:** Each section is an autonomous module under `src/sections//` returning `(markdown, error)`. `push_job` orchestrates four modules (RSS / GitHub / HN run in parallel via `asyncio.gather`; Insights runs after, consuming the three section outputs), wraps each output with HTML-comment sentinels (``), and writes the assembled push file. Module failure (other than RSS) degrades to omitting that section. State is local files only — `news-data/trending-history.json` deduplicates GitHub repos across days. + +**Tech Stack:** Python 3.12 / asyncio / aiohttp / BeautifulSoup4 (new) / croniter / markdownify / DeepSeek (OpenAI-compatible) LLM API / Algolia HN public API / GitHub REST API v3. + +**Spec reference:** `docs/extra-sections-design.md` + +--- + +## Phase 0: Prerequisites + +### Task 0: Add BeautifulSoup4 dependency + +**Files:** +- Modify: `pyproject.toml` + +- [ ] **Step 1: Add bs4 to dependencies** + +Open `pyproject.toml` and add `"beautifulsoup4>=4.12.0",` to the `dependencies` list (after `"aiohttp>=3.9.0",`). + +- [ ] **Step 2: Sync deps** + +Run: `uv sync` +Expected: package installed, no errors. + +- [ ] **Step 3: Verify import works** + +Run: `uv run python -c "from bs4 import BeautifulSoup; print('ok')"` +Expected: `ok` + +- [ ] **Step 4: Commit** + +```bash +git add pyproject.toml uv.lock +git commit -m "build: add beautifulsoup4 for HTML parsing" +``` + +--- + +## Phase 1: Storage Layer Foundation + +The storage layer adds sentinel-aware section extraction, a `TrendingHistory` class for GH dedup, and a `profile` field on push frontmatter. These changes are independent of any new section module and must land first. + +### Task 1: Add `extract_section` to storage.py + +Parses `` boundaries. Backward-compatible: old push files without sentinel return whole body when `section="rss"`, empty otherwise. + +**Files:** +- Modify: `src/storage.py` (append after `_extract_push_titles`) +- Test: `tests/pytest/test_storage_sections.py` (create) + +- [ ] **Step 1: Write the failing test** + +Create `tests/pytest/test_storage_sections.py`: + +```python +"""测试新增的 sentinel 切片与 section-aware 读取""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src")) + +from storage import extract_section + + +class TestExtractSection: + def test_extract_section_with_sentinel(self): + md = ( + "intro\n" + "\n" + "RSS body\n" + "\n" + "\n" + "\n" + "GH body\n" + "\n" + ) + assert extract_section(md, "rss").strip() == "RSS body" + assert extract_section(md, "github").strip() == "GH body" + assert extract_section(md, "hackernews") == "" + + def test_extract_section_legacy_file_rss(self): + legacy = "# AI Daily\n### 1️⃣ foo\n### 2️⃣ bar\n" + assert extract_section(legacy, "rss") == legacy + + def test_extract_section_legacy_file_non_rss(self): + legacy = "# AI Daily\n### 1️⃣ foo\n" + assert extract_section(legacy, "github") == "" + assert extract_section(legacy, "hackernews") == "" + assert extract_section(legacy, "insights") == "" + + def test_extract_section_missing_end_marker(self): + broken = "\ncontent only\n" + assert extract_section(broken, "rss") == "" +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/pytest/test_storage_sections.py -v` +Expected: ImportError or `AttributeError: module 'storage' has no attribute 'extract_section'` + +- [ ] **Step 3: Implement `extract_section`** + +Append to `src/storage.py`: + +```python +_SECTION_RE_CACHE: Dict[str, re.Pattern] = {} + + +def _section_re(section: str) -> re.Pattern: + """获取/缓存 sentinel 正则。section 名做转义,允许字母数字下划线""" + if section not in _SECTION_RE_CACHE: + s = re.escape(section) + pattern = rf"(.*?)" + _SECTION_RE_CACHE[section] = re.compile(pattern, flags=re.DOTALL) + return _SECTION_RE_CACHE[section] + + +def extract_section(push_md: str, section: str) -> str: + """从 push 文件内容中切出 之间的 markdown。 + + 向后兼容: + - 新文件(带 sentinel): 返回 sentinel 边界内的原文(不去边界空行) + - 老文件(无 sentinel) 且 section == 'rss': 返回整个 push_md + - 老文件(无 sentinel) 且 section != 'rss': 返回空字符串 + - sentinel 残缺(只有 BEGIN 没有 END): 返回空字符串 + """ + match = _section_re(section).search(push_md) + if match: + return match.group(1) + + # 老文件兜底:rss 段视为整个 body + has_any_sentinel = "\n" + "### 1️⃣ RSS Title One\n" + "### 2️⃣ RSS Title Two\n" + "\n\n" + "\n" + "### GH Repo Title\n" + "\n", + encoding="utf-8", + ) + rss_titles = load_recent_section_titles("rss", 3, str(tmp_path)) + assert "RSS Title One" in rss_titles + assert "RSS Title Two" in rss_titles + assert "GH Repo Title" not in rss_titles + + gh_titles = load_recent_section_titles("github", 3, str(tmp_path)) + assert "GH Repo Title" in gh_titles + assert "RSS Title One" not in gh_titles +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/pytest/test_storage_sections.py::TestLoadRecentSectionTitles -v` +Expected: `AttributeError` on the import. + +- [ ] **Step 3: Implement `load_recent_section_titles`** + +Append to `src/storage.py`: + +```python +def load_recent_section_titles( + section: str, days: int, data_dir: str = "news-data" +) -> str: + """加载最近 days 天 push 文件中 section 段的标题清单(供 LLM 查重防风格趋同)。 + + 返回紧凑纯文本,每行一条事件;遇到老文件(无 sentinel)按 extract_section 的兜底语义处理。 + """ + data_path = Path(data_dir) + if not data_path.exists(): + return "" + + tz = get_timezone() + today = datetime.now(tz).date() + + items: List[tuple] = [] + loaded_files: List[str] = [] + for i in range(days): + d = today - timedelta(days=i) + pattern = f"push-{d.isoformat()}-*.md" + for push_file in sorted(data_path.glob(pattern)): + if push_file.stat().st_size == 0: + continue + try: + with open(push_file, "r", encoding="utf-8") as f: + content = f.read() + except Exception: + continue + section_md = extract_section(content, section) + if not section_md: + continue + items.extend(_extract_push_titles(section_md)) + loaded_files.append(push_file.name) + + if loaded_files: + print( + f" 📂 已加载 {len(loaded_files)} 个 push 文件 (section={section}): " + f"{', '.join(loaded_files)}" + ) + + return "\n".join(f"- [{t}] {title}" if t else f"- {title}" for t, title in items) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/pytest/test_storage_sections.py::TestLoadRecentSectionTitles -v` +Expected: 3 tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add src/storage.py tests/pytest/test_storage_sections.py +git commit -m "feat(storage): add load_recent_section_titles for section-scoped history" +``` + +--- + +### Task 3: Add `TrendingHistory` to storage.py + +Owns `news-data/trending-history.json` read / write / touch / cleanup. Single source of truth for GH dedup state. + +**Files:** +- Modify: `src/storage.py` +- Test: `tests/pytest/test_storage_sections.py` (append) + +- [ ] **Step 1: Write the failing test** + +Append to `tests/pytest/test_storage_sections.py`: + +```python +from storage import TrendingHistory, load_trending_history + + +class TestTrendingHistory: + def test_load_missing_file(self, tmp_path): + path = tmp_path / "trending.json" + h = load_trending_history(str(path)) + assert h.repos == {} + + def test_touch_then_save_then_reload(self, tmp_path): + path = tmp_path / "trending.json" + h = load_trending_history(str(path)) + today = date(2026, 5, 17) + h.touch("https://github.com/a/b", today) + h.touch("https://github.com/c/d", today) + h.save() + + h2 = load_trending_history(str(path)) + assert h2.repos == { + "https://github.com/a/b": "2026-05-17", + "https://github.com/c/d": "2026-05-17", + } + + def test_contains_returns_membership(self, tmp_path): + h = load_trending_history(str(tmp_path / "x.json")) + h.touch("https://github.com/a/b", date(2026, 5, 17)) + assert "https://github.com/a/b" in h + assert "https://github.com/x/y" not in h + + def test_cleanup_removes_expired_entries(self, tmp_path): + path = tmp_path / "trending.json" + path.write_text( + '{"repos": {' + '"https://github.com/old/repo": "2026-05-01", ' + '"https://github.com/new/repo": "2026-05-15"' + '}, "updated_at": "2026-05-15T00:00:00+08:00"}', + encoding="utf-8", + ) + h = load_trending_history(str(path)) + h.cleanup(today=date(2026, 5, 17), keep_days=7) + assert "https://github.com/old/repo" not in h + assert "https://github.com/new/repo" in h + + def test_cleanup_keeps_today_inclusive(self, tmp_path): + h = load_trending_history(str(tmp_path / "x.json")) + h.touch("https://github.com/a/b", date(2026, 5, 10)) + # 2026-05-10 + 7 days = 2026-05-17 (last_seen 2026-05-10 仍在 keep 区间) + h.cleanup(today=date(2026, 5, 17), keep_days=7) + assert "https://github.com/a/b" in h + # 再过 1 天就出区间 + h.cleanup(today=date(2026, 5, 18), keep_days=7) + assert "https://github.com/a/b" not in h +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/pytest/test_storage_sections.py::TestTrendingHistory -v` +Expected: import error. + +- [ ] **Step 3: Implement `TrendingHistory` + loader** + +Append to `src/storage.py`: + +```python +class TrendingHistory: + """GitHub trending 已查阅 repo 索引。 + + repos 字段:url → last_seen_date (ISO YYYY-MM-DD)。 + 每次早报 cleanup 一次,touch 完所有今日 URL 后 save。 + """ + + def __init__(self, path: str, repos: Dict[str, str]): + self._path = path + self.repos: Dict[str, str] = dict(repos) + + def __contains__(self, url: str) -> bool: + return url in self.repos + + def touch(self, url: str, today: date) -> None: + self.repos[url] = today.isoformat() + + def cleanup(self, today: date, keep_days: int) -> None: + cutoff = today - timedelta(days=keep_days) + self.repos = { + url: d + for url, d in self.repos.items() + if _parse_iso_date_safe(d) is not None + and _parse_iso_date_safe(d) >= cutoff + } + + def save(self) -> None: + path = Path(self._path) + path.parent.mkdir(parents=True, exist_ok=True) + payload = { + "repos": self.repos, + "updated_at": datetime.now(get_timezone()).isoformat(), + } + with open(path, "w", encoding="utf-8") as f: + json.dump(payload, f, ensure_ascii=False, indent=2) + + +def _parse_iso_date_safe(s: str) -> Optional[date]: + try: + return date.fromisoformat(s) + except (ValueError, TypeError): + return None + + +def load_trending_history(path: str) -> TrendingHistory: + """读取 trending-history.json;不存在返回空实例。""" + p = Path(path) + if not p.exists() or p.stat().st_size == 0: + return TrendingHistory(path, {}) + try: + with open(p, "r", encoding="utf-8") as f: + data = json.load(f) + return TrendingHistory(path, data.get("repos", {})) + except (json.JSONDecodeError, OSError): + print(f"⚠️ trending-history 读取失败,使用空索引: {path}") + return TrendingHistory(path, {}) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/pytest/test_storage_sections.py::TestTrendingHistory -v` +Expected: 5 tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add src/storage.py tests/pytest/test_storage_sections.py +git commit -m "feat(storage): add TrendingHistory for GH repo dedup state" +``` + +--- + +### Task 4: Add `profile` field to `save_push_file` + +Push frontmatter gets a `profile: "morning"|"default"` tag so downstream tools can filter. + +**Files:** +- Modify: `src/storage.py:363-378` (the existing `save_push_file`) +- Test: `tests/pytest/test_storage_sections.py` (append) + +- [ ] **Step 1: Write the failing test** + +Append to `tests/pytest/test_storage_sections.py`: + +```python +from storage import save_push_file + + +class TestSavePushFileProfile: + def test_default_profile_when_not_specified(self, tmp_path): + f = tmp_path / "push-x.md" + save_push_file(str(f), "body content", source_count=1, total_entries=1) + text = f.read_text(encoding="utf-8") + assert 'profile: "default"' in text + assert "body content" in text + + def test_morning_profile(self, tmp_path): + f = tmp_path / "push-x.md" + save_push_file( + str(f), "body", source_count=2, total_entries=3, profile="morning" + ) + text = f.read_text(encoding="utf-8") + assert 'profile: "morning"' in text +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/pytest/test_storage_sections.py::TestSavePushFileProfile -v` +Expected: FAIL (current `save_push_file` doesn't accept `profile`). + +- [ ] **Step 3: Update `save_push_file` signature** + +Open `src/storage.py`. Replace the `save_push_file` function (currently around line 363-378) with: + +```python +def save_push_file( + filepath: str, + content: str, + source_count: int, + total_entries: int, + profile: str = "default", +): + """保存推送文件（Markdown格式） + + Args: + profile: "morning" | "default" ← 早报或常规;写入 frontmatter,便于按 profile 分析 + """ + path = Path(filepath) + path.parent.mkdir(parents=True, exist_ok=True) + + push_time = datetime.now(get_timezone()) + frontmatter = ( + "---\n" + f'pushDate: "{push_time.isoformat()}"\n' + f'profile: "{profile}"\n' + f"sourceCount: {source_count}\n" + f"totalEntries: {total_entries}\n" + "---\n\n" + ) + + with open(path, "w", encoding="utf-8") as f: + f.write(frontmatter + content) +``` + +- [ ] **Step 4: Verify existing storage tests still pass** + +Run: `uv run pytest tests/pytest/test_storage.py tests/pytest/test_storage_sections.py -v` +Expected: all pass (existing callers pass no `profile`, default "default" kicks in). + +- [ ] **Step 5: Commit** + +```bash +git add src/storage.py tests/pytest/test_storage_sections.py +git commit -m "feat(storage): add profile field to push frontmatter" +``` + +--- + +### Task 5: Extend `cleanup_old_files` to prune expired entries in `trending-history.json` + +The fetch job's daily cleanup should not delete the trending-history file (it's cumulative state), but should remove entries older than `keep_days`. + +**Files:** +- Modify: `src/storage.py:419-451` (the existing `cleanup_old_files`) +- Test: `tests/pytest/test_storage_sections.py` (append) + +- [ ] **Step 1: Write the failing test** + +Append to `tests/pytest/test_storage_sections.py`: + +```python +from storage import cleanup_old_files + + +class TestCleanupOldFilesTrendingHistory: + def test_prunes_trending_history_entries_not_file(self, tmp_path): + path = tmp_path / "trending-history.json" + old_date = (datetime.now().date() - timedelta(days=30)).isoformat() + fresh_date = datetime.now().date().isoformat() + path.write_text( + '{"repos": {' + f'"https://github.com/a/b": "{old_date}", ' + f'"https://github.com/c/d": "{fresh_date}"' + '}, "updated_at": "..."}', + encoding="utf-8", + ) + cleanup_old_files(days=7, data_dir=str(tmp_path)) + # 文件应保留 + assert path.exists() + # 过期条目应被剪枝 + import json as _j + data = _j.loads(path.read_text(encoding="utf-8")) + assert "https://github.com/a/b" not in data["repos"] + assert "https://github.com/c/d" in data["repos"] +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/pytest/test_storage_sections.py::TestCleanupOldFilesTrendingHistory -v` +Expected: FAIL (current cleanup doesn't touch trending-history). + +- [ ] **Step 3: Update `cleanup_old_files`** + +Open `src/storage.py`. After the existing `for pattern in [...]` loop (around line 428-447), but **before** the final `if deleted_count > 0:` print, insert: + +```python + # trending-history.json: 剪枝过期条目,保留文件本身 + trending_path = data_path / "trending-history.json" + if trending_path.exists() and trending_path.stat().st_size > 0: + try: + history = load_trending_history(str(trending_path)) + before = len(history.repos) + history.cleanup(today=datetime.now().date(), keep_days=days) + after = len(history.repos) + if after < before: + history.save() + print(f" ✂️ trending-history 剪枝: {before} → {after} 条") + except Exception as e: + print(f" ⚠️ trending-history 剪枝失败: {e}") +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/pytest/test_storage_sections.py::TestCleanupOldFilesTrendingHistory tests/pytest/test_storage.py -v` +Expected: all pass. + +- [ ] **Step 5: Commit** + +```bash +git add src/storage.py tests/pytest/test_storage_sections.py +git commit -m "feat(storage): prune expired entries in trending-history.json on cleanup" +``` + +--- + +## Phase 2: RSS Module Migration + +Migrate the existing RSS push flow into the new `sections` package without changing behavior. + +### Task 6: Create `src/sections/rss/section.py` + +`run_rss_section(config, now)` wraps existing `collect_entries_for_push` + `compose_digest` and returns `(markdown, error)`. + +**Files:** +- Create: `src/sections/__init__.py` (empty) +- Create: `src/sections/rss/__init__.py` (empty) +- Create: `src/sections/rss/section.py` +- Test: `tests/pytest/test_sections_rss.py` + +- [ ] **Step 1: Write the failing test** + +Create `tests/pytest/test_sections_rss.py`: + +```python +"""测试 RSS 模块返回 (markdown, error) 契约""" + +import sys +from pathlib import Path +from unittest.mock import patch, AsyncMock + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from src.sections.rss.section import run_rss_section + + +@pytest.mark.asyncio +async def test_returns_markdown_when_entries_present(sample_config, tmp_path): + with patch( + "src.sections.rss.section.collect_entries_for_push", + return_value=([{"link": "x", "title": "t", "score": 80}], []), + ), patch( + "src.sections.rss.section.compose_digest", + new=AsyncMock(return_value="# digest body"), + ), patch( + "src.sections.rss.section.load_recent_push_titles", return_value="" + ), patch( + "src.sections.rss.section.get_last_push_file", return_value=None + ): + md, err = await run_rss_section(sample_config, now=None) + + assert md == "# digest body" + assert err is None + + +@pytest.mark.asyncio +async def test_returns_empty_when_no_entries(sample_config): + with patch( + "src.sections.rss.section.collect_entries_for_push", return_value=([], []) + ), patch( + "src.sections.rss.section.get_last_push_file", return_value=None + ): + md, err = await run_rss_section(sample_config, now=None) + + assert md == "" + assert err is None + + +@pytest.mark.asyncio +async def test_returns_error_on_compose_failure(sample_config): + with patch( + "src.sections.rss.section.collect_entries_for_push", + return_value=([{"link": "x"}], []), + ), patch( + "src.sections.rss.section.compose_digest", + new=AsyncMock(side_effect=RuntimeError("LLM down")), + ), patch( + "src.sections.rss.section.load_recent_push_titles", return_value="" + ), patch( + "src.sections.rss.section.get_last_push_file", return_value=None + ): + md, err = await run_rss_section(sample_config, now=None) + + assert md == "" + assert "LLM down" in err +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/pytest/test_sections_rss.py -v` +Expected: ModuleNotFoundError. + +- [ ] **Step 3: Implement module** + +Create `src/sections/__init__.py`: + +```python +"""板块模块包。每个子模块导出 run__section(config, now) -> (markdown, error)""" +``` + +Create `src/sections/rss/__init__.py`: + +```python +from src.sections.rss.section import run_rss_section + +__all__ = ["run_rss_section"] +``` + +Create `src/sections/rss/section.py`: + +```python +"""RSS 板块:沿用现有 collect_entries_for_push + compose_digest 流程 + +迁移自 src/main.py::run_push_job 中 RSS digest 部分,行为保持一致。 +""" + +from datetime import datetime +from typing import Dict, Optional, Tuple + +from src.llm import compose_digest +from src.storage import ( + extract_push_time, + get_last_push_file, + load_recent_push_titles, +) + + +async def run_rss_section( + config: Dict, now: Optional[datetime] = None +) -> Tuple[str, Optional[str]]: + """生成 RSS digest markdown 段(不含 sentinel)。 + + 返回: + (markdown, error) + - 无新内容时返回 ("", None) + - compose_digest 失败时返回 ("", error_message) + """ + # 延迟 import 避免循环引用:collect_entries_for_push 仍在 main.py + from src.main import collect_entries_for_push + + last_push_file = get_last_push_file() + last_push_time = extract_push_time(last_push_file) if last_push_file else None + + min_score = config["filter"]["min_score"] + context_days = config["filter"]["context_days"] + + to_push, context = collect_entries_for_push( + last_push_time=last_push_time, + context_days=context_days, + min_score=min_score, + ) + + if not to_push: + print("ℹ️ RSS: 无新消息") + return "", None + + push_context_days = config["filter"].get("push_context_days", 5) + recent = load_recent_push_titles(push_context_days) + + try: + md = await compose_digest(to_push, context, config["llm"], recent_push_context=recent) + return md, None + except Exception as e: + msg = f"compose_digest 失败: {e}" + print(f"⚠️ RSS: {msg}") + return "", msg +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/pytest/test_sections_rss.py -v` +Expected: 3 tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add src/sections/ tests/pytest/test_sections_rss.py +git commit -m "feat(sections): extract RSS digest flow into run_rss_section" +``` + +--- + +## Phase 3: GitHub Module + +### Task 7: Save a real GitHub trending HTML fixture + +Save a real snapshot for deterministic parser tests. The structure may drift; this is the canonical "what we built against". + +**Files:** +- Create: `tests/pytest/fixtures/github_trending.html` + +- [ ] **Step 1: Download a real page** + +```bash +mkdir -p tests/pytest/fixtures +curl -fsSL -A "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" \ + "https://github.com/trending" \ + -o tests/pytest/fixtures/github_trending.html +``` + +- [ ] **Step 2: Verify the fixture has expected markers** + +Run: `grep -c 'class="Box-row"' tests/pytest/fixtures/github_trending.html` +Expected: a number ≥ 10 (typically 25). If 0, GitHub markup changed — adjust selectors in Task 8 accordingly. + +- [ ] **Step 3: Commit fixture** + +```bash +git add tests/pytest/fixtures/github_trending.html +git commit -m "test(github): snapshot github.com/trending fixture" +``` + +--- + +### Task 8: Implement `trending_scraper.py` + +Async `fetch_trending_page` + `parse_trending_html` returning `[{url, full_name, description, language, stars_today, stars_total}]`. + +**Files:** +- Create: `src/sections/github/__init__.py` (empty) +- Create: `src/sections/github/trending_scraper.py` +- Test: `tests/pytest/test_sections_github_scraper.py` + +- [ ] **Step 1: Write the failing test** + +Create `tests/pytest/test_sections_github_scraper.py`: + +```python +"""测试 GitHub trending HTML 解析""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from src.sections.github.trending_scraper import parse_trending_html + + +def test_parse_trending_html_returns_repo_dicts(): + fixture = ( + Path(__file__).parent / "fixtures" / "github_trending.html" + ).read_text(encoding="utf-8") + + repos = parse_trending_html(fixture) + + assert len(repos) > 0 + first = repos[0] + assert first["url"].startswith("https://github.com/") + assert "/" in first["full_name"] + assert isinstance(first["stars_today"], int) + assert isinstance(first["stars_total"], int) + # description / language 可为空字符串但必须是 str + assert isinstance(first["description"], str) + assert isinstance(first["language"], str) + + +def test_parse_trending_html_dedupes_by_url(): + fixture = ( + Path(__file__).parent / "fixtures" / "github_trending.html" + ).read_text(encoding="utf-8") + repos = parse_trending_html(fixture) + urls = [r["url"] for r in repos] + assert len(urls) == len(set(urls)) + + +def test_parse_trending_html_empty_input(): + assert parse_trending_html("") == [] + assert parse_trending_html("no repos") == [] +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/pytest/test_sections_github_scraper.py -v` +Expected: ModuleNotFoundError. + +- [ ] **Step 3: Implement scraper** + +Create `src/sections/github/__init__.py` (empty): + +```python +``` + +Create `src/sections/github/trending_scraper.py`: + +```python +"""GitHub Trending 单页 HTML 抓取与解析。 + +数据源: https://github.com/trending (无 language / since 过滤) +""" + +import re +from typing import Dict, List + +import aiohttp +from bs4 import BeautifulSoup + +TRENDING_URL = "https://github.com/trending" +USER_AGENT = ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" +) +_NUM_RE = re.compile(r"[\d,]+") + + +def _parse_int(s: str) -> int: + m = _NUM_RE.search(s or "") + if not m: + return 0 + return int(m.group(0).replace(",", "")) + + +async def fetch_trending_page(timeout: int = 10) -> str: + """抓取 trending 页 HTML;非 200 抛 RuntimeError""" + async with aiohttp.ClientSession( + headers={"User-Agent": USER_AGENT} + ) as session: + async with session.get( + TRENDING_URL, timeout=aiohttp.ClientTimeout(total=timeout) + ) as resp: + if resp.status != 200: + raise RuntimeError( + f"GitHub trending 返回 {resp.status}: {await resp.text()[:200]}" + ) + return await resp.text() + + +def parse_trending_html(html: str) -> List[Dict]: + """解析 trending HTML,返回去重后的 repo 字典数组。""" + if not html: + return [] + + soup = BeautifulSoup(html, "html.parser") + seen_urls = set() + repos: List[Dict] = [] + + for article in soup.select("article.Box-row"): + h2 = article.find("h2") + a = h2.find("a") if h2 else None + if not a or not a.get("href"): + continue + + href = a["href"].strip() + full_name = href.lstrip("/") + url = f"https://github.com/{full_name}" + if url in seen_urls: + continue + seen_urls.add(url) + + # description + desc_tag = article.find("p") + description = (desc_tag.get_text(strip=True) if desc_tag else "") or "" + + # language + lang_tag = article.find("span", attrs={"itemprop": "programmingLanguage"}) + language = (lang_tag.get_text(strip=True) if lang_tag else "") or "" + + # stars_total: 第一个指向 /stargazers 的链接 + stars_total = 0 + star_a = article.find("a", href=re.compile(r"/stargazers$")) + if star_a: + stars_total = _parse_int(star_a.get_text(strip=True)) + + # stars_today: 末尾的 "N stars today" span (class 多变,按文本) + stars_today = 0 + for span in article.find_all("span"): + t = span.get_text(strip=True) + if "stars today" in t or "stars this week" in t or "stars this month" in t: + stars_today = _parse_int(t) + break + + repos.append( + { + "url": url, + "full_name": full_name, + "description": description, + "language": language, + "stars_today": stars_today, + "stars_total": stars_total, + } + ) + + return repos +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/pytest/test_sections_github_scraper.py -v` +Expected: 3 tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add src/sections/github/ tests/pytest/test_sections_github_scraper.py +git commit -m "feat(github): add trending page scraper with HTML fixture test" +``` + +--- + +### Task 9: Implement `repo_enricher.py` + +Two REST API calls per repo (`/repos/{o}/{r}` + `/repos/{o}/{r}/readme`). Token via env var optional. + +**Files:** +- Create: `src/sections/github/repo_enricher.py` +- Test: `tests/pytest/test_sections_github_enricher.py` + +- [ ] **Step 1: Write the failing test** + +Create `tests/pytest/test_sections_github_enricher.py`: + +```python +"""测试 GitHub REST API enrich 字段映射、archived 过滤、token 鉴权头""" + +import base64 +import os +import sys +from pathlib import Path +from unittest.mock import patch, AsyncMock, MagicMock + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from src.sections.github.repo_enricher import enrich_repo, _auth_headers + + +def test_auth_headers_with_token(monkeypatch): + monkeypatch.setenv("GITHUB_TOKEN", "ghp_secret") + headers = _auth_headers(token_env="GITHUB_TOKEN") + assert headers["Authorization"] == "Bearer ghp_secret" + assert headers["Accept"] == "application/vnd.github+json" + + +def test_auth_headers_without_token(monkeypatch): + monkeypatch.delenv("GITHUB_TOKEN", raising=False) + headers = _auth_headers(token_env="GITHUB_TOKEN") + assert "Authorization" not in headers + assert headers["Accept"] == "application/vnd.github+json" + + +@pytest.mark.asyncio +async def test_enrich_repo_merges_metadata_and_readme(): + readme_body = "# Title\n\nProject description here." + readme_b64 = base64.b64encode(readme_body.encode("utf-8")).decode("ascii") + + metadata_payload = { + "description": "real desc", + "topics": ["llm", "rag"], + "license": {"spdx_id": "MIT"}, + "pushed_at": "2026-05-16T10:00:00Z", + "archived": False, + } + readme_payload = {"content": readme_b64, "encoding": "base64"} + + async def fake_get_json(session, url, **kwargs): + if url.endswith("/readme"): + return readme_payload + return metadata_payload + + base = { + "url": "https://github.com/o/r", + "full_name": "o/r", + "description": "from trending", + "language": "Python", + "stars_today": 100, + "stars_total": 5000, + } + + with patch( + "src.sections.github.repo_enricher._get_json", new=AsyncMock(side_effect=fake_get_json) + ): + enriched = await enrich_repo( + session=MagicMock(), repo=base, token_env="GITHUB_TOKEN", readme_max_chars=200 + ) + + assert enriched["topics"] == ["llm", "rag"] + assert enriched["license"] == "MIT" + assert enriched["pushed_at"] == "2026-05-16T10:00:00Z" + assert "Project description" in enriched["readme_excerpt"] + # trending 已有字段保留 + assert enriched["stars_today"] == 100 + + +@pytest.mark.asyncio +async def test_enrich_repo_returns_none_when_archived(): + metadata_payload = {"archived": True, "topics": [], "pushed_at": "x"} + + async def fake_get_json(session, url, **kwargs): + if url.endswith("/readme"): + return {"content": ""} + return metadata_payload + + base = {"url": "https://github.com/o/r", "full_name": "o/r"} + with patch( + "src.sections.github.repo_enricher._get_json", new=AsyncMock(side_effect=fake_get_json) + ): + result = await enrich_repo( + session=MagicMock(), repo=base, token_env="GITHUB_TOKEN", readme_max_chars=200 + ) + assert result is None + + +@pytest.mark.asyncio +async def test_enrich_repo_truncates_readme(): + readme_body = "x" * 5000 + readme_b64 = base64.b64encode(readme_body.encode("utf-8")).decode("ascii") + + async def fake_get_json(session, url, **kwargs): + if url.endswith("/readme"): + return {"content": readme_b64, "encoding": "base64"} + return {"archived": False, "topics": [], "pushed_at": "p"} + + base = {"url": "https://github.com/o/r", "full_name": "o/r"} + with patch( + "src.sections.github.repo_enricher._get_json", new=AsyncMock(side_effect=fake_get_json) + ): + enriched = await enrich_repo( + session=MagicMock(), repo=base, token_env="GITHUB_TOKEN", readme_max_chars=100 + ) + assert len(enriched["readme_excerpt"]) == 100 +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/pytest/test_sections_github_enricher.py -v` +Expected: ModuleNotFoundError. + +- [ ] **Step 3: Implement enricher** + +Create `src/sections/github/repo_enricher.py`: + +```python +"""GitHub REST API enrich:metadata + README → enriched repo dict + +匿名调用受 60 req/hr 限,设置 GITHUB_TOKEN 环境变量后走 5000 req/hr。 +""" + +import asyncio +import base64 +import os +from typing import Dict, List, Optional, Tuple + +import aiohttp + +API_BASE = "https://api.github.com" + + +def _auth_headers(token_env: str = "GITHUB_TOKEN") -> Dict[str, str]: + headers = {"Accept": "application/vnd.github+json"} + token = os.environ.get(token_env) + if token: + headers["Authorization"] = f"Bearer {token}" + return headers + + +async def _get_json( + session: aiohttp.ClientSession, url: str, timeout: int = 10 +) -> Optional[Dict]: + async with session.get( + url, timeout=aiohttp.ClientTimeout(total=timeout) + ) as resp: + if resp.status == 404: + return None + if resp.status != 200: + raise RuntimeError(f"GitHub API {resp.status} for {url}") + return await resp.json() + + +async def enrich_repo( + session: aiohttp.ClientSession, + repo: Dict, + token_env: str = "GITHUB_TOKEN", + readme_max_chars: int = 3000, + timeout: int = 10, +) -> Optional[Dict]: + """单 repo 双调用 enrich。返回 None 表示该 repo 应剔除(archived 或 metadata 不可达)。 + + 任一调用失败 raise → 调用方按 return_exceptions 模式聚合错误。 + """ + full_name = repo["full_name"] + meta_url = f"{API_BASE}/repos/{full_name}" + readme_url = f"{API_BASE}/repos/{full_name}/readme" + + meta, readme = await asyncio.gather( + _get_json(session, meta_url, timeout=timeout), + _get_json(session, readme_url, timeout=timeout), + ) + + if meta is None: + return None + if meta.get("archived"): + return None + + license_spdx = "" + if isinstance(meta.get("license"), dict): + license_spdx = meta["license"].get("spdx_id") or "" + + readme_excerpt = "" + if readme and readme.get("content"): + try: + raw = base64.b64decode(readme["content"]).decode("utf-8", errors="replace") + readme_excerpt = raw[:readme_max_chars] + except Exception: + readme_excerpt = "" + + return { + **repo, + "topics": meta.get("topics") or [], + "license": license_spdx, + "pushed_at": meta.get("pushed_at") or "", + "readme_excerpt": readme_excerpt, + } + + +async def enrich_repos( + candidates: List[Dict], + token_env: str = "GITHUB_TOKEN", + readme_max_chars: int = 3000, + timeout: int = 10, +) -> Tuple[List[Dict], List[str]]: + """并发 enrich 多个 repo。返回 (enriched_list_with_archived_filtered, errors)""" + errors: List[str] = [] + headers = _auth_headers(token_env) + + async with aiohttp.ClientSession(headers=headers) as session: + results = await asyncio.gather( + *[ + enrich_repo(session, r, token_env, readme_max_chars, timeout) + for r in candidates + ], + return_exceptions=True, + ) + + enriched: List[Dict] = [] + for r, candidate in zip(results, candidates): + if isinstance(r, Exception): + errors.append(f"enrich {candidate['full_name']} 失败: {r}") + elif r is not None: + enriched.append(r) + return enriched, errors +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/pytest/test_sections_github_enricher.py -v` +Expected: 5 tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add src/sections/github/repo_enricher.py tests/pytest/test_sections_github_enricher.py +git commit -m "feat(github): add repo_enricher using REST API metadata + readme" +``` + +--- + +### Task 10: Implement `src/sections/github/section.py` + +Glue: scrape → cleanup history → filter → touch new → save history → enrich → LLM summarize. + +**Files:** +- Create: `src/sections/github/section.py` +- Test: `tests/pytest/test_sections_github_section.py` + +- [ ] **Step 1: Write the failing test** + +Create `tests/pytest/test_sections_github_section.py`: + +```python +"""测试 GitHub 板块编排:抓取 → history 过滤 → enrich → LLM 总结""" + +import sys +from datetime import date +from pathlib import Path +from unittest.mock import patch, AsyncMock + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from src.sections.github.section import run_github_section + + +def _cfg(history_file: str, max_deep_dive: int = 10) -> dict: + return { + "filter": {"keep_days": 7}, + "sections": { + "github_trending": { + "enabled": True, + "max_items": 3, + "max_deep_dive": max_deep_dive, + "readme_max_chars": 3000, + "history_file": history_file, + "request_timeout": 10, + "tokenName": "GITHUB_TOKEN", + } + }, + "llm": { + "model": "x", + "baseUrl": "http://x", + "apiKeyName": "DEEPSEEK_API_KEY", + "prompts": {"section_github": "prompts/section_github.md"}, + }, + } + + +@pytest.mark.asyncio +async def test_disabled_returns_empty(tmp_path): + cfg = _cfg(str(tmp_path / "h.json")) + cfg["sections"]["github_trending"]["enabled"] = False + md, err = await run_github_section(cfg, now=None) + assert md == "" + assert err is None + + +@pytest.mark.asyncio +async def test_no_candidates_after_history_returns_empty(tmp_path): + history_path = tmp_path / "h.json" + # 预置 history,使得所有今日 scrape 出来的 repo 都已存在 + history_path.write_text( + '{"repos": {"https://github.com/a/b": "2026-05-16"}, "updated_at": "x"}', + encoding="utf-8", + ) + cfg = _cfg(str(history_path)) + + with patch( + "src.sections.github.section.fetch_trending_page", new=AsyncMock(return_value="") + ), patch( + "src.sections.github.section.parse_trending_html", + return_value=[{"url": "https://github.com/a/b", "full_name": "a/b"}], + ): + md, err = await run_github_section(cfg, now=None) + + assert md == "" + assert err is None + + +@pytest.mark.asyncio +async def test_happy_path_enriches_and_summarizes(tmp_path): + history_path = tmp_path / "h.json" + cfg = _cfg(str(history_path), max_deep_dive=10) + + repos = [ + { + "url": "https://github.com/o1/r1", + "full_name": "o1/r1", + "description": "d1", + "language": "Python", + "stars_today": 100, + "stars_total": 1000, + } + ] + enriched = [{**repos[0], "topics": ["llm"], "license": "MIT", "pushed_at": "p", "readme_excerpt": "rm"}] + + with patch( + "src.sections.github.section.fetch_trending_page", new=AsyncMock(return_value="") + ), patch( + "src.sections.github.section.parse_trending_html", return_value=repos + ), patch( + "src.sections.github.section.enrich_repos", + new=AsyncMock(return_value=(enriched, [])), + ), patch( + "src.sections.github.section.summarize_github_trending", + new=AsyncMock(return_value=("## GH section md", None)), + ): + md, err = await run_github_section(cfg, now=None) + + assert md == "## GH section md" + assert err is None + # history 应已写入今日 scrape 出的 URL + import json as _j + saved = _j.loads(history_path.read_text(encoding="utf-8")) + assert "https://github.com/o1/r1" in saved["repos"] + + +@pytest.mark.asyncio +async def test_truncates_candidates_to_max_deep_dive(tmp_path): + cfg = _cfg(str(tmp_path / "h.json"), max_deep_dive=2) + repos = [ + {"url": f"https://github.com/o/r{i}", "full_name": f"o/r{i}"} for i in range(5) + ] + captured = {} + + async def fake_enrich(candidates, **kwargs): + captured["count"] = len(candidates) + return [], [] + + with patch( + "src.sections.github.section.fetch_trending_page", new=AsyncMock(return_value="") + ), patch( + "src.sections.github.section.parse_trending_html", return_value=repos + ), patch( + "src.sections.github.section.enrich_repos", new=AsyncMock(side_effect=fake_enrich) + ): + await run_github_section(cfg, now=None) + + assert captured["count"] == 2 + + +@pytest.mark.asyncio +async def test_scrape_failure_returns_error(tmp_path): + cfg = _cfg(str(tmp_path / "h.json")) + with patch( + "src.sections.github.section.fetch_trending_page", + new=AsyncMock(side_effect=RuntimeError("HTTP 500")), + ): + md, err = await run_github_section(cfg, now=None) + assert md == "" + assert "HTTP 500" in err +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/pytest/test_sections_github_section.py -v` +Expected: ModuleNotFoundError. + +- [ ] **Step 3: Implement section orchestrator** + +Create `src/sections/github/section.py`: + +```python +"""GitHub Trending 板块入口。 + +流程:trending 抓取 → history 过滤 → 候选写回 history → deep-dive → LLM 总结 +""" + +from datetime import datetime +from typing import Dict, Optional, Tuple + +from src.config import get_timezone +from src.sections.github.repo_enricher import enrich_repos +from src.sections.github.trending_scraper import ( + fetch_trending_page, + parse_trending_html, +) +from src.storage import load_trending_history + + +async def run_github_section( + config: Dict, now: Optional[datetime] = None +) -> Tuple[str, Optional[str]]: + cfg = config.get("sections", {}).get("github_trending", {}) + if not cfg.get("enabled", False): + return "", None + + # 延迟 import 避免循环 + from src.llm import summarize_github_trending + + today = (now or datetime.now(get_timezone())).date() + keep_days = config["filter"]["keep_days"] + timeout = cfg.get("request_timeout", 10) + max_deep_dive = cfg.get("max_deep_dive", 10) + readme_max_chars = cfg.get("readme_max_chars", 3000) + history_path = cfg.get("history_file", "news-data/trending-history.json") + token_env = cfg.get("tokenName", "GITHUB_TOKEN") + + # 1. 抓取 + try: + html = await fetch_trending_page(timeout=timeout) + except Exception as e: + return "", f"GH 抓取失败: {e}" + + all_repos = parse_trending_html(html) + if not all_repos: + return "", None + + # 2. history 加载 + 清理 + history = load_trending_history(history_path) + history.cleanup(today=today, keep_days=keep_days) + + # 3. 候选筛选 + candidates = [] + for repo in all_repos: + if repo["url"] in history: + history.touch(repo["url"], today) + else: + candidates.append(repo) + + # 4. 候选写回 history + 持久化 + for repo in candidates: + history.touch(repo["url"], today) + history.save() + + if not candidates: + return "", None + if len(candidates) > max_deep_dive: + candidates = candidates[:max_deep_dive] + + # 5. 并发 enrich + enriched, enrich_errors = await enrich_repos( + candidates, + token_env=token_env, + readme_max_chars=readme_max_chars, + timeout=timeout, + ) + for e in enrich_errors: + print(f"⚠️ GH enrich: {e}") + if not enriched: + return "", None + + # 6. LLM 总结 + md, err = await summarize_github_trending(enriched, config["llm"]) + if err: + return "", f"summarize_github_trending: {err}" + return md or "", None +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/pytest/test_sections_github_section.py -v` +Expected: 5 tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add src/sections/github/section.py tests/pytest/test_sections_github_section.py +git commit -m "feat(github): orchestrate trending → history → enrich → LLM" +``` + +--- + +### Task 11: Add `summarize_github_trending` to `src/llm.py` + `prompts/section_github.md` + +**Files:** +- Modify: `src/llm.py` (append new async function) +- Create: `prompts/section_github.md` +- Test: `tests/pytest/test_sections_github_section.py` (already mocks the function; add direct test) + +- [ ] **Step 1: Write the failing test** + +Create `tests/pytest/test_llm_extra_sections.py`: + +```python +"""测试新增 LLM 函数 (summarize_github_trending 等)""" + +import sys +from pathlib import Path +from unittest.mock import patch, AsyncMock + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src")) + +from llm import summarize_github_trending + + +@pytest.mark.asyncio +async def test_summarize_github_trending_happy_path(tmp_path): + prompt_path = tmp_path / "section_github.md" + prompt_path.write_text("Repos: {repos_json}\nmax_items={max_items}", encoding="utf-8") + + config = { + "model": "x", + "baseUrl": "http://x", + "apiKeyName": "DEEPSEEK_API_KEY", + "prompts": {"section_github": str(prompt_path)}, + "sections": {"github_trending": {"max_items": 3}}, + } + enriched = [{"full_name": "o/r", "readme_excerpt": "rm"}] + + with patch("llm.call_llm", new=AsyncMock(return_value="## md")): + md, err = await summarize_github_trending(enriched, config) + + assert md == "## md" + assert err is None + + +@pytest.mark.asyncio +async def test_summarize_github_trending_llm_failure_returns_error(tmp_path): + prompt_path = tmp_path / "section_github.md" + prompt_path.write_text("x {repos_json} {max_items}", encoding="utf-8") + config = { + "model": "x", + "baseUrl": "http://x", + "apiKeyName": "DEEPSEEK_API_KEY", + "prompts": {"section_github": str(prompt_path)}, + "sections": {"github_trending": {"max_items": 3}}, + } + with patch("llm.call_llm", new=AsyncMock(side_effect=RuntimeError("boom"))): + md, err = await summarize_github_trending([{"full_name": "o/r"}], config) + assert md == "" + assert "boom" in err +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/pytest/test_llm_extra_sections.py::test_summarize_github_trending_happy_path -v` +Expected: ImportError. + +- [ ] **Step 3: Implement function + prompt** + +Append to `src/llm.py`: + +```python +async def summarize_github_trending( + enriched_repos: List[Dict], config: Dict +) -> Tuple[str, Optional[str]]: + """GH 板块总结:从 enriched 候选中选 1-max_items + 写 markdown。不传历史上下文。""" + prompt_path = config.get("prompts", {}).get( + "section_github", "prompts/section_github.md" + ) + max_items = ( + config.get("sections", {}).get("github_trending", {}).get("max_items", 3) + ) + prompt = load_prompt( + prompt_path, + repos_json=json.dumps(enriched_repos, ensure_ascii=False, indent=2), + max_items=max_items, + ) + try: + return await call_llm(prompt, config), None + except Exception as e: + msg = f"summarize_github_trending 失败: {e}" + print(f"⚠️ {msg}") + return "", msg +``` + +Create `prompts/section_github.md`: + +```markdown +你是开源情报分析师。从下列 GitHub Trending 候选项目中,挑出 **1-{max_items} 个**最值得关注的 AI 相关项目并行文。 + +## 关注领域(正面列表) +- **AI Agent**:智能体架构、工具链、多智能体、自主规划、Agent 框架 +- **AI 模型**:训练、推理、微调、量化部署、模型服务、语音/多模态/视觉模型 +- **AI 基础设施**:GPU 调度、芯片硬件、数据中心、推理优化、分布式训练、向量数据库、RAG 框架 +- **大厂/前沿动态**:Apple、Google、Meta、OpenAI、Anthropic、Microsoft、xAI 等公司的官方动作与战略 +- **AI 集成的开发者工具**:API 网关、自动化脚本、低代码平台等明确与 AI 协同的工具 +- **创新性开源产品**:日增长显著且有清晰用户价值 + +## 排除(负面列表) +- 嵌入式开发(Arduino、ESP32、树莓派、单片机) +- 底层系统编程(内存分配器、编译器、链接器,与 AI 工作负载无关时) +- 通用开发工具(命名规范、代码风格、纯前端模板、UI 组件库、管理后台模板、静态网站主题) +- 学习资源(纯教程仓库、面试题合集、Roadmap,除非含实用代码的深度技术指南) +- 配置文件集合(Dotfiles、配置模板) +- 与 AI/科技无关的内容(电子书、资源搬运、刷榜项目) +- 纯娱乐/高风险误用(deepfake 等无明确基础设施价值) + +## 输入数据 +JSON 数组,字段:url / full_name / description / language / stars_today / stars_total / topics / license / pushed_at / readme_excerpt + +```json +{repos_json} +``` + +## 选项规则 +- 优先信号:`stars_today` 高 + `topics` 含 AI 标签(agent/llm/rag/inference/training 等) + readme 描述明确 +- 跳过:`archived=true`(若漏过)、纯 awesome-list、个人 dotfiles +- 一句话价值定位需点明"解决什么问题",避免营销语("震撼""炸裂""革命性"等禁用) + +## 输出格式(严格 Markdown,不要任何引导语) + +```markdown +## ⭐ GitHub 趋势 + +- **owner/repo** ⭐{{stars_today}} — 一句话价值定位 [link]({{url}}) +- ... +``` + +若候选中没有任何符合关注领域的项目,直接输出 `## ⭐ GitHub 趋势\n\n- 今日无显著 AI 相关趋势`,不要硬编。 +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/pytest/test_llm_extra_sections.py::test_summarize_github_trending_happy_path tests/pytest/test_llm_extra_sections.py::test_summarize_github_trending_llm_failure_returns_error -v` +Expected: 2 pass. + +- [ ] **Step 5: Commit** + +```bash +git add src/llm.py prompts/section_github.md tests/pytest/test_llm_extra_sections.py +git commit -m "feat(llm): add summarize_github_trending with prompt" +``` + +--- + +## Phase 4: Hacker News Module + +### Task 12: Save HN frontpage HTML fixture + +**Files:** +- Create: `tests/pytest/fixtures/hn_frontpage.html` + +- [ ] **Step 1: Download** + +```bash +curl -fsSL -A "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36" \ + "https://news.ycombinator.com/news" \ + -o tests/pytest/fixtures/hn_frontpage.html +``` + +- [ ] **Step 2: Verify** + +Run: `grep -c 'class="athing"' tests/pytest/fixtures/hn_frontpage.html` +Expected: ≥ 25 (usually 30). + +- [ ] **Step 3: Commit** + +```bash +git add tests/pytest/fixtures/hn_frontpage.html +git commit -m "test(hn): snapshot HN frontpage fixture" +``` + +--- + +### Task 13: Implement `frontpage_scraper.py` + +**Files:** +- Create: `src/sections/hackernews/__init__.py` (empty) +- Create: `src/sections/hackernews/frontpage_scraper.py` +- Test: `tests/pytest/test_sections_hackernews_scraper.py` + +- [ ] **Step 1: Write the failing test** + +Create `tests/pytest/test_sections_hackernews_scraper.py`: + +```python +"""测试 HN 首页 HTML 解析""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from src.sections.hackernews.frontpage_scraper import parse_frontpage_html + + +def test_parse_frontpage_returns_stories(): + fixture = ( + Path(__file__).parent / "fixtures" / "hn_frontpage.html" + ).read_text(encoding="utf-8") + + stories = parse_frontpage_html(fixture) + + assert len(stories) >= 25 + s = stories[0] + assert s["id"] + assert s["title"] + assert s["url"] + assert isinstance(s["points"], int) + assert isinstance(s["comments"], int) + assert s["comments_url"].startswith("https://news.ycombinator.com/item?id=") + + +def test_parse_frontpage_detects_show_hn_internal_url(): + # 构造一个最小内部链接故事 (Ask HN / Show HN) + html = """ + + + + + + + +
+ + Ask HN: what's new? + +
+ + 50 points + by alice + 2 hours ago + | 5 comments + +
+ """ + stories = parse_frontpage_html(html) + assert len(stories) == 1 + s = stories[0] + assert s["id"] == "111" + assert s["url"].startswith("https://news.ycombinator.com/item?id=") + assert s["site"] == "" + assert s["points"] == 50 + assert s["comments"] == 5 +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/pytest/test_sections_hackernews_scraper.py -v` +Expected: ModuleNotFoundError. + +- [ ] **Step 3: Implement** + +Create `src/sections/hackernews/__init__.py` (empty). + +Create `src/sections/hackernews/frontpage_scraper.py`: + +```python +"""HN 首页 HTML 抓取与解析。 + +数据源: https://news.ycombinator.com/news (30 条) +""" + +import re +from typing import Dict, List + +import aiohttp +from bs4 import BeautifulSoup + +FRONTPAGE_URL = "https://news.ycombinator.com/news" +USER_AGENT = ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" +) +_NUM_RE = re.compile(r"\d+") + + +def _first_int(text: str) -> int: + m = _NUM_RE.search(text or "") + return int(m.group(0)) if m else 0 + + +async def fetch_frontpage(timeout: int = 10) -> str: + async with aiohttp.ClientSession(headers={"User-Agent": USER_AGENT}) as session: + async with session.get( + FRONTPAGE_URL, timeout=aiohttp.ClientTimeout(total=timeout) + ) as resp: + if resp.status != 200: + raise RuntimeError(f"HN frontpage 返回 {resp.status}") + return await resp.text() + + +def parse_frontpage_html(html: str) -> List[Dict]: + """解析首页 HTML,返回 [{id, title, url, site, points, comments, comments_url}]""" + if not html: + return [] + soup = BeautifulSoup(html, "html.parser") + stories: List[Dict] = [] + + for athing in soup.select("tr.athing"): + item_id = athing.get("id") + if not item_id: + continue + + title_a = athing.select_one("span.titleline > a") + if not title_a: + continue + title = title_a.get_text(strip=True) + href = title_a.get("href", "") + # 内部链接(Ask HN / Show HN) + if href.startswith("item?id="): + url = f"https://news.ycombinator.com/{href}" + site = "" + else: + url = href + site_tag = athing.select_one("span.sitestr") + site = site_tag.get_text(strip=True) if site_tag else "" + + # 同 id 的下一个 tr 是 subtext + sub_tr = athing.find_next_sibling("tr") + points = 0 + comments = 0 + comments_url = f"https://news.ycombinator.com/item?id={item_id}" + if sub_tr: + score = sub_tr.select_one("span.score") + if score: + points = _first_int(score.get_text(strip=True)) + # 最后一个 a[href^="item?id="] 是评论链接 + comment_a = None + for a in sub_tr.find_all("a", href=re.compile(r"^item\?id=")): + comment_a = a + if comment_a: + comments = _first_int(comment_a.get_text(strip=True)) + comments_url = f"https://news.ycombinator.com/{comment_a['href']}" + + stories.append( + { + "id": item_id, + "title": title, + "url": url, + "site": site, + "points": points, + "comments": comments, + "comments_url": comments_url, + } + ) + + return stories +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/pytest/test_sections_hackernews_scraper.py -v` +Expected: 2 pass. + +- [ ] **Step 5: Commit** + +```bash +git add src/sections/hackernews/__init__.py src/sections/hackernews/frontpage_scraper.py tests/pytest/test_sections_hackernews_scraper.py +git commit -m "feat(hn): add frontpage scraper with fixture test" +``` + +--- + +### Task 14: Implement `item_enricher.py` + +Algolia for comments + post text; `html_to_markdown` for external link content. + +**Files:** +- Create: `src/sections/hackernews/item_enricher.py` +- Test: `tests/pytest/test_sections_hackernews_enricher.py` + +- [ ] **Step 1: Write the failing test** + +Create `tests/pytest/test_sections_hackernews_enricher.py`: + +```python +"""测试 HN enrich(Algolia + 外链正文)""" + +import sys +from pathlib import Path +from unittest.mock import patch, AsyncMock, MagicMock + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from src.sections.hackernews.item_enricher import enrich_story + + +@pytest.mark.asyncio +async def test_enrich_external_link_story(): + story = { + "id": "111", + "title": "T", + "url": "https://example.com/post", + "site": "example.com", + "points": 100, + "comments": 5, + "comments_url": "https://news.ycombinator.com/item?id=111", + } + algolia_payload = { + "text": None, + "children": [ + {"text": "

comment one

"}, + {"text": "

comment two

"}, + {"text": "

comment three

"}, + {"text": "

comment four

"}, + ], + } + + async def fake_algolia(session, item_id, **kw): + return algolia_payload + + async def fake_link(session, url, **kw): + return "

link body

" + + with patch( + "src.sections.hackernews.item_enricher._fetch_algolia_item", + new=AsyncMock(side_effect=fake_algolia), + ), patch( + "src.sections.hackernews.item_enricher._fetch_url_html", + new=AsyncMock(side_effect=fake_link), + ): + enriched = await enrich_story( + session=MagicMock(), + story=story, + top_comments=3, + comment_max_chars=500, + link_content_max_chars=3000, + algolia_base="https://hn.algolia.com/api/v1", + timeout=10, + ) + + assert len(enriched["top_comments"]) == 3 + assert "comment one" in enriched["top_comments"][0] + assert "link body" in enriched["link_content"] + + +@pytest.mark.asyncio +async def test_enrich_show_hn_uses_root_text_no_external_fetch(): + story = { + "id": "222", + "title": "Show HN: T", + "url": "https://news.ycombinator.com/item?id=222", + "site": "", + "points": 200, + "comments": 10, + "comments_url": "https://news.ycombinator.com/item?id=222", + } + algolia_payload = { + "text": "

post body text

", + "children": [{"text": "

c1

"}], + } + link_calls = [] + + async def fake_algolia(session, item_id, **kw): + return algolia_payload + + async def fake_link(session, url, **kw): + link_calls.append(url) + return "should not be called" + + with patch( + "src.sections.hackernews.item_enricher._fetch_algolia_item", + new=AsyncMock(side_effect=fake_algolia), + ), patch( + "src.sections.hackernews.item_enricher._fetch_url_html", + new=AsyncMock(side_effect=fake_link), + ): + enriched = await enrich_story( + session=MagicMock(), + story=story, + top_comments=3, + comment_max_chars=500, + link_content_max_chars=3000, + algolia_base="https://hn.algolia.com/api/v1", + timeout=10, + ) + + assert link_calls == [] + assert "post body text" in enriched["link_content"] + + +@pytest.mark.asyncio +async def test_enrich_truncates_comments_and_link(): + story = { + "id": "333", + "title": "T", + "url": "https://example.com/a", + "site": "example.com", + "points": 100, + "comments": 2, + "comments_url": "x", + } + long_comment = "

" + ("y" * 2000) + "

" + long_link = "" + ("z" * 5000) + "" + + async def fake_algolia(session, item_id, **kw): + return {"text": None, "children": [{"text": long_comment}]} + + async def fake_link(session, url, **kw): + return long_link + + with patch( + "src.sections.hackernews.item_enricher._fetch_algolia_item", + new=AsyncMock(side_effect=fake_algolia), + ), patch( + "src.sections.hackernews.item_enricher._fetch_url_html", + new=AsyncMock(side_effect=fake_link), + ): + enriched = await enrich_story( + session=MagicMock(), + story=story, + top_comments=3, + comment_max_chars=100, + link_content_max_chars=200, + algolia_base="https://hn.algolia.com/api/v1", + timeout=10, + ) + + assert len(enriched["top_comments"][0]) <= 100 + assert len(enriched["link_content"]) <= 200 + + +@pytest.mark.asyncio +async def test_enrich_failure_returns_partial(): + story = { + "id": "444", + "title": "T", + "url": "https://example.com/x", + "site": "example.com", + "points": 100, + "comments": 2, + "comments_url": "x", + } + + async def fake_algolia(session, item_id, **kw): + raise RuntimeError("algolia down") + + async def fake_link(session, url, **kw): + return "ok" + + with patch( + "src.sections.hackernews.item_enricher._fetch_algolia_item", + new=AsyncMock(side_effect=fake_algolia), + ), patch( + "src.sections.hackernews.item_enricher._fetch_url_html", + new=AsyncMock(side_effect=fake_link), + ): + enriched = await enrich_story( + session=MagicMock(), + story=story, + top_comments=3, + comment_max_chars=500, + link_content_max_chars=3000, + algolia_base="https://hn.algolia.com/api/v1", + timeout=10, + ) + + # 算法失败 → top_comments 留空,link_content 仍获取 + assert enriched["top_comments"] == [] + assert "ok" in enriched["link_content"] +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/pytest/test_sections_hackernews_enricher.py -v` +Expected: ModuleNotFoundError. + +- [ ] **Step 3: Implement** + +Create `src/sections/hackernews/item_enricher.py`: + +```python +"""HN 单 story enrich:Algolia 评论 + 外链正文。 + +Algolia API: GET /api/v1/items/{id} +- root.text 是 Show HN / Ask HN 的 post 正文 +- root.children[] 是顶层评论(按 HN ranking 排序) +""" + +import asyncio +from typing import Dict, List, Optional, Tuple + +import aiohttp + +from src.processor import html_to_markdown + +USER_AGENT = ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" +) + + +def _is_internal_hn_url(url: str) -> bool: + return url.startswith("https://news.ycombinator.com/item?id=") + + +async def _fetch_algolia_item( + session: aiohttp.ClientSession, item_id: str, algolia_base: str, timeout: int +) -> Dict: + url = f"{algolia_base}/items/{item_id}" + async with session.get( + url, timeout=aiohttp.ClientTimeout(total=timeout) + ) as resp: + if resp.status != 200: + raise RuntimeError(f"Algolia /items/{item_id} 返回 {resp.status}") + return await resp.json() + + +async def _fetch_url_html( + session: aiohttp.ClientSession, url: str, timeout: int +) -> str: + async with session.get( + url, timeout=aiohttp.ClientTimeout(total=timeout) + ) as resp: + if resp.status != 200: + raise RuntimeError(f"外链 {url} 返回 {resp.status}") + return await resp.text() + + +async def enrich_story( + session: aiohttp.ClientSession, + story: Dict, + top_comments: int, + comment_max_chars: int, + link_content_max_chars: int, + algolia_base: str, + timeout: int, +) -> Dict: + """对单 story enrich。任一子任务失败 → 对应字段留空,不抛。""" + item_id = story["id"] + is_internal = _is_internal_hn_url(story["url"]) + + # 并发:Algolia + 外链(仅外链类) + tasks = [_fetch_algolia_item(session, item_id, algolia_base, timeout)] + if not is_internal: + tasks.append(_fetch_url_html(session, story["url"], timeout)) + + results = await asyncio.gather(*tasks, return_exceptions=True) + algolia_result = results[0] + external_html_result = results[1] if not is_internal else None + + # 评论解析 + comments_list: List[str] = [] + post_text = "" + if not isinstance(algolia_result, Exception) and algolia_result: + post_text = algolia_result.get("text") or "" + children = algolia_result.get("children") or [] + for child in children[:top_comments]: + raw = (child or {}).get("text") or "" + if not raw: + continue + md = html_to_markdown(raw) + comments_list.append(md[:comment_max_chars]) + + # link_content + link_content = "" + if is_internal: + # Show HN / Ask HN:post 自身正文 + if post_text: + link_content = html_to_markdown(post_text)[:link_content_max_chars] + else: + if not isinstance(external_html_result, Exception) and external_html_result: + link_content = html_to_markdown( + external_html_result, base_url=story["url"] + )[:link_content_max_chars] + + return { + **story, + "link_content": link_content, + "top_comments": comments_list, + } + + +async def enrich_stories( + stories: List[Dict], + top_comments: int, + comment_max_chars: int, + link_content_max_chars: int, + algolia_base: str = "https://hn.algolia.com/api/v1", + timeout: int = 10, +) -> Tuple[List[Dict], List[str]]: + """并发 enrich 多个 stories。""" + errors: List[str] = [] + async with aiohttp.ClientSession(headers={"User-Agent": USER_AGENT}) as session: + results = await asyncio.gather( + *[ + enrich_story( + session, + s, + top_comments, + comment_max_chars, + link_content_max_chars, + algolia_base, + timeout, + ) + for s in stories + ], + return_exceptions=True, + ) + enriched: List[Dict] = [] + for r, src in zip(results, stories): + if isinstance(r, Exception): + errors.append(f"enrich story {src['id']} 失败: {r}") + else: + enriched.append(r) + return enriched, errors +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/pytest/test_sections_hackernews_enricher.py -v` +Expected: 4 pass. + +- [ ] **Step 5: Commit** + +```bash +git add src/sections/hackernews/item_enricher.py tests/pytest/test_sections_hackernews_enricher.py +git commit -m "feat(hn): add item enricher using Algolia + html_to_markdown" +``` + +--- + +### Task 15: Implement `src/sections/hackernews/section.py` + +Glue: scrape → light LLM select → enrich → LLM summarize. + +**Files:** +- Create: `src/sections/hackernews/section.py` +- Test: `tests/pytest/test_sections_hackernews_section.py` + +- [ ] **Step 1: Write the failing test** + +Create `tests/pytest/test_sections_hackernews_section.py`: + +```python +import sys +from pathlib import Path +from unittest.mock import patch, AsyncMock + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from src.sections.hackernews.section import run_hackernews_section + + +def _cfg() -> dict: + return { + "filter": {"keep_days": 7}, + "sections": { + "hackernews": { + "enabled": True, + "select_k": 1, + "top_comments": 20, + "comment_max_chars": 500, + "link_content_max_chars": 3000, + "request_timeout": 10, + "algolia_base": "https://hn.algolia.com/api/v1", + } + }, + "llm": { + "model": "x", + "baseUrl": "http://x", + "apiKeyName": "DEEPSEEK_API_KEY", + "prompts": { + "section_hackernews_select": "prompts/section_hackernews_select.md", + "section_hackernews": "prompts/section_hackernews.md", + }, + }, + } + + +@pytest.mark.asyncio +async def test_disabled_returns_empty(): + cfg = _cfg() + cfg["sections"]["hackernews"]["enabled"] = False + md, err = await run_hackernews_section(cfg, now=None) + assert md == "" + assert err is None + + +@pytest.mark.asyncio +async def test_select_empty_returns_silent(): + cfg = _cfg() + with patch( + "src.sections.hackernews.section.fetch_frontpage", new=AsyncMock(return_value="") + ), patch( + "src.sections.hackernews.section.parse_frontpage_html", + return_value=[{"id": "1", "title": "x"}], + ), patch( + "src.sections.hackernews.section.select_ai_related_hn", + new=AsyncMock(return_value=([], None)), + ): + md, err = await run_hackernews_section(cfg, now=None) + assert md == "" + assert err is None + + +@pytest.mark.asyncio +async def test_happy_path(): + cfg = _cfg() + front = [{"id": "1", "title": "AI thing", "url": "https://e.com/a", "site": "e.com", "points": 100, "comments": 5, "comments_url": "x"}] + enriched = [{**front[0], "link_content": "body", "top_comments": ["c1"]}] + + with patch( + "src.sections.hackernews.section.fetch_frontpage", new=AsyncMock(return_value="") + ), patch( + "src.sections.hackernews.section.parse_frontpage_html", return_value=front + ), patch( + "src.sections.hackernews.section.select_ai_related_hn", + new=AsyncMock(return_value=(["1"], None)), + ), patch( + "src.sections.hackernews.section.enrich_stories", + new=AsyncMock(return_value=(enriched, [])), + ), patch( + "src.sections.hackernews.section.summarize_hackernews", + new=AsyncMock(return_value=("## HN md", None)), + ): + md, err = await run_hackernews_section(cfg, now=None) + assert md == "## HN md" + assert err is None + + +@pytest.mark.asyncio +async def test_scrape_failure_returns_error(): + cfg = _cfg() + with patch( + "src.sections.hackernews.section.fetch_frontpage", + new=AsyncMock(side_effect=RuntimeError("net")), + ): + md, err = await run_hackernews_section(cfg, now=None) + assert md == "" + assert "net" in err +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/pytest/test_sections_hackernews_section.py -v` +Expected: ModuleNotFoundError. + +- [ ] **Step 3: Implement section** + +Create `src/sections/hackernews/section.py`: + +```python +"""HN 板块入口。流程:首页 → 轻 LLM 选 K → enrich → 最终 LLM 行文""" + +from datetime import datetime +from typing import Dict, Optional, Tuple + +from src.sections.hackernews.frontpage_scraper import ( + fetch_frontpage, + parse_frontpage_html, +) +from src.sections.hackernews.item_enricher import enrich_stories + + +async def run_hackernews_section( + config: Dict, now: Optional[datetime] = None +) -> Tuple[str, Optional[str]]: + cfg = config.get("sections", {}).get("hackernews", {}) + if not cfg.get("enabled", False): + return "", None + + from src.llm import select_ai_related_hn, summarize_hackernews + + timeout = cfg.get("request_timeout", 10) + select_k = cfg.get("select_k", 1) + top_comments = cfg.get("top_comments", 20) + comment_max_chars = cfg.get("comment_max_chars", 500) + link_content_max_chars = cfg.get("link_content_max_chars", 3000) + algolia_base = cfg.get("algolia_base", "https://hn.algolia.com/api/v1") + + # 1. 抓首页 + try: + html = await fetch_frontpage(timeout=timeout) + except Exception as e: + return "", f"HN 首页抓取失败: {e}" + + front = parse_frontpage_html(html) + if not front: + return "", None + + # 2. 轻 LLM 初筛 + selected_ids, select_err = await select_ai_related_hn(front, k=select_k, config=config["llm"]) + if select_err: + return "", f"select_ai_related_hn: {select_err}" + if not selected_ids: + return "", None + + selected = [s for s in front if s["id"] in set(selected_ids)] + if not selected: + return "", None + + # 3. enrich + enriched, enrich_errors = await enrich_stories( + selected, + top_comments=top_comments, + comment_max_chars=comment_max_chars, + link_content_max_chars=link_content_max_chars, + algolia_base=algolia_base, + timeout=timeout, + ) + for e in enrich_errors: + print(f"⚠️ HN enrich: {e}") + if not enriched: + return "", None + + # 4. LLM 总结 + md, err = await summarize_hackernews(enriched, config["llm"]) + if err: + return "", f"summarize_hackernews: {err}" + return md or "", None +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/pytest/test_sections_hackernews_section.py -v` +Expected: 4 pass. + +- [ ] **Step 5: Commit** + +```bash +git add src/sections/hackernews/section.py tests/pytest/test_sections_hackernews_section.py +git commit -m "feat(hn): orchestrate frontpage → select → enrich → LLM" +``` + +--- + +### Task 16: Add `select_ai_related_hn` + `summarize_hackernews` to `src/llm.py` + 2 prompt files + +**Files:** +- Modify: `src/llm.py` +- Create: `prompts/section_hackernews_select.md` +- Create: `prompts/section_hackernews.md` +- Test: `tests/pytest/test_llm_extra_sections.py` (append) + +- [ ] **Step 1: Write the failing test** + +Append to `tests/pytest/test_llm_extra_sections.py`: + +```python +from llm import select_ai_related_hn, summarize_hackernews + + +@pytest.mark.asyncio +async def test_select_ai_related_hn_parses_id_array(tmp_path): + prompt_path = tmp_path / "select.md" + prompt_path.write_text("k={k} candidates={candidates_json}", encoding="utf-8") + config = { + "model": "x", + "baseUrl": "http://x", + "apiKeyName": "DEEPSEEK_API_KEY", + "prompts": {"section_hackernews_select": str(prompt_path)}, + } + with patch("llm.call_llm", new=AsyncMock(return_value='["111", "222"]')): + ids, err = await select_ai_related_hn( + [{"id": "111"}, {"id": "222"}, {"id": "333"}], k=2, config=config + ) + assert ids == ["111", "222"] + assert err is None + + +@pytest.mark.asyncio +async def test_select_ai_related_hn_empty_array(tmp_path): + prompt_path = tmp_path / "select.md" + prompt_path.write_text("{k}{candidates_json}", encoding="utf-8") + config = { + "model": "x", + "baseUrl": "http://x", + "apiKeyName": "DEEPSEEK_API_KEY", + "prompts": {"section_hackernews_select": str(prompt_path)}, + } + with patch("llm.call_llm", new=AsyncMock(return_value="[]")): + ids, err = await select_ai_related_hn([{"id": "1"}], k=1, config=config) + assert ids == [] + assert err is None + + +@pytest.mark.asyncio +async def test_summarize_hackernews_happy(tmp_path): + prompt_path = tmp_path / "hn.md" + prompt_path.write_text("{stories_json}", encoding="utf-8") + config = { + "model": "x", + "baseUrl": "http://x", + "apiKeyName": "DEEPSEEK_API_KEY", + "prompts": {"section_hackernews": str(prompt_path)}, + } + with patch("llm.call_llm", new=AsyncMock(return_value="## HN summary")): + md, err = await summarize_hackernews( + [{"id": "1", "title": "t", "link_content": "x", "top_comments": []}], config + ) + assert md == "## HN summary" + assert err is None +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/pytest/test_llm_extra_sections.py -v -k "select_ai_related_hn or summarize_hackernews"` +Expected: ImportError. + +- [ ] **Step 3: Implement functions + prompts** + +Append to `src/llm.py`: + +```python +async def select_ai_related_hn( + candidates: List[Dict], k: int, config: Dict +) -> Tuple[List[str], Optional[str]]: + """轻 LLM:从 HN 首页候选元数据中挑 k 个 AI 相关 id。 + + 输入候选只含 id/title/site/points/comments 字段(不含正文)。 + """ + prompt_path = config.get("prompts", {}).get( + "section_hackernews_select", "prompts/section_hackernews_select.md" + ) + slim = [ + { + "id": c.get("id"), + "title": c.get("title", ""), + "site": c.get("site", ""), + "points": c.get("points", 0), + "comments": c.get("comments", 0), + } + for c in candidates + ] + prompt = load_prompt( + prompt_path, + k=k, + candidates_json=json.dumps(slim, ensure_ascii=False, indent=2), + ) + try: + response = await call_llm(prompt, config) + except Exception as e: + msg = f"select_ai_related_hn 失败: {e}" + print(f"⚠️ {msg}") + return [], msg + + try: + ids = _parse_llm_json_response(response) + except ValueError as e: + msg = f"select_ai_related_hn 解析失败: {e}" + print(f"⚠️ {msg}") + return [], msg + + if not isinstance(ids, list): + return [], "select_ai_related_hn 返回非数组" + return [str(x) for x in ids][:k], None + + +async def summarize_hackernews( + enriched_stories: List[Dict], config: Dict +) -> Tuple[str, Optional[str]]: + """对输入的 K 个 enriched stories 行文。不传历史上下文。""" + prompt_path = config.get("prompts", {}).get( + "section_hackernews", "prompts/section_hackernews.md" + ) + prompt = load_prompt( + prompt_path, + stories_json=json.dumps(enriched_stories, ensure_ascii=False, indent=2), + ) + try: + return await call_llm(prompt, config), None + except Exception as e: + msg = f"summarize_hackernews 失败: {e}" + print(f"⚠️ {msg}") + return "", msg +``` + +Create `prompts/section_hackernews_select.md`: + +```markdown +你是 HN 早间选题人。从下列 30 条 HN 首页元数据中,挑出 **{k}** 个最符合关注领域的 story id。 + +## 关注领域(正面列表) +- AI Agent(智能体架构、工具链、多智能体、自主规划) +- AI 模型(训练、推理、微调、应用、语音/多模态) +- AI 基础设施(芯片、硬件、数据中心、推理优化、向量数据库、RAG 框架) +- 大厂/前沿动态(Apple、Google、Meta、OpenAI、Anthropic、Microsoft、xAI) +- AI 集成的开发者工具(API 网关、自动化、低代码与 AI 协同) + +## 排除(负面列表) +- 嵌入式开发(Arduino、ESP32、树莓派、单片机) +- 底层系统编程(内存分配器、编译器、链接器,与 AI 无关时) +- 通用开发工具(命名规范、代码风格) +- 与 AI/科技无关的内容 + +## 决策原则 +- title + site 不足以确定 AI 相关时,**宁可漏选不可错选**(错选会让最终板块写出与 AI Daily 调性无关的内容) +- 若全部 30 条都不符合,返回空数组 `[]` + +## 输入 +```json +{candidates_json} +``` + +## 输出 +**只输出 JSON 数组**,如:`["12345", "67890"]` 或 `[]` +严禁输出任何解释性文字、markdown 包装、或自然语言句子。 +``` + +Create `prompts/section_hackernews.md`: + +```markdown +你是 HN 早间编辑。对输入的 enriched stories **全部行文**(不再二次挑选)。 + +## 输入(JSON 数组,字段含 link_content 与 top_comments) +```json +{stories_json} +``` + +## 内容要求(每条 story) +1. 提炼原文 `link_content` 的核心(背景 / 要点 / 结论) +2. 汇总 `top_comments` 中有价值的观点(支持 / 反对 / 补充),不是简单复述 +3. 若评论中出现明显反驳原文的观点,必须保留并标注 + +## 输出格式(严格 Markdown,不要任何引导语) + +```markdown +## 🟧 Hacker News 热议 + +### {{title}} ({{points}} pts · {{comments}} comments) + +**📌 内容总结** + +- 要点 1 +- 要点 2 +- 要点 3(可选) + +**💬 HN 讨论** + +- 观点 1(含反对/补充) +- 观点 2(可选) + +🔗 [原文]({{url}}) | [HN 讨论页]({{comments_url}}) +``` + +## 风格约束 +- 客观、犀利、克制 +- 避免与 RSS digest 句式雷同;不做行业宏大叙事 +- 禁用词汇:震撼、炸裂、革命性、现象级 +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/pytest/test_llm_extra_sections.py -v` +Expected: 5 tests pass (including the earlier GH tests). + +- [ ] **Step 5: Commit** + +```bash +git add src/llm.py prompts/section_hackernews_select.md prompts/section_hackernews.md tests/pytest/test_llm_extra_sections.py +git commit -m "feat(llm): add select_ai_related_hn and summarize_hackernews with prompts" +``` + +--- + +## Phase 5: Insights Module + +### Task 17: Implement `insights/section.py` + `generate_trend_insights` + prompt + +**Files:** +- Create: `src/sections/insights/__init__.py` (empty) +- Create: `src/sections/insights/section.py` +- Modify: `src/llm.py` +- Create: `prompts/insights.md` +- Test: `tests/pytest/test_sections_insights.py` + +- [ ] **Step 1: Write the failing test** + +Create `tests/pytest/test_sections_insights.py`: + +```python +import sys +from pathlib import Path +from unittest.mock import patch, AsyncMock + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from src.sections.insights.section import run_insights_section + + +def _cfg() -> dict: + return { + "filter": {"push_context_days": 5}, + "sections": {"insights": {"enabled": True}}, + "llm": { + "model": "x", + "baseUrl": "http://x", + "apiKeyName": "DEEPSEEK_API_KEY", + "prompts": {"insights": "prompts/insights.md"}, + }, + } + + +@pytest.mark.asyncio +async def test_disabled_returns_empty(): + cfg = _cfg() + cfg["sections"]["insights"]["enabled"] = False + md, err = await run_insights_section("rss", "gh", "hn", cfg, now=None) + assert md == "" + assert err is None + + +@pytest.mark.asyncio +async def test_marks_empty_sections_for_llm(): + cfg = _cfg() + captured = {} + + async def fake_gen(sections, recent_insights, config): + captured["sections"] = sections + return "insights md", None + + with patch( + "src.sections.insights.section.load_recent_section_titles", return_value="" + ), patch( + "src.sections.insights.section.generate_trend_insights", + new=AsyncMock(side_effect=fake_gen), + ): + md, err = await run_insights_section("", "gh md", "", cfg, now=None) + + assert md == "insights md" + assert captured["sections"]["rss"] == "(本次无内容)" + assert captured["sections"]["github"] == "gh md" + assert captured["sections"]["hackernews"] == "(本次无内容)" +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/pytest/test_sections_insights.py -v` +Expected: ModuleNotFoundError. + +- [ ] **Step 3: Implement** + +Create `src/sections/insights/__init__.py` (empty). + +Create `src/sections/insights/section.py`: + +```python +"""Insights 板块:基于 RSS/GH/HN 三段成品 + 近 N 天 insights 历史做跨板块小结""" + +from datetime import datetime +from typing import Dict, Optional, Tuple + +from src.storage import load_recent_section_titles + +EMPTY_MARKER = "(本次无内容)" + + +async def run_insights_section( + rss_md: str, + gh_md: str, + hn_md: str, + config: Dict, + now: Optional[datetime] = None, +) -> Tuple[str, Optional[str]]: + cfg = config.get("sections", {}).get("insights", {}) + if not cfg.get("enabled", False): + return "", None + + from src.llm import generate_trend_insights + + days = config["filter"].get("push_context_days", 5) + recent = load_recent_section_titles("insights", days) + + sections = { + "rss": rss_md or EMPTY_MARKER, + "github": gh_md or EMPTY_MARKER, + "hackernews": hn_md or EMPTY_MARKER, + } + + md, err = await generate_trend_insights(sections, recent, config["llm"]) + if err: + return "", f"generate_trend_insights: {err}" + return md or "", None +``` + +Append to `src/llm.py`: + +```python +async def generate_trend_insights( + sections: Dict[str, str], recent_insights: str, config: Dict +) -> Tuple[str, Optional[str]]: + """输入三段成品 + 近期 insights 标题清单,返回洞察段 markdown。""" + prompt_path = config.get("prompts", {}).get("insights", "prompts/insights.md") + prompt = load_prompt( + prompt_path, + rss=sections.get("rss", ""), + github=sections.get("github", ""), + hackernews=sections.get("hackernews", ""), + recent_insights=recent_insights or "", + ) + try: + return await call_llm(prompt, config), None + except Exception as e: + msg = f"generate_trend_insights 失败: {e}" + print(f"⚠️ {msg}") + return "", msg +``` + +Create `prompts/insights.md`: + +```markdown +你是 AI 行业观察员。基于今日三段已生成的成品做一段**跨板块趋势小结**。 + +## 今日素材(三段成品) + +### RSS 板块 +{rss} + +### GitHub 板块 +{github} + +### Hacker News 板块 +{hackernews} + +## 近 N 天 insights 段标题(仅供风格参考与防趋同,严禁措辞模仿) + +{recent_insights} + + +## 任务 +基于今日三段成品,做跨板块小结。可参考的切入角度(不必全部覆盖,按今日素材最突出的张力来组织): +- 跨板块的"交叉信号"(同一话题在 RSS / GH / HN 中同时出现) +- 与近几天对比"新升温"或"退潮"的关键词 +- "反直觉发现"(违反常识、值得停下来想一下的一条) +- 行业结构信号(资本 / 监管 / 算力 / 应用层等) + +## 输出要求 +- 直接输出 markdown 段,不带任何引导语 +- 起始用 `## 💡 今日洞察` +- 总长度 200-400 字之间 +- 不要简单复述其他板块的具体新闻;洞察的价值在"连接"而非"清单" +- 避免与 RSS digest 句式雷同;禁用宏大叙事词汇("深水区""临界点""下半场""博弈"等) +- 每条洞察须能溯源到今日素材中的具体信号,而不是凭空总结 +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/pytest/test_sections_insights.py -v` +Expected: 2 pass. + +- [ ] **Step 5: Commit** + +```bash +git add src/sections/insights/ prompts/insights.md src/llm.py tests/pytest/test_sections_insights.py +git commit -m "feat(insights): add cross-section trend summary module" +``` + +--- + +## Phase 6: push_job Orchestration + +### Task 18: Add `is_morning_push` to `src/main.py` + +**Files:** +- Modify: `src/main.py` +- Test: `tests/pytest/test_morning_detection.py` + +- [ ] **Step 1: Write the failing test** + +Create `tests/pytest/test_morning_detection.py`: + +```python +"""测试早报判定:cron + 容差""" + +import sys +from datetime import datetime, timedelta, timezone +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from src.main import is_morning_push + + +TZ = timezone(timedelta(hours=8)) + + +def _cfg(cron, tol=5): + return { + "schedule": { + "morning_cron": cron, + "morning_match_tolerance_minutes": tol, + "timezone_hours": 8, + } + } + + +def test_returns_false_when_no_morning_cron_configured(): + assert is_morning_push(datetime(2026, 5, 17, 8, 0, tzinfo=TZ), {"schedule": {}}) is False + + +def test_match_exact_time(): + cfg = _cfg("0 8 * * *") + now = datetime(2026, 5, 17, 8, 0, tzinfo=TZ) + assert is_morning_push(now, cfg) is True + + +def test_match_within_tolerance(): + cfg = _cfg("0 8 * * *", tol=5) + assert is_morning_push(datetime(2026, 5, 17, 8, 4, tzinfo=TZ), cfg) is True + assert is_morning_push(datetime(2026, 5, 17, 7, 56, tzinfo=TZ), cfg) is True + + +def test_outside_tolerance(): + cfg = _cfg("0 8 * * *", tol=5) + assert is_morning_push(datetime(2026, 5, 17, 8, 6, tzinfo=TZ), cfg) is False + assert is_morning_push(datetime(2026, 5, 17, 9, 0, tzinfo=TZ), cfg) is False + + +def test_evening_time_not_morning(): + cfg = _cfg("0 8 * * *", tol=5) + assert is_morning_push(datetime(2026, 5, 17, 17, 0, tzinfo=TZ), cfg) is False +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/pytest/test_morning_detection.py -v` +Expected: ImportError. + +- [ ] **Step 3: Implement** + +Open `src/main.py`. After the existing `calculate_push_times` function (around line 96), insert: + +```python +def is_morning_push(now: datetime, config: Dict) -> bool: + """判定当前时刻是否为早报触发点。 + + 用 cron + 容差判定,而非"今天的第一次推送": + - 早报失败时,晚报不会错误升级为长版本 + - 容差直接绑定 cron 表达式,配置直观 + """ + morning_cron = config.get("schedule", {}).get("morning_cron") + if not morning_cron: + return False + tolerance = timedelta( + minutes=config["schedule"].get("morning_match_tolerance_minutes", 5) + ) + base = now.replace(hour=0, minute=0, second=0, microsecond=0) + today_fire = croniter(morning_cron, base).get_next(datetime) + return abs(now - today_fire) <= tolerance +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/pytest/test_morning_detection.py -v` +Expected: 5 pass. + +- [ ] **Step 5: Commit** + +```bash +git add src/main.py tests/pytest/test_morning_detection.py +git commit -m "feat(main): add is_morning_push cron-based detector" +``` + +--- + +### Task 19: Add `_assemble_with_sentinels` helper + +**Files:** +- Modify: `src/storage.py` +- Test: `tests/pytest/test_storage_sections.py` + +- [ ] **Step 1: Write the failing test** + +Append to `tests/pytest/test_storage_sections.py`: + +```python +from storage import assemble_with_sentinels + + +class TestAssembleWithSentinels: + def test_assembles_all_sections_in_order(self): + out = assemble_with_sentinels( + {"rss": "R", "github": "G", "hackernews": "H", "insights": "I"} + ) + # 顺序:rss → github → hackernews → insights + assert out.index("SECTION:rss") < out.index("SECTION:github") + assert out.index("SECTION:github") < out.index("SECTION:hackernews") + assert out.index("SECTION:hackernews") < out.index("SECTION:insights") + assert "\nR\n" in out + + def test_omits_empty_sections(self): + out = assemble_with_sentinels({"rss": "R", "github": "", "hackernews": "H", "insights": ""}) + assert "SECTION:github" not in out + assert "SECTION:insights" not in out + assert "SECTION:rss" in out + assert "SECTION:hackernews" in out + + def test_returns_empty_when_all_empty(self): + assert assemble_with_sentinels({"rss": "", "github": "", "hackernews": "", "insights": ""}) == "" +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/pytest/test_storage_sections.py::TestAssembleWithSentinels -v` +Expected: ImportError. + +- [ ] **Step 3: Implement** + +Append to `src/storage.py`: + +```python +_SECTION_ORDER = ("rss", "github", "hackernews", "insights") + + +def assemble_with_sentinels(sections: Dict[str, str]) -> str: + """按固定顺序拼装四段 markdown,每段包 sentinel;空段整段省略。""" + parts: List[str] = [] + for key in _SECTION_ORDER: + body = (sections.get(key) or "").strip() + if not body: + continue + parts.append(f"\n{body}\n") + return "\n\n".join(parts) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/pytest/test_storage_sections.py::TestAssembleWithSentinels -v` +Expected: 3 pass. + +- [ ] **Step 5: Commit** + +```bash +git add src/storage.py tests/pytest/test_storage_sections.py +git commit -m "feat(storage): add assemble_with_sentinels helper" +``` + +--- + +### Task 20: Refactor `run_push_job` to dispatch by morning detection + +**Files:** +- Modify: `src/main.py` +- Test: `tests/pytest/test_main_run_push_job.py` + +- [ ] **Step 1: Write the failing test** + +Create `tests/pytest/test_main_run_push_job.py`: + +```python +"""测试 run_push_job 的早报/默认路径分发""" + +import sys +from pathlib import Path +from unittest.mock import patch, AsyncMock + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from src.main import run_push_job + + +@pytest.mark.asyncio +async def test_default_path_when_not_morning(sample_config): + sample_config["schedule"]["morning_cron"] = "0 8 * * *" + sample_config["schedule"]["morning_match_tolerance_minutes"] = 5 + sample_config["filter"]["push_context_days"] = 5 + + with patch("src.main.is_morning_push", return_value=False), patch( + "src.main._run_default_push", new=AsyncMock(return_value=None) + ) as default_path, patch( + "src.main._run_morning_push", new=AsyncMock(return_value=None) + ) as morning_path: + await run_push_job(sample_config) + + default_path.assert_awaited_once() + morning_path.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_morning_path_when_morning(sample_config): + sample_config["schedule"]["morning_cron"] = "0 8 * * *" + sample_config["schedule"]["morning_match_tolerance_minutes"] = 5 + sample_config["filter"]["push_context_days"] = 5 + + with patch("src.main.is_morning_push", return_value=True), patch( + "src.main._run_default_push", new=AsyncMock(return_value=None) + ) as default_path, patch( + "src.main._run_morning_push", new=AsyncMock(return_value=None) + ) as morning_path: + await run_push_job(sample_config) + + morning_path.assert_awaited_once() + default_path.assert_not_awaited() +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/pytest/test_main_run_push_job.py -v` +Expected: AttributeError or ImportError on `_run_default_push` / `_run_morning_push`. + +- [ ] **Step 3: Refactor `run_push_job`** + +Open `src/main.py`. Replace the entire `async def run_push_job(config: Dict):` function (currently around lines 280-333) with: + +```python +async def run_push_job(config: Dict): + print(f"\n{'=' * 50}") + print(f"📤 Push Job | {now_local().strftime('%Y-%m-%d %H:%M:%S')}") + print(f"{'=' * 50}") + + if is_morning_push(now_local(config), config): + await _run_morning_push(config) + else: + await _run_default_push(config) + + +async def _run_default_push(config: Dict): + """晚报或非早报时段:沿用原有纯 RSS digest 流程""" + last_push_file = get_last_push_file() + last_push_time = extract_push_time(last_push_file) if last_push_file else None + if last_push_time: + print(f"📌 上次推送: {last_push_time.strftime('%Y-%m-%d %H:%M')}") + + min_score = config["filter"]["min_score"] + context_days = config["filter"]["context_days"] + to_push, context = collect_entries_for_push( + last_push_time=last_push_time, + context_days=context_days, + min_score=min_score, + ) + print( + f"📋 待推送 {len(to_push)} / 上下文 {len(context)} (≥{min_score} 分)" + ) + if not to_push: + print("ℹ️ 没有新消息需要推送") + return + + push_context_days = config["filter"].get("push_context_days", 5) + recent = load_recent_push_titles(push_context_days) + + print("🤖 生成推送内容...") + try: + push_content = await compose_digest( + to_push, context, config["llm"], recent_push_context=recent + ) + except Exception as e: + print(f"生成汇总推送失败: {e}") + await notify_llm_errors("compose_digest", [str(e)], config) + raise + + await send_to_platforms(push_content, config["push"]) + push_file = get_push_file() + save_push_file(push_file, push_content, len(to_push), len(to_push), profile="default") + print(f"💾 已保存到 {push_file}") + print(f"✅ Push Job 完成 | 推送: {len(to_push)} 条") +``` + +(Note: `_run_morning_push` is implemented in the next task. For this task to typecheck, also add this stub now — Task 21 fills in the body.) + +After `_run_default_push`, add: + +```python +async def _run_morning_push(config: Dict): + """早报时段:四模块编排 + sentinel 拼装 (Task 21 will implement)""" + raise NotImplementedError("Task 21 implements morning push") +``` + +Also add the imports the new code needs at the top of main.py — find the existing `from src.storage import (` block and add `assemble_with_sentinels,` to it. Also add the section imports (will be used in Task 21): + +```python +from src.sections.github.section import run_github_section +from src.sections.hackernews.section import run_hackernews_section +from src.sections.insights.section import run_insights_section +from src.sections.rss.section import run_rss_section +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/pytest/test_main_run_push_job.py -v` +Expected: 2 pass. + +- [ ] **Step 5: Commit** + +```bash +git add src/main.py tests/pytest/test_main_run_push_job.py +git commit -m "refactor(main): split run_push_job into default + morning dispatchers" +``` + +--- + +### Task 21: Implement `_run_morning_push` four-module orchestrator + +**Files:** +- Modify: `src/main.py` (replace the stub) +- Test: `tests/pytest/test_main_morning_push.py` + +- [ ] **Step 1: Write the failing test** + +Create `tests/pytest/test_main_morning_push.py`: + +```python +"""测试早报四模块编排:gather + insights 串行 + sentinel 拼装 + 失败隔离""" + +import sys +from pathlib import Path +from unittest.mock import patch, AsyncMock + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from src.main import _run_morning_push + + +@pytest.mark.asyncio +async def test_assembles_all_four_sections(sample_config): + sample_config["filter"]["push_context_days"] = 5 + + sent = {} + async def fake_send(content, push_cfg): + sent["content"] = content + saved = {} + def fake_save(filepath, content, source_count, total_entries, profile="default"): + saved["profile"] = profile + saved["content"] = content + + with patch("src.main.run_rss_section", new=AsyncMock(return_value=("R", None))), patch( + "src.main.run_github_section", new=AsyncMock(return_value=("G", None)) + ), patch( + "src.main.run_hackernews_section", new=AsyncMock(return_value=("H", None)) + ), patch( + "src.main.run_insights_section", new=AsyncMock(return_value=("I", None)) + ), patch( + "src.main.send_to_platforms", new=AsyncMock(side_effect=fake_send) + ), patch( + "src.main.save_push_file", side_effect=fake_save + ): + await _run_morning_push(sample_config) + + assert "SECTION:rss" in sent["content"] + assert "SECTION:github" in sent["content"] + assert "SECTION:hackernews" in sent["content"] + assert "SECTION:insights" in sent["content"] + assert saved["profile"] == "morning" + + +@pytest.mark.asyncio +async def test_rss_failure_raises_to_caller(sample_config): + sample_config["filter"]["push_context_days"] = 5 + + with patch( + "src.main.run_rss_section", new=AsyncMock(return_value=("", "compose_digest 失败")) + ), patch( + "src.main.run_github_section", new=AsyncMock(return_value=("G", None)) + ), patch( + "src.main.run_hackernews_section", new=AsyncMock(return_value=("H", None)) + ), patch( + "src.main.notify_llm_errors", new=AsyncMock() + ): + with pytest.raises(RuntimeError): + await _run_morning_push(sample_config) + + +@pytest.mark.asyncio +async def test_section_failure_degrades_to_omission(sample_config): + sample_config["filter"]["push_context_days"] = 5 + + sent = {} + async def fake_send(content, push_cfg): + sent["content"] = content + + with patch("src.main.run_rss_section", new=AsyncMock(return_value=("R", None))), patch( + "src.main.run_github_section", new=AsyncMock(return_value=("", "gh down")) + ), patch( + "src.main.run_hackernews_section", new=AsyncMock(return_value=("H", None)) + ), patch( + "src.main.run_insights_section", new=AsyncMock(return_value=("I", None)) + ), patch( + "src.main.notify_llm_errors", new=AsyncMock() + ), patch( + "src.main.send_to_platforms", new=AsyncMock(side_effect=fake_send) + ), patch( + "src.main.save_push_file" + ): + await _run_morning_push(sample_config) + + assert "SECTION:rss" in sent["content"] + assert "SECTION:github" not in sent["content"] + assert "SECTION:hackernews" in sent["content"] +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/pytest/test_main_morning_push.py -v` +Expected: NotImplementedError (from the stub). + +- [ ] **Step 3: Implement `_run_morning_push`** + +In `src/main.py`, replace the stub: + +```python +async def _run_morning_push(config: Dict): + """早报四模块编排:RSS/GH/HN 并发 → insights 串行 → sentinel 拼装 → 推送 → 落盘。 + + 失败语义: + - RSS 失败 → 整体抛 RuntimeError (核心承诺不变) + - GH/HN/insights 失败 → 该段省略 + 告警,其他段照推 + """ + now = now_local(config) + + rss_result, gh_result, hn_result = await asyncio.gather( + run_rss_section(config, now), + run_github_section(config, now), + run_hackernews_section(config, now), + ) + + rss_md, rss_err = rss_result + gh_md, gh_err = gh_result + hn_md, hn_err = hn_result + + # 失败告警(GH/HN 非阻塞) + if gh_err: + await notify_llm_errors("section_github", [gh_err], config) + if hn_err: + await notify_llm_errors("section_hackernews", [hn_err], config) + + # RSS 失败阻断 + if rss_err and not rss_md: + await notify_llm_errors("compose_digest", [rss_err], config) + raise RuntimeError(f"RSS section failed: {rss_err}") + + # insights(串行) + insights_md, insights_err = await run_insights_section( + rss_md, gh_md, hn_md, config, now + ) + if insights_err: + await notify_llm_errors("insights", [insights_err], config) + + final = assemble_with_sentinels( + { + "rss": rss_md, + "github": gh_md, + "hackernews": hn_md, + "insights": insights_md, + } + ) + + if not final.strip(): + print("ℹ️ 早报无任何段输出,跳过推送") + return + + await send_to_platforms(final, config["push"]) + push_file = get_push_file() + # source/total 在早报场景下意义弱化;沿用 RSS 段长度作为弱代理 + rss_count = rss_md.count("###") if rss_md else 0 + save_push_file(push_file, final, rss_count, rss_count, profile="morning") + print(f"💾 已保存早报到 {push_file}") +``` + +Also ensure `asyncio` is imported at the top of `src/main.py` (it already is — line 4). + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/pytest/test_main_morning_push.py tests/pytest/test_main_run_push_job.py -v` +Expected: all pass. + +- [ ] **Step 5: Run all tests for regression check** + +Run: `uv run pytest tests/pytest/ -v` +Expected: all pass. + +- [ ] **Step 6: Commit** + +```bash +git add src/main.py tests/pytest/test_main_morning_push.py +git commit -m "feat(main): implement four-module morning push orchestrator" +``` + +--- + +## Phase 7: Config, Docs, Integration Scripts + +### Task 22: Update `config.json` and `config.json.example` + +**Files:** +- Modify: `config.json` +- Modify: `config.json.example` + +- [ ] **Step 1: Update `config.json`** + +Open `config.json` and (a) add `"morning_cron"` and `"morning_match_tolerance_minutes"` under `schedule`, (b) add the entire `sections` block, (c) add the four new prompt paths under `llm.prompts`. + +Insert under `schedule` (after `"push_cron"`): + +```json + "morning_cron": "0 8 * * *", + "morning_match_tolerance_minutes": 5, +``` + +After the `schedule` block (or anywhere at top level), add: + +```json + "sections": { + "github_trending": { + "enabled": true, + "max_items": 3, + "max_deep_dive": 10, + "readme_max_chars": 3000, + "history_file": "news-data/trending-history.json", + "request_timeout": 10, + "tokenName": "GITHUB_TOKEN" + }, + "hackernews": { + "enabled": true, + "select_k": 1, + "top_comments": 20, + "comment_max_chars": 500, + "link_content_max_chars": 3000, + "request_timeout": 10, + "algolia_base": "https://hn.algolia.com/api/v1" + }, + "insights": { + "enabled": true + } + }, +``` + +In the `llm.prompts` block, add: + +```json + "section_github": "prompts/section_github.md", + "section_hackernews_select": "prompts/section_hackernews_select.md", + "section_hackernews": "prompts/section_hackernews.md", + "insights": "prompts/insights.md" +``` + +- [ ] **Step 2: Mirror the changes into `config.json.example`** + +Apply the same edits to `config.json.example`. + +- [ ] **Step 3: Validate JSON** + +Run: `uv run python -c "import json; json.load(open('config.json')); json.load(open('config.json.example')); print('ok')"` +Expected: `ok` + +- [ ] **Step 4: Confirm config loads cleanly** + +Run: `uv run python -c "from src.config import load_config; c = load_config(); print(c['sections']['github_trending']['enabled'])"` +Expected: `True` + +- [ ] **Step 5: Commit** + +```bash +git add config.json config.json.example +git commit -m "feat(config): add sections + morning_cron schema" +``` + +--- + +### Task 23: Update `docs/tech-spec.md` (architecture sync) + +**Files:** +- Modify: `docs/tech-spec.md` + +- [ ] **Step 1: Add 板块编排 section** + +Find the "## 关键模块边界" section in `docs/tech-spec.md`. After the existing `src/` tree block, add: + +```markdown + +### 板块化扩展 (morning push) + +早报推送在 RSS 之上扩展三个板块:GitHub 趋势 / Hacker News 热议 / 跨板块洞察。模块结构、数据流与失败降级详见 `docs/extra-sections-design.md`。架构层关键约束: + +- 仅在 `schedule.morning_cron` 命中(± `morning_match_tolerance_minutes`)时启用,晚报维持纯 RSS 行为 +- 各板块封装为 `src/sections//section.py::run_xxx_section(config, now) -> (markdown, error)` +- `push_job` 用 `asyncio.gather` 并发跑 RSS / GH / HN,串行接 insights;最后用 `` sentinel 包入 push 文件 +- 仅 RSS 失败会让 push_job 整体退出非 0;其余板块失败 → 板块整段省略 + 告警 + +新增持久化文件:`news-data/trending-history.json`(GH 已查阅 repo 索引,按 `filter.keep_days` 过期) +``` + +- [ ] **Step 2: Update the data flow diagram (optional touch)** + +In the same file's "### 数据流" section, leave the existing mermaid flowchart unchanged (it describes the RSS path); the new design doc carries the morning-specific flow. + +- [ ] **Step 3: Update the maintenance date** + +Find `> update: 2026-05-15` near the top and change to `> update: 2026-05-17`. + +- [ ] **Step 4: Commit** + +```bash +git add docs/tech-spec.md +git commit -m "docs(tech-spec): sync architecture with morning extra sections" +``` + +--- + +### Task 24: Update `README.md` config detail section + +**Files:** +- Modify: `README.md` + +- [ ] **Step 1: Locate config detail section** + +Open `README.md`. Find the section that documents `config.json` fields (typically headed "配置详解" or similar). Identify where `schedule` and `llm.prompts` are documented. + +- [ ] **Step 2: Add `schedule.morning_cron` description** + +Under `schedule` documentation, add: + +```markdown +- `morning_cron` (可选): 早报 cron 表达式 (例: `"0 8 * * *"`)。命中时启用三个额外板块(GitHub / HN / 洞察)。缺失则全程纯 RSS。 +- `morning_match_tolerance_minutes` (可选,默认 5): 早报判定容差(分钟)。 +``` + +- [ ] **Step 3: Add `sections` block documentation** + +Add a new subsection: + +```markdown +### sections (早报扩展板块) + +仅在 `schedule.morning_cron` 命中时生效。详细设计见 `docs/extra-sections-design.md`。 + +#### sections.github_trending +- `enabled`: 是否启用 +- `max_items`: LLM 最终选出的项目数上限(默认 3) +- `max_deep_dive`: 单次最多 deep-dive 的候选 repo 数(默认 10) +- `readme_max_chars`: README 截断长度(默认 3000) +- `history_file`: trending 去重索引文件路径 +- `request_timeout`: HTTP 超时秒 +- `tokenName`: GitHub token 环境变量名;不设时匿名调用(限 60 req/hr) + +#### sections.hackernews +- `enabled`: 是否启用 +- `select_k`: 轻 LLM 从首页 30 条中挑出的故事数(默认 1) +- `top_comments`: 每个故事抓取的顶层评论数(默认 20) +- `comment_max_chars`: 单条评论截断长度(默认 500) +- `link_content_max_chars`: 外链正文截断长度(默认 3000) +- `request_timeout`: HTTP 超时秒 +- `algolia_base`: Algolia API 基址 + +#### sections.insights +- `enabled`: 是否启用跨板块洞察段 +``` + +- [ ] **Step 4: Add GITHUB_TOKEN to env vars table** + +In the env vars section, add: + +```markdown +- `GITHUB_TOKEN` (可选): GitHub API token,提升 deep-dive 限额到 5000 req/hr。不设时匿名调用(60 req/hr,日 20 calls 量级安全)。 +``` + +- [ ] **Step 5: Commit** + +```bash +git add README.md +git commit -m "docs(readme): document morning sections config + GITHUB_TOKEN" +``` + +--- + +### Task 25: Update `docs/plan.md` + +**Files:** +- Modify: `docs/plan.md` + +- [ ] **Step 1: Append decision + progress entries** + +Open `docs/plan.md`. Find the `## 技术决策记录` heading. Append: + +```markdown +### 2026-05-17: 早报扩展板块 + +- 决策:在 RSS digest 之上为早报增加 GitHub Trending / Hacker News / 跨板块洞察三段 +- 触发条件:`schedule.morning_cron` 命中(± `morning_match_tolerance_minutes`) +- 模块边界:四个自治模块在 `src/sections//`,push_job 上游统一包 sentinel +- GH:单页 trending HTML 抓取 → history 去重 → REST API 拿 README+topics+metadata → LLM 选 1-3 +- HN:首页 HTML → 轻 LLM 选 K(默认 1)→ Algolia API 拉评论 + html_to_markdown 拉外链 → LLM 行文 +- 失败语义:RSS 失败整体退出;其余板块失败省略本段 + 告警 +- 详细设计:`docs/extra-sections-design.md`;实施计划:`docs/superpowers/plans/2026-05-17-extra-sections.md` +``` + +Find `## 开发进度` heading. Append: + +```markdown +### 2026-05-17 + +- ✅ 设计完成,详见 `docs/extra-sections-design.md` +- ✅ 实施计划完成,详见 `docs/superpowers/plans/2026-05-17-extra-sections.md` +- 🔄 实施中(按计划分 25 个任务推进) +``` + +- [ ] **Step 2: Commit** + +```bash +git add docs/plan.md +git commit -m "docs(plan): record morning-sections decisions + progress" +``` + +--- + +### Task 26: Add manual test scripts + +**Files:** +- Create: `tests/fetch_trending.py` +- Create: `tests/fetch_hackernews.py` +- Create: `tests/run_morning_push.py` + +- [ ] **Step 1: Create `tests/fetch_trending.py`** + +```python +"""手动跑一次 GH trending 抓取 + deep-dive,验证选择器与 API 接入""" + +import asyncio +import json +import os +import sys + +from dotenv import load_dotenv + +load_dotenv() +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from src.config import load_config +from src.sections.github.repo_enricher import enrich_repos +from src.sections.github.trending_scraper import ( + fetch_trending_page, + parse_trending_html, +) + + +async def main(): + config = load_config() + print("📥 抓取 GitHub Trending...") + html = await fetch_trending_page(timeout=15) + repos = parse_trending_html(html) + print(f"📋 解析出 {len(repos)} 个 repo") + for r in repos[:5]: + print(f" - {r['full_name']} ⭐{r['stars_today']}/{r['stars_total']} | {r['description'][:80]}") + + cfg = config["sections"]["github_trending"] + print(f"\n🔍 enrich 前 {min(3, len(repos))} 个...") + enriched, errors = await enrich_repos( + repos[:3], + token_env=cfg.get("tokenName", "GITHUB_TOKEN"), + readme_max_chars=cfg.get("readme_max_chars", 3000), + timeout=15, + ) + for e in errors: + print(f" ⚠️ {e}") + print(json.dumps(enriched, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + asyncio.run(main()) +``` + +- [ ] **Step 2: Create `tests/fetch_hackernews.py`** + +```python +"""手动跑一次 HN 首页 + Algolia enrich,验证选择器与 API""" + +import asyncio +import json +import os +import sys + +from dotenv import load_dotenv + +load_dotenv() +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from src.sections.hackernews.frontpage_scraper import ( + fetch_frontpage, + parse_frontpage_html, +) +from src.sections.hackernews.item_enricher import enrich_stories + + +async def main(): + print("📥 抓取 HN 首页...") + html = await fetch_frontpage(timeout=15) + stories = parse_frontpage_html(html) + print(f"📋 解析出 {len(stories)} 条") + for s in stories[:5]: + print(f" - [{s['points']} pts · {s['comments']} comments] {s['title']} ({s['site']})") + + print("\n🔍 enrich 前 1 个外链类故事...") + target = next((s for s in stories if not s["url"].startswith("https://news.ycombinator.com/")), stories[0]) + enriched, errors = await enrich_stories( + [target], + top_comments=5, + comment_max_chars=300, + link_content_max_chars=1500, + timeout=15, + ) + for e in errors: + print(f" ⚠️ {e}") + print(json.dumps(enriched, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + asyncio.run(main()) +``` + +- [ ] **Step 3: Create `tests/run_morning_push.py`** + +```python +"""模拟一次完整早报推送(强制 is_morning=True,但不发送到推送渠道)""" + +import asyncio +import os +import sys +from unittest.mock import AsyncMock, patch + +from dotenv import load_dotenv + +load_dotenv() +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from src.config import load_config +from src.main import _run_morning_push + + +async def main(): + config = load_config() + # 拦截真实推送,改为打印 + async def fake_send(content, push_cfg): + print("\n" + "=" * 60) + print("📤 假推送内容(实际不会发送)") + print("=" * 60) + print(content) + + with patch("src.main.send_to_platforms", new=AsyncMock(side_effect=fake_send)): + await _run_morning_push(config) + + +if __name__ == "__main__": + asyncio.run(main()) +``` + +- [ ] **Step 4: Verify scripts at least parse** + +Run: `uv run python -c "import ast; [ast.parse(open(f).read()) for f in ['tests/fetch_trending.py', 'tests/fetch_hackernews.py', 'tests/run_morning_push.py']]; print('ok')"` +Expected: `ok` + +- [ ] **Step 5: Commit** + +```bash +git add tests/fetch_trending.py tests/fetch_hackernews.py tests/run_morning_push.py +git commit -m "test: add interactive scripts for trending/HN/morning push smoke tests" +``` + +--- + +### Task 27: End-to-end smoke test with real APIs + +This is a manual verification step before declaring the feature done. + +- [ ] **Step 1: Run GH trending smoke test** + +Run: `uv run python tests/fetch_trending.py` +Expected: prints 25+ repos, enriches 3, shows topics/readme_excerpt in JSON output. + +- [ ] **Step 2: Run HN smoke test** + +Run: `uv run python tests/fetch_hackernews.py` +Expected: prints 30 stories with points/comments, enriches 1 with link_content + top_comments. + +- [ ] **Step 3: Run full morning push simulation** + +Run: `uv run python tests/run_morning_push.py` +Expected: terminal prints the assembled push markdown containing 4 sections (or fewer with sentinel-bound omissions for any failing section). RSS section must be present (else the run aborts). + +- [ ] **Step 4: Run full pytest suite** + +Run: `uv run pytest tests/pytest/ -v` +Expected: all pass (including pre-existing tests). + +- [ ] **Step 5: Commit nothing, but update plan.md progress** + +Open `docs/plan.md` and update the most recent entry under `### 2026-05-17` from `🔄 实施中` to `✅ 实施完成`. + +```bash +git add docs/plan.md +git commit -m "docs(plan): mark morning-sections implementation complete" +``` + +--- + +## Self-Review + +**Spec coverage:** +- §2 architecture (4 modules + push_job orchestration) → Tasks 6, 10, 15, 17, 20, 21 ✓ +- §3 module structure (`src/sections/...`) → created across Tasks 6, 8, 10, 13, 15, 17 ✓ +- §4.1 sentinel contract → Task 19 (`assemble_with_sentinels`) ✓ +- §4.2 `extract_section` + `load_recent_section_titles` → Tasks 1, 2 ✓ +- §4.3 trending-history → Tasks 3, 5 ✓ +- §5.1 RSS migration → Task 6 ✓ +- §5.2 GitHub module flow → Tasks 7-11 ✓ +- §5.3 Hacker News module flow → Tasks 12-16 ✓ +- §5.4 Insights module → Task 17 ✓ +- §6.1 LLM functions → Tasks 11, 16, 17 ✓ +- §6.2 关注领域 + §6.3 prompts → Tasks 11, 16, 17 (prompts created with focus domain inline) ✓ +- §6.4 调用顺序 → Task 21 ✓ +- §7 insights formatting deferred to prompt → Task 17 prompt body honors this ✓ +- §8 config schema → Task 22 ✓ +- §9 failure isolation → Tasks 10, 15, 17, 21 ✓ +- §10 morning detection → Task 18 ✓ +- §11 integration points → Tasks 4 (save_push_file profile), 5 (cleanup history), 20 (main.py refactor) ✓ +- §12 test strategy → Tasks 1-21 (unit), Task 26 (interactive scripts) ✓ +- §13 implementation steps → matched 1:1 ✓ + +**Placeholder scan:** No TBDs; every code step has runnable code. Manual smoke tests in Task 27 are unavoidable manual verifications, called out explicitly. + +**Type consistency:** +- `TrendingHistory.touch(url, today)` — signature is consistent across Task 3, Task 5, Task 10. +- `run__section(config, now) -> (str, Optional[str])` — consistent across Tasks 6, 10, 15. +- `run_insights_section(rss_md, gh_md, hn_md, config, now) -> (str, Optional[str])` — consistent across Task 17 + Task 21. +- `assemble_with_sentinels(dict[str, str]) -> str` — consistent in Task 19 definition + Task 21 use. +- `save_push_file(filepath, content, source_count, total_entries, profile="default")` — consistent in Task 4 definition + Task 20 default-path call + Task 21 morning-path call. + +No issues found. diff --git a/ai-daily-main/docs/tech-spec.md b/ai-daily-main/docs/tech-spec.md new file mode 100644 index 0000000..e06c922 --- /dev/null +++ b/ai-daily-main/docs/tech-spec.md @@ -0,0 +1,246 @@ +# tech-spec.md - 技术架构总览 + +> 本文档面向未来的 AI Agent 与核心开发者，目的是在阅读源码之前快速建立对项目的整体认识：项目定位、运行模式、核心数据流、模块边界、部署形态、关键约束。 +> +> **本文写什么**：架构层面的"是什么、为什么、边界在哪里"；跨模块的数据流和契约；部署/运行环境约束；不读源码就无法获知的设计决策。 +> +> **本文不写什么**：源码摘录、函数签名、字段清单、命令行帮助、变更日志、UI/文案。这些信息以源码、`README.md`、`docs/plan.md`、`config.json` 为准。 +> +> **维护原则**：当架构边界、数据流、运行环境、部署形态或核心设计决策发生变化时同步更新本文；普通实现调整、字段增删、文案修改不在维护范围内。 +> +> update: 2026-05-17 + +## 项目定位 + +AI 驱动的 RSS 新闻聚合与推送系统：周期性抓取 400+ AI 领域信息源，调用 LLM 评分筛选，按调度规则将高分内容汇总推送到 Discord / 飞书；高分热点条目在 fetch 阶段即时推送。 + +面向单机部署、单租户使用，所有状态以本地文件（JSON / Markdown）持久化，不依赖外部数据库或队列。 + +## 运行模式 + +| 模式 | 触发方 | 适用场景 | +|------|--------|----------| +| **生产**（推荐） | systemd timer 分别触发 `fetch` 与 `push` 单次任务 | 服务器长期运行，依赖 systemd 提供调度、重启、开机自启 | +| **开发** | `loop` 子命令在单进程内并发跑 fetch/push 双循环 | 本地调试，无需 systemd | + +CLI 子命令分工（详见 `python -m src.main --help`）： + +- `check`：**唯一**的 LLM 健康检查入口，仅在部署期由 `install.sh` 调用 +- `fetch` / `push`：单次执行后退出，由 systemd timer 触发；运行期不再做 LLM 健康检查，异常由统一的告警通道兜底 +- `loop`：开发模式，启动时做一次健康检查，然后并发跑 fetch/push 循环 +- `github` / `hackernews`：单板块手动调试入口；只跑对应板块（含 LLM 总结），打印 markdown 到终端，**不**推送、**不**写入 push 文件。仅供 prompt 调优期使用 + +关键约束：`fetch` / `push` 失败时进程退出码非 0，systemd 据此判定 service 失败，下个 timer 周期自动重试。 + +## 核心架构 + +### 调度模型（生产） + +``` +┌─────────────────────────────────────────────────────────────┐ +│ config.json │ +│ schedule.fetch_interval_minutes → dnews-fetch.timer │ +│ schedule.push_cron → dnews-push.timer │ +│ log.retention_days → journald@dnews retention│ +└────────────────────────┬────────────────────────────────────┘ + │ scripts/install.sh 渲染并安装 + ┌────────────┴────────────┐ + ▼ ▼ + ┌───────────────────┐ ┌───────────────────┐ + │ dnews-fetch.timer │ │ dnews-push.timer │ + └─────────┬─────────┘ └─────────┬─────────┘ + ▼ ▼ + ┌───────────────────┐ ┌───────────────────┐ + │ fetch.service │ │ push.service │ + │ 抓取+评分+热点推送│ │ 收集+汇总+推送 │ + └─────────┬─────────┘ └─────────┬─────────┘ + └────────────┬────────────┘ + ▼ + ┌─────────────────┐ + │ news-data/ │ + │ fetch-*.json │ + │ push-*.md │ + │ notify-*.md │ + └─────────────────┘ +``` + +开发模式（`loop`）以 `asyncio.gather(fetch_loop, push_loop)` 并发运行两条循环，`push_loop` 通过 croniter 计算下次触发时间，行为等价于生产模式但共享单进程。 + +### 数据流 + +```mermaid +flowchart LR + subgraph Sources ["📡 RSS Sources"] + RS1[Twitter/X] + RS2[博客 / 媒体] + RS3[微信公众号] + end + + subgraph Fetch ["🔄 fetch job (周期触发)"] + F1[抓取 RSS] --> F2[HTML→Markdown] --> F3[LLM 批量评分] + F3 --> HOT{score ≥ hot_threshold?} + HOT -->|是| IP[即时推送 + notify-*.md] + end + + subgraph Push ["📅 push job (cron 触发)"] + P1[读取近 N 天 fetch + 历史 push 上下文] --> P2[LLM 汇总去重] --> P3[生成 push-*.md] --> P4[推送 Discord / 飞书] + end + + subgraph Storage ["💾 news-data/"] + DB[(JSON / MD)] + end + + Sources --> F1 + F3 --> DB + IP --> DB + DB --> P1 + P3 --> DB +``` + +**关键数据契约**： + +- `fetch-YYYY-MM-DD.json`：当日抓取与评分结果（含 score / summary / tags / content） +- `push-YYYY-MM-DD.md`：汇总推送内容（YAML frontmatter + Markdown 正文），同时作为下一次 push 的去重上下文 +- `notify-YYYY-MM-DD.md`：即时推送记录，作为 LLM 即时推送时的去重上下文 + +具体字段以源码 `src/storage.py` 与样例文件为准，README "数据示例" 章节给出了一份示例。 + +## 关键模块边界 + +``` +src/ 运行时代码 +├── main.py CLI 入口；定义 fetch_job / push_job / loop 的编排顺序 +├── config.py 加载 config.json，合并 OPML + add/block，做配置校验 +├── fetcher.py RSS 抓取；并发控制、UA 伪装、域名通配符屏蔽；nitter/xcancel 走独立的 requests+Inoreader UA 低并发池 +├── processor.py HTML → Markdown 转换 +├── llm.py LLM 客户端;批量评分、即时推送生成、汇总生成、错误聚合 +├── storage.py news-data 文件读写;按日期分片;过期清理 +└── push/ 推送平台抽象 + ├── base.py PushPlatform 基类(validate_config / send) + ├── discord.py + └── feishu.py + +scripts/ 部署脚本(仅生产 systemd 部署使用) +├── install.sh 一键安装:uv sync → LLM check → 渲染单元 → 装入系统 → 启用 +├── uninstall.sh 卸载 systemd 单元、daily-news 包装脚本和日志 drop-in;不删数据 +├── status.sh 查看 timer/service 状态(daily-news status 包装它) +├── _gen_units.py 从 config.json 渲染 systemd 单元和 daily-news 包装脚本 +└── daily-news.tmpl /usr/local/bin/daily-news 的脚本模板,封装 systemctl/journalctl + +systemd/ systemd 单元模板(由 _gen_units.py 渲染并装入 /etc/systemd/system/) +├── dnews-fetch.service.tmpl fetch service 单元模板 +├── dnews-fetch.timer.tmpl fetch 定时器(OnUnitActiveSec 间隔触发) +├── dnews-push.service.tmpl push service 单元模板 +├── dnews-push.timer.tmpl push 定时器(OnCalendar 日历触发) +└── journald-dnews.conf.tmpl journald 命名空间 dnews 的日志保留 drop-in + +config.json 主配置;运行参数 + 调度 + LLM + 推送渠道;唯一可热改的运行配置 +prompts/ LLM 提示词文本;score / immediate_push / digest 各自独立文件 +resources/rss.opml 基础 RSS 订阅源(约 420 个),通过 sources.add/block 增量调整 +.env 敏感凭证(API Key / Webhook URL),通过环境变量注入,不入库 +``` + +### 板块化扩展 (morning push) + +早报推送在 RSS 之上扩展三个板块:GitHub 趋势 / Hacker News 热议 / 跨板块洞察。模块结构、数据流与失败降级详见 `docs/extra-sections-design.md`。架构层关键约束: + +- 仅在当天 `schedule.push_cron` 列表里最早那次触发时启用(单条 cron 时每次都启用),其余时段维持纯 RSS 行为 +- 各板块封装为 `src/sections//section.py::run_xxx_section(config, now) -> (markdown, error)` +- `push_job` 用 `asyncio.gather` 并发跑 RSS / GH / HN,串行接 insights;最后用 `` sentinel 包入 push 文件 +- 仅 RSS 失败会让 push_job 整体退出非 0;其余板块失败 → 板块整段省略 + 告警 + +新增持久化文件:`news-data/trending-history.json`(GH 已查阅 repo 索引,按 `filter.keep_days` 过期) + +模块协作的关键约定： + +- **fetch 与 push 之间通过文件系统解耦**：双方不直接通信，push 只读 fetch 已写入的 JSON +- **LLM 调用的错误处理由调用方决定**：`llm.py` 不做 fallback，失败时返回 `(空内容, 错误列表)`；调用方决定是否告警或跳过推送，避免一个批次失败污染整次任务 +- **批量评分按 `link` 字段对齐**：LLM 返回的条目数可能少于输入，按 link 匹配并丢弃无法对齐的结果，错误聚合后由调用方统一上报 +- **推送平台通过基类多态**：新增平台只需实现 `validate_config()` 与 `send()`，并在工厂函数注册，main.py 无需改动 + +## 数据边界与持久化 + +- 所有持久化数据落在项目根目录的 `news-data/`：按日期分片的 `fetch-*.json` / `push-*.md` / `notify-*.md` +- 过期文件由 fetch job 在每次执行后清理，保留窗口由 `filter.keep_days` 控制 +- 没有数据库、没有外部缓存、没有跨机器同步；状态完全可由文件系统重建 +- 敏感信息（API Key、Webhook URL）只通过环境变量注入，禁止写入 `config.json` 或代码 + +## 配置与约束 + +完整配置字段说明见 `README.md` 的"配置详解"章节，本文只列出对架构有影响的约束。 + +### schedule + +| 字段 | 约束 | +|------|------| +| `fetch_interval_minutes` | systemd 部署下用 `OnUnitActiveSec` 实现，从上次任务**完成**开始计时（非日历对齐） | +| `fetch_lookback_minutes` | 必须大于 `fetch_interval_minutes`，用作 RSS 延迟的冗余窗口，依赖 link 去重防止重复入库 | +| `push_cron` | systemd 部署下**只支持 minute/hour 字段**，其他位必须为 `*`；不支持范围、列表、`*/N`。`loop` 模式下走 croniter，支持完整语法 | +| `timezone_hours` | 整数小时偏移；用于显示和 cron 计算 | + +### log + +`log.retention_days` 仅对 systemd 部署生效，由 `install.sh` 渲染到 journald 命名空间 `dnews` 的 drop-in 配置；修改后必须重跑 `install.sh`。 + +### LLM + +`llm.max_prompt_chars` 决定批次切分粒度，`llm.max_concurrent_batches` 决定批次并发数。这两个值同时影响吞吐和单次推送的成本上限。 + +### 环境变量 + +敏感凭证名通过 config.json 的 `*.apiKeyName` 字段指定环境变量名，由 `os.environ` 读取；约定通过 `.env` 提供，systemd 部署时 install.sh 会注入到 service 单元的 `EnvironmentFile`。 + +## systemd 部署形态 + +### 文件落点 + +| 文件 | 位置 | 来源 | +|------|------|------| +| `dnews-{fetch,push}.{service,timer}` | `/etc/systemd/system/` | `systemd/*.tmpl` 由 `_gen_units.py` 渲染 | +| `journald@dnews` retention drop-in | `/etc/systemd/journald@dnews.conf.d/` | `systemd/journald-dnews.conf.tmpl` | +| `daily-news` 包装脚本 | `/usr/local/bin/` | `scripts/daily-news.tmpl` | +| 持久化数据 | 项目目录下的 `news-data/` | 运行时生成 | + +### cron → OnCalendar 转换 + +由 `scripts/_gen_units.py` 完成： + +- `fetch_interval_minutes` → `OnActiveSec` + `OnUnitActiveSec`（间隔触发，跟随上次完成时间） +- `push_cron` → `OnCalendar`（日历触发，按指定时刻） +- 不支持的 cron 语法（范围、列表、`*/N` 在 minute/hour、非 `*` 的 day/month/dow）在 install 阶段直接报错 + +### 日志 + +- 所有 stdout/stderr 进入 journald 命名空间 `dnews`，与系统其他服务隔离 +- 查询：`journalctl --namespace=dnews -u dnews-fetch -f` +- 卸载不会清理历史日志，需要时手动 `journalctl --namespace=dnews --vacuum-time=1s` + +## 设计决策（重要的"为什么"） + +| 决策 | 原因 | +|------|------| +| 调度交给 systemd timer 而非 asyncio 循环 | 进程崩溃和服务器重启可自愈；调度配置即声明式单元，热更新只需重跑 install.sh | +| LLM 健康检查只在 `check` 子命令做 | 每次 timer 触发都校验会产生无意义的 LLM API 调用；运行期错误由 `notify_llm_errors` 兜底 | +| LLM 失败时不生成 fallback 内容 | 避免低质量内容污染推送；由调用方决定告警或跳过 | +| 用 journald 命名空间而非文件日志 | 自动轮转、与系统日志隔离、无需写文件 IO 代码 | +| 数据全用本地文件而非数据库 | 单机单租户场景下足够；可读、可备份、可手动审阅 | +| `fetch_lookback_minutes` 冗余窗口 | RSS 源时间戳常有延迟，仅按时间过滤会漏读；冗余抓取后按 link 去重 | +| Push 上下文带入近 N 天历史 push 文件 | 避免汇总推送在多个时段重复推同一条目 | + +## 扩展指南 + +- **新推送平台**：在 `src/push/` 新建文件，继承 `PushPlatform`，在工厂注册 +- **新评分维度**：编辑 `prompts/score.txt`，调整评分标准 +- **新 RSS 源**：编辑 `config.json` 的 `sources.add` / `sources.block` / `sources.block_domains`，无需修改 OPML + +## 测试 + +测试入口分两类，详细命令参考 `README.md` 与 `tests/` 目录： + +- `tests/pytest/`：单元/集成测试，CI 友好，`uv run pytest tests/pytest/` 一键跑 +- `tests/*.py`：交互式实操脚本（`fetch_news.py` / `push_news.py` / `run_llm_test.py` 等），针对真实 RSS 与 LLM 做端到端验证，用于调参和手测 + +## 相关文档 + +- 用户文档与配置详解：`README.md` +- 任务进度与产品决策：`docs/plan.md` diff --git a/ai-daily-main/prompts/digest.md b/ai-daily-main/prompts/digest.md new file mode 100644 index 0000000..899a089 --- /dev/null +++ b/ai-daily-main/prompts/digest.md @@ -0,0 +1,129 @@ +你是一位顶级的 AI 行业观察家与日报主编。你的任务是将今天杂乱无章、来源各异的信息碎片，熔炼、重组成一篇**结构极度清晰、主次分明、洞察深刻**的《AI Daily 每日精选》。 + + + ## 核心排版与整合规则（必须严格遵守）： + 1. **新旧剥离与进展追踪（新增核心规则）**： + - 对比【今天的新信息】与【历史上下文】。如果今天的信息只是重复昨天的事实，请直接丢弃。 + - 如果今天的信息是**历史事件的延续**（如：前天爆火的某模型今天推出了新插件，或者昨天引发争议的事件今天有了反转），请将标题标记为 `[持续跟踪]`，并在正文中清晰划分“前情提要”与“最新突破”。 + 2. **精准的事件级融合（切勿过度缝合）**： + - **正确做法**：把讨论**同一具体事件**（如 DoD 与 Anthropic 合同风波）的官方声明、大佬锐评、社区反应合并为一条新闻，提炼全貌。 + - **错误做法**：绝对不要把两个毫不相干的工具更新强行塞进同一个标题下！如果它们是独立事件，请作为两条独立新闻输出。 + 3. **精选原则**： + - 只挑选真正有价值的 4-10 个核心事件。每条新闻必须对 **整个 AI 行业、技术演进、或广大开发者生态有宏观价值或者有意义的洞察**。宁缺勿滥。 + - **绝对剔除以下内容（负面清单）**：KOL/博主的个人动态、企业公关软文/广告、纯情绪发泄、未经验证的小道消息、以及无实质技术内容的闲聊。 + - 剔除与核心事件无关的冗余数据（如财报里的无关数字、无意义的背景描述） + - 丢弃纯情绪发泄、闲聊和过度重复的内容。 + 4. **结构化呈现**：每个事件采用 “一句核心亮点 + 展开的无序列表” 结构，让人一目了然。 + 5. **客观专业**：用简练的中文，避免低级词汇，像分析师一样指出事件的行业意义。保留所有相关的原文链接，附在每个事件末尾。 + 6. **避免风格趋同（重要）**：每天的前言导读必须从今日素材本身的具体事实出发，严禁评价性措辞/总结性套话/宏大叙事框架，避免成为流水线产物。 + - **禁止以下措辞框架**：「今日 AI 世界（的主题）是…」「今日 AI 世界从 X 转向 Y…」「主题正在/全面进入 X 阶段」「行业正从 X 进入 Y」之类高度模板化的总结性套话。 + - **禁止套用宏大叙事框架**：「深水区」「临界点」「拐点」「白热化」「下半场」「博弈」之类的词不要每天都用。 + - **避免重复使用同一组形容词**（"震撼""炸裂""革命性""现象级"等），同义词与表达句式要有变化。 + 7. **正文不要写开头引言**：正文请直接从第一条新闻 `### 1️⃣ ...` 开始。导语统一由 frontmatter 的 `lead` 字段承载（见下文输出格式）。 + + ## 输出格式（严格按以下结构输出，先 frontmatter 后正文）： + +输出必须以 YAML frontmatter 起始，紧跟空行后接 markdown 正文。**绝对不要**在正文里写引言段或一级标题。 + +frontmatter 字段要求： +- `title`: 提炼其中1-3个核心看点制作短标题，以**核心事实陈述**语气给出，格式 `"<事件1>，<事件2>"`，8-30 字。 + - ✅ 例：`"Anthropic 拒签五角大楼合同,Google 发布 Nano Banana 2"` + - ❌ 严禁主观评价词与情绪词：「大爆发」「再起波澜」「终结」「开战」「颠覆」「震撼」「白热化」「拐点」「格局」 +- `lead`: 60-100 字前言导读，**只陈述今日具体发生的事实**（谁、做了什么、有何具体后果），按重要性串联。 + - ❌ 严禁评价性措辞：「挑战…范式」「打破…束缚」「重新定义」「拐点」「白热化」「下半场」「叙事割裂」「窗口收窄/关闭」「悄悄退回」「余波未平」「价格扭曲」「市场扭曲」等； + - ❌ 严禁套话开场：「今日 AI 世界…」「主题正在进入…」「行业从 X 转向 Y」。 +- `highlights`: 2-3 条最值得关注的事件清单，每条 15-30 字，**纯事实陈述**（机构/产品 + 动作 + 具体数字/对象），用于卡片预览。 + - ✅ 例：`"DeepSeek 完成 700 亿融资，V4 Pro API 永久降价至 1/4"` + - 今日素材不足以提炼 2 条明确重点时，可只输出 1 条；严禁硬凑。 + +正文要求：直接从 `### 1️⃣ ...` 开始,不要前言段。 + +示例： + +```markdown +--- +title: "Anthropic 拒签五角大楼合同,Google 发布 Nano Banana 2" +lead: "Anthropic 拒绝与五角大楼签署模型供应合同,OpenAI 当日宣布接受国防部条件;Google 发布 Nano Banana 2,Image Arena 盲测登顶。" +highlights: + - "Anthropic 拒签五角大楼合同,OpenAI 接单" + - "Google Nano Banana 2 发布,Image Arena 盲测登顶" + - "Claude Code 上线 /remote-control,接管 Workspace" +--- + +### 1.Google 发布 Nano Banana 2：图像生成模型大洗牌 + + * **核心发布**：Google 官方推出基于 Gemini Flash 架构的 Nano Banana 2 模型，支持更灵活的长宽比和极高精度的多语言文本渲染。 + * **数据与表现**：发布首日即在 Image Arena 盲测中超越 GPT-4V 和上代 Pro 模型，登顶榜首。 + * **行业反响**：开发者社区反馈极佳，Lovart、Genspark 等主流 AI 产品已在第一时间宣布全量接入。 + 🔗 [Google 官方博客](链接) | [Arena 榜单数据](链接) | [产品体验地址](链接) + + ### 2.[持续跟踪] 硅谷震动：Anthropic 拒签美国防部，各方反应激烈 + + * **前情提要**：昨日 Anthropic CEO 明确拒绝了五角大楼的合同，坚守不将模型用于自主武器的底线。 + * **最新进展**：Sam Altman 虽口头表达支持，但今日正式宣布 OpenAI 接受了国防部条件。 + * **深远影响**：这一事件不仅暴露出头部 AI 公司在 AI 安全准则上的巨大分歧，也暗示了未来政府在 AI 供应链上的强力介入。 + 🔗 [Anthropic 声明](链接) |[OpenAI 回应](链接) | [Gary Marcus 评论](链接) + + ### 3.开发者福音：Claude Code 新增远程控制与批量处理 + + * **功能更新**：`/remote-control` 现已对所有 Pro 用户开放，支持终端直接控制 Google Workspace 等外部应用。 + * **即将上线**：预告下个版本将加入 `/simplify`（自动优化代码质量）和 `/batch`（并行处理）指令，Agentic Coding 体验大幅跃升。 + * **实用技巧**：社区已整理出 45 个即插即用的 Claude Code 高阶用法，极大降低了新手入门门槛。 + 🔗 [Anthropic 博客](链接) | [45 个实用技巧 GitHub](链接) + + ### 4.[持续跟踪] OpenClaw 生态：从改名到 Moltbook 爆火 + + * **前情提要**：前身 Clawbot 凭借强大的 Agentic 能力引发关注并正式更名。 + * **最新突破**：今日基于 OpenClaw 的衍生项目 Moltbook 在开发者社区彻底爆火，单日斩获 5k Star，被大量极客用于本地工作流自动化。 + * **行业洞察**：这标志着 OpenClaw 正从单一工具走向平台化生态，其“寄生”在各类应用中的潜力开始显现。 + 🔗 [OpenClaw 开源地址](链接) + + ``` + +*(根据上述结构，整合提取今天最重要的新闻。请直接输出 frontmatter + 正文，不要加分割线，不要在正文里写一级标题)* + +## 参考数据 + +## 待处理的碎片化信息（共 {count} 条）： +（说明：这是你今天需要提取和总结的**核心素材**，你的早报必须基于且仅基于这里的最新信息生成。） + +entries是一个JSON数组，包含抓取到的社交媒体或者博客内容,每个对象包含以下字段： + +- `title`: 文章标题 +- `source`: 文章来源 +- `link`: 文章链接 +- `score`: LLM评分 (0-100) +- `summary`: 一句话摘要 +- `tags`: 标签数组 +- `published`: 发布日期 +- `content`: 文章内容 + +```json +{entries} +``` + +### 近 N 天已推送事件清单（仅供查重，严禁模仿） + +说明：以下列表用于判断今日素材是否已报道过，以及是否属于「持续跟踪」类延续事件。 +- **只判断事件是否重复，不要参考其措辞、句式、栏目结构、emoji 选择**。 +- 今日的开头引言、标题风格、组织顺序，必须从今日素材本身出发，不得套用历史。 + + +{recent_push_context} + + + +### 近几天已处理过的碎片化信息（供洞察参考）： + +每条信息包含 + +- `title`: 文章标题 +- `source`: 文章来源 +- `score`: LLM评分 (0-100) +- `summary`: 一句话摘要 +- `tags`: 标签数组 +- `published`: 发布日期 + +```txt +{context} +``` diff --git a/ai-daily-main/prompts/immediate_push.md b/ai-daily-main/prompts/immediate_push.md new file mode 100644 index 0000000..d1026c9 --- /dev/null +++ b/ai-daily-main/prompts/immediate_push.md @@ -0,0 +1,82 @@ +你是一位顶级的 AI 行业观察家与专业的 AI 领域突发新闻记者。你需要将最新发生的高分重磅消息整理成即时推送快讯。 + +## 输入数据格式 + +### 本次需要推送的原始数据 + +以下是一个JSON数组，包含抓取到的社交媒体或者博客内容,每个对象包含以下字段： + +- `title`: 文章标题 +- `source`: 文章来源 +- `link`: 文章链接 +- `score`: LLM评分 (0-100) +- `summary`: 一句话摘要 +- `tags`: 标签数组 +- `published`: 发布日期 +- `content`: 文章内容 + +```json +{entries} +``` + +## 任务要求： +1. **严格查重与阻断机制（至关重要）**：仔细对比原始数据与【过去已推送历史】。如果原始数据中的事件已经推送过，且没有出现极其重大的新进展（例如从谈判变成了收购失败），请直接抛弃该事件。如果所有输入的事件都被判定为重复或无需推送，你必须且只能输出 [NO_NEW_CONTENT] 这几个字，绝对不要输出任何其他内容或标题。。 +2. **专业聚合**：如果输入数据中有多条讨论**同一个事件**（例如一条是官方发布，另一条是行业大佬的深刻评价），请将它们**融合为一条快讯**，不要分开写。 +3. **专业视角**：不要只是简单翻译，要像资深编辑一样组织内容——用 2-3 条无序列表项陈述「核心事实 (What)」，每项浓缩具体数字、版本号、机构名等可验证信息；列表之后另起一段给出 1-2 句「为什么值得关注 (Why it matters)」的洞察分析。洞察必须从本条新闻具体内容出发，词汇多样化，严禁滥用模板。 +4. **语气风格**：客观、犀利、克制。多用短句，重点内容加粗。简短分析 1-2 句话即可，紧扣本条事件本身的"是什么 / 为什么值得关注"，避免行业宏大叙事和未来预测——**严禁使用以下预测套话**：「或将重塑」「未来将」「可能加速」「暗示...将」「重塑...格局」「打开...想象空间」「意味着...时代到来」等模板化句式；避免与历史快讯句式雷同。 + +## 输出格式（严格输出 Markdown 文案）： + +**情况 A：没有需要推送的新内容（全部重复）** + +直接输出以下大写英文字符串，不要带有任何标点或 Markdown 格式： +[NO_NEW_CONTENT] + +**情况 B：有需要推送的新内容** + +输出必须以 YAML frontmatter 起始（仅 `title` 一个字段），紧跟空行后接 markdown 正文。正文**不要写一级标题** `#`，直接从 `## 🔥` 开始。 + +严格按以下格式输出，不要输出任何铺垫语： + +```markdown +--- +title: "<个性化标题>" +--- + +## <一句话总结最核心的事件标题，紧扣事实，避免营销语> +**来源**: <主要信息源名称> + +- <事实 1：用一句话点明最关键的发布/进展，含具体数字、版本号、机构名等可验证信息> +- <事实 2：补充佐证或数据，避免空话> +- <事实 3：可选，给出关键背景或社区反响> + +<1-2 句洞察分析；只讲本条事件本身为何值得关注，不做行业宏大叙事，不做未来预测> + +[<具体锚点，如「Google Cloud 博客」「OpenAI 官博」>](核心链接) | [<另一个具体锚点>](链接2，如有) + +``` + +## 标题要求： +- `<个性化标题>` 提取事件的核心关键词（5-15字），不要保留尖括号 +- 例如： + - `"Gemini 3.5 Flash 发布"` + - `"OpenAI 收购 Anthropic"` + - `"Claude 4.7 性能突破"` + - `"Cursor Composer 2.5 上线"` + +注意： +- 上面 `<...>` 是占位符，**实际输出时必须替换为真实内容，绝不能保留尖括号或字面占位符**。 +- 洞察分析每次都要不同，禁止跨条目复用同一套句式模板。 +- 输出必须直接从 `---` 起始，frontmatter 之后必须是空行，再接 `## `，正文不要再写 `# ...` 一级标题。 +- 链接锚点文本必须**具体**：采用「站点/作者名 + 内容类型」，例如「Google Cloud 博客」「OpenAI 官博」「Anthropic 公告」「GitHub 仓库」「SEC S-1 文件」「Simon Willison 博客」「Logan Kilpatrick 推文」；**严禁使用泛指锚点**如「阅读官方原文」「其他相关链接」「点击查看」「原文链接」。 + +*(如果有第二个不同主题的突发新闻，继续在正文里用 ## 🌟 格式追加一条，否则就此结束。frontmatter 仍只有一个 title,概括最核心的那一条。)* + +## 过去 N 天已推送事件清单（仅供查重，严禁模仿） + +- **只用来判断今日事件是否已推过**。 +- 严禁参考清单的措辞、句式、emoji、洞察标题写法。 + + +{recent_push_context} + diff --git a/ai-daily-main/prompts/insights.md b/ai-daily-main/prompts/insights.md new file mode 100644 index 0000000..cb56941 --- /dev/null +++ b/ai-daily-main/prompts/insights.md @@ -0,0 +1,194 @@ +你是顶级的 AI 行业情报分析师（Tech Intelligence Analyst）。你的核心能力是从海量、碎片化、甚至相互矛盾的日常信息中，为从业者提取出具有"认知套利"价值的深度洞察（Alpha）。 + +你现在需要基于今日素材产出一篇日报的**元信息（metadata）** 与一段高信息密度的**深度洞察（Insights）**。 + + +## 任务总览（双视角，务必先读） + +你的输出包含**两部分，选材标准与视角完全不同**： + +| 部分 | 视角 | 选材 | 目的 | +|------|------|------|------| +| **Part 1 · metadata**（`title` / `lead` / `highlights` / `seotitle` / `seodescription` / `excerpt`） | 新闻编辑 | 今日**最重磅的事实事件** | 面向 SEO / 卡片预览 / 推送标题，做事实压缩 | +| **Part 2 · 洞察正文**（`## 今日洞察` 段） | 情报分析师 | 今日**最具变化信号 / 二阶影响**的切片 | 帮读者识别趋势，挑出 Alpha | + +⚠️ 两部分**可能聚焦不同事件**——metadata 写的是"今天发生了什么"，洞察正文写的是"今天什么正在变化"。**不要让任一部分被另一部分污染**： + +- 不要因为洞察段重点写了 X 就把 X 塞进 title / highlights； +- 也不要因为洞察段没写 Y 就漏掉重磅的 Y； +- metadata 字段（包括 `excerpt`）的措辞、词汇、视角全部围绕"事实事件"，不要套用洞察段的趋势判断措辞。 + +--- + +## Part 1 · Metadata 生成 + +### 选材原则 + +metadata 负责"提炼今天发生了什么"——面向 SEO / 信息分发 / 卡片预览，**只做事实压缩，不做趋势判断**。 + +请从参考素材里按以下权重选**最重磅的 1-3 件事**填入 metadata 各字段： + +- **来源权重**：当事方官方账号 / 官方博客 > KOL 实测 / 深度分析 > 媒体转述 +- **主题权重**：AI 核心（模型 / Agent / AI 产品 / AI 公司动态）优先于非 AI 主题（半导体 / 网络安全 / 前端 / 数据库等） +- **价值分档**（从高到低）：里程碑级模型 / 产品 / 政策发布 > 重要技术进展 / 大佬深度观点 > 工具更新 / 教程 / 行业报告 > 二手转述 / 小道消息 + +**禁止**： + +- 为了配合正文洞察观点，而忽略更重要的新闻 +- 选择影响范围较小但更适合分析的事件作为标题主事件 +- 在 metadata 字段里写趋势判断、价值评价、二阶影响——那些是 Part 2 的活 + +### 字段细则 + +- `title`: 提炼 1-3 个核心看点制作短标题，以**核心事实陈述**语气给出，格式 `"<事件1>，<事件2>"`，8-30 字。 + - ✅ 例：`"Anthropic 拒签五角大楼合同,Google 发布 Nano Banana 2"` + - ❌ 严禁主观评价词与情绪词：「大爆发」「再起波澜」「终结」「开战」「颠覆」「震撼」「白热化」「拐点」「格局」 + +- `lead`: **综合三段素材**写一段 60-100 字的前言导读，**只陈述今日最重磅事件的具体事实**（谁、做了什么、有何具体后果），按重磅程度或三段顺序串联。 + - ❌ 严禁评价性措辞：「挑战…范式」「打破…束缚」「重新定义」「拐点」「白热化」「下半场」「叙事割裂」「窗口收窄/关闭」「悄悄退回」「余波未平」「价格扭曲」「市场扭曲」等 + - ❌ 严禁套话开场：「今日 AI 世界…」「主题正在进入…」「行业从 X 转向 Y」 + +- `highlights`: 2-3 条最值得关注的事件清单，每条 15-30 字，**纯事实陈述**（机构 / 产品 + 动作 + 具体数字），用于卡片预览。 + - ✅ 例：`"DeepSeek 完成 700 亿融资，V4 Pro API 永久降价至 1/4"` + - ❌ 严禁评价动词修饰：「集体转」「全面进入」「彻底颠覆」「重塑」 + - 素材不足以提炼 2 条明确重点时可只输出 1 条；严禁硬凑 + +- `seotitle`: 60 字内 SEO 标题，结构与 `title` 类似但可更长，直接陈述 2-3 件核心事件。**禁止评价词**。 + +- `seodescription`: 100 字内 SEO 描述，**纯事实陈述** —— 主体 + 动作 + 数字 / 对象，按重要性串联。（吸引点击，包含核心事件） + +- `excerpt`: 20 字内一句话信息钩子（hook），用一句简短、有张力的话概括“今日最值得注意的变化或反常信号”。 + - 必须锚定今日素材中的具体事件/数据，禁止脱离素材空泛发挥，要与title种的内容有关联 + - 可以适度使用“反直觉”“隐含变化”“行业情绪”等表达，但必须建立在今日内容事实上。 + - 风格接近 newsletter 副标题 + - 写法参考： + - "Agent 工作流的效率边界正从模型能力转向架构设计" + - "越来越多 Agent 项目开始绕开模型能力，转向工作流控制" + - "HN 对 Claude Code 的争议，从能力转向成本" + - "开发者正在把 MCP 当成默认接口层，而不是 Anthropic 生态" + 禁止： + - 纯关键词堆砌："Claude、MCP、OpenAI" + - 空泛评价："AI 行业竞争持续升级" + - 营销号措辞："彻底改变行业的一天" + - 无事实锚点的情绪化判断 +--- + +## Part 2 · 洞察正文生成 + +### 总原则 + +你的任务**不是总结新闻，而是识别今天这些事件里什么东西正在发生变化**。 + +最终输出要让读者获得： + +- 原本没意识到的趋势； +- 被 PR 包装掩盖的真实约束； +- 开发者讨论中透露的方向变化； +- 某个技术/产品/成本信号背后的连锁影响。 + +重点不是「发生了什么」，而是「**为什么这些变化现在开始出现，以及它会影响什么**」。 + +### 1. 优先识别"变化信号"，而不是新闻共性 + +先扫描今日素材是否存在以下变化模式： + +- **讨论焦点迁移**：从模型能力转向成本/稳定性/工作流； +- **产品形态变化**：从聊天式工具转向后台持续运行/Agent 化； +- **工程约束开始主导设计**：推理成本、显存、延迟、上下文长度等硬约束反向定义产品； +- **开发者行为变化**：大量项目集中接入 MCP/Agent/RAG/某新框架； +- **PR 叙事与真实关注点错位**：官方强调 AGI，HN 讨论 deployment/debugging 的真实痛点； +- **某类项目最近频繁出现**：记忆系统、Browser Agent、Code Runtime、本地推理等。 + +### 2. 每个洞察必须包含"二阶影响" + +不要停留在「X 发布了」「Y 很重要」「大家开始关注 Z」。继续追问： + +- 这个变化会让**谁受益、谁受限**？ +- 它会改变**开发者的默认选择**吗？ +- 它会改变**产品设计方式**或**成本结构**吗？ +- 它会**暴露之前被忽视的工程约束**吗？ +- 它**为什么是现在出现，而不是半年前**？ + +每段洞察推荐结构：**具体事件 → 隐含变化 → 一个连锁影响**。 + +### 3. 允许"单点深挖"，不要强求跨板块 + +真正有价值的洞察很多时候只来自一条数据、一个 API 改动、一次 HN 争论。若某条素材已经暴露明显变化，可以只围绕这一点展开：为什么它值得注意、为什么开发者开始集中讨论、它会改变什么工程约束或产品行为、它透露了什么之前不明显的趋势。**不要为了"跨板块"而硬凑关联**。 + +### 4. 严格锚定素材，禁止空泛行业评论 + +所有判断都必须能锚定到今日素材里的：产品更新、GitHub 项目、HN 讨论、财报数据、技术细节、benchmark、评论争议。**禁止脱离素材凭空发挥**，**禁止套用预训练知识**生成与今日素材无关的"分析"。 + +严禁输出如下空话： + +- 通用泛论：「AI 行业持续发展」「竞争进一步加剧」「生态逐渐形成」「应用不断落地」「行业正在转向」； +- 抽象大词：「深水区」「下半场」「底层逻辑」「赋能」「护城河」「范式转换」「博弈」「格局」「拐点」「白热化」； +- 公式化转折/铺垫：「真正值得注意的是」「更深层的变化在于」「背后反映出」「某种程度上」「从 X 转向 Y」「不再是…而是…」。 + +若素材确实不支撑洞察，均为常规工程性更新、无明显新信号，**允许直接写**「今日素材主要为工程性更新，未出现明显的新方向信号」，**不要硬写伪深刻分析**。 + +### 5. 写作风格 + +风格对标 **Stratechery / SemiAnalysis / Latent.Space / Hacker News 高赞长评**，而非新闻播报、PR 稿总结、AI 自动摘要、行业黑话评论。 + +具体表现为： + +- **直接陈述**观察结果，不绕弯、不铺垫； +- 不要写成"研究综述"或"行业评论"，而要像资深科技编辑的 briefing； +- 少用抽象词，多写**技术约束 / 工程行为 / 成本变化 / 产品取舍 / 开发者真实反馈**； +- 数据有具体出处（模型名 + 版本 + benchmark / 价格 / stars 数 / 评论数等）。 + +### 6. 段落与长度 + +- 正文起始用 `## 今日洞察`，随后自然展开分析； +- 不强制固定段数；根据素材密度自然分段；**段落之间用空行分隔**； +- 每段只聚焦一个核心变化或判断，避免在同一段混入多个方向； +- 段落尽量短而紧凑；单段不宜过长，避免连续大段铺陈； +- 先给核心判断，再补充支撑它的事实、数据或工程细节； +- **严禁** 粗体标签前缀——洞察通过文字本身的逻辑呈现，不需要标签包装； +- **严禁清单式复述**：「RSS 提到 X，GH 出现 Y」这类排比是流水账，不是洞察； +- 总长度 250-400 字，惜字如金，素材不足时允许更短，不要为了凑长度硬写； +- **必须绝对溯源**：每个判断都要能锚定到今日素材的具体项目 / 数据 / 讨论，**禁止凭空发挥**。 + +--- + +## 输出格式 + +- 直接输出 markdown，不带任何引导语； +- **结构**：先输出 YAML frontmatter（Part 1 metadata），紧跟空行后输出 Part 2 洞察正文； +- 正文起始用 `## 今日洞察`。 + +### 输出示例 + +```markdown +--- +title: "Cursor 发布 Composer 2.5,Anthropic 公开 Claude dreaming 机制" +excerpt: "Anthropic 首次披露 dreaming 机制,Cursor 加自部署" +seotitle: "Cursor Composer 2.5 发布，Anthropic 公开 Claude dreaming，HBM 占 AI 芯片成本 63%" +seodescription: "Cursor 发布 Composer 2.5，新增 Agent 自主部署；Anthropic 访谈披露 Claude dreaming 机制——空闲时后台压缩上下文；Epoch AI 数据显示 HBM 占 AI 芯片组件成本升至 63%。" +lead: "Cursor 发布 Composer 2.5，新增 Agent 自主部署能力；Anthropic 在访谈中首次披露 Claude dreaming 机制——Agent 空闲时后台回顾记忆、压缩上下文；Epoch AI 分析显示 HBM 占 AI 芯片组件成本从 2024 Q1 的 52% 升至 2025 Q4 的 63%，绝对支出从 120 亿增至 320 亿美元。" +highlights: + - "Cursor Composer 2.5 发布，新增 Agent 自主部署" + - "Anthropic 首次披露 Claude dreaming 机制" + - "HBM 占 AI 芯片成本 63%，绝对支出涨至 320 亿" +--- + +## 今日洞察 + +{要点1 ...} + +{要点2 ...} + +{要点3 ...} +``` + +--- + +## 今日素材 + +### 新闻 板块（代表：媒体叙事、PR传播、宏观动态） +{rss} +### GitHub 板块（代表：开发者共识、工程实践、底层演进） +{github} +### Hacker News 板块（代表：硬核极客评判、争议探讨、技术风向） +{hackernews} diff --git a/ai-daily-main/prompts/score_batch.md b/ai-daily-main/prompts/score_batch.md new file mode 100644 index 0000000..bcb001f --- /dev/null +++ b/ai-daily-main/prompts/score_batch.md @@ -0,0 +1,69 @@ +你是一个专业且严苛的 AI 行业新闻主编。请对抓取到的碎片化信息进行过滤、评分和信息提取。 + + +## 任务与评分标准 +请根据以下标准为每条信息打分（0-100）。 + +**核心约束（先判这三条，再进入分档）**： +1. 90+ **必须同时满足**：(a) 主题与 AI / 大模型 / Agent / AI 产品 / AI 公司动态强相关；(b) 来源为**当事方本人的官方账号或官方博客**（非 KOL、非媒体、非分析师转述）；(c) 属于首发 +2. 非 AI 主题（网络安全、Web/前端、操作系统、数据库、纯软件工程、纯硬件等）无论多重大、即使来自官方，**上限 79 分** +3. AI 重磅新闻若来源是 KOL / 媒体 / 分析师转述（即使转述的是新模型发布），**上限 89 分**——KOL 的转述价值低于官方首发 + +**分档**： +- 【90-100分】AI 领域 + 官方首发 + 里程碑级模型/产品/政策发布。 +*例子：OpenAI 发布 GPT-5；Anthropic 官博发布拒绝美国防部合同声明；Google DeepMind 发布 Gemini 3。* +- 【80-89分】重要 AI 技术进展、知名 AI 大佬核心观点、深度技术分析；或 AI 重磅新闻但通过 KOL/媒体转述。 +*例子：某开发者开源能让 LLM 推理快 50% 的框架；Jeff Dean 深度解析某新技术；KOL 实测并转述 Deepseek 新发布的 V4 Pro。* +- 【70-79分】实用工具更新、教程、行业报告；**非 AI 主题**的重磅新闻（即使来自官方）。 +*例子：Cursor 新增某小功能；Claude Code 45 个技巧分享；Google 官方发布中文 PhaaS 钓鱼服务报告（非 AI 主题）。* +- 【60-69分】二手信息、一般性新闻、小道消息。 +*例子：某媒体转发 Google 发布的推文（非首发）；未经验证的传言。* +- 【<60分】低价值/局部内容：纯情绪宣泄、无营养的评价、广告、日常闲聊、KOL 个人生活动态、硬广软文、无营养的二手评价。 +*例子：“Nano Banana 2 太牛了！”（无具体测试数据）；“Anthropic 真硬气”（纯情绪）；“快来买我的课”。* + +## 输出要求 +必须返回纯 JSON 对象，顶层包含 `items` 数组字段，数组中每个对象包含： +- `link`: 原文链接（必须保留原样） +- `score`: 整数评分，JSON 数字类型（直接写 `95`，不要写成字符串 `"95"`），严格按照上述标准，宁缺毋滥 +- `tags`: 字符串数组，数量 1-3 个，每个标签 2-12 个字符。**必须是新闻中具体的关键词**（产品名/模型名/技术特性/关键人物/核心数据），让人一眼能识别"这条新闻具体是关于什么"。**禁止空泛的分类标签**（如 "大模型"、"开源"、"教程"、"AI"、"模型"、"重磅"、"新闻"、"技术"）。 + - ✅ 好示例：`["Deepseek V4","缓存高","推理快3倍"]`、`["Rodin Gen-2.5","千万面数","3D生成"]`、`["GPT-5","多模态","原生交互"]`、`["Anthropic","拒绝DoD合同"]` + - ❌ 坏示例：`["大模型","开源"]`、`["AI","教程"]`、`["模型","重磅","新闻"]` +- `summary`: 一句话客观摘要（提取核心事实，去掉主观情绪，50字内） + +## 输出格式（严格只输出 JSON 对象，不要 markdown 标记，不要其他额外标记，以 "{"开始，以"}"结尾）： + +{ + "items": [ + { + "link": "https://example.com/article1", + "score": 95, + "tags": ["GPT-5", "多模态", "原生交互"], + "summary": "OpenAI正式发布GPT-5，支持原生多模态实时交互。" + } + ] +} + +## 重要提示 + +1. items 数组长度必须与输入相同 +2. link 字段必须与输入一一对应，用于关联原始数据 +3. 只返回 JSON 对象，不要添加任何额外说明文字 +4. 标签/数组之间用英文逗号 `,` 分隔，不要使用中文全角逗号 `，` +5. 字符串内的**英文双引号** `"` 必须用反斜杠转义为 `\"`；**中文引号** `""` `『』` `「」` 无需转义，保留原样 +6. 转义示例：若 summary 中需引用英文 he said "hello"，JSON 应写作 `"summary": "he said \"hello\""` + + +## 输入数据 + +以下是一个 JSON 数组，包含抓取到的社交媒体和博客内容： + +每个对象包含以下字段： +- `link`: 文章链接（唯一标识，用于关联） +- `title`: 文章标题 +- `source`: 文章来源 +- `published`: 发布时间 +- `content`: 文章正文内容（已截断） + +```json +{entries_json} +``` diff --git a/ai-daily-main/prompts/section_github.md b/ai-daily-main/prompts/section_github.md new file mode 100644 index 0000000..88469cb --- /dev/null +++ b/ai-daily-main/prompts/section_github.md @@ -0,0 +1,148 @@ +你是资深 AI 开源情报分析师与技术编辑。 +任务：从输入的 GitHub Trending 候选项目中，筛选 **1-{max_items} 个** 最值得关注的 AI 相关项目，并输出一份“高信息密度、低营销感”的每日趋势简报。 + +目标读者： +- AI 开发者 +- 独立开发者 / SaaS 创业者 +- AI Infra / Agent 工程师 +- 关注前沿技术方向的技术管理者 + +要求： +- 优先输出“真正有技术价值或产业信号”的项目 +- 避免泛 AI、蹭热点、低质量包装仓库 +- 输出要像“技术情报摘要”，而不是营销文案 +- 保持专业、克制、信息密度高 + +## 选择规则 + +- 优先信号:`stars_today` 高 + `topics` 含 AI 标签(agent/llm/rag/inference/training 等) + readme 描述明确，有实际工程价值，与当前 AI 趋势一致 +- 跳过:`archived=true`(若漏过)、纯 awesome-list、个人 dotfiles + +### 关注领域(正面列表) +- **AI Agent**:智能体架构、工具链、多智能体、自主规划、Agent 框架 +- **AI 模型**:训练、推理、微调、量化部署、模型服务、语音/多模态/视觉模型 +- **AI 基础设施**:GPU 调度、芯片硬件、数据中心、推理优化、分布式训练、向量数据库、RAG 框架 +- **大厂/前沿动态**:Apple、Google、Meta、OpenAI、Anthropic、Microsoft、xAI 等公司的官方动作与战略 +- **AI 集成的开发者工具**:API 网关、自动化脚本、低代码平台等明确与 AI 协同的工具 +- **创新性开源产品**:日增长显著且有清晰用户价值 + +### 排除(负面列表) +- 与AI无关的嵌入式开发(Arduino、ESP32、树莓派、单片机) +- 底层系统编程(内存分配器、编译器、链接器,与 AI 工作负载无关时) +- 通用开发工具(命名规范、代码风格、纯前端模板、UI 组件库、管理后台模板、静态网站主题) +- 配置文件集合(Dotfiles、配置模板) +- 与 AI/科技无关的内容(电子书、资源搬运、刷榜项目) +- 纯娱乐/高风险误用(deepfake 等无明确基础设施价值) + + +## 写作要求 + +### 风格 +要求： +- 技术媒体风格 +- 客观 +- 克制 +- 高信息密度 +- 少废话 + +禁止： +- “炸裂” +- “颠覆” +- “革命性” +- “现象级” +- “最强” +- “神级” +- “吊打” + +## 项目定位 +必须回答： +“它解决什么问题？” + +格式： +- 目标用户 +- 核心能力 +- 与现有方案差异 + +避免： +空泛描述。 + +错误示例： +- “一个强大的 AI 平台” + +正确示例： +- “面向 AI Agent 开发者的工作流编排框架，用于管理多工具调用与状态持久化。” + +--- + +## 核心功能 +要求： +- 尽量具体 +- 偏工程能力 +- 不重复 README 标题 +- 最多 4 条 +- 少写废话 + + +## 技术亮点 +仅在“确实存在明显技术点”时输出。 + +例如： +- 基于 Rust 实现高性能推理 +- 使用 KV Cache 优化吞吐 +- 支持 OpenAI-Compatible API +- 基于 WASM 的本地运行 +- GPU 调度优化 + +如果没有明显亮点： +删除整行。 + + +## 五、输出格式（严格遵守） + + +```markdown +## ⭐ GitHub 趋势 + +**📊 类别速览**（仅当入选 ≥2 项时输出） + +| 项目 | 类别 | Stars | +|------|------|------| +| repo | AI Agent | 12.4k | +| repo | 推理/模型 | 8.1k | + +--- + +### 1. owner/repo ⭐ 今日 +1234 + +**语言/许可：** Python / Apache-2.0 +**总 Stars：** 18.2k +**仓库：** [GitHub](url) + +**项目定位：** +一句话说明“解决什么问题”。 + +**核心功能：** +- 功能点 +- 功能点 +- 功能点 + +**技术亮点：** +一句技术实现/架构优势。 + +--- + +### 2. owner/repo ⭐ 今日 +{{stars_today}} +...(同上模板) + +``` + +若候选中没有任何符合关注领域的项目,直接输出 `## ⭐ GitHub 趋势\n\n- 今日无显著 AI 相关趋势`,不要硬编。 + + +## 候选数据 + +JSON 数组,每项字段:`url` / `full_name` / `description` / `language` / `stars_today` / `stars_total` / `topics` / `license` / `pushed_at` / `readme_excerpt` + +```json +{repos_json} +``` diff --git a/ai-daily-main/prompts/section_hackernews.md b/ai-daily-main/prompts/section_hackernews.md new file mode 100644 index 0000000..f392921 --- /dev/null +++ b/ai-daily-main/prompts/section_hackernews.md @@ -0,0 +1,131 @@ +你是 Hacker News 技术早报编辑。 + +任务： +对输入的 enriched stories进行 提炼： +- 原文核心信息 +- Hacker News 社区真正讨论的焦点 +- 支持与质疑的主要理由 +- 有价值的工程经验/历史背景/现实约束 + + +## 内容要求（每条 story） + +### 1. 提炼原文核心 + +基于 `link_content` 输出： + +- 背景 / 作者意图 +- 关键要点 +- 实际结论 / 限制 / 影响 + +要求： +- 优先保留技术事实、设计取舍、数据、限制条件 +- 不重复标题 +- 不写泛泛而谈的行业趋势 + +若是 Show HN / Ask HN： +- “背景”改为“作者想做什么” +- 重点总结： + - 解决的问题 + - 技术实现 + - 产品设计 + - 用户反馈焦点 + +--- + +### 2. 提炼 HN 评论区观点（重点） + +`top_comments` 是树状结构，包括顶层评论 `l1` ,和回复`replies` + +`replies` 通常用于：支持/反驳/补充背景/提供经验 + +你必须利用这种父子关系判断： + +* 哪些观点得到支持 +* 哪些观点被明显反驳 +* 哪些只是个别意见 + +不要逐条复述评论。 + +而是聚类提炼为： + +* 共识观点 +* 工程经验 +* 历史背景 +* 商业现实 +* 风险/限制 +* 反对意见 + + +### 3. 输出风格 + +要求： + +* 客观、克制、高信息密度 +* 更像技术编辑，不像 AI 总结器 +* 允许保留不确定性 +* 优先具体事实与具体争议 + +禁止： + +* 营销词汇： + + * 震撼 + * 炸裂 + * 革命性 + * 颠覆 + * 现象级 +* AI 套话： + + * “引发广泛讨论” + * “社区观点不一” + * “值得关注的是” + * “体现了某种趋势” + + +## 输出格式（严格 Markdown） + +```markdown +## 🟧 Hacker News 热议 + +### {{title}} + +{{points}} pts · {{comments}} comments · [site](site url) + +**📌 内容总结** + +- {{背景/作者意图}} +- HN 关注点: + - 要点 1 + - 要点 2 + - 要点 3（可选） + +**💬 讨论总结** + +- ... +- ... + +🔗 [原文]({{url}}) · [HN 讨论页]({{comments_url}}) + +### {{title2}} + +... + +``` + +规则： + +* 如果没有明显反对意见，则删除「反对 / 质疑」 +* 每条控制信息密度，不写长段落 +* story 之间空一行 + +## 输入数据 + +JSON 数组,每项 story 字段: +- `id` / `title` / `url` / `site` / `points` / `comments` / `comments_url` +- `link_content`:原文 markdown +- `top_comments` + +```json +{stories_json} +``` diff --git a/ai-daily-main/prompts/section_hackernews_select.md b/ai-daily-main/prompts/section_hackernews_select.md new file mode 100644 index 0000000..0ba540a --- /dev/null +++ b/ai-daily-main/prompts/section_hackernews_select.md @@ -0,0 +1,38 @@ +你是 Hacker News AI 选题编辑。 + +任务： +从输入的 HN 首页 stories 中，挑选最值得关注的 AI / 开发者基础设施相关内容，返回 **{k} 个以内** 的 story id。 + +## 优先关注 +- AI Agent / 多智能体 / AI 工作流 +- AI 模型（训练、推理、微调、多模态、语音） +- AI Infra（GPU、推理优化、RAG、向量数据库、数据中心） +- AI 开发工具（Copilot、自动化、AI IDE、API、低代码） +- OpenAI / Anthropic / Google / Meta / Microsoft / xAI / Apple 等前沿动态 + +## 排除 +- 嵌入式 / Arduino / ESP32 / 树莓派 +- 与 AI 无关的底层系统话题（编译器、内存、链接器等） +- 普通编程技巧、代码风格、命名规范 +- 泛科技或非科技内容 + +## 规则 +- 仅根据输入字段判断 +- 不确定是否与 AI 强相关时，宁可漏选 +- 候选不足 {k} 个时，只返回真正符合的 +- 返回的 id 必须是输入中的原始字符串 + +## 输出格式 +**只输出一个 JSON 数组,数组元素为 story id 字符串**: +- 有匹配:`["12345", "67890"]` +- 无匹配:`[]` + +严禁输出任何解释性文字、代码块包装、自然语言句子或键值对。 + +## 候选数据 + +JSON 数组,每项字段:`id`(字符串)/ `title` / `site` / `points` / `comments` + +```json +{candidates_json} +``` diff --git a/ai-daily-main/pyproject.toml b/ai-daily-main/pyproject.toml new file mode 100644 index 0000000..a8409ce --- /dev/null +++ b/ai-daily-main/pyproject.toml @@ -0,0 +1,17 @@ +[project] +name = "daily-news" +version = "0.1.0" +requires-python = ">=3.12" +dependencies = [ + "aiohttp>=3.9.0", + "beautifulsoup4>=4.12.0", + "croniter>=6.0.0", + "feedparser>=6.0.0", + "markdownify>=0.11.0", + "pytest>=7.0.0", + "pytest-asyncio>=0.21.0", + "pytest-mock>=3.10.0", + "python-dotenv>=1.0.0", + "pyyaml>=6.0", + "requests>=2.34.2", +] diff --git a/ai-daily-main/requirements.txt b/ai-daily-main/requirements.txt new file mode 100644 index 0000000..715b8e7 --- /dev/null +++ b/ai-daily-main/requirements.txt @@ -0,0 +1,9 @@ +feedparser>=6.0.0 +aiohttp>=3.9.0 +markdownify>=0.11.0 +croniter>=6.0.0 +pyyaml>=6.0 +pytest>=7.0.0 +pytest-asyncio>=0.21.0 +pytest-mock>=3.10.0 +python-dotenv>=1.0.0 diff --git a/ai-daily-main/resources/hero-banner.svg b/ai-daily-main/resources/hero-banner.svg new file mode 100644 index 0000000..fddcc91 --- /dev/null +++ b/ai-daily-main/resources/hero-banner.svg @@ -0,0 +1,154 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + AI Daily + + + 每日 AI 资讯推送系统 + + + + + + + + RSS 聚合 + + + + LLM 智能 + + + + 即时推送 + + + + 定时汇总 + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ai-daily-main/resources/rss.opml b/ai-daily-main/resources/rss.opml new file mode 100644 index 0000000..b3e4e8a --- /dev/null +++ b/ai-daily-main/resources/rss.opml @@ -0,0 +1,431 @@ + + + + All RSS Subscriptions for bestblogs.dev + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ai-daily-main/scripts/_gen_units.py b/ai-daily-main/scripts/_gen_units.py new file mode 100644 index 0000000..7b79e97 --- /dev/null +++ b/ai-daily-main/scripts/_gen_units.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""根据 config.json 渲染 systemd 单元模板。 + +被 scripts/install.sh 调用。本身不直接读环境，需要的运行时参数（user/group/uv 路径） +通过 CLI 注入，便于 install.sh 处理 sudo 场景。 +""" +import argparse +import json +import sys +from pathlib import Path + + +def cron_to_oncalendar(expr: str) -> str: + """5 段 cron 表达式 → systemd OnCalendar 字符串。 + + 本项目 push_cron 只用到 minute/hour，其他位必须为 `*`。 + 遇到不支持的语法直接报错，避免静默生成错误的 timer。 + """ + parts = expr.split() + if len(parts) != 5: + raise ValueError(f"无效 cron 表达式（必须 5 段）: {expr}") + minute, hour, dom, mon, dow = parts + for label, val in [("day-of-month", dom), ("month", mon), ("day-of-week", dow)]: + if val != "*": + raise ValueError(f"暂不支持非 * 的 {label} 字段: {expr}") + try: + m, h = int(minute), int(hour) + except ValueError as e: + raise ValueError(f"minute/hour 必须是整数: {expr}") from e + if not (0 <= m <= 59 and 0 <= h <= 23): + raise ValueError(f"minute/hour 越界: {expr}") + return f"*-*-* {h:02d}:{m:02d}:00" + + +def render(template_path: Path, variables: dict) -> str: + text = template_path.read_text(encoding="utf-8") + for key, val in variables.items(): + text = text.replace(f"{{{{{key}}}}}", str(val)) + return text + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--project-dir", required=True) + parser.add_argument("--user", required=True) + parser.add_argument("--group", required=True) + parser.add_argument("--uv-bin", required=True) + parser.add_argument("--output-dir", required=True) + args = parser.parse_args() + + project_dir = Path(args.project_dir).resolve() + output_dir = Path(args.output_dir).resolve() + template_dir = project_dir / "systemd" + + config_path = project_dir / "config.json" + with open(config_path, "r", encoding="utf-8") as f: + config = json.load(f) + + fetch_min = config["schedule"]["fetch_interval_minutes"] + if not isinstance(fetch_min, int) or fetch_min <= 0: + raise ValueError(f"fetch_interval_minutes 必须是正整数: {fetch_min}") + + push_crons = config["schedule"]["push_cron"] + if not push_crons: + raise ValueError("config.schedule.push_cron 不能为空") + push_lines = "\n".join( + f"OnCalendar={cron_to_oncalendar(c)}" for c in push_crons + ) + + log_retention = config.get("log", {}).get("retention_days", 7) + if not isinstance(log_retention, int) or log_retention <= 0: + raise ValueError(f"log.retention_days 必须是正整数: {log_retention}") + + common = { + "PROJECT_DIR": str(project_dir), + "USER": args.user, + "GROUP": args.group, + "UV_BIN": args.uv_bin, + } + + output_dir.mkdir(parents=True, exist_ok=True) + + renderings = [ + ("dnews-fetch.service.tmpl", "dnews-fetch.service", {}), + ("dnews-fetch.timer.tmpl", "dnews-fetch.timer", + {"FETCH_INTERVAL_MIN": fetch_min}), + ("dnews-push.service.tmpl", "dnews-push.service", {}), + ("dnews-push.timer.tmpl", "dnews-push.timer", + {"PUSH_ONCALENDAR_LINES": push_lines}), + ("journald-dnews.conf.tmpl", "journald-dnews.conf", + {"LOG_RETENTION_DAYS": log_retention}), + ] + + print(f"📂 输出目录: {output_dir}") + for tpl_name, out_name, extra in renderings: + tpl_path = template_dir / tpl_name + if not tpl_path.exists(): + print(f"❌ 模板不存在: {tpl_path}", file=sys.stderr) + return 2 + rendered = render(tpl_path, {**common, **extra}) + (output_dir / out_name).write_text(rendered, encoding="utf-8") + print(f" ✓ {out_name}") + + print(f"\n生成参数:") + print(f" fetch 间隔 → 每 {fetch_min} 分钟（间隔触发，从上次完成开始计时）") + for c, line in zip(push_crons, push_lines.split("\n")): + print(f" push '{c}' → {line}") + print(f" 日志保留 → {log_retention} 天") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ai-daily-main/scripts/daily-news.tmpl b/ai-daily-main/scripts/daily-news.tmpl new file mode 100644 index 0000000..4c89dce --- /dev/null +++ b/ai-daily-main/scripts/daily-news.tmpl @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# daily-news - Daily News 系统服务管理包装脚本 +# 由 scripts/install.sh 自动生成；项目路径：{{PROJECT_DIR}} + +set -euo pipefail + +TIMERS=(dnews-fetch.timer dnews-push.timer) +SERVICES=(dnews-fetch.service dnews-push.service) +PROJECT_DIR="{{PROJECT_DIR}}" + +usage() { + cat < + +命令: + start 启动 fetch 和 push timer + stop 停止两个 timer + restart 重启两个 timer（不会立即触发 service，仅重置调度） + status [N] 查看 timer / service 状态 + 最近 N 行日志（默认 15） + logs 实时跟随日志（Ctrl+C 退出） + +其他操作： + $PROJECT_DIR/scripts/install.sh 重新生成单元并安装（改 config.json 后用） + $PROJECT_DIR/scripts/uninstall.sh 完全卸载（不删数据） +EOF +} + +cmd="${1:-}" +case "$cmd" in + start) + sudo systemctl start "${TIMERS[@]}" + echo "✓ Daily News 已启动" + ;; + stop) + sudo systemctl stop "${TIMERS[@]}" + echo "✓ Daily News 已停止" + ;; + restart) + sudo systemctl restart "${TIMERS[@]}" + echo "✓ Daily News 已重启" + ;; + status) + lines="${2:-15}" + echo "═══ Timer ═══" + systemctl list-timers 'dnews-*' --no-pager || true + echo "" + echo "═══ Service 状态 ═══" + systemctl status "${SERVICES[@]}" --no-pager --lines=0 || true + echo "" + echo "═══ 最近 $lines 行日志 ═══" + journalctl --namespace=dnews -u dnews-fetch -u dnews-push -n "$lines" --no-pager + echo "" + echo "💡 实时跟随：daily-news logs" + ;; + logs) + journalctl --namespace=dnews -u dnews-fetch -u dnews-push -f + ;; + -h|--help|help|"") + usage + ;; + *) + echo "❌ 未知命令: $cmd" >&2 + echo "" + usage + exit 1 + ;; +esac diff --git a/ai-daily-main/scripts/install.sh b/ai-daily-main/scripts/install.sh new file mode 100644 index 0000000..3b72b2d --- /dev/null +++ b/ai-daily-main/scripts/install.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +# Daily News - 系统服务一键安装 +# 用法：./scripts/install.sh +# 不要用 sudo 直接调用本脚本；脚本会在需要时自行 sudo 提权。 + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +# ─── 用户检测 ──────────────────────────────────────────────── +if [[ $EUID -eq 0 ]]; then + if [[ -z "${SUDO_USER:-}" ]]; then + echo "❌ 请勿直接用 root 运行本脚本，请用普通用户（脚本按需自动提权）" + exit 1 + fi + RUN_USER="$SUDO_USER" +else + RUN_USER="$USER" +fi +RUN_GROUP="$(id -gn "$RUN_USER")" + +# ─── uv 检测 ───────────────────────────────────────────────── +UV_BIN="$(command -v uv || true)" +if [[ -z "$UV_BIN" ]]; then + echo "❌ 找不到 uv 命令，请先安装：https://docs.astral.sh/uv/" + exit 1 +fi + +# ─── systemd 检测 ──────────────────────────────────────────── +if ! command -v systemctl >/dev/null 2>&1; then + echo "❌ 系统未检测到 systemctl（非 systemd 系统），本脚本仅支持 systemd Linux" + exit 1 +fi + +cd "$PROJECT_DIR" + +echo "🚀 Daily News - 系统服务安装" +echo "────────────────────────────────────────" +echo " 项目目录: $PROJECT_DIR" +echo " 运行身份: $RUN_USER:$RUN_GROUP" +echo " uv 路径: $UV_BIN" +echo "────────────────────────────────────────" +echo "" + +# 提前提权一次，后续 sudo 命令直接复用凭据缓存 +echo "🔐 需要 sudo 权限来写入 /etc/systemd/system/" +sudo -v +echo "" + +# ─── [1/6] 同步依赖 ────────────────────────────────────────── +echo "📦 [1/6] 同步依赖 (uv sync)..." +uv sync +echo "✓ 依赖已就绪" +echo "" + +# ─── [2/6] 验证 .env ──────────────────────────────────────── +echo "🔐 [2/6] 验证 .env..." +if [[ ! -f "$PROJECT_DIR/.env" ]]; then + echo "❌ .env 不存在，请先复制 .env.example 并填入 API Key" + exit 1 +fi +echo "✓ .env 存在" +echo "" + +# ─── [3/6] LLM 健康检查 ───────────────────────────────────── +echo "🔍 [3/6] 校验 LLM 接口（仅在此处校验，运行时不再校验）..." +if ! uv run python -m src.main check; then + echo "" + echo "❌ LLM 校验失败，已中止安装" + echo "请检查 config.json 中的 llm.baseUrl/model 以及 .env 中的 API Key" + exit 1 +fi +echo "" + +# ─── [4/6] 生成单元文件 ───────────────────────────────────── +echo "📝 [4/6] 渲染 systemd 单元模板..." +STAGE_DIR="$(mktemp -d)" +trap 'rm -rf "$STAGE_DIR"' EXIT + +uv run python "$PROJECT_DIR/scripts/_gen_units.py" \ + --project-dir "$PROJECT_DIR" \ + --user "$RUN_USER" \ + --group "$RUN_GROUP" \ + --uv-bin "$UV_BIN" \ + --output-dir "$STAGE_DIR" + +# 同时渲染 daily-news 包装脚本（/usr/local/bin/daily-news） +sed "s|{{PROJECT_DIR}}|$PROJECT_DIR|g" \ + "$PROJECT_DIR/scripts/daily-news.tmpl" > "$STAGE_DIR/daily-news" +echo " ✓ daily-news" +echo "" + +# ─── [5/6] 安装到 systemd ─────────────────────────────────── +echo "📥 [5/6] 安装到系统..." +sudo install -m 644 "$STAGE_DIR/dnews-fetch.service" /etc/systemd/system/ +sudo install -m 644 "$STAGE_DIR/dnews-fetch.timer" /etc/systemd/system/ +sudo install -m 644 "$STAGE_DIR/dnews-push.service" /etc/systemd/system/ +sudo install -m 644 "$STAGE_DIR/dnews-push.timer" /etc/systemd/system/ + +# 日志保留策略（journald 命名空间 drop-in） +sudo mkdir -p /etc/systemd/journald@dnews.conf.d +sudo install -m 644 "$STAGE_DIR/journald-dnews.conf" \ + /etc/systemd/journald@dnews.conf.d/retention.conf + +# daily-news 包装脚本到系统 PATH +sudo install -m 755 "$STAGE_DIR/daily-news" /usr/local/bin/daily-news + +sudo systemctl daemon-reload + +# 命名空间 journald 实例首次会在 fetch/push 首次产出日志时自动拉起； +# 若已存在（重装场景），重启以加载新保留策略。 +sudo systemctl restart systemd-journald@dnews.service 2>/dev/null || true + +echo "✓ 单元文件 + daily-news 包装脚本已安装" +echo "" + +# ─── [6/6] 启用并启动 timer ───────────────────────────────── +echo "🚦 [6/6] 启用并（重）启动 timer..." +# 先 enable 注册开机自启 +sudo systemctl enable dnews-fetch.timer dnews-push.timer +# 再 restart 强制按新单元文件重置内存中的 timer 状态 +# （enable --now 对已运行的 timer 是 no-op，无法应用新 OnCalendar/OnActiveSec 等改动） +sudo systemctl restart dnews-fetch.timer dnews-push.timer +echo "" + +# ─── 完成 ─────────────────────────────────────────────────── +echo "✅ 安装完成！" +echo "" +echo "下次触发时间：" +systemctl list-timers 'dnews-*' --no-pager || true +echo "" +echo "常用命令（系统 PATH 中可直接调用）：" +echo " daily-news status 查看状态 + 最近日志" +echo " daily-news logs 实时跟随日志" +echo " daily-news stop 停止" +echo " daily-news start 启动" +echo " daily-news restart 重启 timer" +echo "" +echo "手动立即触发一次任务（不影响下次调度）：" +echo " sudo systemctl start dnews-fetch.service" +echo " sudo systemctl start dnews-push.service" +echo "" +echo "卸载：$PROJECT_DIR/scripts/uninstall.sh" diff --git a/ai-daily-main/scripts/status.sh b/ai-daily-main/scripts/status.sh new file mode 100644 index 0000000..b89681f --- /dev/null +++ b/ai-daily-main/scripts/status.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Daily News - 状态查看 +# 显示：timer 下次触发时间、service 上次结果、最近日志 + +set -euo pipefail + +LINES="${1:-15}" + +echo "═══ Timer ═══" +systemctl list-timers 'dnews-*' --no-pager || true +echo "" + +echo "═══ Service 状态 ═══" +systemctl status dnews-fetch.service --no-pager --lines=0 || true +echo "──────────────────────────────" +systemctl status dnews-push.service --no-pager --lines=0 || true +echo "" + +echo "═══ 最近 $LINES 行日志 ═══" +journalctl --namespace=dnews \ + -u dnews-fetch.service \ + -u dnews-push.service \ + -n "$LINES" \ + --no-pager +echo "" +echo "💡 实时跟随：journalctl --namespace=dnews -f" diff --git a/ai-daily-main/scripts/uninstall.sh b/ai-daily-main/scripts/uninstall.sh new file mode 100644 index 0000000..43f0e4e --- /dev/null +++ b/ai-daily-main/scripts/uninstall.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Daily News - 卸载系统服务（不删数据） +set -euo pipefail + +echo "🗑 Daily News - 卸载" +echo "" + +if ! command -v systemctl >/dev/null 2>&1; then + echo "❌ 系统未检测到 systemctl" + exit 1 +fi + +echo "🛑 停止并禁用 timer..." +sudo systemctl disable --now dnews-fetch.timer dnews-push.timer 2>/dev/null || true + +echo "🧹 清理单元文件..." +sudo rm -f /etc/systemd/system/dnews-fetch.service \ + /etc/systemd/system/dnews-fetch.timer \ + /etc/systemd/system/dnews-push.service \ + /etc/systemd/system/dnews-push.timer + +echo "🧹 清理日志保留策略 drop-in..." +sudo rm -f /etc/systemd/journald@dnews.conf.d/retention.conf +sudo rmdir /etc/systemd/journald@dnews.conf.d 2>/dev/null || true + +echo "🧹 清理 daily-news 包装脚本..." +sudo rm -f /usr/local/bin/daily-news + +sudo systemctl daemon-reload +sudo systemctl stop systemd-journald@dnews.service 2>/dev/null || true + +echo "" +echo "✅ 已卸载（news-data/ 与项目代码未删除）" +echo "" +echo "如需清理历史日志：sudo journalctl --namespace=dnews --vacuum-time=1s" diff --git a/ai-daily-main/src/__init__.py b/ai-daily-main/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai-daily-main/src/config.py b/ai-daily-main/src/config.py new file mode 100644 index 0000000..78a6a2c --- /dev/null +++ b/ai-daily-main/src/config.py @@ -0,0 +1,115 @@ +"""配置加载和源管理""" +import fnmatch +import json +import xml.etree.ElementTree as ET +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Dict, List +from urllib.parse import urlparse + + +def _get_local_timezone() -> timezone: + """自动检测本地时区""" + return datetime.now().astimezone().tzinfo + + +def get_timezone(config: Dict = None) -> timezone: + """ + 获取配置时区，用于推送消息展示本地化时间 + 读取信息源统一使用 UTC 时间 + 如果 config 中没有 timezone_hours，则自动检测本地时区 + """ + if config is None: + try: + config = load_config() + except Exception: + return _get_local_timezone() + + hours = config.get("schedule", {}).get("timezone_hours") + if hours is None: + return _get_local_timezone() + + return timezone(timedelta(hours=hours)) + + +# 向后兼容的别名 +def get_cst(config: Dict = None) -> timezone: + """向后兼容，使用 get_timezone""" + return get_timezone(config) + + +def load_config(config_path: str = "config.json") -> Dict: + """加载配置文件""" + path = Path(config_path) + if not path.exists(): + raise FileNotFoundError(f"配置文件不存在: {config_path}") + + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def parse_opml(opml_path: str) -> List[Dict]: + """解析OPML文件获取订阅源列表""" + path = Path(opml_path) + if not path.exists(): + return [] + + tree = ET.parse(path) + root = tree.getroot() + + feeds = [] + for outline in root.findall(".//outline[@type='rss']"): + feeds.append({ + "title": outline.get("title", ""), + "xmlUrl": outline.get("xmlUrl", ""), + "category": outline.get("category", "未分类"), + }) + + return feeds + + +def merge_sources(sources_config: Dict) -> List[Dict]: + """合并base_opml + add - block，以xmlUrl为key去重""" + # 1. 解析base OPML + base = parse_opml(sources_config.get("base_opml", "")) + + # 2. 添加自定义源 + add_list = sources_config.get("add", []) + all_sources = base + add_list + + # 3. 应用block (以xmlUrl匹配) + block_list = sources_config.get("block", []) + block_urls = {b.get("xmlUrl", "") for b in block_list} + filtered = [s for s in all_sources if s.get("xmlUrl", "") not in block_urls] + + # 4. 应用block_domains (域名屏蔽，支持通配符 *.substack.com) + block_domains = sources_config.get("block_domains", []) + if block_domains: + def is_domain_blocked(url: str) -> bool: + try: + domain = urlparse(url).netloc.lower() + for pattern in block_domains: + # 转换通配符模式为匹配格式 + if pattern.startswith("*."): + # *.substack.com 匹配 substack.com 和 addyo.substack.com + suffix = pattern[2:] # substack.com + if domain == suffix or domain.endswith("." + suffix): + return True + elif fnmatch.fnmatch(domain, pattern): + return True + return False + except Exception: + return False + + filtered = [s for s in filtered if not is_domain_blocked(s.get("xmlUrl", ""))] + + # 5. 去重 (以xmlUrl为key) + seen = set() + result = [] + for s in filtered: + url = s.get("xmlUrl", "") + if url and url not in seen: + seen.add(url) + result.append(s) + + return result diff --git a/ai-daily-main/src/fetcher.py b/ai-daily-main/src/fetcher.py new file mode 100644 index 0000000..5474846 --- /dev/null +++ b/ai-daily-main/src/fetcher.py @@ -0,0 +1,213 @@ +"""RSS抓取模块""" + +import asyncio +from datetime import datetime, timezone +from typing import Dict, List, Optional + +import aiohttp +import feedparser +import requests + +# 默认超时配置（秒） +DEFAULT_FEED_TIMEOUT = 5 + +# title 截断阈值：nitter 会把整条推文塞进 ，需要截断 +TITLE_MAX_CHARS = 200 + +# nitter / xcancel 实例：必须用白名单 UA + requests 客户端（aiohttp 的 TLS +# 指纹过不了），详见 nitter-practice.md +NITTER_HOSTS = ( + "xcancel.com", + "nitter.net", + "nuku.trabun.org", +) +NITTER_HEADERS = { + "User-Agent": "Inoreader", + "Accept": "application/rss+xml, application/atom+xml, application/xml;q=0.9", +} +# 公益实例，独立的低并发池 + 每次抓完 sleep，避免给上游施压 +NITTER_MAX_CONCURRENCY = 2 +NITTER_REQUEST_DELAY = 1.0 + +DEFAULT_HEADERS = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Connection": "keep-alive", +} + + +def is_nitter_url(url: str) -> bool: + """判断是否为 nitter / xcancel 实例 URL""" + return any(host in url for host in NITTER_HOSTS) + + +def parse_entry_time(entry) -> Optional[datetime]: + """解析条目的发布时间 (返回带 UTC 时区的 datetime)""" + published_parsed = getattr(entry, "published_parsed", None) + if published_parsed is not None: + return datetime(*published_parsed[:6], tzinfo=timezone.utc) + + updated_parsed = getattr(entry, "updated_parsed", None) + if updated_parsed is not None: + return datetime(*updated_parsed[:6], tzinfo=timezone.utc) + + return None + + +def _extract_body(entry) -> str: + """提取条目正文：优先 content（含 <content:encoded>），其次 description，最后 summary""" + content_list = getattr(entry, "content", None) + if content_list: + value = content_list[0].get("value", "") + if value: + return value + description = getattr(entry, "description", "") + if description: + return description + return getattr(entry, "summary", "") or "" + + +def _truncate_title(title: str) -> str: + if len(title) <= TITLE_MAX_CHARS: + return title + return title[:TITLE_MAX_CHARS].rstrip() + "…" + + +def _parse_feed_entries(content, feed_info: Dict, cutoff_time: datetime) -> List[Dict]: + """把 feed 字节/字符串解析为条目列表，按 cutoff 时间过滤""" + feed = feedparser.parse(content) + entries = [] + + for entry in feed.entries: + pub_date = parse_entry_time(entry) + + # RSS 通常按时间倒序排列，一旦发现过期直接跳出 + if pub_date and pub_date < cutoff_time: + break + + entries.append( + { + "title": _truncate_title(entry.get("title", "无标题")), + "link": entry.get("link", ""), + "published": pub_date, + "source": feed_info["title"], + "content": _extract_body(entry), + "tags": [], + "score": 0, + "summary": "", + } + ) + + return entries + + +async def _fetch_nitter_content(url: str, timeout: int) -> Optional[bytes]: + """nitter / xcancel 专用：requests + Inoreader UA，丢线程池避免阻塞 loop""" + + def _sync(): + try: + r = requests.get(url, headers=NITTER_HEADERS, timeout=timeout) + if r.status_code != 200: + print(f"⚠️ HTTP {r.status_code}: {url}") + return None + return r.content + except Exception as e: + print(f"⚠️ nitter 抓取失败 {url}: {e}") + return None + + return await asyncio.to_thread(_sync) + + +async def _fetch_aiohttp_content( + url: str, timeout: int, session: aiohttp.ClientSession = None +) -> Optional[str]: + """普通 RSS 源：aiohttp + 浏览器 UA""" + client_timeout = aiohttp.ClientTimeout(total=timeout) + + if session is not None: + async with session.get( + url, headers=DEFAULT_HEADERS, timeout=client_timeout + ) as resp: + if resp.status != 200: + print(f"⚠️ HTTP {resp.status}: {url}") + return None + return await resp.text() + + async with aiohttp.ClientSession() as sess: + async with sess.get( + url, headers=DEFAULT_HEADERS, timeout=client_timeout + ) as resp: + if resp.status != 200: + print(f"⚠️ HTTP {resp.status}: {url}") + return None + return await resp.text() + + +async def fetch_single_feed_async( + feed_info: Dict, + cutoff_time: datetime, + timeout: int = 5, + session: aiohttp.ClientSession = None, +) -> List[Dict]: + """异步获取单个源的条目""" + try: + if timeout is None: + timeout = DEFAULT_FEED_TIMEOUT + + url = feed_info["xmlUrl"] + + if is_nitter_url(url): + content = await _fetch_nitter_content(url, timeout) + else: + content = await _fetch_aiohttp_content(url, timeout, session) + + if content is None: + return [] + + return _parse_feed_entries(content, feed_info, cutoff_time) + except Exception as e: + print(f"⚠️ 获取失败 {feed_info['title']}: {e}") + return [] + + +async def fetch_all_feeds( + feeds: List[Dict], cutoff_time: datetime, max_workers: int = 10, timeout: int = None +) -> List[Dict]: + """并发获取所有源的条目；nitter/xcancel 走独立的低并发池""" + if timeout is None: + timeout = DEFAULT_FEED_TIMEOUT + + nitter_feeds = [f for f in feeds if is_nitter_url(f.get("xmlUrl", ""))] + normal_feeds = [f for f in feeds if not is_nitter_url(f.get("xmlUrl", ""))] + + normal_sem = asyncio.Semaphore(max_workers) + nitter_sem = asyncio.Semaphore(NITTER_MAX_CONCURRENCY) + + async def fetch_normal(feed): + async with normal_sem: + return await fetch_single_feed_async(feed, cutoff_time, timeout) + + async def fetch_nitter(feed): + async with nitter_sem: + result = await fetch_single_feed_async(feed, cutoff_time, timeout) + # 公益实例：抓完 sleep，把同一 worker 串内的请求拉开 + await asyncio.sleep(NITTER_REQUEST_DELAY) + return result + + ordered_feeds = normal_feeds + nitter_feeds + tasks = [fetch_normal(f) for f in normal_feeds] + [ + fetch_nitter(f) for f in nitter_feeds + ] + + results = await asyncio.gather(*tasks, return_exceptions=True) + + all_entries = [] + for feed, result in zip(ordered_feeds, results): + if isinstance(result, Exception): + print(f"⚠️ 获取失败 {feed['title']}: {result}") + else: + all_entries.extend(result) + + return all_entries diff --git a/ai-daily-main/src/llm.py b/ai-daily-main/src/llm.py new file mode 100644 index 0000000..731a663 --- /dev/null +++ b/ai-daily-main/src/llm.py @@ -0,0 +1,634 @@ +"""LLM模块 - 评分和汇总""" + +import asyncio +import json +import os +import re +from datetime import datetime +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +from src.markdown_utils import normalize_str_list, parse_frontmatter + + +def load_prompt(prompt_path: str, **kwargs) -> str: + """加载提示词模板并填充变量""" + path = Path(prompt_path) + if not path.exists(): + raise FileNotFoundError(f"提示词文件不存在: {prompt_path}") + + with open(path, "r", encoding="utf-8") as f: + template = f.read() + + # 先把模板中的 {{ 和 }} 替换成占位符，避免与format冲突 + template = template.replace("{{", "\x00LEFT_BRACE\x00").replace( + "}}", "\x00RIGHT_BRACE\x00" + ) + + # 替换变量 + for key, value in kwargs.items(): + template = template.replace(f"{{{key}}}", str(value)) + + # 恢复 {{ 和 }} + template = template.replace("\x00LEFT_BRACE\x00", "{").replace( + "\x00RIGHT_BRACE\x00", "}" + ) + + return template + + +async def call_llm( + prompt: str, config: Dict, response_format: Optional[Dict] = None +) -> str: + """调用LLM API - 统一使用OpenAI兼容接口""" + model = config.get("model", "gpt-4o-mini") + base_url = config.get("baseUrl", "https://api.openai.com/v1") + api_key_name = config.get("apiKeyName", "OPENAI_API_KEY") + + api_key = os.environ.get(api_key_name) + if not api_key: + raise ValueError(f"未设置{api_key_name}环境变量") + + import aiohttp + + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + } + + payload = { + "model": model, + "messages": [{"role": "user", "content": prompt}], + "temperature": 0.3, + } + if response_format is not None: + payload["response_format"] = response_format + + url = f"{base_url}/chat/completions" + + async with aiohttp.ClientSession() as session: + async with session.post(url, headers=headers, json=payload) as resp: + if resp.status != 200: + text = await resp.text() + raise RuntimeError(f"LLM API错误: {resp.status} - {text}") + + data = await resp.json() + return data["choices"][0]["message"]["content"] + + +async def check_llm_available(config: Dict, timeout_seconds: int = 15) -> str: + """启动时检查 LLM 接口可用性""" + prompt = "Reply with OK only." + + try: + response = await asyncio.wait_for( + call_llm(prompt, config), timeout=timeout_seconds + ) + except asyncio.TimeoutError as exc: + raise RuntimeError(f"LLM可用性检查超时({timeout_seconds}s)") from exc + except Exception as exc: + raise RuntimeError(f"LLM可用性检查失败: {exc}") from exc + + response_text = response.strip() + if not response_text: + raise RuntimeError("LLM可用性检查返回空响应") + + return response_text + + +def _build_batch_prompt(entries: List[Dict], prompt_path: str = None) -> str: + """构建批量评分prompt""" + # 构建entries JSON列表（只包含必要字段） + entries_for_llm = [] + for e in entries: + entries_for_llm.append( + { + "link": e.get("link", ""), + "title": e.get("title", "无标题"), + "source": e.get("source", "未知来源"), + "published": e.get("published", ""), + "content": e.get("content", "")[:2000], # 限制内容长度 + } + ) + + entries_json = json.dumps(entries_for_llm, ensure_ascii=False, indent=2) + + # 从文件加载提示词模板，如果未指定则使用默认路径 + if prompt_path is None: + prompt_path = "prompts/score_batch.md" + + return load_prompt(prompt_path, entries_json=entries_json) + + +def _parse_llm_json_response(response: str) -> List[Dict]: + """解析LLM返回的JSON响应""" + text = response.strip() + + # 尝试去除markdown代码块 + if text.startswith("```json"): + text = text[7:] + elif text.startswith("```"): + text = text[3:] + + if text.endswith("```"): + text = text[:-3] + + text = text.strip() + + # 尝试查找JSON数组 + if text.startswith("[") and text.endswith("]"): + try: + return json.loads(text) + except json.JSONDecodeError: + print("⚠️ 直接解析JSON失败，尝试从文本中提取JSON数组") + pass + + # 尝试从文本中提取JSON数组 + match = re.search(r"\[.*\]", text, re.DOTALL) + if match: + try: + return json.loads(match.group()) + except json.JSONDecodeError: + print("⚠️ 从文本中提取JSON数组失败:", text) + pass + + raise ValueError(f"无法从响应中解析JSON: {response[:200]}...") + + +def _parse_score_response(response: str) -> List[Dict]: + """解析评分LLM响应。 + + json_object 模式下应返回 {"items": [...]} 形式的对象; + 兼容直接数组与 markdown 包裹作为兜底路径。 + """ + text = response.strip() + + if text.startswith("```json"): + text = text[7:] + elif text.startswith("```"): + text = text[3:] + if text.endswith("```"): + text = text[:-3] + text = text.strip() + + parsed = None + try: + parsed = json.loads(text) + except json.JSONDecodeError: + for pattern in (r"\{.*\}", r"\[.*\]"): + match = re.search(pattern, text, re.DOTALL) + if match: + try: + parsed = json.loads(match.group()) + break + except json.JSONDecodeError: + continue + + if parsed is None: + print(f"无法从响应中解析JSON: {response}") + raise ValueError(f"无法从响应中解析JSON: {response[:200]}...") + + if isinstance(parsed, list): + return parsed + + if isinstance(parsed, dict): + for key in ("items", "results", "data", "scores"): + if isinstance(parsed.get(key), list): + return parsed[key] + list_values = [v for v in parsed.values() if isinstance(v, list)] + if len(list_values) == 1: + return list_values[0] + + print(f"无法从响应中提取评分数组: {response}") + raise ValueError(f"无法从响应中提取评分数组: {response[:200]}...") + + +def _split_entries_for_batch( + entries: List[Dict], max_prompt_chars: int = 10000 +) -> List[List[Dict]]: + """将entries分成多个批次，每批不超过max_prompt_chars字符""" + if not entries: + return [] + + batches = [] + current_batch = [] + current_chars = 0 + + # 预留prompt模板和JSON包装的空间 + overhead = len(_build_batch_prompt([])) + 500 + + for entry in entries: + # 估算该entry在JSON中的字符数 + entry_chars = len( + json.dumps( + { + "link": entry.get("link", ""), + "title": entry.get("title", "")[:100], + "source": entry.get("source", ""), + "published": entry.get("published", ""), + "content": entry.get("content", "")[:2000], + }, + ensure_ascii=False, + ) + ) + + # 如果当前批次加上这个entry会超出限制，且当前批次不为空，则创建新批次 + if current_chars + entry_chars + overhead > max_prompt_chars and current_batch: + batches.append(current_batch) + current_batch = [entry] + current_chars = entry_chars + else: + current_batch.append(entry) + current_chars += entry_chars + + # 添加最后一个批次 + if current_batch: + batches.append(current_batch) + + return batches + + +def _reconcile_batch_results( + entries: List[Dict], results: List[Dict], batch_index: int +) -> Tuple[List[Dict], List[str]]: + """对单批评分结果按 link 过滤，保留可回收结果""" + entry_links = {entry.get("link") for entry in entries if entry.get("link")} + matched_results = [] + result_links = set() + + for item in results: + if not isinstance(item, dict): + continue + + link = item.get("link") + if link: + result_links.add(link) + if link in entry_links: + matched_results.append(item) + + errors = [] + if len(results) != len(entries) or len(matched_results) != len(entries): + missing_links = sorted(entry_links - result_links) + error_message = ( + "批次{batch} 评分结果异常: 输入{input_count}, 返回{output_count}, " + "匹配{matched_count}, 未评分链接({missing_count}): {missing}" + ).format( + batch=batch_index + 1, + input_count=len(entries), + output_count=len(results), + matched_count=len(matched_results), + missing_count=len(missing_links), + missing=missing_links, + ) + print(f"⚠️ {error_message}") + errors.append(error_message) + + return matched_results, errors + + +async def _score_single_batch( + entries: List[Dict], config: Dict, batch_index: int = 0 +) -> Tuple[List[Dict], List[str]]: + """对单批entries进行评分""" + # 从config获取批量评分提示词路径 + prompt_path = config.get("prompts", {}).get("score_batch", "prompts/score_batch.md") + prompt = _build_batch_prompt(entries, prompt_path) + + try: + response = await call_llm( + prompt, config, response_format={"type": "json_object"} + ) + results = _parse_score_response(response) + + if not isinstance(results, list): + raise ValueError(f"LLM返回的不是数组: {type(results)}") + + return _reconcile_batch_results(entries, results, batch_index) + + except Exception as e: + error_message = f"批次{batch_index + 1} 评分失败: {e}" + print(f"⚠️ {error_message}") + return [], [error_message] + + +async def score_batch( + entries: List[Dict], config: Dict +) -> Tuple[List[Dict], List[str]]: + """ + 批量评分 - 智能分批处理 + + 根据数据量自动决定分批策略： + - 小批量：一次性发送 + - 大批量：分成多个批次并行处理 + """ + if not entries: + return [], [] + + # 获取分批配置 + max_prompt_chars = config.get("max_prompt_chars", 10000) + max_concurrent_batches = config.get("max_concurrent_batches", 3) + + # 分批 + batches = _split_entries_for_batch(entries, max_prompt_chars) + print(f"📦 分成 {len(batches)} 个批次评分 (共 {len(entries)} 条)") + + # 如果只有一批，直接处理 + if len(batches) == 1: + scores, errors = await _score_single_batch(batches[0], config, batch_index=0) + return _merge_scores(entries, scores), errors + + # 多批并行处理（限制并发数） + semaphore = asyncio.Semaphore(max_concurrent_batches) + + async def score_with_limit(batch_index: int, batch: List[Dict]): + async with semaphore: + return await _score_single_batch(batch, config, batch_index=batch_index) + + # 并发处理所有批次 + batch_tasks = [ + score_with_limit(batch_index, batch) + for batch_index, batch in enumerate(batches) + ] + batch_results = await asyncio.gather(*batch_tasks) + + # 合并所有评分结果 + all_scores = [] + all_errors = [] + for scores, errors in batch_results: + all_scores.extend(scores) + all_errors.extend(errors) + + return _merge_scores(entries, all_scores), all_errors + + +def _merge_scores(entries: List[Dict], scores: List[Dict]) -> List[Dict]: + """将评分结果合并到原始entries中""" + # 构建link到score的映射 + score_map = {s.get("link"): s for s in scores if s.get("link")} + + merged = [] + for entry in entries: + link = entry.get("link") + score_data = score_map.get(link, {}) + + # 确保 score 为整数类型 + score_value = score_data.get("score", entry.get("score")) + if isinstance(score_value, str): + try: + score_value = int(score_value) + except (ValueError, TypeError): + score_value = 0 + + merged.append( + { + **entry, + "tags": score_data.get("tags", entry.get("tags", [])), + "score": score_value, + "summary": score_data.get("summary", entry.get("summary", "")), + } + ) + + return merged + + +async def generate_immediate_push( + entries: List[Dict], config: Dict, recent_push_context: str = "" +) -> Tuple[str, Optional[str]]: + """生成即时推送内容 + + Args: + entries: 原始entries列表（调用方已筛选好高分条目） + config: LLM配置 + recent_push_context: 近期推送上下文，用于去重 + """ + prompt_path = config.get("prompts", {}).get( + "immediate_push", "prompts/immediate_push.txt" + ) + + # 直接使用传入的entries，转为JSON格式传给prompt + prompt = load_prompt( + prompt_path, + count=len(entries), + entries=json.dumps(entries, ensure_ascii=False, indent=2), + recent_push_context=recent_push_context, + ) + + try: + return await call_llm(prompt, config), None + except Exception as e: + error_message = f"生成即时推送失败: {e}" + print(f"⚠️ {error_message}") + return "", error_message + + +async def compose_digest( + entries: List[Dict], + context: List[Dict], + config: Dict, + recent_push_context: str = "", +) -> str: + """生成定时汇总推送内容 + + Args: + entries: 原始entries列表 + context: 历史碎片化信息（用于去重参考），只保留 title, published, tags, summary, source + config: LLM配置 + recent_push_context: 近期汇总推送上下文，用于去重 + """ + prompt_path = config.get("prompts", {}).get("digest", "prompts/digest.md") + + # context 只保留必要字段，拼接成字符串 + context_text = [] + for c in context: + tags_str = ", ".join(c.get("tags", [])) if c.get("tags") else "" + context_text.append( + f"[score: {c.get('score', 0)}] title:{c.get('title', '')}\n" + f"published: {c.get('published', '')}\n" + f"tags: {tags_str}\n" + f"source: {c.get('source', '')}\n" + f"summary: {c.get('summary', '')}" + ) + + prompt = load_prompt( + prompt_path, + count=len(entries), + entries=json.dumps(entries, ensure_ascii=False, indent=2), + context="\n\n".join(context_text), + recent_push_context=recent_push_context, + date=datetime.now().strftime("%Y-%m-%d"), + ) + + try: + return await call_llm(prompt, config) + except Exception: + raise + + +async def summarize_github_trending( + enriched_repos: List[Dict], config: Dict +) -> Tuple[str, Optional[str]]: + """GH 板块总结:从 enriched 候选中选 1-max_items + 写 markdown。不传历史上下文。""" + prompt_path = config.get("prompts", {}).get( + "section_github", "prompts/section_github.md" + ) + max_items = ( + config.get("sections", {}).get("github_trending", {}).get("max_items", 3) + ) + prompt = load_prompt( + prompt_path, + repos_json=json.dumps(enriched_repos, ensure_ascii=False, indent=2), + max_items=max_items, + ) + try: + return await call_llm(prompt, config), None + except Exception as e: + msg = f"summarize_github_trending 失败: {e}" + print(f"⚠️ {msg}") + return "", msg + + +async def select_ai_related_hn( + candidates: List[Dict], k: int, config: Dict +) -> Tuple[List[str], Optional[str]]: + """轻 LLM:从 HN 首页候选元数据中挑 k 个 AI 相关 id。 + + 输入候选只含 id/title/site/points/comments 字段(不含正文)。 + """ + prompt_path = config.get("prompts", {}).get( + "section_hackernews_select", "prompts/section_hackernews_select.md" + ) + slim = [ + { + "id": c.get("id"), + "title": c.get("title", ""), + "site": c.get("site", ""), + "points": c.get("points", 0), + "comments": c.get("comments", 0), + } + for c in candidates + ] + prompt = load_prompt( + prompt_path, + k=k, + candidates_json=json.dumps(slim, ensure_ascii=False, indent=2), + ) + try: + response = await call_llm(prompt, config) + except Exception as e: + msg = f"select_ai_related_hn 失败: {e}" + print(f"⚠️ {msg}") + return [], msg + + try: + ids = _parse_llm_json_response(response) + except ValueError as e: + msg = f"select_ai_related_hn 解析失败: {e}" + print(f"⚠️ {msg}") + return [], msg + + if not isinstance(ids, list): + return [], "select_ai_related_hn 返回非数组" + return [str(x) for x in ids][:k], None + + +async def summarize_hackernews( + enriched_stories: List[Dict], config: Dict +) -> Tuple[str, Optional[str]]: + """对输入的 K 个 enriched stories 行文(K 由 select_k 决定)。不传历史上下文。""" + prompt_path = config.get("prompts", {}).get( + "section_hackernews", "prompts/section_hackernews.md" + ) + prompt = load_prompt( + prompt_path, + stories_json=json.dumps(enriched_stories, ensure_ascii=False, indent=2), + ) + try: + return await call_llm(prompt, config), None + except Exception as e: + msg = f"summarize_hackernews 失败: {e}" + print(f"⚠️ {msg}") + return "", msg + + +async def generate_trend_insights( + sections: Dict[str, str], config: Dict +) -> Tuple[str, Optional[str]]: + """输入三段成品,返回洞察段 markdown(含 frontmatter)。""" + prompt_path = config.get("prompts", {}).get("insights", "prompts/insights.md") + prompt = load_prompt( + prompt_path, + rss=sections.get("rss", ""), + github=sections.get("github", ""), + hackernews=sections.get("hackernews", ""), + ) + try: + return await call_llm(prompt, config), None + except Exception as e: + msg = f"generate_trend_insights 失败: {e}" + print(f"⚠️ {msg}") + return "", msg + + +def parse_insights_with_metadata(llm_output: str, date: str) -> Tuple[str, Dict]: + """解析 insights LLM 输出,返回 (insights_md, metadata)。 + + metadata 字段:title / excerpt / seotitle / seodescription / lead / highlights / + profile / date。缺失字段补默认值。 + """ + meta, body = parse_frontmatter(llm_output) + insights_md = body if meta else llm_output + + metadata = { + "title": meta.get("title") or f"📰 AI Daily 每日精选 | {date}", + "excerpt": meta.get("excerpt", ""), + "seotitle": meta.get("seotitle", ""), + "seodescription": meta.get("seodescription", ""), + "lead": meta.get("lead", ""), + "highlights": normalize_str_list(meta.get("highlights")), + "profile": "morning", + "date": date, + } + return insights_md, metadata + + +def parse_digest_with_metadata(llm_output: str, date: str) -> Tuple[str, Dict]: + """解析 digest LLM 输出,返回 (digest_md, metadata)。 + + metadata 字段:title / lead / highlights / profile / date。 + 无 frontmatter 时回退到 "🌙 AI Daily 晚报 | {date}" 标题。 + """ + meta, body = parse_frontmatter(llm_output) + digest_md = body if meta else llm_output + + metadata = { + "title": meta.get("title") or f"🌙 AI Daily 晚报 | {date}", + "lead": meta.get("lead", ""), + "highlights": normalize_str_list(meta.get("highlights")), + "profile": "default", + "date": date, + } + return digest_md, metadata + + +def parse_immediate_push_with_metadata( + llm_output: str, default_title: str +) -> Tuple[str, Dict]: + """解析即时推送 LLM 输出,返回 (body, metadata)。 + + metadata 仅含 title / profile。无 frontmatter 时降级到旧式 `# ` 标题提取, + 再降级到 default_title。 + """ + meta, body = parse_frontmatter(llm_output) + + if meta and meta.get("title"): + return body, {"title": meta["title"], "profile": "hotspot"} + + # 兼容旧格式:从正文一级标题提取 + match = re.search(r"^\s*#\s+(.+?)\s*\n(.*)$", llm_output, re.DOTALL | re.MULTILINE) + if match: + return match.group(2).rstrip(), { + "title": match.group(1).strip(), + "profile": "hotspot", + } + + return llm_output, {"title": default_title, "profile": "hotspot"} diff --git a/ai-daily-main/src/main.py b/ai-daily-main/src/main.py new file mode 100644 index 0000000..d090a6b --- /dev/null +++ b/ai-daily-main/src/main.py @@ -0,0 +1,714 @@ +"""AI每日资讯推送系统 - 主程序""" + +import argparse +import asyncio +import os +import sys +from datetime import date, datetime, timedelta, timezone +from typing import Dict, List, Optional + +# 加载 .env 文件 +from dotenv import load_dotenv + +load_dotenv() + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from croniter import croniter + +from src.config import get_timezone, load_config, merge_sources +from src.fetcher import fetch_all_feeds +from src.llm import ( + check_llm_available, + generate_immediate_push, + parse_immediate_push_with_metadata, + score_batch, +) +from src.processor import html_to_markdown +from src.push import send_to_platforms +from src.sections.github.section import run_github_section +from src.sections.hackernews.section import run_hackernews_section +from src.sections.insights.section import run_insights_section +from src.sections.rss.section import run_rss_section +from src.storage import ( + append_entries, + assemble_with_sentinels, + cleanup_old_files, + get_fetch_file, + get_notify_file, + get_push_file, + load_existing_links, + load_recent_notify_content, + load_recent_push_content, + read_entries, + save_notify_file, + save_push_file, +) + + +async def notify_llm_errors(stage: str, errors: List[str], config: Dict): + """发送简单的 LLM 异常通知""" + if not errors: + return + + lines = [ + "## LLM异常", + "", + f"stage: {stage}", + f"time: {now_local(config).strftime('%Y-%m-%d %H:%M:%S')}", + "", + ] + lines.extend(f"- {error}" for error in errors) + + try: + await send_to_platforms("\n".join(lines), config["push"]) + except Exception as e: + print(f"⚠️ LLM异常通知发送失败: {e}") + + +def now_local(config: Dict = None) -> datetime: + """获取配置时区的当前时间""" + return datetime.now(get_timezone(config)) + + +def parse_time_to_local(time_str: str, config: Dict = None) -> Optional[datetime]: + """解析时间字符串为配置时区的datetime""" + try: + dt = datetime.fromisoformat(time_str.replace("Z", "+00:00")) + return dt.astimezone(get_timezone(config)) + except (ValueError, TypeError): + return None + + +def calculate_push_times( + cron_list: List[str], offset_days: int = 0, config: Dict = None +) -> List[datetime]: + base_date = datetime.now(get_timezone(config)).date() + timedelta(days=offset_days) + times = [] + for cron in cron_list: + try: + minute, hour, _, _, _ = cron.split() + t = datetime.combine( + base_date, + datetime.strptime(f"{hour}:{minute}", "%H:%M").time(), + tzinfo=get_timezone(config), + ) + times.append(t) + except ValueError: + continue + return sorted(times) + + +def is_morning_push(now: datetime, config: Dict) -> bool: + """判定当前 push 是否为「早报」(触发 GH/HN/insights 三段)。 + + 规则:在 `schedule.push_cron` 列表里,now 离哪条 cron 最近,就归为那条; + 最近的那条若是当天最早的 cron,则视为早报。 + + 特例: + - `push_cron` 为空 → 不视为早报 + - `push_cron` 只有一条 → 该条即"最早"也即"最近",任何触发都视为早报 + """ + cron_list = config.get("schedule", {}).get("push_cron", []) + if not cron_list: + return False + if len(cron_list) == 1: + return True + + base = now.replace(hour=0, minute=0, second=0, microsecond=0) + today_fires = [croniter(c, base).get_next(datetime) for c in cron_list] + closest = min(today_fires, key=lambda f: abs(now - f)) + return closest == min(today_fires) + + +def collect_entries_for_push( + last_push_time: Optional[datetime], + context_days: int = 2, + min_score: int = 60, + data_dir: str = "news-data", +) -> tuple[List[Dict], List[Dict]]: + """ + 收集推送所需的条目，返回 (待推送条目, 上下文条目) + + 逻辑： + 1. 获取 context_days 天内的所有条目 + 2. 按 min_score 过滤 + 3. push_time = max(last_push_time, now - 24h) + 4. 晚于 push_time 的 → 待推送条目 + 5. 早于 push_time 的 → 上下文条目（用于LLM去重参考） + """ + tz = get_timezone() + now = datetime.now(tz) + + # 获取 context_days 天的所有条目 + all_entries = [] + today = now.date() + for i in range(context_days): + d = today - timedelta(days=i) + fetch_file = get_fetch_file(d, data_dir) + for entry in read_entries(fetch_file): + all_entries.append(entry) + + print( + f"📋 收集总条目: {len(all_entries)} 条 , context_days: {context_days}, min_score:{min_score}" + ) + + # 按 min_score 过滤 + qualified_entries = [e for e in all_entries if (e.get("score") or 0) >= min_score] + print(f"📋 过滤后条目: {len(qualified_entries)} 条 ") + + # 计算推送时间边界：max(last_push_time, now - 24h) + past_24h = now - timedelta(hours=24) + push_cutoff = ( + last_push_time if last_push_time and last_push_time > past_24h else past_24h + ) + + print(f"推送时间边界: {push_cutoff.strftime('%Y-%m-%d %H:%M:%S')}") + + # 分割条目 + to_push = [] + context = [] + # context 只供 LLM 做去重/历史参考,不需要 content/link/fetched_at 等大字段 + CONTEXT_FIELDS = ("title", "source", "score", "summary", "tags", "published") + + for entry in qualified_entries: + entry_time = parse_time_to_local(entry.get("fetched_at", "")) + if entry_time and entry_time > push_cutoff: + to_push.append(entry) + else: + context.append({k: entry.get(k) for k in CONTEXT_FIELDS}) + + # 上下文按分数排序，取前50 + context = sorted(context, key=lambda x: x.get("score", 0), reverse=True)[:50] + + return to_push, context + + +async def run_fetch_job(config: Dict): + print(f"\n{'=' * 50}") + print(f"🔄 Fetch Job | {now_local().strftime('%Y-%m-%d %H:%M:%S')}") + print(f"{'=' * 50}") + + interval = config["schedule"]["fetch_interval_minutes"] + lookback = config["schedule"].get("fetch_lookback_minutes", 120) + lookback = max(lookback, interval) + threshold = lookback + interval + cutoff = datetime.now(timezone.utc) - timedelta(minutes=lookback) + + sources = merge_sources(config["sources"]) + print(f"📂 共 {len(sources)} 个订阅源") + + if not sources: + print("⚠️ 没有可用的订阅源") + return + + max_workers = config.get("fetch", {}).get("max_workers", 20) + timeout = config.get("fetch", {}).get("timeout", 30) + entries = await fetch_all_feeds( + sources, cutoff, max_workers=max_workers, timeout=timeout + ) + print(f"📥 抓取到 {len(entries)} 条原始消息") + + if not entries: + return + + for entry in entries: + entry["content"] = html_to_markdown( + entry.get("content", ""), entry.get("link", "") + ) + + fetch_file = get_fetch_file() + existing_links = load_existing_links(fetch_file, threshold) + new_entries = [ + e for e in entries if e.get("link") and e["link"] not in existing_links + ] + print(f"🆕 新消息 {len(new_entries)} 条 | 链接数：{len(existing_links)}") + + if not new_entries: + return + + print("🤖 LLM评分中...") + # 预处理：将所有 datetime 转换为字符串，避免 JSON 序列化错误 + for entry in new_entries: + if isinstance(entry.get("published"), datetime): + entry["published"] = ( + entry["published"].astimezone(get_timezone(config)).isoformat() + ) + + scored, score_errors = await score_batch(new_entries, config["llm"]) + if score_errors: + print(f"⚠️ [score_batch] {len(score_errors)} 个错误: {score_errors[0]}") + await notify_llm_errors("score_batch", score_errors, config) + + is_new_file = not os.path.exists(fetch_file) + if is_new_file: + cleanup_old_files(days=config["filter"]["keep_days"]) + + # 添加 fetched_at 时间戳 + for entry in scored: + entry["fetched_at"] = now_local().isoformat() + if isinstance(entry.get("published"), datetime): + entry["published"] = ( + entry["published"].astimezone(get_timezone(config)).isoformat() + ) + + # 批量保存到 JSON 文件 + from datetime import date + + meta = {"date": date.today().isoformat()} + append_entries(fetch_file, scored, meta) + + print(f"💾 已保存到 {fetch_file}") + + hot_threshold = config["filter"]["hot_threshold"] + no_content_marker = config["filter"].get("no_content_marker", "[NO_NEW_CONTENT]") + hot_entries = [e for e in scored if (e.get("score") or 0) >= hot_threshold] + if hot_entries: + print(f"🔥 发现 {len(hot_entries)} 条热点消息，即时推送...") + + # 加载近期已推送内容（仅供 LLM 查重，避免风格趋同） + context_days = config["filter"]["context_days"] + recent_notify = load_recent_notify_content(context_days) + recent_push = load_recent_push_content(context_days) + recent_context = ( + f"=== 近期即时推送 ===\n{recent_notify}\n\n" + f"=== 近期汇总推送 ===\n{recent_push}" + ) + + push_content, immediate_push_error = await generate_immediate_push( + hot_entries, config["llm"], recent_push_context=recent_context + ) + + if immediate_push_error: + print(f"⚠️ [generate_immediate_push] {immediate_push_error}") + await notify_llm_errors( + "generate_immediate_push", [immediate_push_error], config + ) + + if not push_content: + print("⚠️ 即时推送内容生成失败，跳过本次热点推送") + print( + f"✅ Fetch Job 完成 | 新消息: {len(scored)} 条 | 热点: {len(hot_entries)} 条" + ) + return + + # 检查是否有实际内容需要推送 + if no_content_marker in push_content: + print(f"ℹ️ 无新内容需要推送 (LLM判定为重复内容)") + else: + # 提取标题并构建 metadata + now = now_local(config) + timestamp = now.strftime("%Y-%m-%d %H:%M") + content_without_title, metadata = parse_immediate_push_with_metadata( + push_content, f"🚨 AI Daily 快讯 | {timestamp}" + ) + metadata["pushTime"] = now.isoformat() + + await send_to_platforms( + content_without_title, + config["push"], + "🚨 AI Daily 快讯 | " + metadata["title"], + metadata=metadata, + ) + # 保存即时推送内容到notify文件 + notify_file = get_notify_file() + save_notify_file(notify_file, content_without_title, metadata) + print(f"💾 已保存即时推送到 {notify_file}") + + print(f"✅ Fetch Job 完成 | 新消息: {len(scored)} 条 | 热点: {len(hot_entries)} 条") + + +async def run_push_job(config: Dict): + print(f"\n{'=' * 50}") + print(f"📤 Push Job | {now_local().strftime('%Y-%m-%d %H:%M:%S')}") + print(f"{'=' * 50}") + + if is_morning_push(now_local(config), config): + await _run_morning_push(config) + else: + await _run_default_push(config) + + +async def _run_default_push(config: Dict): + """晚报或非早报时段:沿用原有纯 RSS digest 流程,委托给 run_rss_section""" + now = now_local(config) + rss_md, metadata, rss_err = await run_rss_section(config, now) + + if rss_err and not rss_md: + print(f"⚠️ [compose_digest] {rss_err}") + await notify_llm_errors("compose_digest", [rss_err], config) + raise RuntimeError(f"RSS section failed: {rss_err}") + + if not rss_md: + # run_rss_section 在无新消息时已打印 "ℹ️ RSS: 无新消息" + return + + # metadata 缺失兜底:parse_digest_with_metadata 失败 / LLM 输出无 frontmatter 时可能返回 None + if not metadata: + date_str = now.strftime("%Y-%m-%d") + metadata = { + "title": f"🌙 AI Daily 晚报 | {date_str}", + "lead": "", + "highlights": [], + "profile": "default", + "date": date_str, + } + metadata.setdefault("pushTime", now.isoformat()) + + await send_to_platforms( + rss_md, + config["push"], + title="📰 AI Daily 每日精选 | " + metadata["title"], + metadata=metadata, + ) + push_file = get_push_file() + rss_count = rss_md.count("###") + save_push_file( + push_file, + rss_md, + rss_count, + rss_count, + profile="default", + metadata=metadata, + ) + print(f"💾 已保存到 {push_file}") + print(f"✅ Push Job 完成 | 推送条目: {rss_count}") + + +async def _run_morning_push(config: Dict): + """早报四模块编排:RSS/GH/HN 并发 → insights 串行 → sentinel 拼装 → 推送 → 落盘。 + + 失败语义: + - RSS 失败 → 整体抛 RuntimeError (核心承诺不变) + - GH/HN/insights 失败 → 该段省略 + 告警,其他段照推 + """ + now = now_local(config) + + rss_result, gh_result, hn_result = await asyncio.gather( + run_rss_section(config, now), + run_github_section(config, now), + run_hackernews_section(config, now), + ) + + # 早报场景:digest 的 metadata 通常会被 insights 段覆盖, + # 但保留以便在 insights 失败时作为兜底来源(title 关键词 / lead / highlights) + rss_md, digest_meta, rss_err = rss_result + gh_md, gh_err = gh_result + hn_md, hn_err = hn_result + + if gh_err: + print(f"⚠️ [section_github] {gh_err}") + await notify_llm_errors("section_github", [gh_err], config) + if hn_err: + print(f"⚠️ [section_hackernews] {hn_err}") + await notify_llm_errors("section_hackernews", [hn_err], config) + + if rss_err and not rss_md: + print(f"⚠️ [compose_digest] {rss_err}") + await notify_llm_errors("compose_digest", [rss_err], config) + raise RuntimeError(f"RSS section failed: {rss_err}") + + insights_md, metadata, insights_err = await run_insights_section( + rss_md, gh_md, hn_md, config, now + ) + if insights_err: + print(f"⚠️ [insights] {insights_err}") + await notify_llm_errors("insights", [insights_err], config) + + # 如果 insights 失败,优先用 digest metadata 兜底;两者都缺再走默认 + if not metadata: + date_str = now.strftime("%Y-%m-%d") + fallback = digest_meta or {} + digest_title = fallback.get("title", "") + + title = digest_title if digest_title else f"📰 AI Daily 每日精选 | {date_str}" + metadata = { + "date": date_str, + "pushTime": now.isoformat(), + "title": title, + "excerpt": "", + "seotitle": "", + "seodescription": "", + "lead": fallback.get("lead", ""), + "highlights": fallback.get("highlights", []), + "profile": "morning", + } + else: + metadata.setdefault("pushTime", now.isoformat()) + + final = assemble_with_sentinels( + { + "rss": rss_md, + "github": gh_md, + "hackernews": hn_md, + "insights": insights_md, + } + ) + + if not final.strip(): + print("ℹ️ 早报无任何段输出,跳过推送") + return + + await send_to_platforms( + final, + config["push"], + title="📰 AI Daily 每日精选 | " + metadata["title"], + metadata=metadata, + ) + push_file = get_push_file() + rss_count = rss_md.count("###") if rss_md else 0 + save_push_file( + push_file, final, rss_count, rss_count, profile="morning", metadata=metadata + ) + print(f"💾 已保存早报到 {push_file}") + + +async def fetch_loop(config: Dict): + """Fetch循环 - 修复时间漂移并支持优雅退出""" + import time + + interval_seconds = config["schedule"]["fetch_interval_minutes"] * 60 + print(f"🔄 Fetch循环已启动 | 严格间隔: {interval_seconds / 60}分钟") + + while True: + start_time = time.monotonic() # 使用 monotonic 避免系统时间修改影响 + + try: + await run_fetch_job(config) + except asyncio.CancelledError: + print("⚠️ Fetch循环被外部取消，正在安全退出...") + break # 允许外部取消任务 + except Exception as e: + print(f"❌ Fetch Job 失败: {e}") + + # 计算任务耗时 + elapsed = time.monotonic() - start_time + # 计算还需要睡多久（如果任务耗时超过间隔，则不睡，立刻进入下一次） + sleep_time = max(0.0, interval_seconds - elapsed) + + if sleep_time > 0: + print(f"⏰ 下次抓取: {sleep_time / 60:.1f}分钟后") + + try: + await asyncio.sleep(sleep_time) + except asyncio.CancelledError: + print("⚠️ 睡眠被中断，Fetch循环安全退出...") + break + + +async def push_loop(config: Dict): + """Push循环 - 无状态 croniter + 原生异步睡眠""" + cron_list = config["schedule"]["push_cron"] + tz = get_timezone(config) + + # 1. 启动前预校验 cron 表达式，过滤掉无效配置 + valid_crons = [] + for cron in cron_list: + if croniter.is_valid(cron): + valid_crons.append(cron) + else: + print(f"⚠️ 忽略无效的 cron 表达式: '{cron}'") + + if not valid_crons: + print("❌ 没有有效的推送时间配置，Push循环退出") + return + + print(f"📤 Push循环已启动 | 定时: {', '.join(valid_crons)} | 时区: {tz}") + + # 2. 主循环 + while True: + try: + now = datetime.now(tz) + + # 💡 核心优化：无状态计算。 + # 每次都基于此刻的真实时间，动态计算所有有效 cron 的下一次时间，取最近的一个。 + # 这样无论 run_push_job 执行多久，或者系统休眠过，永远都不会算错。 + next_push = min( + croniter(cron, now).get_next(datetime) for cron in valid_crons + ) + + wait_seconds = (next_push - datetime.now(tz)).total_seconds() + + if wait_seconds > 0: + print( + f"⏰ 下次推送: {next_push.strftime('%Y-%m-%d %H:%M:%S')} (等待 {wait_seconds / 60:.1f} 分钟)" + ) + + # 💡 核心优化：直接 Sleep。asyncio 天生支持被 CancelledError 瞬间打断 + await asyncio.sleep(wait_seconds) + + # 到达推送时间，执行推送 + print(f"📤 执行推送: {datetime.now(tz).strftime('%Y-%m-%d %H:%M:%S')}") + await run_push_job(config) + + # 增加 1 秒缓冲：防止 run_push_job 执行过快（不到 1 秒）， + # 导致下一个循环的 now 仍停留在当前秒，croniter 算出重复的时间点。 + await asyncio.sleep(1) + + except asyncio.CancelledError: + print("⚠️ Push循环收到取消信号，安全退出...") + break # 直接 break 退出循环即可 + except Exception as e: + print(f"❌ Push 循环异常: {e}") + # 遇到未知异常时休眠 60 秒，防止死循环疯狂报错打满日志 + await asyncio.sleep(60) + + +async def cmd_check(config: Dict) -> int: + """校验 LLM 接口可达性（部署期使用，运行期不再校验）""" + print("🔍 校验 LLM 接口...") + try: + await check_llm_available(config["llm"]) + except Exception as e: + print(f"❌ LLM 接口不可用: {e}") + return 1 + print("✅ LLM 接口可用") + return 0 + + +async def cmd_fetch(config: Dict) -> int: + """单次抓取（systemd timer 调用）""" + try: + await run_fetch_job(config) + return 0 + except Exception as e: + print(f"❌ Fetch 任务失败: {e}") + return 1 + + +async def cmd_push(config: Dict) -> int: + """单次推送（systemd timer 调用）""" + try: + await run_push_job(config) + return 0 + except Exception as e: + print(f"❌ Push 任务失败: {e}") + return 1 + + +async def cmd_loop(config: Dict) -> int: + """长跑模式（本地开发/调试用）""" + print("🔍 检查 LLM 接口可用性...") + try: + await check_llm_available(config["llm"]) + print("✅ LLM 接口可用") + except Exception as e: + print(f"❌ LLM 接口不可用: {e}") + return 1 + await asyncio.gather(fetch_loop(config), push_loop(config)) + return 0 + + +async def cmd_rss(config: Dict) -> int: + """单独跑一次 RSS digest 板块,打印结果不推送""" + print("📰 RSS Digest 单板块运行") + try: + md, meta, err = await run_rss_section(config, now=now_local(config)) + except Exception as e: + print(f"❌ RSS 板块失败: {e}") + return 1 + if err: + print(f"❌ {err}") + return 1 + if not md: + print("ℹ️ 本次无内容") + return 0 + print("\n" + "=" * 60) + print("📑 metadata:") + if meta: + import json as _json + + print(_json.dumps(meta, ensure_ascii=False, indent=2)) + else: + print("(none)") + print("=" * 60) + print(md) + print("=" * 60) + return 0 + + +async def cmd_github(config: Dict) -> int: + """单独跑一次 GitHub trending 板块,打印结果不推送""" + print("⭐ GitHub Trending 单板块运行") + try: + md, err = await run_github_section(config, now=now_local(config)) + except Exception as e: + print(f"❌ GitHub 板块失败: {e}") + return 1 + if err: + print(f"❌ {err}") + return 1 + if not md: + print("ℹ️ 本次无内容") + return 0 + print("\n" + "=" * 60) + print(md) + print("=" * 60) + return 0 + + +async def cmd_hackernews(config: Dict) -> int: + """单独跑一次 Hacker News 板块,打印结果不推送""" + print("🟧 Hacker News 单板块运行") + try: + md, err = await run_hackernews_section(config, now=now_local(config)) + except Exception as e: + print(f"❌ Hacker News 板块失败: {e}") + return 1 + if err: + print(f"❌ {err}") + return 1 + if not md: + print("ℹ️ 本次无内容") + return 0 + print("\n" + "=" * 60) + print(md) + print("=" * 60) + return 0 + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + prog="daily-news", + description="AI 每日资讯推送系统", + ) + sub = parser.add_subparsers(dest="command", required=True) + sub.add_parser("check", help="校验 LLM 接口可达性") + sub.add_parser("fetch", help="单次抓取并退出") + sub.add_parser("push", help="单次推送并退出") + sub.add_parser("loop", help="长跑模式（开发/调试用）") + sub.add_parser("rss", help="单独跑一次 RSS Digest 板块（仅打印,不推送）") + sub.add_parser("github", help="单独跑一次 GitHub Trending 板块（仅打印,不推送）") + sub.add_parser("hackernews", help="单独跑一次 Hacker News 板块（仅打印,不推送）") + return parser.parse_args() + + +def main() -> int: + print("🚀 AI每日资讯推送系统") + args = _parse_args() + + try: + config = load_config() + print("✅ 配置加载成功") + except Exception as e: + print(f"❌ 加载配置失败: {e}") + return 1 + + handlers = { + "check": cmd_check, + "fetch": cmd_fetch, + "push": cmd_push, + "loop": cmd_loop, + "rss": cmd_rss, + "github": cmd_github, + "hackernews": cmd_hackernews, + } + return asyncio.run(handlers[args.command](config)) + + +if __name__ == "__main__": + try: + sys.exit(main()) + except KeyboardInterrupt: + print("\n👋 程序已退出") + sys.exit(0) diff --git a/ai-daily-main/src/markdown_utils.py b/ai-daily-main/src/markdown_utils.py new file mode 100644 index 0000000..94f9e60 --- /dev/null +++ b/ai-daily-main/src/markdown_utils.py @@ -0,0 +1,83 @@ +"""Markdown / YAML frontmatter 公共辅助。 + +供 storage、llm 等模块复用,避免重复实现。本模块零业务依赖,仅依赖 yaml/json/re。 +""" + +import json +import re +from typing import Any, Dict, List, Tuple + +import yaml + +__all__ = [ + "yaml_value", + "dump_frontmatter", + "parse_frontmatter", + "normalize_str_list", +] + + +def yaml_value(v: Any) -> str: + """把单个值序列化为 YAML 合法的 token,借道 JSON 语法。 + + 依据:JSON 是 YAML 1.2 的真子集,任何 json.dumps 的输出都是合法 YAML 标量/序列/映射。 + 始终带引号的字符串可以避免 PyYAML 的若干怪癖(折行、未引号字符串歧义、unicode 转义)。 + """ + if isinstance(v, (dict, list, str)): + return json.dumps(v, ensure_ascii=False) + if isinstance(v, bool): + return "true" if v else "false" + if v is None: + return "" + return str(v) + + +def dump_frontmatter(meta: Dict) -> str: + """把扁平 metadata dict 序列化为 frontmatter 文本(不含包围的 `---`)。 + + 保留插入顺序(title 在前,bookkeeping 字段在后)。仅支持扁平结构 —— 当前所有 + metadata 都是扁平的,无需处理嵌套。 + """ + return "".join(f"{k}: {yaml_value(v)}\n" for k, v in meta.items()) + + +_FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n?(.*)$", re.DOTALL) + + +def parse_frontmatter(text: str) -> Tuple[Dict, str]: + """从 markdown 文本中分离 YAML frontmatter 与正文。 + + 返回 (metadata_dict, body)。无 frontmatter / YAML 解析失败 / 非 dict 时返回 + ({}, 原文) —— 保证降级路径不丢内容,调用方可凭 metadata_dict 是否为空判定。 + """ + match = _FRONTMATTER_RE.match(text.strip()) + if not match: + return {}, text + + try: + meta = yaml.safe_load(match.group(1)) or {} + except yaml.YAMLError: + print("frontmatter 数据段解析失败") + return {}, text + + if not isinstance(meta, dict): + return {}, text + + return meta, match.group(2).strip() + + +def normalize_str_list(value: Any) -> List[str]: + """将 str/list/None 规整为非空字符串列表。 + + 用于兜底 LLM 输出的列表型字段(如 frontmatter 中的 highlights):单字符串包裹为单元素列表, + null/非列表/非字符串返回空列表,列表中夹杂的空白项过滤掉。不做数量截断。 + """ + if not value: + return [] + if isinstance(value, str): + items = [value] + elif isinstance(value, list): + items = value + else: + return [] + return [str(x).strip() for x in items if str(x).strip()] diff --git a/ai-daily-main/src/processor.py b/ai-daily-main/src/processor.py new file mode 100644 index 0000000..dd0f314 --- /dev/null +++ b/ai-daily-main/src/processor.py @@ -0,0 +1,37 @@ +"""内容处理模块 - HTML转Markdown""" + +import re +from urllib.parse import urljoin + +from markdownify import markdownify as md + + +def html_to_markdown(html: str, base_url: str = "") -> str: + """ + 将HTML转换为Markdown，保留链接和图片 + 使用markdownify库，并进行后处理优化 + """ + # markdownify会自动处理<img>为![](url)，<a>为[text](url) + markdown = md(html, heading_style="ATX") + + # 处理相对链接 + if base_url: + + def replace_rel_link(m): + prefix, path, suffix = m.groups() + if path.startswith(("http://", "https://", "data:")): + return m.group(0) + abs_url = urljoin(base_url, path) + return f"{prefix}{abs_url}{suffix}" + + markdown = re.sub(r"(!?\[.*?\]\()(.*?)(\))", replace_rel_link, markdown) + + # 后处理优化 + # 1. 直接匹配移除 xgo.ing 推广链接 + markdown = markdown.replace("[⚡ Powered by xgo.ing](https://xgo.ing)", "") + markdown = markdown.replace("[⚡ Powered by xgo.ing](https://xgo.ing/)", "") + + # 2. 清理多余空行 + markdown = re.sub(r"\n{3,}", "\n\n", markdown) + + return markdown.strip() diff --git a/ai-daily-main/src/push/__init__.py b/ai-daily-main/src/push/__init__.py new file mode 100644 index 0000000..457efbb --- /dev/null +++ b/ai-daily-main/src/push/__init__.py @@ -0,0 +1,42 @@ +"""推送平台模块""" + +from typing import Dict, Optional + +from .base import PushPlatform +from .discord import DiscordPlatform +from .feishu import FeishuPlatform +from .custom import CustomPlatform + + +def create_platform(name: str, config: Dict) -> Optional[PushPlatform]: + """工厂函数，创建推送平台实例""" + platforms = { + "discord": DiscordPlatform, + "feishu": FeishuPlatform, + "custom": CustomPlatform, + } + + if name not in platforms: + raise ValueError(f"未知推送平台: {name}") + + platform_class = platforms[name] + platform = platform_class(config) + + if not platform.validate_config(config): + return None + + return platform + + +async def send_to_platforms(content: str, push_config: Dict, title: str = None, metadata: Optional[Dict] = None): + """发送内容到所有已启用且配置有效的平台""" + for platform_name, platform_conf in push_config.items(): + platform = create_platform(platform_name, platform_conf) + if platform is None: + continue + + try: + await platform.send(content, title, metadata) + print(f"✅ 已推送到 {platform_name}") + except Exception as e: + print(f"❌ 推送到 {platform_name} 失败: {e}") diff --git a/ai-daily-main/src/push/base.py b/ai-daily-main/src/push/base.py new file mode 100644 index 0000000..95cb082 --- /dev/null +++ b/ai-daily-main/src/push/base.py @@ -0,0 +1,26 @@ +"""推送平台基类""" +from abc import ABC, abstractmethod +from typing import Dict + + +class PushPlatform(ABC): + """推送平台抽象基类""" + + def __init__(self, config: Dict): + self.config = config + + @abstractmethod + def validate_config(self, config: Dict) -> bool: + """验证配置是否有效""" + pass + + @abstractmethod + async def send(self, content: str, title: str = None, metadata: Dict = None): + """发送内容 + + Args: + content: 正文内容 + title: 标题（可选，兼容旧接口） + metadata: 元信息（可选，新增参数） + """ + pass diff --git a/ai-daily-main/src/push/custom.py b/ai-daily-main/src/push/custom.py new file mode 100644 index 0000000..e5b27f5 --- /dev/null +++ b/ai-daily-main/src/push/custom.py @@ -0,0 +1,54 @@ +"""自定义 API 推送平台""" +import os +from typing import Dict, Optional +import aiohttp +from .base import PushPlatform + + +class CustomPlatform(PushPlatform): + """自定义 API 推送平台""" + + def validate_config(self, config: Dict) -> bool: + """验证配置""" + if not config.get("enabled", False): + return False + + api_key_name = config.get("apiKeyName") + token_key_name = config.get("tokenKeyName") + + if not api_key_name or not token_key_name: + print("❌ Custom 平台配置缺少 apiKeyName 或 tokenKeyName") + return False + + url = os.getenv(api_key_name) + token = os.getenv(token_key_name) + + if not url or not token: + print(f"❌ 环境变量 {api_key_name} 或 {token_key_name} 未设置") + return False + + return True + + async def send(self, content: str, title: str = None, metadata: Optional[Dict] = None): + """发送到自定义 API""" + api_key_name = self.config.get("apiKeyName") + token_key_name = self.config.get("tokenKeyName") + + url = os.getenv(api_key_name) + token = os.getenv(token_key_name) + + payload = { + "content": content, + "metadata": metadata + } + + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json" + } + + async with aiohttp.ClientSession() as session: + async with session.post(url, json=payload, headers=headers) as resp: + if resp.status != 200: + error_text = await resp.text() + raise Exception(f"API 返回错误 {resp.status}: {error_text}") diff --git a/ai-daily-main/src/push/discord.py b/ai-daily-main/src/push/discord.py new file mode 100644 index 0000000..c84b2f5 --- /dev/null +++ b/ai-daily-main/src/push/discord.py @@ -0,0 +1,60 @@ +"""Discord推送平台""" + +import os +from typing import Dict + +import aiohttp + +from .base import PushPlatform + + +class DiscordPlatform(PushPlatform): + """Discord Webhook推送""" + + def __init__(self, config: Dict): + super().__init__(config) + self.api_key_name = config.get("apiKeyName", "DISCORD_WEBHOOK_URL") + self.webhook_url = os.environ.get(self.api_key_name, "") + + def validate_config(self, config: Dict) -> bool: + """检查Discord配置是否有效""" + if not config.get("enabled", False): + return False + api_key_name = config.get("apiKeyName", "DISCORD_WEBHOOK_URL") + webhook = os.environ.get(api_key_name, "") + return bool(webhook and webhook.startswith("https://discord.com/api/webhooks/")) + + async def send(self, content: str, title: str = None, metadata: Dict = None): + """发送到Discord（忽略 metadata）""" + full_content = f"# {title}\n\n{content}" if title else content + chunks = self._split_content(full_content, limit=2000) + + async with aiohttp.ClientSession() as session: + for chunk in chunks: + payload = {"content": chunk} + async with session.post(self.webhook_url, json=payload) as resp: + if resp.status != 204: + text = await resp.text() + raise RuntimeError(f"Discord推送失败: {resp.status} - {text}") + + def _split_content(self, content: str, limit: int = 2000) -> list: + """Discord限制2000字符，需要分割""" + if len(content) <= limit: + return [content] + + chunks = [] + lines = content.split("\n") + current = "" + + for line in lines: + if len(current) + len(line) + 1 > limit: + if current: + chunks.append(current) + current = line + else: + current += "\n" + line if current else line + + if current: + chunks.append(current) + + return chunks diff --git a/ai-daily-main/src/push/feishu.py b/ai-daily-main/src/push/feishu.py new file mode 100644 index 0000000..7f58bda --- /dev/null +++ b/ai-daily-main/src/push/feishu.py @@ -0,0 +1,92 @@ +"""飞书推送平台""" + +import os +from typing import Dict + +import aiohttp + +from .base import PushPlatform + + +class FeishuPlatform(PushPlatform): + """飞书 Webhook 推送""" + + def __init__(self, config: Dict): + super().__init__(config) + self.api_key_name = config.get("apiKeyName", "FEISHU_WEBHOOK_URL") + self.webhook_url = os.environ.get(self.api_key_name, "") + + def validate_config(self, config: Dict) -> bool: + """检查飞书配置是否有效""" + if not config.get("enabled", False): + return False + api_key_name = config.get("apiKeyName", "FEISHU_WEBHOOK_URL") + webhook = os.environ.get(api_key_name, "") + return bool(webhook) + + async def send(self, content: str, title: str = None, metadata: Dict = None): + """发送到飞书（忽略 metadata）""" + chunks = self._split_content(content, limit=8000) + + async with aiohttp.ClientSession() as session: + for chunk in chunks: + payload = self._build_payload(chunk, title) + async with session.post(self.webhook_url, json=payload) as resp: + if resp.status != 200: + text = await resp.text() + raise RuntimeError(f"飞书推送失败: {resp.status} - {text}") + data = await resp.json() + if data.get("code") != 0: + raise RuntimeError(f"飞书推送失败: {data.get('msg')}") + + def _build_payload(self, content: str, title: str = None) -> Dict: + """ + 构建飞书卡片消息 payload，支持 Markdown， + 参考 https://open.feishu.cn/document/feishu-cards/card-json-v2-structure + """ + + header = {} + if title: + header = { + "title": {"content": title, "tag": "plain_text"}, + "template": "blue", + } + + return { + "msg_type": "interactive", + "card": { + "schema": "2.0", # 【重点1】显式声明使用 V2 版本结构 + "header": header, + "body": { # 【重点2】V2 中，所有的内容元素都必须放在 body 里面 + "elements": [ + { + "tag": "markdown", + "content": content, + "text_align": "left", # 可选：left / center / right + }, + ], + }, + }, + } + + def _split_content(self, content: str, limit: int = 8000) -> list: + """飞书卡片消息 markdown 元素限制 8000 字符""" + if len(content) <= limit: + return [content] + + chunks = [] + lines = content.split("\n") + current = "" + + for line in lines: + if len(current) + len(line) + 1 > limit: + if current: + chunks.append(current) + current = line + else: + current += "\n" + line if current else line + + if current: + chunks.append(current) + + return chunks diff --git a/ai-daily-main/src/sections/__init__.py b/ai-daily-main/src/sections/__init__.py new file mode 100644 index 0000000..fb82036 --- /dev/null +++ b/ai-daily-main/src/sections/__init__.py @@ -0,0 +1 @@ +"""板块模块包。每个子模块导出 run_<board>_section(config, now) -> (markdown, error)""" diff --git a/ai-daily-main/src/sections/github/__init__.py b/ai-daily-main/src/sections/github/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai-daily-main/src/sections/github/repo_enricher.py b/ai-daily-main/src/sections/github/repo_enricher.py new file mode 100644 index 0000000..2fb610f --- /dev/null +++ b/ai-daily-main/src/sections/github/repo_enricher.py @@ -0,0 +1,111 @@ +"""GitHub REST API enrich:metadata + README → enriched repo dict + +匿名调用受 60 req/hr 限,设置 GITHUB_TOKEN 环境变量后走 5000 req/hr。 +""" + +import asyncio +import base64 +import os +from typing import Dict, List, Optional, Tuple + +import aiohttp + +API_BASE = "https://api.github.com" + + +def _auth_headers(token_env: str = "GITHUB_TOKEN") -> Dict[str, str]: + headers = {"Accept": "application/vnd.github+json"} + token = os.environ.get(token_env) + if token: + headers["Authorization"] = f"Bearer {token}" + print(f"🔑 GH: 已配置 {token_env},鉴权调用 (5000 req/hr)") + else: + print(f"⚠️ GH: 未配置 {token_env},匿名调用 (60 req/hr)") + return headers + + +async def _get_json( + session: aiohttp.ClientSession, url: str, timeout: int = 10 +) -> Optional[Dict]: + async with session.get( + url, timeout=aiohttp.ClientTimeout(total=timeout) + ) as resp: + if resp.status == 404: + return None + if resp.status != 200: + raise RuntimeError(f"GitHub API {resp.status} for {url}") + return await resp.json() + + +async def enrich_repo( + session: aiohttp.ClientSession, + repo: Dict, + token_env: str = "GITHUB_TOKEN", + readme_max_chars: int = 3000, + timeout: int = 10, +) -> Optional[Dict]: + """单 repo 双调用 enrich。返回 None 表示该 repo 应剔除(archived 或 metadata 不可达)。 + + 任一调用失败 raise → 调用方按 return_exceptions 模式聚合错误。 + """ + full_name = repo["full_name"] + meta_url = f"{API_BASE}/repos/{full_name}" + readme_url = f"{API_BASE}/repos/{full_name}/readme" + + meta, readme = await asyncio.gather( + _get_json(session, meta_url, timeout=timeout), + _get_json(session, readme_url, timeout=timeout), + ) + + if meta is None: + return None + if meta.get("archived"): + return None + + license_spdx = "" + if isinstance(meta.get("license"), dict): + license_spdx = meta["license"].get("spdx_id") or "" + + readme_excerpt = "" + if readme and readme.get("content"): + try: + raw = base64.b64decode(readme["content"]).decode("utf-8", errors="replace") + readme_excerpt = raw[:readme_max_chars] + except Exception: + readme_excerpt = "" + + return { + **repo, + "topics": meta.get("topics") or [], + "license": license_spdx, + "pushed_at": meta.get("pushed_at") or "", + "readme_excerpt": readme_excerpt, + } + + +async def enrich_repos( + candidates: List[Dict], + token_env: str = "GITHUB_TOKEN", + readme_max_chars: int = 3000, + timeout: int = 10, +) -> Tuple[List[Dict], List[str]]: + """并发 enrich 多个 repo。返回 (enriched_list_with_archived_filtered, errors)""" + errors: List[str] = [] + headers = _auth_headers(token_env) + + async with aiohttp.ClientSession(headers=headers) as session: + results = await asyncio.gather( + *[ + enrich_repo(session, r, token_env, readme_max_chars, timeout) + for r in candidates + ], + return_exceptions=True, + ) + + enriched: List[Dict] = [] + for r, candidate in zip(results, candidates): + if isinstance(r, Exception): + errors.append(f"enrich {candidate['full_name']} 失败: {r}") + elif r is not None: + enriched.append(r) + return enriched, errors diff --git a/ai-daily-main/src/sections/github/section.py b/ai-daily-main/src/sections/github/section.py new file mode 100644 index 0000000..1fea357 --- /dev/null +++ b/ai-daily-main/src/sections/github/section.py @@ -0,0 +1,104 @@ +"""GitHub Trending 板块入口。 + +流程:trending 抓取 → history 过滤 → 候选写回 history → deep-dive → LLM 总结 +""" + +from datetime import datetime +from typing import Dict, Optional, Tuple + +from src.config import get_timezone +from src.sections.github.repo_enricher import enrich_repos +from src.sections.github.trending_scraper import ( + fetch_trending_page, + parse_trending_html, +) +from src.storage import load_trending_history + + +async def run_github_section( + config: Dict, now: Optional[datetime] = None +) -> Tuple[str, Optional[str]]: + cfg = config.get("sections", {}).get("github_trending", {}) + if not cfg.get("enabled", False): + return "", None + + # 延迟 import,Task 11 才提供 summarize_github_trending + from src.llm import summarize_github_trending + + today = (now or datetime.now(get_timezone())).date() + keep_days = config["filter"]["keep_days"] + timeout = cfg.get("request_timeout", 10) + max_deep_dive = cfg.get("max_deep_dive", 10) + readme_max_chars = cfg.get("readme_max_chars", 3000) + history_path = cfg.get("history_file", "news-data/trending-history.json") + token_env = cfg.get("tokenName", "GITHUB_TOKEN") + + # 1. 抓取 + print("📥 GH: 抓取 trending 页...") + try: + html = await fetch_trending_page(timeout=timeout) + except Exception as e: + return "", f"GH 抓取失败: {e}" + + all_repos = parse_trending_html(html) + print(f"📋 GH: 解析 {len(all_repos)} 个 repo") + if not all_repos: + return "", None + + # 2. history 加载 + 清理 + history = load_trending_history(history_path) + before_cleanup = len(history.repos) + history.cleanup(today=today, keep_days=keep_days) + after_cleanup = len(history.repos) + if before_cleanup != after_cleanup: + print( + f"🧹 GH: history 清理过期 {before_cleanup - after_cleanup} 条 (剩 {after_cleanup})" + ) + + # 3. 候选筛选(按 spec §4.3 语义) + candidates = [] + already_seen = 0 + for repo in all_repos: + if repo["url"] in history: + history.touch(repo["url"], today) + already_seen += 1 + else: + candidates.append(repo) + print(f"🔍 GH: history 过滤掉 {already_seen} 条已见,新候选 {len(candidates)} 条") + + if not candidates: + print("ℹ️ GH: 无新候选,跳过") + return "", None + if len(candidates) > max_deep_dive: + print(f"✂️ GH: 候选 {len(candidates)} 超 max_deep_dive={max_deep_dive},截断") + candidates = candidates[:max_deep_dive] + + # 4. 候选写回 history + 持久化 + for repo in candidates: + history.touch(repo["url"], today) + history.save() + + # 5. 并发 enrich + print(f"🌐 GH: 并发 enrich {len(candidates)} 个 repo (REST API)...") + enriched, enrich_errors = await enrich_repos( + candidates, + token_env=token_env, + readme_max_chars=readme_max_chars, + timeout=timeout, + ) + for e in enrich_errors: + print(f"⚠️ GH enrich: {e}") + print( + f"✅ GH: enrich 成功 {len(enriched)} / 失败 {len(enrich_errors)} / " + f"输入 {len(candidates)}" + ) + if not enriched: + return "", None + + # 6. LLM 总结 + print(f"🤖 GH: summarize {len(enriched)} 个候选...") + md, err = await summarize_github_trending(enriched, config["llm"]) + if err: + return "", f"summarize_github_trending: {err}" + print(f"✅ GH: 板块输出 {len(md or '')} chars") + return md or "", None diff --git a/ai-daily-main/src/sections/github/trending_scraper.py b/ai-daily-main/src/sections/github/trending_scraper.py new file mode 100644 index 0000000..91e162b --- /dev/null +++ b/ai-daily-main/src/sections/github/trending_scraper.py @@ -0,0 +1,98 @@ +"""GitHub Trending 单页 HTML 抓取与解析。 + +数据源: https://github.com/trending (无 language / since 过滤) +""" + +import re +from typing import Dict, List + +import aiohttp +from bs4 import BeautifulSoup + +TRENDING_URL = "https://github.com/trending" +USER_AGENT = ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" +) +_NUM_RE = re.compile(r"[\d,]+") + + +def _parse_int(s: str) -> int: + m = _NUM_RE.search(s or "") + if not m: + return 0 + return int(m.group(0).replace(",", "")) + + +async def fetch_trending_page(timeout: int = 10) -> str: + """抓取 trending 页 HTML;非 200 抛 RuntimeError""" + async with aiohttp.ClientSession( + headers={"User-Agent": USER_AGENT} + ) as session: + async with session.get( + TRENDING_URL, timeout=aiohttp.ClientTimeout(total=timeout) + ) as resp: + if resp.status != 200: + text = await resp.text() + raise RuntimeError( + f"GitHub trending 返回 {resp.status}: {text[:200]}" + ) + return await resp.text() + + +def parse_trending_html(html: str) -> List[Dict]: + """解析 trending HTML,返回去重后的 repo 字典数组。""" + if not html: + return [] + + soup = BeautifulSoup(html, "html.parser") + seen_urls = set() + repos: List[Dict] = [] + + for article in soup.select("article.Box-row"): + h2 = article.find("h2") + a = h2.find("a") if h2 else None + if not a or not a.get("href"): + continue + + href = a["href"].strip() + full_name = href.lstrip("/") + url = f"https://github.com/{full_name}" + if url in seen_urls: + continue + seen_urls.add(url) + + # description + desc_tag = article.find("p") + description = (desc_tag.get_text(strip=True) if desc_tag else "") or "" + + # language + lang_tag = article.find("span", attrs={"itemprop": "programmingLanguage"}) + language = (lang_tag.get_text(strip=True) if lang_tag else "") or "" + + # stars_total: 第一个指向 /stargazers 的链接 + stars_total = 0 + star_a = article.find("a", href=re.compile(r"/stargazers$")) + if star_a: + stars_total = _parse_int(star_a.get_text(strip=True)) + + # stars_today: 末尾的 "N stars today" span + stars_today = 0 + for span in article.find_all("span"): + t = span.get_text(strip=True) + if "stars today" in t or "stars this week" in t or "stars this month" in t: + stars_today = _parse_int(t) + break + + repos.append( + { + "url": url, + "full_name": full_name, + "description": description, + "language": language, + "stars_today": stars_today, + "stars_total": stars_total, + } + ) + + return repos diff --git a/ai-daily-main/src/sections/hackernews/__init__.py b/ai-daily-main/src/sections/hackernews/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai-daily-main/src/sections/hackernews/frontpage_scraper.py b/ai-daily-main/src/sections/hackernews/frontpage_scraper.py new file mode 100644 index 0000000..4979098 --- /dev/null +++ b/ai-daily-main/src/sections/hackernews/frontpage_scraper.py @@ -0,0 +1,91 @@ +"""HN 首页 HTML 抓取与解析。 + +数据源: https://news.ycombinator.com/news (30 条) +""" + +import re +from typing import Dict, List + +import aiohttp +from bs4 import BeautifulSoup + +FRONTPAGE_URL = "https://news.ycombinator.com/news" +USER_AGENT = ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" +) +_NUM_RE = re.compile(r"\d+") + + +def _first_int(text: str) -> int: + m = _NUM_RE.search(text or "") + return int(m.group(0)) if m else 0 + + +async def fetch_frontpage(timeout: int = 10) -> str: + async with aiohttp.ClientSession(headers={"User-Agent": USER_AGENT}) as session: + async with session.get( + FRONTPAGE_URL, timeout=aiohttp.ClientTimeout(total=timeout) + ) as resp: + if resp.status != 200: + raise RuntimeError(f"HN frontpage 返回 {resp.status}") + return await resp.text() + + +def parse_frontpage_html(html: str) -> List[Dict]: + """解析首页 HTML,返回 [{id, title, url, site, points, comments, comments_url}] + + 注:HN frontpage 的 athing 行的实际 class 是 'athing submission' (多类), + 用 CSS 选择器 'tr.athing' 仍然匹配。 + """ + if not html: + return [] + soup = BeautifulSoup(html, "html.parser") + stories: List[Dict] = [] + + for athing in soup.select("tr.athing"): + item_id = athing.get("id") + if not item_id: + continue + + title_a = athing.select_one("span.titleline > a") + if not title_a: + continue + title = title_a.get_text(strip=True) + href = title_a.get("href", "") + if href.startswith("item?id="): + url = f"https://news.ycombinator.com/{href}" + site = "" + else: + url = href + site_tag = athing.select_one("span.sitestr") + site = site_tag.get_text(strip=True) if site_tag else "" + + sub_tr = athing.find_next_sibling("tr") + points = 0 + comments = 0 + comments_url = f"https://news.ycombinator.com/item?id={item_id}" + if sub_tr: + score = sub_tr.select_one("span.score") + if score: + points = _first_int(score.get_text(strip=True)) + comment_a = None + for a in sub_tr.find_all("a", href=re.compile(r"^item\?id=")): + comment_a = a + if comment_a: + comments = _first_int(comment_a.get_text(strip=True)) + comments_url = f"https://news.ycombinator.com/{comment_a['href']}" + + stories.append( + { + "id": item_id, + "title": title, + "url": url, + "site": site, + "points": points, + "comments": comments, + "comments_url": comments_url, + } + ) + + return stories diff --git a/ai-daily-main/src/sections/hackernews/item_enricher.py b/ai-daily-main/src/sections/hackernews/item_enricher.py new file mode 100644 index 0000000..2b0d502 --- /dev/null +++ b/ai-daily-main/src/sections/hackernews/item_enricher.py @@ -0,0 +1,237 @@ +"""HN 单 story enrich:Algolia 评论树 + 外链正文。 + +Algolia API: GET /api/v1/items/{id} +- root.text 是 Show HN / Ask HN 的 post 正文 +- root.children[] 是顶层评论(按 HN ranking 排序) +- 每条 child 自己还有 children[],承载嵌套回复 + +enrich 策略:取 L1 + 每个 L1 下前 N 条 L2 回复,合并为 tree JSON: + [{"l1": "顶层评论", "replies": ["回复 1", "回复 2"]}, ...] + +外链正文优先走 Jina Reader (https://r.jina.ai/<url>, 返回 markdown), +失败回退到直接 GET + html_to_markdown。Jina API key 可选(配置后免费额度更高), +环境变量名通过 `sections.hackernews.jinaTokenName` 配置(默认 JINA_API_KEY)。 +""" + +import asyncio +import os +from typing import Dict, List, Optional, Tuple + +import aiohttp + +from src.processor import html_to_markdown + +USER_AGENT = ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" +) + +JINA_READER_BASE = "https://r.jina.ai" + + +def _is_internal_hn_url(url: str) -> bool: + return url.startswith("https://news.ycombinator.com/item?id=") + + +async def _fetch_algolia_item( + session: aiohttp.ClientSession, item_id: str, algolia_base: str, timeout: int +) -> Dict: + url = f"{algolia_base}/items/{item_id}" + async with session.get( + url, timeout=aiohttp.ClientTimeout(total=timeout) + ) as resp: + if resp.status != 200: + raise RuntimeError(f"Algolia /items/{item_id} 返回 {resp.status}") + return await resp.json() + + +async def _fetch_url_html( + session: aiohttp.ClientSession, url: str, timeout: int +) -> str: + async with session.get( + url, timeout=aiohttp.ClientTimeout(total=timeout) + ) as resp: + if resp.status != 200: + raise RuntimeError(f"外链 {url} 返回 {resp.status}") + return await resp.text() + + +async def _fetch_via_jina( + session: aiohttp.ClientSession, + url: str, + timeout: int, + jina_token_env: str = "JINA_API_KEY", +) -> str: + """通过 Jina Reader 拉取外链 markdown。`jina_token_env` 指定 API key 环境变量名,配置后免费额度更高。""" + jina_url = f"{JINA_READER_BASE}/{url}" + headers = {"Accept": "text/markdown"} + api_key = os.environ.get(jina_token_env) if jina_token_env else None + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + async with session.get( + jina_url, headers=headers, timeout=aiohttp.ClientTimeout(total=timeout) + ) as resp: + if resp.status != 200: + raise RuntimeError(f"Jina Reader {url} 返回 {resp.status}") + return await resp.text() + + +async def _fetch_external_markdown( + session: aiohttp.ClientSession, + url: str, + timeout: int, + jina_token_env: str = "JINA_API_KEY", +) -> str: + """获取外链正文 markdown:先走 Jina Reader,失败回退到直接 GET + html_to_markdown。""" + try: + return await _fetch_via_jina(session, url, timeout, jina_token_env=jina_token_env) + except Exception: + html = await _fetch_url_html(session, url, timeout) + return html_to_markdown(html, base_url=url) + + +def _collect_comments_tree( + root: Dict, + top_comments: int, + top_l2_per_l1: int, + comment_max_chars: int, + comments_total_chars: int, +) -> List[Dict]: + """从 Algolia 根节点提取评论树,返回 [{l1, replies}]。 + + 规则: + - L1 上限 `top_comments`,每个 L1 下取前 `top_l2_per_l1` 条 L2 作为 replies + - 每条 text 过 html_to_markdown,单条截断到 `comment_max_chars` + - 累计字符达 `comments_total_chars` 时立即停止(防离群 story 撑爆 prompt) + - 跳过空 text;空 replies 仍保留 `replies: []`,schema 一致 + """ + out: List[Dict] = [] + consumed = 0 + l1_children = (root.get("children") or [])[:top_comments] + for l1 in l1_children: + if consumed >= comments_total_chars: + break + l1_raw = (l1 or {}).get("text") or "" + if not l1_raw: + continue + l1_md = html_to_markdown(l1_raw)[:comment_max_chars] + consumed += len(l1_md) + replies: List[str] = [] + for l2 in ((l1 or {}).get("children") or [])[:top_l2_per_l1]: + if consumed >= comments_total_chars: + break + l2_raw = (l2 or {}).get("text") or "" + if not l2_raw: + continue + l2_md = html_to_markdown(l2_raw)[:comment_max_chars] + consumed += len(l2_md) + replies.append(l2_md) + out.append({"l1": l1_md, "replies": replies}) + return out + + +async def enrich_story( + session: aiohttp.ClientSession, + story: Dict, + top_comments: int, + top_l2_per_l1: int, + comment_max_chars: int, + comments_total_chars: int, + link_content_max_chars: int, + algolia_base: str, + timeout: int, + jina_token_env: str = "JINA_API_KEY", +) -> Dict: + """对单 story enrich。任一子任务失败 → 对应字段留空,不抛。""" + item_id = story["id"] + is_internal = _is_internal_hn_url(story["url"]) + + tasks = [ + _fetch_algolia_item( + session, item_id, algolia_base=algolia_base, timeout=timeout + ) + ] + if not is_internal: + tasks.append( + _fetch_external_markdown( + session, story["url"], timeout=timeout, jina_token_env=jina_token_env + ) + ) + + results = await asyncio.gather(*tasks, return_exceptions=True) + algolia_result = results[0] + external_markdown_result = results[1] if not is_internal else None + + comments_tree: List[Dict] = [] + post_text = "" + if not isinstance(algolia_result, Exception) and algolia_result: + post_text = algolia_result.get("text") or "" + comments_tree = _collect_comments_tree( + algolia_result, + top_comments=top_comments, + top_l2_per_l1=top_l2_per_l1, + comment_max_chars=comment_max_chars, + comments_total_chars=comments_total_chars, + ) + + link_content = "" + if is_internal: + if post_text: + link_content = html_to_markdown(post_text)[:link_content_max_chars] + else: + if ( + not isinstance(external_markdown_result, Exception) + and external_markdown_result + ): + link_content = external_markdown_result[:link_content_max_chars] + + return { + **story, + "link_content": link_content, + "top_comments": comments_tree, + } + + +async def enrich_stories( + stories: List[Dict], + top_comments: int, + top_l2_per_l1: int, + comment_max_chars: int, + comments_total_chars: int, + link_content_max_chars: int, + algolia_base: str = "https://hn.algolia.com/api/v1", + timeout: int = 10, + jina_token_env: str = "JINA_API_KEY", +) -> Tuple[List[Dict], List[str]]: + """并发 enrich 多个 stories。""" + errors: List[str] = [] + if os.environ.get(jina_token_env): + print(f"🔑 HN: 已配置 {jina_token_env},Jina Reader 鉴权调用") + else: + print(f"⚠️ HN: 未配置 {jina_token_env},Jina Reader 匿名调用 (额度受限)") + async with aiohttp.ClientSession(headers={"User-Agent": USER_AGENT}) as session: + results = await asyncio.gather( + *[ + enrich_story( + session, + s, + top_comments=top_comments, + top_l2_per_l1=top_l2_per_l1, + comment_max_chars=comment_max_chars, + comments_total_chars=comments_total_chars, + link_content_max_chars=link_content_max_chars, + algolia_base=algolia_base, + timeout=timeout, + jina_token_env=jina_token_env, + ) + for s in stories + ], + return_exceptions=True, + ) + enriched: List[Dict] = [] + for r, src in zip(results, stories): + if isinstance(r, Exception): + errors.append(f"enrich story {src['id']} 失败: {r}") + else: + enriched.append(r) + return enriched, errors diff --git a/ai-daily-main/src/sections/hackernews/section.py b/ai-daily-main/src/sections/hackernews/section.py new file mode 100644 index 0000000..208342e --- /dev/null +++ b/ai-daily-main/src/sections/hackernews/section.py @@ -0,0 +1,90 @@ +"""HN 板块入口。流程:首页 → 轻 LLM 选 K → enrich → 最终 LLM 行文""" + +from datetime import datetime +from typing import Dict, Optional, Tuple + +from src.sections.hackernews.frontpage_scraper import ( + fetch_frontpage, + parse_frontpage_html, +) +from src.sections.hackernews.item_enricher import enrich_stories + + +async def run_hackernews_section( + config: Dict, now: Optional[datetime] = None +) -> Tuple[str, Optional[str]]: + cfg = config.get("sections", {}).get("hackernews", {}) + if not cfg.get("enabled", False): + return "", None + + # 延迟 import (Task 16 提供这两个函数) + from src.llm import select_ai_related_hn, summarize_hackernews + + timeout = cfg.get("request_timeout", 10) + select_k = cfg.get("select_k", 1) + top_comments = cfg.get("top_comments", 30) + top_l2_per_l1 = cfg.get("top_l2_per_l1", 3) + comment_max_chars = cfg.get("comment_max_chars", 2000) + comments_total_chars = cfg.get("comments_total_chars", 60000) + link_content_max_chars = cfg.get("link_content_max_chars", 50000) + algolia_base = cfg.get("algolia_base", "https://hn.algolia.com/api/v1") + jina_token_env = cfg.get("jinaTokenName", "JINA_API_KEY") + + # 1. 抓首页 + print("📥 HN: 抓取首页...") + try: + html = await fetch_frontpage(timeout=timeout) + except Exception as e: + return "", f"HN 首页抓取失败: {e}" + + front = parse_frontpage_html(html) + print(f"📋 HN: 解析 {len(front)} 条 frontpage stories") + if not front: + return "", None + + # 2. 轻 LLM 初筛 + print(f"🤖 HN: 轻 LLM 初筛 (k={select_k})...") + selected_ids, select_err = await select_ai_related_hn(front, k=select_k, config=config["llm"]) + if select_err: + return "", f"select_ai_related_hn: {select_err}" + if not selected_ids: + print("ℹ️ HN: 初筛未挑出 AI 相关内容,跳过") + return "", None + + selected = [s for s in front if s["id"] in set(selected_ids)] + if not selected: + return "", None + print( + f"🎯 HN: 初筛选出 {len(selected)} 个: " + f"{', '.join(s['id'] for s in selected)}" + ) + + # 3. enrich + print(f"🌐 HN: enrich {len(selected)} 个 story (Algolia + 外链)...") + enriched, enrich_errors = await enrich_stories( + selected, + top_comments=top_comments, + top_l2_per_l1=top_l2_per_l1, + comment_max_chars=comment_max_chars, + comments_total_chars=comments_total_chars, + link_content_max_chars=link_content_max_chars, + algolia_base=algolia_base, + timeout=timeout, + jina_token_env=jina_token_env, + ) + for e in enrich_errors: + print(f"⚠️ HN enrich: {e}") + print( + f"✅ HN: enrich 成功 {len(enriched)} / 失败 {len(enrich_errors)} / " + f"输入 {len(selected)}" + ) + if not enriched: + return "", None + + # 4. LLM 总结 + print(f"🤖 HN: summarize {len(enriched)} 个 enriched story...") + md, err = await summarize_hackernews(enriched, config["llm"]) + if err: + return "", f"summarize_hackernews: {err}" + print(f"✅ HN: 板块输出 {len(md or '')} chars") + return md or "", None diff --git a/ai-daily-main/src/sections/insights/__init__.py b/ai-daily-main/src/sections/insights/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai-daily-main/src/sections/insights/section.py b/ai-daily-main/src/sections/insights/section.py new file mode 100644 index 0000000..0df9b38 --- /dev/null +++ b/ai-daily-main/src/sections/insights/section.py @@ -0,0 +1,37 @@ +"""Insights 板块:基于 RSS/GH/HN 三段成品做跨板块小结""" + +from datetime import datetime +from typing import Dict, Optional, Tuple + +EMPTY_MARKER = "(本次无内容)" + + +async def run_insights_section( + rss_md: str, + gh_md: str, + hn_md: str, + config: Dict, + now: Optional[datetime] = None, +) -> Tuple[str, Optional[Dict], Optional[str]]: + cfg = config.get("sections", {}).get("insights", {}) + if not cfg.get("enabled", False): + return "", None, None + + from src.llm import generate_trend_insights, parse_insights_with_metadata + + sections = { + "rss": rss_md or EMPTY_MARKER, + "github": gh_md or EMPTY_MARKER, + "hackernews": hn_md or EMPTY_MARKER, + } + + md, err = await generate_trend_insights(sections, config["llm"]) + if err: + return "", None, f"generate_trend_insights: {err}" + + if now is None: + now = datetime.now() + date_str = now.strftime("%Y-%m-%d") + insights_md, metadata = parse_insights_with_metadata(md or "", date_str) + + return insights_md, metadata, None diff --git a/ai-daily-main/src/sections/rss/__init__.py b/ai-daily-main/src/sections/rss/__init__.py new file mode 100644 index 0000000..6e882c2 --- /dev/null +++ b/ai-daily-main/src/sections/rss/__init__.py @@ -0,0 +1,3 @@ +from src.sections.rss.section import run_rss_section + +__all__ = ["run_rss_section"] diff --git a/ai-daily-main/src/sections/rss/section.py b/ai-daily-main/src/sections/rss/section.py new file mode 100644 index 0000000..2f76ea6 --- /dev/null +++ b/ai-daily-main/src/sections/rss/section.py @@ -0,0 +1,62 @@ +"""RSS 板块:沿用现有 collect_entries_for_push + compose_digest 流程 + +迁移自 src/main.py::run_push_job 中 RSS digest 部分,行为保持一致。 +""" + +from datetime import datetime +from typing import Dict, Optional, Tuple + +from src.llm import compose_digest, parse_digest_with_metadata +from src.storage import ( + extract_push_time, + get_last_push_file, + load_recent_push_content, +) + + +async def run_rss_section( + config: Dict, now: Optional[datetime] = None +) -> Tuple[str, Optional[Dict], Optional[str]]: + """生成 RSS digest markdown 段(不含 sentinel)。 + + 返回: + (markdown_body, metadata, error) + - 无新内容时返回 ("", None, None) + - compose_digest 失败时返回 ("", None, error_message) + - metadata 字段:title / lead / highlights / profile=default / date + 早报场景下调用方可丢弃 metadata(由 insights 段覆盖) + """ + # 延迟 import 避免循环:Task 20-21 后 main.py 会反向 import run_rss_section + from src.main import collect_entries_for_push + + last_push_file = get_last_push_file() + last_push_time = extract_push_time(last_push_file) if last_push_file else None + + min_score = config["filter"]["min_score"] + context_days = config["filter"]["context_days"] + + to_push, context = collect_entries_for_push( + last_push_time=last_push_time, + context_days=context_days, + min_score=min_score, + ) + + if not to_push: + print("ℹ️ RSS: 无新消息") + return "", None, None + + push_context_days = config["filter"].get("push_context_days", 5) + recent = load_recent_push_content(push_context_days) + + try: + raw = await compose_digest( + to_push, context, config["llm"], recent_push_context=recent + ) + except Exception as e: + msg = f"compose_digest 失败: {e}" + print(f"⚠️ RSS: {msg}") + return "", None, msg + + date_str = (now or datetime.now()).strftime("%Y-%m-%d") + body, metadata = parse_digest_with_metadata(raw or "", date_str) + return body, metadata, None diff --git a/ai-daily-main/src/storage.py b/ai-daily-main/src/storage.py new file mode 100644 index 0000000..799b421 --- /dev/null +++ b/ai-daily-main/src/storage.py @@ -0,0 +1,546 @@ +"""数据存储模块 - JSON文件读写""" + +import json +import re +from datetime import date, datetime, timedelta +from pathlib import Path +from typing import Dict, List, Optional + +from src.config import get_timezone +from src.markdown_utils import dump_frontmatter, parse_frontmatter + + +def get_fetch_file(d: date = None, data_dir: str = "news-data") -> str: + """获取fetch文件路径 (使用配置时区)""" + if d is None: + d = datetime.now(get_timezone()).date() + return f"{data_dir}/fetch-{d.isoformat()}.json" + + +def get_push_file(push_time: datetime = None, data_dir: str = "news-data") -> str: + """生成push文件路径""" + if push_time is None: + push_time = datetime.now(get_timezone()) + time_str = push_time.strftime("%Y-%m-%d-%H-%M-%S") + return f"{data_dir}/push-{time_str}.md" + + +def get_notify_file(d: date = None, data_dir: str = "news-data") -> str: + """获取notify文件路径 (使用配置时区)""" + if d is None: + d = datetime.now(get_timezone()).date() + return f"{data_dir}/notify-{d.isoformat()}.md" + + +def save_notify_file( + filepath: str, + content: str, + metadata: Dict = None, +): + """保存即时推送文件（Markdown格式），同一天的内容追加到同一文件""" + path = Path(filepath) + path.parent.mkdir(parents=True, exist_ok=True) + + notify_time = datetime.now(get_timezone()).isoformat() + + if metadata: + frontmatter_dict = metadata.copy() + else: + frontmatter_dict = {"pushTime": notify_time} + + frontmatter = dump_frontmatter(frontmatter_dict) + + new_content = f"---\n{frontmatter}---\n\n{content}\n\n------\n" + + with open(path, "a", encoding="utf-8") as f: + f.write(new_content) + + +_SECTION_RE_CACHE: Dict[str, re.Pattern] = {} + + +def _section_re(section: str) -> re.Pattern: + """获取/缓存 sentinel 正则。section 名做转义,允许字母数字下划线""" + if section not in _SECTION_RE_CACHE: + s = re.escape(section) + pattern = ( + rf"<!--\s*SECTION:{s}\s*BEGIN\s*-->(.*?)<!--\s*SECTION:{s}\s*END\s*-->" + ) + _SECTION_RE_CACHE[section] = re.compile(pattern, flags=re.DOTALL) + return _SECTION_RE_CACHE[section] + + +def extract_section(push_md: str, section: str) -> str: + """从 push 文件内容中切出 <!-- SECTION:{section} BEGIN/END --> 之间的 markdown。 + + 向后兼容: + - 新文件(带 sentinel): 返回 sentinel 边界内的原文(不去边界空行) + - 老文件(无 sentinel) 且 section == 'rss': 返回整个 push_md + - 老文件(无 sentinel) 且 section != 'rss': 返回空字符串 + - sentinel 残缺(只有 BEGIN 没有 END): 返回空字符串 + """ + match = _section_re(section).search(push_md) + if match: + return match.group(1) + + # 老文件兜底:rss 段视为整个 body + has_any_sentinel = "<!-- SECTION:" in push_md + if section == "rss" and not has_any_sentinel: + return push_md + return "" + + +def load_recent_notify_content( + context_days: int = 3, data_dir: str = "news-data" +) -> str: + """加载最近 context_days 天 notify 文件正文（去除 frontmatter，仅供 LLM 查重） + + notify 文件由多个推送块用 `------` 分隔，每块带各自 frontmatter；这里逐块剥离 + frontmatter 后用 `------` 重新拼接，保留事件全文。 + """ + data_path = Path(data_dir) + if not data_path.exists(): + return "" + + tz = get_timezone() + today = datetime.now(tz).date() + + blocks: List[str] = [] + loaded_files = [] + for i in range(context_days): + d = today - timedelta(days=i) + notify_file = data_path / f"notify-{d.isoformat()}.md" + if not notify_file.exists() or notify_file.stat().st_size == 0: + continue + try: + with open(notify_file, "r", encoding="utf-8") as f: + content = f.read() + except Exception: + continue + for block in content.split("------"): + if not block.strip(): + continue + _, body = parse_frontmatter(block) + if body: + blocks.append(body) + loaded_files.append(notify_file.name) + + if loaded_files: + print( + f" 📂 已加载 {len(loaded_files)} 个 notify 文件: {', '.join(loaded_files)}" + ) + + return "\n\n------\n\n".join(blocks) + + +def load_recent_push_content( + context_days: int = 3, data_dir: str = "news-data", section: str = "rss" +) -> str: + """加载最近 context_days 天 push 文件中指定 section 的正文(去除 frontmatter,仅供 LLM 查重)。 + + Args: + section: sentinel 段名,默认 "rss"。老文件(无 sentinel) 且 section == "rss" + 时会兜底返回整个 body(由 extract_section 处理),其它 section 在 + 老文件上返回空。 + """ + data_path = Path(data_dir) + if not data_path.exists(): + return "" + + tz = get_timezone() + today = datetime.now(tz).date() + + bodies: List[str] = [] + loaded_files = [] + for i in range(context_days): + d = today - timedelta(days=i) + pattern = f"push-{d.isoformat()}-*.md" + for push_file in sorted(data_path.glob(pattern)): + if push_file.stat().st_size == 0: + continue + try: + with open(push_file, "r", encoding="utf-8") as f: + content = f.read() + except Exception: + continue + section_md = extract_section(content, section) + if not section_md: + continue + # 老文件兜底路径会把整篇文件还回来,此时仍需剥离 frontmatter; + # 新文件 sentinel 内不含 frontmatter,parse_frontmatter 会原样返回。 + _, body = parse_frontmatter(section_md) + body = body or section_md + body = body.strip() + if body: + bodies.append(body) + loaded_files.append(push_file.name) + + if loaded_files: + print( + f" 📂 已加载 {len(loaded_files)} 个 push 文件 (section={section}): " + f"{', '.join(loaded_files)}" + ) + + return "\n\n------\n\n".join(bodies) + + +def get_last_push_file(data_dir: str = "news-data") -> Optional[str]: + """从news-data目录找到最新的push文件""" + data_path = Path(data_dir) + if not data_path.exists(): + return None + + push_files = sorted(data_path.glob("push-*.md")) + return str(push_files[-1]) if push_files else None + + +def extract_push_time(filepath: str) -> Optional[datetime]: + """从push文件名提取时间""" + try: + basename = Path(filepath).name + time_str = basename.replace("push-", "").replace(".md", "") + dt = datetime.strptime(time_str, "%Y-%m-%d-%H-%M-%S") + return dt.replace(tzinfo=get_timezone()) + except (ValueError, AttributeError): + return None + + +def read_entries(filepath: str) -> List[Dict]: + """读取fetch文件，返回entries列表""" + path = Path(filepath) + if not path.exists(): + return [] + + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + + return data.get("entries", []) + + +def read_fetch_data(filepath: str) -> Dict: + """读取完整的fetch文件数据（包含meta和entries）""" + path = Path(filepath) + if not path.exists(): + return {"meta": {}, "entries": []} + + # 检查文件是否为空 + if path.stat().st_size == 0: + return {"meta": {}, "entries": []} + + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def save_fetch_file(filepath: str, meta: Dict, entries: List[Dict]): + """保存fetch文件（JSON格式）""" + path = Path(filepath) + path.parent.mkdir(parents=True, exist_ok=True) + + data = {"meta": meta, "entries": entries} + + with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, ensure_ascii=False, indent=2) + + +def append_entries(filepath: str, new_entries: List[Dict], meta: Dict = None): + """追加条目到fetch文件""" + path = Path(filepath) + + # 读取现有数据 + if path.exists(): + data = read_fetch_data(filepath) + else: + data = {"meta": meta or {}, "entries": []} + + # 更新meta（如果提供了） + if meta: + data["meta"].update(meta) + + # 去重：基于link字段 + existing_links = {e.get("link") for e in data["entries"]} + for entry in new_entries: + if entry.get("link") not in existing_links: + data["entries"].append(entry) + existing_links.add(entry.get("link")) + + # 保存 + save_fetch_file(filepath, data["meta"], data["entries"]) + return len(new_entries) + + +def format_entry(entry: Dict) -> str: + """格式化单条条目为Markdown字符串""" + tags = entry.get("tags", []) + tags_str = json.dumps(tags, ensure_ascii=False) if tags else "[]" + score = entry.get("score", "") + summary = entry.get("summary", "") + + return f"""## {entry["title"]} + +--- +source: {entry["source"]} +link: {entry["link"]} +published: {entry["published"]} +fetched_at: {entry["fetched_at"]} +tags: {tags_str} +score: {score} +summary: {summary} +--- + +{entry["content"]} + +------ +""" + + +def json_to_md(data: Dict) -> str: + """ + 将JSON格式的fetch数据转换为Markdown格式，便于阅读 + + Args: + data: {"meta": {...}, "entries": [...]} + + Returns: + Markdown格式的字符串 + """ + meta = data.get("meta", {}) + entries = data.get("entries", []) + + lines = [] + + # 文件头部YAML frontmatter + if meta.get("date"): + lines.append("---") + lines.append(f'date: "{meta["date"]}"') + lines.append("---") + lines.append("") + + # 条目 + for entry in entries: + lines.append(format_entry(entry)) + + return "\n".join(lines) + + +def convert_fetch_json_to_md(json_filepath: str, md_filepath: str = None) -> str: + """ + 将fetch JSON文件转换为Markdown文件 + + Args: + json_filepath: JSON文件路径 + md_filepath: 输出MD文件路径，默认为同名.md + + Returns: + 生成的Markdown内容 + """ + data = read_fetch_data(json_filepath) + md_content = json_to_md(data) + + if md_filepath: + path = Path(md_filepath) + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + f.write(md_content) + + return md_content + + +def save_push_file( + filepath: str, + content: str, + source_count: int, + total_entries: int, + profile: str = "default", + metadata: Dict = None, +): + """保存推送文件（Markdown格式） + + Args: + profile: "morning" | "default" ← 早报或常规;写入 frontmatter,便于按 profile 分析 + metadata: 元信息（可选），如果提供则使用 metadata，否则使用默认格式 + """ + path = Path(filepath) + path.parent.mkdir(parents=True, exist_ok=True) + + if metadata: + # 使用提供的 metadata + frontmatter_dict = metadata.copy() + # 添加推送时间和统计信息 + frontmatter_dict["pushDate"] = datetime.now(get_timezone()).isoformat() + frontmatter_dict["sourceCount"] = source_count + frontmatter_dict["totalEntries"] = total_entries + else: + # 降级：使用默认格式 + push_time = datetime.now(get_timezone()) + frontmatter_dict = { + "profile": profile, + "pushDate": push_time.isoformat(), + "sourceCount": source_count, + "totalEntries": total_entries, + } + + frontmatter = dump_frontmatter(frontmatter_dict) + full_content = f"---\n{frontmatter}---\n\n{content}" + + with open(path, "w", encoding="utf-8") as f: + f.write(full_content) + + +def load_existing_links(filepath: str, threshold: int = 150) -> set: + """加载文件中已有的链接（用于去重） + + 如果当天时间已超过 threshold 分钟，则只需加载当天文件； + 否则需要同时加载当天和昨天的文件（用于处理跨天边界情况）。 + + Args: + filepath: 当天的 fetch 文件路径 + threshold: 阈值（分钟），超过此时间只加载当天文件 + """ + tz = get_timezone() + now = datetime.now(tz) + current_minutes = now.hour * 60 + now.minute + + need_yesterday = current_minutes < threshold + + if not need_yesterday: + if not filepath or not Path(filepath).exists(): + return set() + entries = read_entries(filepath) + return {e.get("link") for e in entries if e.get("link")} + + all_links = set() + if filepath and Path(filepath).exists(): + all_links.update( + {e.get("link") for e in read_entries(filepath) if e.get("link")} + ) + + yesterday = (now - timedelta(days=1)).date() + yesterday_file = get_fetch_file(yesterday) + if Path(yesterday_file).exists(): + all_links.update( + {e.get("link") for e in read_entries(yesterday_file) if e.get("link")} + ) + + return all_links + + +def cleanup_old_files(days: int = 7, data_dir: str = "news-data"): + """清理超过days天的旧文件""" + data_path = Path(data_dir) + if not data_path.exists(): + return + + cutoff = datetime.now() - timedelta(days=days) + deleted_count = 0 + + for pattern in ["fetch-*.json", "fetch-*.md", "push-*.md", "notify-*.md"]: + for file in data_path.glob(pattern): + try: + date_str = ( + file.name.replace("fetch-", "") + .replace("push-", "") + .replace("notify-", "") + .replace(".json", "") + .replace(".md", "") + ) + date_parts = date_str.split("-") + if len(date_parts) >= 3: + file_date = date( + int(date_parts[0]), int(date_parts[1]), int(date_parts[2]) + ) + if file_date < cutoff.date(): + file.unlink() + deleted_count += 1 + print(f" 🗑️ 删除旧文件: {file.name}") + except (ValueError, OSError): + continue + + # trending-history.json: 剪枝过期条目,保留文件本身 + trending_path = data_path / "trending-history.json" + if trending_path.exists() and trending_path.stat().st_size > 0: + try: + history = load_trending_history(str(trending_path)) + before = len(history.repos) + history.cleanup(today=datetime.now().date(), keep_days=days) + after = len(history.repos) + if after < before: + history.save() + print(f" ✂️ trending-history 剪枝: {before} → {after} 条") + except Exception as e: + print(f" ⚠️ trending-history 剪枝失败: {e}") + + if deleted_count > 0: + print(f" ✅ 清理完成: 删除了 {deleted_count} 个旧文件") + + +class TrendingHistory: + """GitHub trending 已查阅 repo 索引。 + + repos 字段:url → last_seen_date (ISO YYYY-MM-DD)。 + 每次早报 cleanup 一次,touch 完所有今日 URL 后 save。 + """ + + def __init__(self, path: str, repos: Dict[str, str]): + self._path = path + self.repos: Dict[str, str] = dict(repos) + + def __contains__(self, url: str) -> bool: + return url in self.repos + + def touch(self, url: str, today: date) -> None: + self.repos[url] = today.isoformat() + + def cleanup(self, today: date, keep_days: int) -> None: + cutoff = today - timedelta(days=keep_days) + self.repos = { + url: d + for url, d in self.repos.items() + if _parse_iso_date_safe(d) is not None and _parse_iso_date_safe(d) >= cutoff + } + + def save(self) -> None: + path = Path(self._path) + path.parent.mkdir(parents=True, exist_ok=True) + payload = { + "repos": self.repos, + "updated_at": datetime.now(get_timezone()).isoformat(), + } + with open(path, "w", encoding="utf-8") as f: + json.dump(payload, f, ensure_ascii=False, indent=2) + + +def _parse_iso_date_safe(s: str) -> Optional[date]: + try: + return date.fromisoformat(s) + except (ValueError, TypeError): + return None + + +def load_trending_history(path: str) -> TrendingHistory: + """读取 trending-history.json;不存在返回空实例。""" + p = Path(path) + if not p.exists() or p.stat().st_size == 0: + return TrendingHistory(path, {}) + try: + with open(p, "r", encoding="utf-8") as f: + data = json.load(f) + return TrendingHistory(path, data.get("repos", {})) + except (json.JSONDecodeError, OSError): + print(f"⚠️ trending-history 读取失败,使用空索引: {path}") + return TrendingHistory(path, {}) + + +_SECTION_ORDER = ("rss", "github", "hackernews", "insights") + + +def assemble_with_sentinels(sections: Dict[str, str]) -> str: + """按固定顺序拼装四段 markdown,每段包 sentinel;空段整段省略。""" + parts: List[str] = [] + for key in _SECTION_ORDER: + body = (sections.get(key) or "").strip() + if not body: + continue + parts.append( + f"<!-- SECTION:{key} BEGIN -->\n{body}\n<!-- SECTION:{key} END -->" + ) + return "\n\n".join(parts) diff --git a/ai-daily-main/systemd/dnews-fetch.service.tmpl b/ai-daily-main/systemd/dnews-fetch.service.tmpl new file mode 100644 index 0000000..73a8457 --- /dev/null +++ b/ai-daily-main/systemd/dnews-fetch.service.tmpl @@ -0,0 +1,15 @@ +[Unit] +Description=Daily News - Fetch Job +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User={{USER}} +Group={{GROUP}} +WorkingDirectory={{PROJECT_DIR}} +EnvironmentFile={{PROJECT_DIR}}/.env +ExecStart={{UV_BIN}} run --project {{PROJECT_DIR}} python -m src.main fetch +LogNamespace=dnews +TimeoutStartSec=20min +Nice=10 diff --git a/ai-daily-main/systemd/dnews-fetch.timer.tmpl b/ai-daily-main/systemd/dnews-fetch.timer.tmpl new file mode 100644 index 0000000..4026786 --- /dev/null +++ b/ai-daily-main/systemd/dnews-fetch.timer.tmpl @@ -0,0 +1,11 @@ +[Unit] +Description=Daily News - Fetch Timer + +[Timer] +OnActiveSec=1s +OnUnitActiveSec={{FETCH_INTERVAL_MIN}}min +AccuracySec=1s +Unit=dnews-fetch.service + +[Install] +WantedBy=timers.target diff --git a/ai-daily-main/systemd/dnews-push.service.tmpl b/ai-daily-main/systemd/dnews-push.service.tmpl new file mode 100644 index 0000000..0629aab --- /dev/null +++ b/ai-daily-main/systemd/dnews-push.service.tmpl @@ -0,0 +1,15 @@ +[Unit] +Description=Daily News - Push Job +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User={{USER}} +Group={{GROUP}} +WorkingDirectory={{PROJECT_DIR}} +EnvironmentFile={{PROJECT_DIR}}/.env +ExecStart={{UV_BIN}} run --project {{PROJECT_DIR}} python -m src.main push +LogNamespace=dnews +TimeoutStartSec=10min +Nice=10 diff --git a/ai-daily-main/systemd/dnews-push.timer.tmpl b/ai-daily-main/systemd/dnews-push.timer.tmpl new file mode 100644 index 0000000..d21426c --- /dev/null +++ b/ai-daily-main/systemd/dnews-push.timer.tmpl @@ -0,0 +1,10 @@ +[Unit] +Description=Daily News - Push Timer + +[Timer] +{{PUSH_ONCALENDAR_LINES}} +AccuracySec=1s +Unit=dnews-push.service + +[Install] +WantedBy=timers.target diff --git a/ai-daily-main/systemd/journald-dnews.conf.tmpl b/ai-daily-main/systemd/journald-dnews.conf.tmpl new file mode 100644 index 0000000..5fe9e3d --- /dev/null +++ b/ai-daily-main/systemd/journald-dnews.conf.tmpl @@ -0,0 +1,2 @@ +[Journal] +MaxRetentionSec={{LOG_RETENTION_DAYS}}day diff --git a/ai-daily-main/tests/__init__.py b/ai-daily-main/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai-daily-main/tests/fetch_hackernews.py b/ai-daily-main/tests/fetch_hackernews.py new file mode 100644 index 0000000..d697ba6 --- /dev/null +++ b/ai-daily-main/tests/fetch_hackernews.py @@ -0,0 +1,45 @@ +"""手动跑一次 HN 首页 + Algolia enrich,验证选择器与 API""" + +import asyncio +import json +import os +import sys + +from dotenv import load_dotenv + +load_dotenv() +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from src.sections.hackernews.frontpage_scraper import ( + fetch_frontpage, + parse_frontpage_html, +) +from src.sections.hackernews.item_enricher import enrich_stories + + +async def main(): + print("📥 抓取 HN 首页...") + html = await fetch_frontpage(timeout=15) + stories = parse_frontpage_html(html) + print(f"📋 解析出 {len(stories)} 条") + for s in stories[:5]: + print(f" - [{s['points']} pts · {s['comments']} comments] {s['title']} ({s['site']})") + + print("\n🔍 enrich 前 1 个外链类故事...") + target = next((s for s in stories if not s["url"].startswith("https://news.ycombinator.com/")), stories[0]) + enriched, errors = await enrich_stories( + [target], + top_comments=5, + top_l2_per_l1=3, + comment_max_chars=2000, + comments_total_chars=60000, + link_content_max_chars=1500, + timeout=15, + ) + for e in errors: + print(f" ⚠️ {e}") + print(json.dumps(enriched, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/ai-daily-main/tests/fetch_news.py b/ai-daily-main/tests/fetch_news.py new file mode 100644 index 0000000..2ccda49 --- /dev/null +++ b/ai-daily-main/tests/fetch_news.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +"""获取新闻脚本 - 模块化测试第一步：获取RSS并存储""" + +import argparse +import asyncio +import os +import sys +from collections import defaultdict +from datetime import datetime, timedelta, timezone +from pathlib import Path +from urllib.parse import urlparse + +# 添加项目根目录到路径 +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from dotenv import load_dotenv + +load_dotenv() + +from src.config import get_timezone, load_config, merge_sources +from src.fetcher import fetch_all_feeds +from src.processor import html_to_markdown +from src.storage import append_entries, get_fetch_file, load_existing_links + + +def parse_args(): + """解析命令行参数""" + parser = argparse.ArgumentParser(description="RSS新闻获取测试") + parser.add_argument( + "--hours", "-H", type=int, default=1, help="获取过去多少小时的新闻 (默认: 1)" + ) + parser.add_argument("--minutes", "-m", type=int, help="获取过去多少分钟的新闻") + parser.add_argument( + "--output-dir", + "-o", + type=str, + default="tests/news-data", + help="输出目录 (默认: tests/news-data)", + ) + parser.add_argument( + "--max-per-domain", + type=int, + default=30, + help="同一域名最大保留源数量 (默认: 30)", + ) + return parser.parse_args() + + +def get_cutoff_time(args) -> datetime: + """根据参数计算截止时间 (返回 UTC 时间)""" + now = datetime.now(timezone.utc) + if args.minutes: + return now - timedelta(minutes=args.minutes) + return now - timedelta(hours=args.hours) + + +def limit_sources_by_domain(sources: list, max_per_domain: int = 30) -> list: + """限制同一域名的源数量""" + domain_sources = defaultdict(list) + + for source in sources: + url = source.get("xmlUrl", "") + try: + domain = urlparse(url).netloc.lower() + # 移除 www 前缀 + if domain.startswith("www."): + domain = domain[4:] + except Exception: + domain = "unknown" + domain_sources[domain].append(source) + + limited_sources = [] + domain_counts = {} + + for domain, src_list in domain_sources.items(): + kept = src_list[:max_per_domain] + limited_sources.extend(kept) + domain_counts[domain] = {"total": len(src_list), "kept": len(kept)} + + return limited_sources, domain_counts + + +async def fetch_news(): + """主函数：获取RSS新闻并存储""" + args = parse_args() + tz = get_timezone() + + print("=" * 60) + print("📰 RSS新闻获取测试 (Step 1)") + print("=" * 60) + + # 1. 加载配置 + print("\n📋 加载配置...") + config = load_config() + all_sources = merge_sources(config["sources"]) + print(f" OPML 解析完成: {len(all_sources)} 个源") + + # 2. 域名限制 + print(f"\n🔍 域名限制 (每域名最多 {args.max_per_domain} 个)...") + sources, domain_stats = limit_sources_by_domain(all_sources, args.max_per_domain) + + # 打印域名统计 + total_domains = len(domain_stats) + limited_domains = sum( + 1 for d in domain_stats.values() if d["total"] > args.max_per_domain + ) + + print(f" 域名总数: {total_domains}") + print(f" 受限域名: {limited_domains}") + print(f" 最终保留: {len(sources)} 个源") + + # 显示受限域名详情 + for domain, stats in sorted(domain_stats.items(), key=lambda x: -x[1]["total"])[:5]: + if stats["total"] > args.max_per_domain: + print(f" - {domain}: {stats['total']} → {stats['kept']}") + + # 3. 计算时间窗口 + cutoff = get_cutoff_time(args) + print(f"\n⏰ 时间窗口") + print(f" UTC: {cutoff.strftime('%Y-%m-%d %H:%M')}") + print( + f" Local: {(datetime.now(tz) - (datetime.now(timezone.utc) - cutoff)).strftime('%Y-%m-%d %H:%M')}" + ) + + # 4. 获取RSS数据 + print(f"\n📡 开始获取...") + max_workers = config.get("fetch", {}).get("max_workers", 10) + timeout = config.get("fetch", {}).get("timeout", 5) + entries = await fetch_all_feeds( + sources, cutoff, max_workers=max_workers, timeout=timeout + ) + + # 5. 统计结果 + print(f"\n📊 获取统计") + print(f" 读取源数: {len(all_sources)}") + print(f" 保留源数: {len(sources)}") + print(f" 获取条目: {len(entries)}") + + if not entries: + print("\n⚠️ 没有获取到新消息") + return 0 + + # 6. 转换HTML到Markdown + print("\n📝 处理内容...") + for entry in entries: + entry["content"] = html_to_markdown( + entry.get("content", ""), entry.get("link", "") + ) + + # 7. 保存到文件 + print(f"\n💾 保存到文件...") + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + # 使用今天的日期作为文件名（JSON格式） + today = datetime.now(tz).date() + fetch_file = output_dir / f"fetch-{today.isoformat()}.json" + + # 加载已有链接去重 + existing_links = ( + load_existing_links(str(fetch_file)) if fetch_file.exists() else set() + ) + new_entries = [ + e for e in entries if e.get("link") and e["link"] not in existing_links + ] + + # 添加时间戳并格式化 + for entry in entries: + entry["fetched_at"] = datetime.now(tz).isoformat() + if isinstance(entry.get("published"), datetime): + entry["published"] = entry["published"].astimezone(tz).isoformat() + + # 使用新的 append_entries 批量保存 + meta = {"date": today.isoformat()} + append_entries(str(fetch_file), entries, meta) + + print(f" 文件: {fetch_file}") + print(f" 保存: {len(entries)} 条") + print(f" 新增: {len(new_entries)} 条") + print(f" 重复: {len(entries) - len(new_entries)} 条") + + print("\n" + "=" * 60) + print("✅ Step 1 完成: RSS获取并存储") + print("=" * 60) + + return len(entries) + + +if __name__ == "__main__": + try: + count = asyncio.run(fetch_news()) + sys.exit(0 if count > 0 else 1) + except KeyboardInterrupt: + print("\n\n👋 已取消") + sys.exit(130) + except Exception as e: + print(f"\n❌ 错误: {e}") + import traceback + + traceback.print_exc() + sys.exit(1) diff --git a/ai-daily-main/tests/fetch_nitter.py b/ai-daily-main/tests/fetch_nitter.py new file mode 100644 index 0000000..9d0f079 --- /dev/null +++ b/ai-daily-main/tests/fetch_nitter.py @@ -0,0 +1,46 @@ +"""手动跑一次 xcancel/nitter 抓取，验证白名单 UA + requests 路径 + +用法：python tests/fetch_nitter.py +""" + +import asyncio +import os +import sys +from datetime import datetime, timedelta, timezone + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from src.fetcher import fetch_all_feeds + +FEEDS = [ + { + "title": "Alibaba_Qwen", + "xmlUrl": "https://rss.xcancel.com/Alibaba_Qwen/rss", + }, + { + "title": "trq212", + "xmlUrl": "https://rss.xcancel.com/trq212/rss", + }, +] + + +async def main(): + cutoff = datetime.now(timezone.utc) - timedelta(days=7) + print(f"📥 抓取 {len(FEEDS)} 个 nitter 源（cutoff={cutoff.isoformat()}）...") + + entries = await fetch_all_feeds(FEEDS, cutoff) + + print(f"📋 拿到 {len(entries)} 条") + for e in entries[:10]: + published = e["published"].isoformat() if e["published"] else "?" + print(f" - [{published}] {e['source']} {e['title']}") + print(f" {e['link']}") + print(f" {e['content']}\n\n") + + if not entries: + print("⚠️ 一条都没拿到 —— 检查 UA / TLS 路径是否生效") + sys.exit(1) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/ai-daily-main/tests/fetch_trending.py b/ai-daily-main/tests/fetch_trending.py new file mode 100644 index 0000000..621079b --- /dev/null +++ b/ai-daily-main/tests/fetch_trending.py @@ -0,0 +1,44 @@ +"""手动跑一次 GH trending 抓取 + deep-dive,验证选择器与 API 接入""" + +import asyncio +import json +import os +import sys + +from dotenv import load_dotenv + +load_dotenv() +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from src.config import load_config +from src.sections.github.repo_enricher import enrich_repos +from src.sections.github.trending_scraper import ( + fetch_trending_page, + parse_trending_html, +) + + +async def main(): + config = load_config() + print("📥 抓取 GitHub Trending...") + html = await fetch_trending_page(timeout=15) + repos = parse_trending_html(html) + print(f"📋 解析出 {len(repos)} 个 repo") + for r in repos[:5]: + print(f" - {r['full_name']} ⭐{r['stars_today']}/{r['stars_total']} | {r['description'][:80]}") + + cfg = config["sections"]["github_trending"] + print(f"\n🔍 enrich 前 {min(3, len(repos))} 个...") + enriched, errors = await enrich_repos( + repos[:3], + token_env=cfg.get("tokenName", "GITHUB_TOKEN"), + readme_max_chars=cfg.get("readme_max_chars", 3000), + timeout=15, + ) + for e in errors: + print(f" ⚠️ {e}") + print(json.dumps(enriched, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/ai-daily-main/tests/manual_trigger_fetch_loop_llm_error.py b/ai-daily-main/tests/manual_trigger_fetch_loop_llm_error.py new file mode 100644 index 0000000..4417597 --- /dev/null +++ b/ai-daily-main/tests/manual_trigger_fetch_loop_llm_error.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +"""手工触发 LLM 错误告警。 + +用法： + source ../.venv/bin/activate + python tests/manual_trigger_fetch_loop_llm_error.py --scenario score + python tests/manual_trigger_fetch_loop_llm_error.py --scenario immediate + python tests/manual_trigger_fetch_loop_llm_error.py --scenario digest + python tests/manual_trigger_fetch_loop_llm_error.py --scenario all + +场景说明： +- score: 走 fetch_loop -> run_fetch_job -> score_batch，触发评分错误通知 +- immediate: 走 run_fetch_job -> generate_immediate_push，触发即时推送生成错误通知 +- digest: 走 run_push_job -> compose_digest，触发汇总生成错误通知 +- all: 依次触发以上三类真实通知 +""" + +import argparse +import asyncio +import sys +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from src.config import load_config +from src.main import fetch_loop, run_fetch_job, run_push_job + +MOCK_FETCH_ENTRIES = [ + { + "title": "Manual Mock News 1", + "link": "https://example.com/mock-1", + "published": "2026-03-08T20:00:00+08:00", + "source": "Manual Mock Source", + "content": "<p>Mock content 1</p>", + "tags": [], + "score": 0, + "summary": "", + }, + { + "title": "Manual Mock News 2", + "link": "https://example.com/mock-2", + "published": "2026-03-08T20:05:00+08:00", + "source": "Manual Mock Source", + "content": "<p>Mock content 2</p>", + "tags": [], + "score": 0, + "summary": "", + }, +] + +MOCK_HOT_SCORED_ENTRIES = [ + { + "title": "Manual Hot News", + "link": "https://example.com/hot-1", + "published": "2026-03-08T21:00:00+08:00", + "source": "Manual Mock Source", + "content": "mock content", + "tags": ["AI"], + "score": 95, + "summary": "manual hot summary", + } +] + +MOCK_DIGEST_ENTRIES = [ + { + "title": "Manual Digest News", + "link": "https://example.com/digest-1", + "published": "2026-03-08T19:00:00+08:00", + "fetched_at": "2026-03-08T21:00:00+08:00", + "source": "Manual Mock Source", + "content": "mock digest content", + "tags": ["AI"], + "score": 88, + "summary": "manual digest summary", + } +] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="手工触发 LLM 错误告警") + parser.add_argument( + "--scenario", + choices=["score", "immediate", "digest", "all"], + default="score", + help="要触发的错误场景，默认 score", + ) + return parser.parse_args() + + +async def _trigger_score_error(config): + mock_sources = [ + { + "title": "Manual Mock Feed", + "xmlUrl": "https://example.com/rss.xml", + "category": "AI", + } + ] + + tmp_dir = Path("test/news-data") + tmp_dir.mkdir(parents=True, exist_ok=True) + fetch_file = tmp_dir / "manual-fetch-loop-error.json" + + async def fake_fetch_all_feeds(*args, **kwargs): + return MOCK_FETCH_ENTRIES + + async def fake_sleep(seconds: float): + if seconds <= 0: + return + raise asyncio.CancelledError() + + with ( + patch("src.main.merge_sources", return_value=mock_sources), + patch("src.main.fetch_all_feeds", side_effect=fake_fetch_all_feeds), + patch("src.main.load_existing_links", return_value=set()), + patch("src.main.get_fetch_file", return_value=str(fetch_file)), + patch( + "src.llm.call_llm", + side_effect=RuntimeError("manual mock llm scoring error"), + ), + patch("src.main.asyncio.sleep", side_effect=fake_sleep), + ): + await fetch_loop(config) + + +async def _trigger_immediate_push_error(config): + tmp_dir = Path("test/news-data") + tmp_dir.mkdir(parents=True, exist_ok=True) + fetch_file = tmp_dir / "manual-immediate-push-error.json" + + async def fake_fetch_all_feeds(*args, **kwargs): + return [dict(entry) for entry in MOCK_HOT_SCORED_ENTRIES] + + with ( + patch( + "src.main.merge_sources", + return_value=[ + { + "title": "Manual", + "xmlUrl": "https://example.com/rss.xml", + "category": "AI", + } + ], + ), + patch("src.main.fetch_all_feeds", side_effect=fake_fetch_all_feeds), + patch("src.main.load_existing_links", return_value=set()), + patch("src.main.get_fetch_file", return_value=str(fetch_file)), + patch( + "src.main.score_batch", + return_value=([dict(entry) for entry in MOCK_HOT_SCORED_ENTRIES], []), + ), + patch( + "src.main.generate_immediate_push", + return_value=("", "manual mock immediate push error"), + ), + ): + await run_fetch_job(config) + + +async def _trigger_digest_error(config): + with ( + patch( + "src.main.collect_entries_for_push", + return_value=([dict(entry) for entry in MOCK_DIGEST_ENTRIES], []), + ), + patch("src.main.get_last_push_file", return_value=None), + patch( + "src.main.compose_digest", + side_effect=RuntimeError("manual mock compose digest error"), + ), + ): + await run_push_job(config) + + +async def _run_selected_scenarios(scenario: str) -> None: + config = load_config() + + if scenario in {"score", "all"}: + print("\n🚨 触发 score_batch 错误通知") + await _trigger_score_error(config) + + if scenario in {"immediate", "all"}: + print("\n🚨 触发 generate_immediate_push 错误通知") + await _trigger_immediate_push_error(config) + + if scenario in {"digest", "all"}: + print("\n🚨 触发 compose_digest 错误通知") + await _trigger_digest_error(config) + + +def main() -> int: + args = parse_args() + + print("🚨 即将触发真实的 LLM 错误提醒") + print(f" - 场景: {args.scenario}") + print(" - 推送: 使用当前 config.json 和环境变量中的真实渠道") + + try: + asyncio.run(_run_selected_scenarios(args.scenario)) + except KeyboardInterrupt: + print("\n已取消") + return 130 + + print("✅ 脚本执行完成；如果推送配置正确，你应该已经收到对应错误提醒。") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ai-daily-main/tests/preview_recent_context.py b/ai-daily-main/tests/preview_recent_context.py new file mode 100644 index 0000000..6af227b --- /dev/null +++ b/ai-daily-main/tests/preview_recent_context.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +"""预览 LLM 查重上下文 —— 直接打印 load_recent_notify_content / load_recent_push_content +返回的内容,便于调试历史去重时实际喂给 LLM 的素材。 + +Usage: + # 默认:读取 config.json 的 filter.context_days,打印 notify + push 两段 + python tests/preview_recent_context.py + + # 指定回溯天数 + python tests/preview_recent_context.py --days 5 + + # 只看一种 + python tests/preview_recent_context.py --type notify + python tests/preview_recent_context.py --type push + + # 自定义数据目录(例如 tests/news-data 里的样本数据) + python tests/preview_recent_context.py --data-dir tests/news-data +""" + +import argparse +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from src.config import load_config +from src.storage import load_recent_notify_content, load_recent_push_content + + +def _resolve_days(cli_days: int | None) -> int: + if cli_days is not None: + return cli_days + try: + config = load_config() + return int(config.get("filter", {}).get("context_days", 3)) + except Exception: + return 3 + + +def _print_section(label: str, content: str) -> None: + sep = "=" * 60 + print(f"\n{sep}\n{label}\n{sep}") + if not content: + print("(空)") + return + print(content) + print( + f"\n--- {label} 字符数: {len(content)} | 块数(按 `------` 切分): " + f"{len([b for b in content.split('------') if b.strip()])} ---" + ) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="预览近 N 天 notify / push 文件正文 (剥离 frontmatter 后)" + ) + parser.add_argument( + "--days", + type=int, + default=None, + help="回溯天数,默认读取 config.filter.context_days(兜底 3)", + ) + parser.add_argument( + "--type", + choices=["notify", "push", "both"], + default="both", + help="预览哪种历史,默认 both", + ) + parser.add_argument( + "--data-dir", + default="news-data", + help="数据目录,默认 news-data", + ) + parser.add_argument( + "--section", + default="rss", + help="push 文件取哪个 sentinel 段,默认 rss(可选 github / hackernews / insights)", + ) + args = parser.parse_args() + + days = _resolve_days(args.days) + print( + f"📅 回溯天数: {days} | 📂 数据目录: {args.data_dir} | " + f"类型: {args.type} | push.section: {args.section}" + ) + + if args.type in ("notify", "both"): + notify_md = load_recent_notify_content(days, args.data_dir) + _print_section("近期即时推送 (notify)", notify_md) + + if args.type in ("push", "both"): + push_md = load_recent_push_content(days, args.data_dir, section=args.section) + _print_section(f"近期汇总推送 (push.{args.section})", push_md) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ai-daily-main/tests/probe_hn_comments.py b/ai-daily-main/tests/probe_hn_comments.py new file mode 100644 index 0000000..e350f89 --- /dev/null +++ b/ai-daily-main/tests/probe_hn_comments.py @@ -0,0 +1,140 @@ +"""Probe HN: 抓取首页前 10 条,统计每个 story 评论树的深度/数量/字符数。 + +用于判断 enrich 策略:是否需要 L2 回复、压缩、过滤短评等。 + +字符数说明: +- "raw" = Algolia 返回的 text 字段(HTML) +- "md" = html_to_markdown 后的字符数(LLM 实际看到的) +""" + +import asyncio +import json +import os +import sys +from typing import Dict, List, Tuple + +import aiohttp + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from src.processor import html_to_markdown +from src.sections.hackernews.frontpage_scraper import ( + fetch_frontpage, + parse_frontpage_html, +) + +ALGOLIA = "https://hn.algolia.com/api/v1/items" +TIMEOUT = 30 + + +async def fetch_item(session: aiohttp.ClientSession, item_id: str) -> Dict: + async with session.get( + f"{ALGOLIA}/{item_id}", + timeout=aiohttp.ClientTimeout(total=TIMEOUT), + ) as resp: + if resp.status != 200: + raise RuntimeError(f"{item_id} -> {resp.status}") + return await resp.json() + + +def walk_tree(node: Dict, depth: int = 0) -> List[Tuple[int, str]]: + """递归遍历评论树,返回 [(depth, text_html), ...]。depth=0 是 story 自身,1 是顶层评论。""" + result: List[Tuple[int, str]] = [] + text = (node or {}).get("text") or "" + if depth > 0 and text: + result.append((depth, text)) + for child in (node or {}).get("children") or []: + result.extend(walk_tree(child, depth + 1)) + return result + + +def stats(samples: List[str]) -> Dict: + if not samples: + return {"count": 0, "raw_chars": 0, "md_chars": 0, "avg_md": 0, "max_md": 0} + md_lens = [len(html_to_markdown(s)) for s in samples] + raw_lens = [len(s) for s in samples] + return { + "count": len(samples), + "raw_chars": sum(raw_lens), + "md_chars": sum(md_lens), + "avg_md": round(sum(md_lens) / len(md_lens)), + "max_md": max(md_lens), + } + + +def by_depth(nodes: List[Tuple[int, str]], d: int) -> List[str]: + return [t for depth, t in nodes if depth == d] + + +async def probe_one( + session: aiohttp.ClientSession, story: Dict +) -> Dict: + item_id = story["id"] + data = await fetch_item(session, item_id) + flat = walk_tree(data) + l1 = by_depth(flat, 1) + l2 = by_depth(flat, 2) + l3plus = [t for d, t in flat if d >= 3] + all_comments = [t for _, t in flat] + return { + "id": item_id, + "title": story["title"][:60], + "page_comments": story["comments"], + "L1": stats(l1), + "L2": stats(l2), + "L3+": stats(l3plus), + "ALL": stats(all_comments), + } + + +def print_row(label: str, s: Dict): + print( + f" {label:5s} count={s['count']:4d} md_total={s['md_chars']:7d} " + f"avg={s['avg_md']:5d} max={s['max_md']:6d}" + ) + + +async def main(): + print("📥 抓取 HN 首页...") + html = await fetch_frontpage(timeout=15) + stories = parse_frontpage_html(html)[:10] + print(f"📋 取前 10 条 (实际 {len(stories)} 条)\n") + + async with aiohttp.ClientSession() as session: + results = await asyncio.gather( + *[probe_one(session, s) for s in stories], + return_exceptions=True, + ) + + summary = {"L1": [], "L2": [], "L3+": [], "ALL": []} + for r in results: + if isinstance(r, Exception): + print(f"❌ {r}\n") + continue + print( + f"#{r['id']} page_comments={r['page_comments']}\n {r['title']}" + ) + print_row("L1", r["L1"]) + print_row("L2", r["L2"]) + print_row("L3+", r["L3+"]) + print_row("ALL", r["ALL"]) + print() + for key in summary: + summary[key].append(r[key]) + + print("=" * 70) + print("📊 10 条 story 汇总(平均/总和)\n") + for key in ["L1", "L2", "L3+", "ALL"]: + rows = summary[key] + total_count = sum(r["count"] for r in rows) + total_md = sum(r["md_chars"] for r in rows) + avg_count_per_story = round(total_count / len(rows), 1) if rows else 0 + avg_md_per_story = round(total_md / len(rows)) if rows else 0 + print( + f" {key:5s} 总数={total_count:4d} 总md字符={total_md:8d} " + f"平均每story count={avg_count_per_story:5.1f} md_chars={avg_md_per_story:6d}" + ) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/ai-daily-main/tests/push_news.py b/ai-daily-main/tests/push_news.py new file mode 100644 index 0000000..0403357 --- /dev/null +++ b/ai-daily-main/tests/push_news.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""推送功能测试 (Step 2) + +测试内容： +1. 读取已获取的新闻数据 +2. 测试推送到所有已启用的平台 +3. 验证推送格式和内容 + +使用方法: + python tests/push_news.py # 默认从 fetch 数据读取发送 (--fake) + python tests/push_news.py --fake # 从 fetch 数据读取发送 + python tests/push_news.py --real # 从 news-data/push-*.md 最新文件发送 +""" + +import argparse +import asyncio +import sys +from datetime import date +from pathlib import Path +from typing import Optional + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from dotenv import load_dotenv + +load_dotenv() + +from src.config import load_config +from src.push import send_to_platforms +from src.storage import ( + convert_fetch_json_to_md, + get_fetch_file, + get_last_push_file, + read_entries, +) + + +def parse_args(): + """解析命令行参数""" + parser = argparse.ArgumentParser(description="推送功能测试") + parser.add_argument( + "--fake", action="store_true", help="从 fetch 数据读取发送（默认）" + ) + parser.add_argument( + "--real", action="store_true", help="从 news-data/push-*.md 最新文件发送" + ) + return parser.parse_args() + + +def load_push_content() -> Optional[str]: + """从 news-data 获取最新的 push 文件内容（去掉 frontmatter）""" + last_file = get_last_push_file(data_dir="tests/news-data") + if not last_file: + print(" ❌ 未找到 push 文件") + return None + + print(f" 📂 读取文件: {last_file}") + + with open(last_file, "r", encoding="utf-8") as f: + content = f.read() + + # 去掉 frontmatter + if content.startswith("---"): + parts = content.split("---", 2) + if len(parts) >= 3: + content = parts[2].strip() + + return content + + +async def test_push(mode: str = "fake"): + """测试推送功能""" + print("=" * 60) + print("📤 推送功能测试 (Step 2)") + print(f" 模式: {'real' if mode == 'real' else 'fake (fetch数据)'}") + print("=" * 60) + + # 加载配置 + print("\n📋 加载配置...") + config = load_config() + + content: Optional[str] = None + title: Optional[str] = None + + if mode == "real": + # 从 push 文件读取 + content = load_push_content() + if content: + # 提取标题 + first_line = content.split("\n")[0] if content else "" + if first_line.startswith("# "): + title = first_line.replace("# ", "").strip() + print(f" 📝 内容长度: {len(content)} 字符") + print(f" 📝 标题: {title}") + else: + # 从 fetch 数据读取（默认） + print("\n📖 读取新闻数据...") + fetch_file = get_fetch_file(date.today(), data_dir="tests/news-data") + entries = read_entries(fetch_file) + print(f" 文件: {fetch_file}") + print(f" 获取到 {len(entries)} 条新闻") + + # 同时生成 Markdown 版本便于阅读 + md_file = str(fetch_file).replace(".json", ".md") + convert_fetch_json_to_md(fetch_file, md_file) + print(f" Markdown版本: {md_file}") + + if not entries: + print("\n⚠️ 没有新闻数据，请先运行 fetch_news.py") + return + + # 构建测试消息 + print("\n📝 构建推送消息...") + content = build_test_message(entries[:5]) + print(f" 消息长度: {len(content)} 字符") + + if not content: + print("\n⚠️ 没有内容可推送") + return + + # 推送到所有已启用平台 + print("\n📤 推送消息...") + try: + await send_to_platforms( + content, config["push"], "📰 AI Daily 每日精选 | {Test:YYYY-MM-DD}" + ) + except Exception as e: + print(f" ❌ 推送失败: {e}") + raise + + print("\n" + "=" * 60) + print("✅ Step 2 完成: 推送测试") + print("=" * 60) + + +def build_test_message(entries: list) -> str: + """构建测试消息""" + lines = [ + "📰 **新闻推送测试**", + "", + f"共获取 {len(entries)} 条新闻：", + "", + ] + + for i, entry in enumerate(entries, 1): + title = entry.get("title", "无标题") + source = entry.get("source", "未知来源") + link = entry.get("link", "") + published = entry.get("published", "") + content = entry.get("content", "")[:100] + + lines.append(f"**{i}. {title}**") + lines.append(f" 📰 来源: {source}") + if published: + lines.append(f" ⏰ 时间: {published}") + if link: + lines.append(f" 🔗 链接: {link}") + if content: + lines.append(f" 📝 内容: {content}...") + lines.append("") + + return "\n".join(lines) + + +if __name__ == "__main__": + args = parse_args() + mode = "real" if args.real else "fake" + asyncio.run(test_push(mode)) diff --git a/ai-daily-main/tests/pytest/__init__.py b/ai-daily-main/tests/pytest/__init__.py new file mode 100644 index 0000000..ae599fb --- /dev/null +++ b/ai-daily-main/tests/pytest/__init__.py @@ -0,0 +1 @@ +# pytest tests for daily-news project diff --git a/ai-daily-main/tests/pytest/conftest.py b/ai-daily-main/tests/pytest/conftest.py new file mode 100644 index 0000000..5a04907 --- /dev/null +++ b/ai-daily-main/tests/pytest/conftest.py @@ -0,0 +1,154 @@ +"""pytest fixtures for daily-news project""" + +import json +import pytest +import sys +from datetime import datetime, timezone, timedelta +from pathlib import Path +from unittest.mock import MagicMock + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + + +@pytest.fixture +def temp_dir(tmp_path): + """临时目录fixture""" + return tmp_path + + +@pytest.fixture +def sample_config(): + """示例配置""" + return { + "sources": { + "base_opml": "resources/rss.opml", + "add": [], + "block": [], + "block_domains": ["*.substack.com"], + }, + "filter": { + "min_score": 60, + "hot_threshold": 90, + "context_days": 2, + "keep_days": 7, + }, + "schedule": { + "fetch_interval_minutes": 30, + "push_cron": ["0 8 * * *", "0 17 * * *"], + "timezone_hours": 8, + }, + "llm": { + "provider": "openai", + "model": "gpt-4o-mini", + "baseUrl": "https://api.openai.com/v1", + "apiKeyName": "OPENAI_API_KEY", + "max_prompt_chars": 10000, + "max_concurrent_batches": 3, + }, + "push": { + "discord": { + "enabled": True, + "webhook_url": "https://discord.com/api/webhooks/test/abc", + }, + "feishu": {"enabled": False, "apiKeyName": "FEISHU_WEBHOOK_URL"}, + }, + } + + +@pytest.fixture +def sample_opml(temp_dir): + """示例OPML文件""" + opml_content = """<?xml version="1.0"?> +<opml version="2.0"> + <body> + <outline title="Feed1" xmlUrl="http://feed1.com/rss" type="rss" category="tech"/> + <outline title="Feed2" xmlUrl="http://feed2.com/rss" type="rss" category="ai"/> + </body> +</opml>""" + opml_file = temp_dir / "test.opml" + opml_file.write_text(opml_content) + return str(opml_file) + + +@pytest.fixture +def sample_entry(): + """示例新闻条目""" + return { + "title": "Test Article Title", + "link": "https://example.com/article", + "published": datetime.now(timezone.utc).isoformat(), + "fetched_at": datetime.now(timezone.utc).isoformat(), + "source": "Test Source", + "content": "<p>Test content</p>", + "summary": "Test summary", + "tags": ["AI", "Tech"], + "score": 85, + } + + +@pytest.fixture +def sample_entries(): + """示例新闻条目列表""" + now = datetime.now(timezone.utc) + return [ + { + "title": "Article 1", + "link": "https://example.com/1", + "published": now.isoformat(), + "fetched_at": now.isoformat(), + "source": "Source1", + "content": "Content 1", + "summary": "Summary 1", + "tags": ["AI"], + "score": 85, + }, + { + "title": "Article 2", + "link": "https://example.com/2", + "published": (now - timedelta(hours=1)).isoformat(), + "fetched_at": now.isoformat(), + "source": "Source2", + "content": "Content 2", + "summary": "Summary 2", + "tags": ["Tech"], + "score": 70, + }, + { + "title": "Article 3", + "link": "https://example.com/3", + "published": (now - timedelta(hours=2)).isoformat(), + "fetched_at": now.isoformat(), + "source": "Source3", + "content": "Content 3", + "summary": "Summary 3", + "tags": ["News"], + "score": 55, + }, + ] + + +@pytest.fixture +def sample_fetch_json(temp_dir, sample_entries): + """示例fetch JSON文件""" + data = { + "meta": {"date": datetime.now().date().isoformat()}, + "entries": sample_entries, + } + json_file = temp_dir / "fetch-test.json" + json_file.write_text(json.dumps(data, ensure_ascii=False, indent=2)) + return str(json_file) + + +@pytest.fixture +def mock_httpx_session(): + """Mock httpx/aiohttp session""" + mock_response = MagicMock() + mock_response.status = 200 + mock_response.text = "<rss></rss>" + + mock_session = MagicMock() + mock_session.__aenter__ = MagicMock(return_value=mock_session) + mock_session.__aexit__ = MagicMock(return_value=None) + mock_session.get = MagicMock(return_value=mock_response) + + return mock_session diff --git a/ai-daily-main/tests/pytest/fixtures/github_trending.html b/ai-daily-main/tests/pytest/fixtures/github_trending.html new file mode 100644 index 0000000..a520f73 --- /dev/null +++ b/ai-daily-main/tests/pytest/fixtures/github_trending.html @@ -0,0 +1,3771 @@ + + +<!DOCTYPE html> +<html + lang="en" + + data-color-mode="auto" data-light-theme="light" data-dark-theme="dark" + data-a11y-animated-images="system" data-a11y-link-underlines="true" + + > + + + + + <head> + <meta charset="utf-8"> + <link rel="dns-prefetch" href="https://github.githubassets.com"> + <link rel="dns-prefetch" href="https://avatars.githubusercontent.com"> + <link rel="dns-prefetch" href="https://github-cloud.s3.amazonaws.com"> + <link rel="dns-prefetch" href="https://user-images.githubusercontent.com/"> + <link rel="preconnect" href="https://github.githubassets.com" crossorigin> + <link rel="preconnect" href="https://avatars.githubusercontent.com"> + + + + <link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/light-2ff56e1b36116ee2.css" /><link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/light_high_contrast-f7f95d7633592089.css" /><link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/dark-2d1fe43dbc9adf1f.css" /><link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/dark_high_contrast-d530ee188d165539.css" /><link data-color-theme="light" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/light-2ff56e1b36116ee2.css" /><link data-color-theme="light_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/light_high_contrast-f7f95d7633592089.css" /><link data-color-theme="light_colorblind" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/light_colorblind-e4bbc49fc7b82570.css" /><link data-color-theme="light_colorblind_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/light_colorblind_high_contrast-cc3f126b45166b83.css" /><link data-color-theme="light_tritanopia" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/light_tritanopia-652ca611ea4e33fe.css" /><link data-color-theme="light_tritanopia_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/light_tritanopia_high_contrast-9e82d635cb6c3f49.css" /><link data-color-theme="dark" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark-2d1fe43dbc9adf1f.css" /><link data-color-theme="dark_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_high_contrast-d530ee188d165539.css" /><link data-color-theme="dark_colorblind" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_colorblind-3ad0ec21150df75b.css" /><link data-color-theme="dark_colorblind_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_colorblind_high_contrast-5691ff467f71a3f6.css" /><link data-color-theme="dark_tritanopia" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_tritanopia-abee7710893cd168.css" /><link data-color-theme="dark_tritanopia_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_tritanopia_high_contrast-eafcf6cd46158360.css" /><link data-color-theme="dark_dimmed" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_dimmed-c7974682a1a84c8d.css" /><link data-color-theme="dark_dimmed_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_dimmed_high_contrast-f8dab3e04f94c501.css" /> + + <style type="text/css"> + :root { + --tab-size-preference: 4; + } + + pre, code { + tab-size: var(--tab-size-preference); + } + </style> + + <link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-primitives-7f694b60439d06c0.css" /> + <link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-bf8570e4081bd07e.css" /> + <link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/global-5efd63e783ac04bb.css" /> + <link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/github-f825c0edd7ad57f8.css" /> + <link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/site-8e580c38ab6535be.css" /> +<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/explore-4806b02ffcc9d15a.css" /> + + + + <script type="application/json" id="client-env">{"locale":"en","featureFlags":["actions_custom_images_storage_billing_ui_visibility","actions_image_version_event","actions_workflow_language_service_allow_concurrency_queue","agent_conflict_resolution","alternate_user_config_repo","arianotify_comprehensive_migration","artifact_ui_v2","billing_discount_threshold_notification","code_scanning_dfa_degraded_experience_notice","codespaces_prebuild_region_target_update","coding_agent_model_selection","coding_agent_model_selection_all_skus","comment_viewer_copy_raw_markdown","contentful_primer_code_blocks","copilot_agent_snippy","copilot_api_agentic_issue_marshal_yaml","copilot_ask_mode_dropdown","copilot_automation_session_author","copilot_chat_attach_multiple_images","copilot_chat_category_rate_limit_messages","copilot_chat_clear_model_selection_for_default_change","copilot_chat_contextual_suggestions_updated","copilot_chat_enable_tool_call_logs","copilot_chat_file_redirect","copilot_chat_input_commands","copilot_chat_opening_thread_switch","copilot_chat_prettify_pasted_code","copilot_chat_reduce_quota_checks","copilot_chat_search_bar_redirect","copilot_chat_vision_in_claude","copilot_chat_vision_preview_gate","copilot_custom_copilots","copilot_custom_copilots_feature_preview","copilot_delete_cli_sessions","copilot_diff_explain_conversation_intent","copilot_diff_reference_context","copilot_duplicate_thread","copilot_extensions_hide_in_dotcom_chat","copilot_extensions_removal_on_marketplace","copilot_features_sql_server_logo","copilot_file_block_ref_matching","copilot_ftp_hyperspace_upgrade_prompt","copilot_icebreakers_experiment_dashboard","copilot_icebreakers_experiment_hyperspace","copilot_immersive_code_block_transition_wrap","copilot_immersive_embedded","copilot_immersive_embedded_deferred_payload","copilot_immersive_embedded_draggable","copilot_immersive_embedded_header_button","copilot_immersive_embedded_implicit_references","copilot_immersive_embedded_skip_copilot_api_token_for_dotcom_context","copilot_immersive_file_block_transition_open","copilot_immersive_file_preview_keep_mounted","copilot_immersive_job_result_preview","copilot_immersive_structured_model_picker","copilot_immersive_task_hyperlinking","copilot_immersive_task_within_chat_thread","copilot_mc_cli_resume_any_users_task","copilot_mission_control_agent_filtering","copilot_mission_control_always_send_integration_id","copilot_mission_control_cli_private_icon","copilot_mission_control_cli_session_status","copilot_mission_control_initial_data_spinner","copilot_mission_control_logs_incremental","copilot_mission_control_task_alive_updates","copilot_mission_control_tasks_repo_filter","copilot_org_policy_page_focus_mode","copilot_redirect_header_button_to_agents","copilot_resource_panel","copilot_scroll_preview_tabs","copilot_share_active_subthread","copilot_spaces_ga","copilot_spaces_individual_policies_ga","copilot_spaces_pagination","copilot_spark_empty_state","copilot_spark_handle_nil_friendly_name","copilot_swe_agent_hide_model_picker_if_only_auto","copilot_swe_agent_pr_comment_model_picker","copilot_swe_agent_use_subagents","copilot_task_api_github_rest_style","copilot_unconfigured_is_inherited","copilot_upgrade_freeze","copilot_usage_metrics_ga","copilot_workbench_slim_line_top_tabs","custom_instructions_file_references","dashboard_indexeddb_caching","dashboard_lists_max_age_filter","dashboard_universe_2025_feedback_dialog","enterprise_managed_settings_for_copilot_clients","filter_support_formcontrol","flex_cta_groups_mvp","global_nav_react","hyperspace_2025_logged_out_batch_1","hyperspace_2025_logged_out_batch_2","hyperspace_2025_logged_out_batch_3","ipm_budget_deep_linking","ipm_global_transactional_message_agents","ipm_global_transactional_message_copilot","ipm_global_transactional_message_issues","ipm_global_transactional_message_prs","ipm_global_transactional_message_repos","ipm_global_transactional_message_spaces","issue_cca_modal_open","issue_cca_multi_assign_modal","issue_cca_task_side_panel","issue_cca_visualization","issue_cca_visualization_session_panel","issue_fields_global_search","issue_type_filter_no_relay","issues_expanded_file_types","issues_lazy_load_comment_box_suggestions","issues_react_chrome_container_query_fix","issues_search_type_gql","landing_pages_ninetailed","landing_pages_web_vitals_tracking","lifecycle_label_name_updates","low_quality_classifier","marketing_pages_search_explore_provider","memex_default_issue_create_repository","memex_live_update_hovercard","memex_mwl_filter_field_delimiter","memex_remove_deprecated_type_issue","merge_status_header_feedback","notifications_menu_defer_labels","oauth_authorize_clickjacking_protection","octocaptcha_origin_optimization","prs_conversations_react","prs_css_anchor_positioning","prs_inbox_deferred_usequeries","react_compiler_issue_viewer","react_compiler_issues_react","react_data_router_serializable_query_deps","repos_contributors_limited_default_range","rules_insights_filter_bar_created","sample_network_conn_type","saved_views_filter_validation_fix","secret_scanning_pattern_alerts_link","security_center_artifact_filters_popover","session_logs_ungroup_reasoning_text","site_features_copilot_universe","site_homepage_collaborate_video","spark_prompt_secret_scanning","spark_server_connection_status","suppress_automated_browser_vitals","user_bypass_actors","viewscreen_sandbox","warn_inaccessible_attachments","webp_support","wiki_editor_iconbuttons","workbench_store_readonly"],"copilotApiOverrideUrl":"https://api.githubcopilot.com"}</script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/high-contrast-cookie-771a5d64a9997172.js"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/wp-runtime-9f875280d59a237d.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/fetch-utilities-18f7f90effa3f0dd.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/28839-28bb58f695d5e365.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/85924-1f0f5f61600f9c8e.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/34646-3b3c3c313ce5ddeb.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/environment-53f425896e39574b.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/runtime-helpers-6e561c87b9671d53.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/2966-db5d6e7392243767.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/96232-2f91b960d23e9fb6.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/41013-ce9f0a483fa6f641.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/51210-07f0116b9064d4ba.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/81058-4b90d5ffbf765928.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/81683-6e791542fe2f37ce.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/46740-4421ca06d57312cc.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/26859-860fabe66e46c274.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/44156-e3af230368d84cf1.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/github-elements-b02944871e808357.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/element-registry-93dd2f58d7d84d31.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/react-core-3a0bc3d3c10a831f.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/react-lib-493beffaa1062d35.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/79039-f2b81734929d0b15.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/61110-f40600dc68e4c6b0.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/2887-d67f71d8e1d3e1d8.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/26533-318ac47648fb7752.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/86483-00384068b148f370.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/49521-900eb9434ea3a3be.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/60481-2e0070b5d23b633a.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/46287-1fd98f482f71d12a.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/33805-370111b2dad9b744.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/89627-12a64f4329866bd1.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/79087-909cf11697b5f298.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/49029-3a132de206358025.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/99328-82a96596275fbd3e.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/behaviors-562601653a2bcbb7.js" defer="defer"></script> +<link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/react-core.06bbbc99d75f3438.module.css" /> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/4244-23afe633cce1a60f.js" defer="defer"></script> +<script crossorigin="anonymous" type="application/javascript" src="https://github.githubassets.com/assets/notifications-global-f606f675f96cab09.js" defer="defer"></script> + + + <title>Trending repositories on GitHub today · GitHub + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + +
+ Skip to content + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + + + +
+ + + + + +
+ + + + + + + + + +
+
+ +
+ +
+ + +
+
+

Trending

+ +

+ See what the GitHub community is most excited about today. +

+
+
+ +
+
+
+ + + +
+
+
+ Spoken Language: + + + Any + + + +
+ + Select a spoken language + + + +
+ +
+
+ +
+
+ +
+ +
+ + Abkhazian + + Afar + + Afrikaans + + Akan + + Albanian + + Amharic + + Arabic + + Aragonese + + Armenian + + Assamese + + Avaric + + Avestan + + Aymara + + Azerbaijani + + Bambara + + Bashkir + + Basque + + Belarusian + + Bengali + + Bihari languages + + Bislama + + Bosnian + + Breton + + Bulgarian + + Burmese + + Catalan, Valencian + + Chamorro + + Chechen + + Chichewa, Chewa, Nyanja + + Chinese + + Chuvash + + Cornish + + Corsican + + Cree + + Croatian + + Czech + + Danish + + Divehi, Dhivehi, Maldivian + + Dutch, Flemish + + Dzongkha + + English + + Esperanto + + Estonian + + Ewe + + Faroese + + Fijian + + Finnish + + French + + Fulah + + Galician + + Georgian + + German + + Greek, Modern + + Guarani + + Gujarati + + Haitian, Haitian Creole + + Hausa + + Hebrew + + Herero + + Hindi + + Hiri Motu + + Hungarian + + Interlingua (International Auxil... + + Indonesian + + Interlingue, Occidental + + Irish + + Igbo + + Inupiaq + + Ido + + Icelandic + + Italian + + Inuktitut + + Japanese + + Javanese + + Kalaallisut, Greenlandic + + Kannada + + Kanuri + + Kashmiri + + Kazakh + + Central Khmer + + Kikuyu, Gikuyu + + Kinyarwanda + + Kirghiz, Kyrgyz + + Komi + + Kongo + + Korean + + Kurdish + + Kuanyama, Kwanyama + + Latin + + Luxembourgish, Letzeburgesch + + Ganda + + Limburgan, Limburger, Limburgish + + Lingala + + Lao + + Lithuanian + + Luba-Katanga + + Latvian + + Manx + + Macedonian + + Malagasy + + Malay + + Malayalam + + Maltese + + Maori + + Marathi + + Marshallese + + Mongolian + + Nauru + + Navajo, Navaho + + North Ndebele + + Nepali + + Ndonga + + Norwegian Bokmål + + Norwegian Nynorsk + + Norwegian + + Sichuan Yi, Nuosu + + South Ndebele + + Occitan + + Ojibwa + + Church Slavic, Old Slavonic, Chu... + + Oromo + + Oriya + + Ossetian, Ossetic + + Punjabi, Panjabi + + Pali + + Persian + + Polish + + Pashto, Pushto + + Portuguese + + Quechua + + Romansh + + Rundi + + Romanian, Moldavian, Moldovan + + Russian + + Sanskrit + + Sardinian + + Sindhi + + Northern Sami + + Samoan + + Sango + + Serbian + + Gaelic, Scottish Gaelic + + Shona + + Sinhala, Sinhalese + + Slovak + + Slovenian + + Somali + + Southern Sotho + + Spanish, Castilian + + Sundanese + + Swahili + + Swati + + Swedish + + Tamil + + Telugu + + Tajik + + Thai + + Tigrinya + + Tibetan + + Turkmen + + Tagalog + + Tswana + + Tonga (Tonga Islands) + + Turkish + + Tsonga + + Tatar + + Twi + + Tahitian + + Uighur, Uyghur + + Ukrainian + + Urdu + + Uzbek + + Venda + + Vietnamese + + Volapük + + Walloon + + Welsh + + Wolof + + Western Frisian + + Xhosa + + Yiddish + + Yoruba + + Zhuang, Chuang + + Zulu +
+
+ +
+ + Loading + +
+ +
+ + +
+ +
+
+ Language: + + + Any + + + +
+ + Select a language + + +
+ +
+ + + +
+ +
+ +
+ + Unknown languages + + 1C Enterprise + + 2-Dimensional Array + + 4D + + ABAP + + ABAP CDS + + ABNF + + ActionScript + + Ada + + Adblock Filter List + + Adobe Font Metrics + + Agda + + AGS Script + + AIDL + + Aiken + + AL + + ALGOL + + Alloy + + Alpine Abuild + + Altium Designer + + AMPL + + AngelScript + + Answer Set Programming + + Ant Build System + + Antlers + + ANTLR + + ApacheConf + + Apex + + API Blueprint + + APL + + Apollo Guidance Computer + + AppleScript + + Arc + + AsciiDoc + + ASL + + ASN.1 + + Classic ASP + + ASP.NET + + AspectJ + + Assembly + + Astro + + Asymptote + + ATS + + Augeas + + AutoHotkey + + AutoIt + + Avro IDL + + Awk + + B (Formal Method) + + B4X + + Ballerina + + BASIC + + Batchfile + + Beef + + Befunge + + Berry + + BibTeX + + BibTeX Style + + Bicep + + Bikeshed + + Bison + + BitBake + + Blade + + BlitzBasic + + BlitzMax + + Bluespec + + Bluespec BH + + Boo + + Boogie + + BQN + + Brainfuck + + BrighterScript + + Brightscript + + Zeek + + Browserslist + + Bru + + BuildStream + + C + + C# + + C++ + + C-ObjDump + + C2hs Haskell + + C3 + + Cabal Config + + Caddyfile + + Cadence + + Cairo + + Cairo Zero + + CameLIGO + + Cangjie + + CAP CDS + + Cap'n Proto + + Carbon + + CartoCSS + + Ceylon + + Chapel + + Charity + + Checksums + + ChucK + + CIL + + Circom + + Cirru + + Clarion + + Clarity + + Classic ASP + + Clean + + Click + + CLIPS + + Clojure + + Closure Templates + + Cloud Firestore Security Rules + + Clue + + CMake + + COBOL + + CODEOWNERS + + CodeQL + + CoffeeScript + + ColdFusion + + ColdFusion CFC + + COLLADA + + Common Lisp + + Common Workflow Language + + Component Pascal + + CoNLL-U + + Cooklang + + Cool + + Rocq Prover + + Cpp-ObjDump + + CQL + + Creole + + crontab + + Crystal + + CSON + + Csound + + Csound Document + + Csound Score + + CSS + + CSV + + Cuda + + CUE + + Cue Sheet + + cURL Config + + Curry + + CWeb + + Cycript + + Cylc + + Cypher + + Cython + + D + + D-ObjDump + + D2 + + Dafny + + Darcs Patch + + Dart + + Daslang + + DataWeave + + Debian Package Control File + + DenizenScript + + desktop + + Dhall + + Diff + + DIGITAL Command Language + + dircolors + + DirectX 3D File + + DM + + DNS Zone + + Dockerfile + + Dogescript + + Dotenv + + DTrace + + Dune + + Dylan + + E + + E-mail + + Eagle + + Earthly + + Easybuild + + EBNF + + eC + + Ecere Projects + + ECL + + ECLiPSe + + Ecmarkup + + Edge + + EdgeQL + + EditorConfig + + Edje Data Collection + + edn + + Eiffel + + EJS + + Elixir + + Elm + + Elvish + + Elvish Transcript + + Emacs Lisp + + EmberScript + + E-mail + + EQ + + Erlang + + Euphoria + + F# + + F* + + Factor + + Fancy + + Fantom + + Faust + + Fennel + + FIGlet Font + + Filebench WML + + Filterscript + + FIRRTL + + fish + + Flix + + Fluent + + FLUX + + Formatted + + Forth + + Fortran + + Fortran Free Form + + FreeBASIC + + FreeMarker + + Frege + + Futhark + + G-code + + Game Maker Language + + GAML + + GAMS + + GAP + + GCC Machine Description + + GDB + + GDScript + + GDShader + + GEDCOM + + Gemfile.lock + + Gemini + + Genero 4gl + + Genero per + + Genie + + Genshi + + Gentoo Ebuild + + Gentoo Eclass + + Gerber Image + + Gettext Catalog + + Gherkin + + Git Attributes + + Git Commit + + Git Config + + Git Revision List + + Gleam + + Glimmer JS + + Glimmer TS + + GLSL + + Glyph + + Glyph Bitmap Distribution Format + + GN + + Gnuplot + + Go + + Go Checksums + + Go Module + + Go Template + + Go Workspace + + Godot Resource + + Golo + + Gosu + + Grace + + Gradle + + Gradle Kotlin DSL + + Grammatical Framework + + Graph Modeling Language + + GraphQL + + Graphviz (DOT) + + Groovy + + Groovy Server Pages + + GSC + + Hack + + Haml + + Handlebars + + HAProxy + + Harbour + + Hare + + Haskell + + Haxe + + HCL + + HIP + + HiveQL + + HLSL + + HOCON + + HolyC + + hoon + + Hosts File + + HTML + + Jinja + + HTML+ECR + + HTML+EEX + + HTML+ERB + + HTML+PHP + + HTML+Razor + + HTTP + + Hurl + + HXML + + Hy + + HyPhy + + iCalendar + + IDL + + Idris + + Ignore List + + IGOR Pro + + ImageJ Macro + + Imba + + Inform 7 + + INI + + Ink + + Inno Setup + + Io + + Ioke + + IRC log + + Isabelle + + Isabelle ROOT + + ISPC + + J + + Jac + + Jai + + Janet + + JAR Manifest + + Jasmin + + Java + + Java Properties + + Java Server Pages + + Java Template Engine + + JavaScript + + JavaScript+ERB + + JCL + + Jest Snapshot + + JetBrains MPS + + JFlex + + Jinja + + Jison + + Jison Lex + + Jolie + + jq + + JSON + + JSON with Comments + + JSON5 + + JSONiq + + JSONLD + + Jsonnet + + Julia + + Julia REPL + + Jupyter Notebook + + Just + + Kaitai Struct + + KakouneScript + + KCL + + KDL + + KerboScript + + KFramework + + KiCad Layout + + KiCad Legacy Layout + + KiCad Schematic + + Kickstart + + Kit + + Koka + + KoLmafia ASH + + Kotlin + + KRL + + Kusto + + kvlang + + LabVIEW + + Lambdapi + + Langium + + Lark + + Lasso + + Latte + + Lean + + Lean 4 + + Leo + + Less + + Lex + + LFE + + LigoLANG + + LilyPond + + Limbo + + Linear Programming + + Linker Script + + Linux Kernel Module + + Liquid + + Liquidsoap + + Literate Agda + + Literate CoffeeScript + + Literate Haskell + + LiveCode Script + + LiveScript + + LLVM + + Logos + + Logtalk + + LOLCODE + + LookML + + LoomScript + + LSL + + LTspice Symbol + + Lua + + Luau + + M + + M3U + + M4 + + M4Sugar + + Macaulay2 + + Makefile + + Mako + + Markdown + + Marko + + Mask + + Wolfram Language + + Mathematical Programming System + + MATLAB + + Maven POM + + Max + + MAXScript + + mcfunction + + mdsvex + + MDX + + Wikitext + + Mercury + + Mermaid + + Meson + + Metal + + MeTTa + + Microsoft Developer Studio Project + + Microsoft Visual Studio Solution + + MiniD + + MiniYAML + + MiniZinc + + MiniZinc Data + + Mint + + Mirah + + mIRC Script + + MLIR + + Modelica + + Modula-2 + + Modula-3 + + Module Management System + + Mojo + + Monkey + + Monkey C + + Moocode + + MoonBit + + MoonScript + + Motoko + + Motorola 68K Assembly + + Move + + MQL4 + + MQL5 + + MTML + + MUF + + mupad + + Muse + + Mustache + + Myghty + + nanorc + + Nasal + + NASL + + NCL + + Nearley + + Nemerle + + NEON + + nesC + + NetLinx + + NetLinx+ERB + + NetLogo + + NewLisp + + Nextflow + + Nginx + + Nickel + + Nim + + Ninja + + Nit + + Nix + + NL + + NMODL + + Noir + + NPM Config + + NSIS + + Nu + + NumPy + + Nunjucks + + Nushell + + NWScript + + OASv2-json + + OASv2-yaml + + OASv3-json + + OASv3-yaml + + Oberon + + ObjDump + + Object Data Instance Notation + + Objective-C + + Objective-C++ + + Objective-J + + ObjectScript + + OCaml + + Odin + + Omgrofl + + OMNeT++ MSG + + OMNeT++ NED + + OMNeT++ MSG + + OMNeT++ NED + + ooc + + Opa + + Opal + + Open Policy Agent + + OpenAPI Specification v2 + + OpenAPI Specification v3 + + OpenCL + + OpenEdge ABL + + OpenQASM + + OpenRC runscript + + OpenSCAD + + OpenStep Property List + + OpenType Feature File + + Option List + + Org + + OverpassQL + + Ox + + Oxygene + + Oz + + P4 + + Pact + + Pan + + Papyrus + + Parrot + + Parrot Assembly + + Parrot Internal Representation + + Pascal + + Pawn + + PDDL + + PEG.js + + Pep8 + + Perl + + PHP + + Pic + + Pickle + + PicoLisp + + PigLatin + + Pike + + Pip Requirements + + Pkl + + PlantUML + + PLpgSQL + + PLSQL + + Pod + + Pod 6 + + PogoScript + + Polar + + Pony + + Portugol + + PostCSS + + PostScript + + POV-Ray SDL + + PowerBuilder + + PowerShell + + Praat + + Prisma + + Processing + + Procfile + + Proguard + + Prolog + + Promela + + Propeller Spin + + Protocol Buffer + + Protocol Buffer Text Format + + Public Key + + Pug + + Puppet + + Pure Data + + PureBasic + + PureScript + + Pyret + + Python + + Python console + + Python traceback + + q + + Q# + + QMake + + QML + + Qt Script + + Quake + + QuakeC + + QuickBASIC + + R + + Racket + + Ragel + + Raku + + RAML + + Rascal + + RAScript + + Raw token data + + RBS + + RDoc + + Readline Config + + REALbasic + + Reason + + ReasonLIGO + + Rebol + + Record Jar + + Red + + Redcode + + Redirect Rules + + Regular Expression + + Ren'Py + + RenderScript + + ReScript + + reStructuredText + + REXX + + Rez + + Rich Text Format + + Ring + + Riot + + RMarkdown + + RobotFramework + + robots.txt + + Roc + + Rocq Prover + + Roff + + Roff Manpage + + RON + + ROS Interface + + Rouge + + RouterOS Script + + RPC + + RPGLE + + RPM Spec + + Ruby + + RUNOFF + + Rust + + Sage + + Sail + + SaltStack + + SAS + + Sass + + Scala + + Scaml + + Scenic + + Scheme + + Scilab + + SCSS + + sed + + Self + + SELinux Policy + + ShaderLab + + Shell + + ShellCheck Config + + ShellSession + + Shen + + Sieve + + Simple File Verification + + Singularity + + Slang + + Slash + + Slice + + Slim + + Slint + + Smali + + Smalltalk + + Smarty + + Smithy + + SmPL + + SMT + + Snakemake + + Solidity + + Soong + + SourcePawn + + SPARQL + + Spline Font Database + + SQF + + SQL + + SQLPL + + Squirrel + + SRecode Template + + SSH Config + + Stan + + Standard ML + + STAR + + Starlark + + Stata + + STL + + STON + + StringTemplate + + Stylus + + SubRip Text + + SugarSS + + SuperCollider + + SurrealQL + + Survex data + + Svelte + + SVG + + Sway + + Sweave + + Swift + + SWIG + + SystemVerilog + + Tact + + Talon + + Tcl + + Tcsh + + Tea + + Teal + + templ + + Terra + + Terraform Template + + TeX + + Texinfo + + Text + + TextGrid + + Textile + + TextMate Properties + + Thrift + + TI Program + + TL-Verilog + + TLA + + TMDL + + Toit + + TOML + + Tor Config + + Tree-sitter Query + + TSPLIB data + + TSQL + + TSV + + TSX + + Turing + + Turtle + + Twig + + TXL + + Type Language + + TypeScript + + TypeSpec + + Typst + + Unified Parallel C + + Unity3D Asset + + Unix Assembly + + Uno + + UnrealScript + + Untyped Plutus Core + + UrWeb + + V + + Vala + + Valve Data Format + + VBA + + VBScript + + vCard + + VCL + + Velocity Template Language + + Vento + + Verilog + + VHDL + + Vim Help File + + Vim Script + + Vim Snippet + + Visual Basic .NET + + Visual Basic .NET + + Visual Basic 6.0 + + Volt + + Vue + + Vyper + + Wavefront Material + + Wavefront Object + + WDL + + Web Ontology Language + + WebAssembly + + WebAssembly Interface Type + + WebIDL + + WebVTT + + Wget Config + + WGSL + + Whiley + + Wikitext + + Win32 Message File + + Windows Registry Entries + + wisp + + Witcher Script + + Wolfram Language + + Wollok + + World of Warcraft Addon Data + + Wren + + X BitMap + + X Font Directory Index + + X PixMap + + X10 + + xBase + + XC + + XCompose + + Xmake + + XML + + XML Property List + + Xojo + + Xonsh + + XPages + + XProc + + XQuery + + XS + + XSLT + + Xtend + + Yacc + + YAML + + YANG + + YARA + + YASnippet + + Yul + + ZAP + + Zeek + + ZenScript + + Zephir + + Zig + + ZIL + + Zimpl + + Zmodel +
+
+
+ + Loading + +
+ +
+ +
+ +
+
+ Date range: + + + Today + + + +
+ + Adjust time span + + +
+ +
+ + + + Today + + + + This week + + + + This month +
+ +
+ +
+
+
+ +
+ + + + + + + + + + + + + + + + +
+
+
+ +
+
+ +
+

Footer

+ + + + +
+
+ + + + + © 2026 GitHub, Inc. + +
+ + +
+
+ + + + + + + + + + + + + + + + + + + + +
+
+
+ + diff --git a/ai-daily-main/tests/pytest/fixtures/hn_frontpage.html b/ai-daily-main/tests/pytest/fixtures/hn_frontpage.html new file mode 100644 index 0000000..b4fbc89 --- /dev/null +++ b/ai-daily-main/tests/pytest/fixtures/hn_frontpage.html @@ -0,0 +1,3 @@ +Hacker News
Hacker Newsnew | past | comments | ask | show | jobs | submitlogin
1.
Zerostack – A Unix-inspired coding agent written in pure Rust (crates.io)
140 points by gidellav 3 hours ago | hide | 42 comments
2.
A nicer voltmeter clock (lcamtuf.substack.com)
50 points by surprisetalk 2 hours ago | hide | 12 comments
3.
MCP Hello Page (hybridlogic.co.uk)
47 points by Dachande663 3 hours ago | hide | 18 comments
4.
A molecule with half-Möbius topology (science.org)
63 points by bryanrasmussen 7 hours ago | hide | discuss
5.
SANA-WM, a 2.6B open-source world model for 1-minute 720p video (nvlabs.github.io)
295 points by mjgil 13 hours ago | hide | 125 comments
6.
Moving away from Tailwind, and learning to structure my CSS (jvns.ca)
430 points by mpweiher 16 hours ago | hide | 275 comments
7.
The Third Hard Problem (mmapped.blog)
25 points by surprisetalk 3 hours ago | hide | 17 comments
8.
Halt and Catch Fire (unstack.io)
77 points by ScottWRobinson 7 hours ago | hide | 51 comments
9.
Accelerando (2005) (antipope.org)
240 points by eamag 13 hours ago | hide | 140 comments
10.
Fisker went bankrupt and owners built an open source car company from the ashes (electrek.co)
31 points by breve 1 hour ago | hide | 3 comments
11.
Content-defined chunking added to Bazel (buildbuddy.io)
24 points by siggi 3 hours ago | hide | 2 comments
12.
Stochastic Parrots: Frequently Unasked Questions (medium.com/emilymenonbender)
17 points by olalonde 2 hours ago | hide | 8 comments
13.
OpenAI and Government of Malta partner to roll out ChatGPT Plus to all citizens (openai.com)
57 points by bookofjoe 5 hours ago | hide | 64 comments
14.
Frontier AI has broken the open CTF format (kabir.au)
337 points by frays 18 hours ago | hide | 326 comments
15.
δ-mem: Efficient Online Memory for Large Language Models (arxiv.org)
193 points by 44za12 16 hours ago | hide | 52 comments
16.
We've made the world too complicated (user8.bearblog.dev)
189 points by James72689 17 hours ago | hide | 180 comments
17.
Fame! A Misunderstanding: A new translation of Albert Camus's complete notebooks (lareviewofbooks.org)
43 points by Caiero 8 hours ago | hide | 7 comments
18.
3D Gaussian Splatting in a Weekend (bfeldman.me)
48 points by b__feldman 7 hours ago | hide | 6 comments
19.
PART Telescopes – Bringing radio astronomy within reach of rural schools (parttelescopes.web.app)
106 points by openrockets 10 hours ago | hide | 28 comments
20.
Kioxia and Dell cram 10 PB into slim 2RU server (blocksandfiles.com)
108 points by rbanffy 8 hours ago | hide | 73 comments
21.
Show HN: Rocksky – Music scrobbling and discovery on the AT Protocol (tangled.org)
57 points by tsiry 8 hours ago | hide | 22 comments
22.
Greek Alphabet Cards (randomquark.com)
96 points by ricochet11 13 hours ago | hide | 47 comments
23.
Futhark by example (2020) (futhark-lang.org)
109 points by tosh 15 hours ago | hide | 27 comments
24.
Nearly 50 Years Later, WKRP in Cincinnati Becomes a Real Radio Station (openculture.com)
108 points by bookofjoe 17 hours ago | hide | 63 comments
25.
I believe there are entire companies right now under AI psychosis (twitter.com/mitchellh)
1877 points by reasonableklout 1 day ago | hide | 1063 comments
26.
Accelerate – Embedded language for high-performance array computations (github.com/acceleratehs)
77 points by tosh 11 hours ago | hide | 17 comments
27.
After 8 years, I rewrote my open-source PyTorch curvature library (github.com/noahgolmant)
74 points by noahgolmant 12 hours ago | hide | 1 comment
28.
Japan’s robot wolf sells out as record bear attacks drive demand (independent.co.uk)
88 points by bookofjoe 6 hours ago | hide | 51 comments
29.Kyber (YC W23) Is Hiring a Founding Marketer (ycombinator.com)
13 hours ago | hide
30.
DeepSeek-V4-Flash means LLM steering is interesting again (seangoedecke.com)
210 points by Brajeshwar 10 hours ago | hide | 68 comments

+
Guidelines | FAQ | Lists | API | Security | Legal | Apply to YC | Contact

+
Search:
\ No newline at end of file diff --git a/ai-daily-main/tests/pytest/test_config.py b/ai-daily-main/tests/pytest/test_config.py new file mode 100644 index 0000000..db876d4 --- /dev/null +++ b/ai-daily-main/tests/pytest/test_config.py @@ -0,0 +1,249 @@ +"""配置模块测试""" + +import json +import pytest +import sys +from datetime import datetime, timezone, timedelta +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src")) + +from config import ( + load_config, + parse_opml, + merge_sources, + get_timezone, +) + + +class TestLoadConfig: + """测试配置加载""" + + def test_load_valid_config(self, temp_dir): + config = { + "sources": {"base_opml": "test.opml", "add": [], "block": []}, + "filter": {"min_score": 60}, + "schedule": {"fetch_interval_minutes": 30}, + "llm": {"provider": "groq", "model": "moonshotai/kimi-k2-instruct"}, + "push": {"discord": {"enabled": False}}, + } + config_file = temp_dir / "config.json" + config_file.write_text(json.dumps(config)) + + result = load_config(str(config_file)) + assert result["filter"]["min_score"] == 60 + assert result["llm"]["provider"] == "groq" + + def test_load_missing_file(self): + with pytest.raises(FileNotFoundError): + load_config("nonexistent.json") + + def test_load_invalid_json(self, temp_dir): + config_file = temp_dir / "config.json" + config_file.write_text("invalid json") + with pytest.raises(json.JSONDecodeError): + load_config(str(config_file)) + + def test_load_config_with_sources(self, temp_dir, sample_opml): + config = { + "sources": { + "base_opml": sample_opml, + "add": [ + { + "title": "Add1", + "xmlUrl": "http://add1.com/rss", + "category": "test", + } + ], + "block": [], + }, + "filter": {"min_score": 60}, + "schedule": {"fetch_interval_minutes": 30, "timezone_hours": 8}, + "llm": {"provider": "test"}, + "push": {"discord": {"enabled": False}}, + } + config_file = temp_dir / "config.json" + config_file.write_text(json.dumps(config)) + + result = load_config(str(config_file)) + assert len(result["sources"]["add"]) == 1 + + +class TestParseOpml: + """测试OPML解析""" + + def test_parse_valid_opml(self, temp_dir): + opml_content = """ + + + + +""" + opml_file = temp_dir / "test.opml" + opml_file.write_text(opml_content) + + feeds = parse_opml(str(opml_file)) + assert len(feeds) == 1 + assert feeds[0]["title"] == "Feed1" + assert feeds[0]["xmlUrl"] == "http://feed1.com/rss" + + def test_parse_missing_file(self): + feeds = parse_opml("nonexistent.opml") + assert feeds == [] + + def test_parse_opml_with_category(self, temp_dir): + opml_content = """ + + + + + +""" + opml_file = temp_dir / "test.opml" + opml_file.write_text(opml_content) + + feeds = parse_opml(str(opml_file)) + assert len(feeds) == 2 + assert feeds[0]["category"] == "技术" + assert feeds[1]["category"] == "AI" + + def test_parse_opml_empty_body(self, temp_dir): + opml_content = """ + + + +""" + opml_file = temp_dir / "test.opml" + opml_file.write_text(opml_content) + + feeds = parse_opml(str(opml_file)) + assert feeds == [] + + +class TestMergeSources: + """测试源合并""" + + def test_merge_base_and_add(self, sample_opml): + config = { + "base_opml": sample_opml, + "add": [ + {"title": "Feed3", "xmlUrl": "http://feed3.com/rss", "category": "test"} + ], + "block": [], + } + sources = merge_sources(config) + assert len(sources) == 3 + + def test_block_by_xmlUrl(self, sample_opml): + config = { + "base_opml": sample_opml, + "add": [], + "block": [{"title": "Block1", "xmlUrl": "http://feed1.com/rss"}], + } + sources = merge_sources(config) + assert all(s["xmlUrl"] != "http://feed1.com/rss" for s in sources) + assert len(sources) == 1 + + def test_deduplicate_by_xmlUrl(self, sample_opml): + config = { + "base_opml": sample_opml, + "add": [ + { + "title": "Duplicate", + "xmlUrl": "http://feed1.com/rss", + "category": "test", + } + ], + "block": [], + } + sources = merge_sources(config) + urls = [s["xmlUrl"] for s in sources] + assert len(urls) == len(set(urls)) + assert len(sources) == 2 + + def test_block_domains_wildcard(self, temp_dir): + opml_content = """ + + + + + +""" + opml_file = temp_dir / "test.opml" + opml_file.write_text(opml_content) + + config = { + "base_opml": str(opml_file), + "add": [], + "block": [], + "block_domains": ["*.substack.com"], + } + sources = merge_sources(config) + assert len(sources) == 1 + assert sources[0]["xmlUrl"] == "https://tech.blog/rss" + + def test_block_domains_exact(self, temp_dir): + opml_content = """ + + + + + +""" + opml_file = temp_dir / "test.opml" + opml_file.write_text(opml_content) + + config = { + "base_opml": str(opml_file), + "add": [], + "block": [], + "block_domains": ["youtube.com"], + } + sources = merge_sources(config) + assert len(sources) == 1 + assert sources[0]["xmlUrl"] == "https://tech.blog/rss" + + def test_block_domains_subdomain(self, temp_dir): + opml_content = """ + + + + + + +""" + opml_file = temp_dir / "test.opml" + opml_file.write_text(opml_content) + + config = { + "base_opml": str(opml_file), + "add": [], + "block": [], + "block_domains": ["*.substack.com"], + } + sources = merge_sources(config) + assert len(sources) == 1 + assert sources[0]["xmlUrl"] == "https://tech.blog/rss" + + +class TestGetTimezone: + """测试时区获取""" + + def test_get_timezone_from_config(self, sample_config): + tz = get_timezone(sample_config) + assert isinstance(tz, timezone) + assert tz.utcoffset(datetime.now()).total_seconds() == 8 * 3600 + + def test_get_timezone_none_config(self): + tz = get_timezone(None) + assert isinstance(tz, timezone) + + def test_get_timezone_no_timezone_hours(self): + config = {"schedule": {}} + tz = get_timezone(config) + assert isinstance(tz, timezone) + + def test_get_timezone_custom_hours(self): + config = {"schedule": {"timezone_hours": -5}} + tz = get_timezone(config) + assert tz.utcoffset(datetime.now()).total_seconds() == -5 * 3600 diff --git a/ai-daily-main/tests/pytest/test_fetcher.py b/ai-daily-main/tests/pytest/test_fetcher.py new file mode 100644 index 0000000..fc3c1b7 --- /dev/null +++ b/ai-daily-main/tests/pytest/test_fetcher.py @@ -0,0 +1,190 @@ +"""RSS抓取模块测试""" + +import pytest +import sys +from datetime import datetime, timezone, timedelta +from pathlib import Path +from unittest.mock import MagicMock, patch, AsyncMock + +sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src")) + +from fetcher import ( + parse_entry_time, + fetch_single_feed_async, + fetch_all_feeds, + DEFAULT_FEED_TIMEOUT, +) + + +class TestParseEntryTime: + """测试时间解析""" + + def test_parse_published_parsed(self): + entry = MagicMock() + entry.published_parsed = (2024, 1, 15, 10, 30, 0, 0, 0, 0) + + result = parse_entry_time(entry) + assert result is not None + assert result.year == 2024 + assert result.month == 1 + assert result.day == 15 + assert result.tzinfo == timezone.utc + + def test_parse_updated_parsed(self): + entry = MagicMock() + entry.published_parsed = None + entry.updated_parsed = (2024, 1, 15, 10, 30, 0, 0, 0, 0) + + result = parse_entry_time(entry) + assert result is not None + assert result.year == 2024 + assert result.month == 1 + assert result.day == 15 + + def test_parse_no_time(self): + entry = MagicMock() + entry.published_parsed = None + entry.updated_parsed = None + + result = parse_entry_time(entry) + assert result is None + + +class TestFetchSingleFeedAsync: + """测试单源抓取""" + + @pytest.mark.asyncio + async def test_fetch_success(self, temp_dir): + rss_content = """ + + + Test Feed + + Article 1 + https://example.com/1 + Mon, 15 Jan 2024 10:00:00 GMT + Test description + + +""" + + feed_info = {"title": "Test Feed", "xmlUrl": "http://test.com/rss"} + cutoff = datetime(2024, 1, 1, tzinfo=timezone.utc) + + mock_response = MagicMock() + mock_response.status = 200 + mock_response.text = AsyncMock(return_value=rss_content) + + mock_session = MagicMock() + mock_session.__aenter__ = AsyncMock(return_value=mock_session) + mock_session.__aexit__ = AsyncMock(return_value=None) + mock_session.get = MagicMock( + return_value=AsyncMock( + __aenter__=AsyncMock(return_value=mock_response), + __aexit__=AsyncMock(return_value=None), + ) + ) + + with patch("aiohttp.ClientSession", return_value=mock_session): + entries = await fetch_single_feed_async( + feed_info, cutoff, session=mock_session + ) + + assert len(entries) == 1 + assert entries[0]["title"] == "Article 1" + assert entries[0]["link"] == "https://example.com/1" + assert entries[0]["source"] == "Test Feed" + + @pytest.mark.asyncio + async def test_fetch_http_error(self): + feed_info = {"title": "Test Feed", "xmlUrl": "http://test.com/rss"} + cutoff = datetime(2024, 1, 1, tzinfo=timezone.utc) + + mock_response = MagicMock() + mock_response.status = 404 + + mock_session = MagicMock() + mock_session.get = MagicMock( + return_value=AsyncMock( + __aenter__=AsyncMock(return_value=mock_response), + __aexit__=AsyncMock(return_value=None), + ) + ) + + entries = await fetch_single_feed_async(feed_info, cutoff, session=mock_session) + assert entries == [] + + @pytest.mark.asyncio + async def test_fetch_timeout(self): + feed_info = {"title": "Test Feed", "xmlUrl": "http://test.com/rss"} + cutoff = datetime(2024, 1, 1, tzinfo=timezone.utc) + + import aiohttp + + mock_session = MagicMock() + mock_session.get = MagicMock(side_effect=aiohttp.ServerTimeoutError()) + + entries = await fetch_single_feed_async(feed_info, cutoff, session=mock_session) + assert entries == [] + + @pytest.mark.asyncio + async def test_fetch_cutoff_filter(self): + feed_info = {"title": "Test Feed", "xmlUrl": "http://test.com/rss"} + cutoff = datetime(2024, 1, 10, tzinfo=timezone.utc) + + import feedparser + + with patch( + "fetcher.fetch_single_feed_async", new_callable=AsyncMock + ) as mock_fetch: + mock_fetch.return_value = [ + { + "title": "New", + "link": "https://example.com/new", + "published": datetime(2024, 1, 15, tzinfo=timezone.utc), + } + ] + + result = await mock_fetch(feed_info, cutoff) + + assert len(result) == 1 + + +class TestFetchAllFeeds: + """测试并发抓取""" + + @pytest.mark.asyncio + async def test_concurrent_limit(self): + feeds = [ + {"title": f"Feed{i}", "xmlUrl": f"http://feed{i}.com/rss"} + for i in range(20) + ] + cutoff = datetime(2024, 1, 1, tzinfo=timezone.utc) + + with patch( + "fetcher.fetch_single_feed_async", new_callable=AsyncMock + ) as mock_fetch: + mock_fetch.return_value = [] + await fetch_all_feeds(feeds, cutoff, max_workers=5) + + assert mock_fetch.call_count == 20 + + @pytest.mark.asyncio + async def test_empty_feeds(self): + cutoff = datetime(2024, 1, 1, tzinfo=timezone.utc) + entries = await fetch_all_feeds([], cutoff) + assert entries == [] + + @pytest.mark.asyncio + async def test_default_timeout(self): + feed_info = {"title": "Test", "xmlUrl": "http://test.com"} + cutoff = datetime.now(timezone.utc) + + with patch( + "fetcher.fetch_single_feed_async", new_callable=AsyncMock + ) as mock_fetch: + mock_fetch.return_value = [] + + await mock_fetch(feed_info, cutoff, timeout=None) + + mock_fetch.assert_called_once() diff --git a/ai-daily-main/tests/pytest/test_llm.py b/ai-daily-main/tests/pytest/test_llm.py new file mode 100644 index 0000000..ddcad13 --- /dev/null +++ b/ai-daily-main/tests/pytest/test_llm.py @@ -0,0 +1,330 @@ +"""LLM模块测试""" + +import json +import pytest +import sys +from datetime import datetime, timezone, timedelta +from pathlib import Path +from unittest.mock import patch, AsyncMock, MagicMock + +sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src")) + +from llm import ( + load_prompt, + _parse_llm_json_response, + _split_entries_for_batch, + _build_batch_prompt, + _merge_scores, + _score_single_batch, + call_llm, + check_llm_available, + generate_immediate_push, + score_batch, +) + + +class TestLoadPrompt: + """测试提示词加载""" + + def test_load_prompt_basic(self, temp_dir): + prompt_file = temp_dir / "test.txt" + prompt_file.write_text("Hello {name}!") + + result = load_prompt(str(prompt_file), name="World") + assert result == "Hello World!" + + def test_load_prompt_missing_file(self): + with pytest.raises(FileNotFoundError): + load_prompt("nonexistent.txt") + + def test_load_prompt_with_braces(self, temp_dir): + prompt_file = temp_dir / "test.txt" + prompt_file.write_text("Hello {name}, curly braces: { }") + + result = load_prompt(str(prompt_file), name="World") + assert result == "Hello World, curly braces: { }" + + def test_load_prompt_multiple_vars(self, temp_dir): + prompt_file = temp_dir / "test.txt" + prompt_file.write_text("{greeting} {name}, you have {count} messages") + + result = load_prompt(str(prompt_file), greeting="Hi", name="Alice", count=5) + assert result == "Hi Alice, you have 5 messages" + + +class TestParseLlmJsonResponse: + """测试LLM响应解析""" + + def test_parse_json_array(self): + response = '[{"link": "https://example.com", "score": 80}]' + result = _parse_llm_json_response(response) + assert len(result) == 1 + assert result[0]["link"] == "https://example.com" + assert result[0]["score"] == 80 + + def test_parse_with_markdown_codeblock(self): + response = """```json +[{"link": "https://example.com", "score": 80}] +```""" + result = _parse_llm_json_response(response) + assert len(result) == 1 + + def test_parse_with_codeblock(self): + response = """``` +[{"link": "https://example.com", "score": 80}] +```""" + result = _parse_llm_json_response(response) + assert len(result) == 1 + + def test_parse_invalid_response(self): + response = "This is not JSON at all" + with pytest.raises(ValueError): + _parse_llm_json_response(response) + + +class TestSplitEntriesForBatch: + """测试条目分批""" + + def test_split_empty(self): + result = _split_entries_for_batch([]) + assert result == [] + + def test_split_single_batch(self): + entries = [ + { + "link": f"https://example.com/{i}", + "title": f"Title{i}", + "content": "x" * 100, + } + for i in range(5) + ] + result = _split_entries_for_batch(entries, max_prompt_chars=10000) + assert len(result) == 1 + + def test_split_multiple_batches(self): + entries = [ + { + "link": f"https://example.com/{i}", + "title": f"Title{i}", + "content": "x" * 5000, + } + for i in range(10) + ] + result = _split_entries_for_batch(entries, max_prompt_chars=10000) + assert len(result) > 1 + + +class TestBuildBatchPrompt: + """测试构建批量提示词""" + + def test_build_batch_prompt_basic(self): + entries = [ + { + "link": "https://example.com/1", + "title": "Title1", + "source": "Source1", + "published": "2024-01-15", + "content": "Content", + } + ] + result = _build_batch_prompt(entries) + assert "Title1" in result + assert "https://example.com/1" in result + + +class TestMergeScores: + """测试评分合并""" + + def test_merge_scores_basic(self): + entries = [ + {"link": "https://example.com/1", "title": "Title1"}, + {"link": "https://example.com/2", "title": "Title2"}, + ] + scores = [ + { + "link": "https://example.com/1", + "score": 85, + "tags": ["AI"], + "summary": "Summary1", + }, + { + "link": "https://example.com/2", + "score": 70, + "tags": ["Tech"], + "summary": "Summary2", + }, + ] + result = _merge_scores(entries, scores) + + assert result[0]["score"] == 85 + assert result[0]["tags"] == ["AI"] + assert result[1]["score"] == 70 + + def test_merge_scores_partial(self): + entries = [ + {"link": "https://example.com/1", "title": "Title1", "score": 50}, + {"link": "https://example.com/2", "title": "Title2", "score": 60}, + ] + scores = [{"link": "https://example.com/1", "score": 85}] + result = _merge_scores(entries, scores) + + assert result[0]["score"] == 85 + assert result[1]["score"] == 60 + + +class TestCallLlm: + """测试LLM调用""" + + @pytest.mark.asyncio + async def test_call_llm_success(self): + config = { + "model": "gpt-4", + "baseUrl": "https://api.openai.com/v1", + "apiKeyName": "OPENAI_API_KEY", + } + + with patch("llm.call_llm", new_callable=AsyncMock) as mock_call: + mock_call.return_value = "Test response" + + result = await mock_call("Test prompt", config) + + assert result == "Test response" + + @pytest.mark.asyncio + async def test_call_llm_missing_key(self): + config = {"model": "gpt-4", "apiKeyName": "MISSING_KEY"} + + with pytest.raises(ValueError, match="未设置MISSING_KEY"): + await call_llm("Test prompt", config) + + +class TestLlmHealthCheck: + """测试LLM可用性检查""" + + @pytest.mark.asyncio + async def test_check_llm_available_success(self, sample_config): + with patch("llm.call_llm", new_callable=AsyncMock) as mock_call: + mock_call.return_value = "OK" + + result = await check_llm_available(sample_config["llm"]) + + assert result == "OK" + + @pytest.mark.asyncio + async def test_check_llm_available_empty_response(self, sample_config): + with patch("llm.call_llm", new_callable=AsyncMock) as mock_call: + mock_call.return_value = " " + + with pytest.raises(RuntimeError, match="返回空响应"): + await check_llm_available(sample_config["llm"]) + + +class TestImmediatePush: + """测试即时推送生成""" + + @pytest.mark.asyncio + async def test_generate_immediate_push_failure_returns_error( + self, sample_entries, sample_config + ): + with patch("llm.load_prompt", return_value="prompt"), patch( + "llm.call_llm", new_callable=AsyncMock + ) as mock_call: + mock_call.side_effect = RuntimeError("boom") + + content, error = await generate_immediate_push( + sample_entries[:1], sample_config["llm"], recent_push_context="" + ) + + assert content == "" + assert error == "生成即时推送失败: boom" + + +class TestScoreBatch: + """测试批量评分""" + + @pytest.mark.asyncio + async def test_score_batch_empty(self, sample_config): + result, errors = await score_batch([], sample_config["llm"]) + assert result == [] + assert errors == [] + + @pytest.mark.asyncio + async def test_score_batch_single(self, sample_entries, sample_config): + entries = sample_entries[:1] + + mock_scores = [ + { + "link": entries[0]["link"], + "score": 85, + "tags": ["AI"], + "summary": "Test summary", + } + ] + + with patch("llm._score_single_batch", new_callable=AsyncMock) as mock_score: + mock_score.return_value = (mock_scores, []) + result, errors = await score_batch(entries, sample_config["llm"]) + + assert len(result) == 1 + assert result[0]["score"] == 85 + assert errors == [] + + @pytest.mark.asyncio + async def test_score_single_batch_failure_returns_empty_results( + self, sample_entries, sample_config + ): + with patch("llm.call_llm", new_callable=AsyncMock) as mock_call: + mock_call.side_effect = RuntimeError("boom") + results, errors = await _score_single_batch( + sample_entries[:2], sample_config["llm"] + ) + + assert results == [] + assert errors == ["批次1 评分失败: boom"] + + @pytest.mark.asyncio + async def test_score_single_batch_reconcile_partial_results( + self, sample_entries, sample_config + ): + entries = sample_entries[:2] + llm_results = [ + { + "link": entries[0]["link"], + "score": 91, + "tags": ["AI"], + "summary": "Matched result", + } + ] + + with patch("llm.call_llm", new_callable=AsyncMock) as mock_call: + mock_call.return_value = json.dumps(llm_results, ensure_ascii=False) + results, errors = await _score_single_batch(entries, sample_config["llm"]) + + assert len(results) == 1 + assert results[0]["score"] == 91 + assert len(errors) == 1 + assert "评分结果异常" in errors[0] + assert "输入2" in errors[0] + assert "返回1" in errors[0] + assert "匹配1" in errors[0] + + @pytest.mark.asyncio + async def test_score_single_batch_keeps_full_results(self, sample_entries, sample_config): + entries = sample_entries[:3] + llm_results = [ + { + "link": entry["link"], + "score": 88, + "tags": ["AI"], + "summary": f"Summary for {index}", + } + for index, entry in enumerate(entries, start=1) + ] + + with patch("llm.call_llm", new_callable=AsyncMock) as mock_call: + mock_call.return_value = json.dumps(llm_results, ensure_ascii=False) + results, errors = await _score_single_batch(entries, sample_config["llm"]) + + assert len(results) == 3 + assert [result["link"] for result in results] == [entry["link"] for entry in entries] + assert errors == [] diff --git a/ai-daily-main/tests/pytest/test_llm_extra_sections.py b/ai-daily-main/tests/pytest/test_llm_extra_sections.py new file mode 100644 index 0000000..5349eb9 --- /dev/null +++ b/ai-daily-main/tests/pytest/test_llm_extra_sections.py @@ -0,0 +1,101 @@ +"""测试新增 LLM 函数 (summarize_github_trending 等)""" + +import sys +from pathlib import Path +from unittest.mock import patch, AsyncMock + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src")) + +from llm import summarize_github_trending, select_ai_related_hn, summarize_hackernews + + +@pytest.mark.asyncio +async def test_summarize_github_trending_happy_path(tmp_path): + prompt_path = tmp_path / "section_github.md" + prompt_path.write_text("Repos: {repos_json}\nmax_items={max_items}", encoding="utf-8") + + config = { + "model": "x", + "baseUrl": "http://x", + "apiKeyName": "DEEPSEEK_API_KEY", + "prompts": {"section_github": str(prompt_path)}, + "sections": {"github_trending": {"max_items": 3}}, + } + enriched = [{"full_name": "o/r", "readme_excerpt": "rm"}] + + with patch("llm.call_llm", new=AsyncMock(return_value="## md")): + md, err = await summarize_github_trending(enriched, config) + + assert md == "## md" + assert err is None + + +@pytest.mark.asyncio +async def test_summarize_github_trending_llm_failure_returns_error(tmp_path): + prompt_path = tmp_path / "section_github.md" + prompt_path.write_text("x {repos_json} {max_items}", encoding="utf-8") + config = { + "model": "x", + "baseUrl": "http://x", + "apiKeyName": "DEEPSEEK_API_KEY", + "prompts": {"section_github": str(prompt_path)}, + "sections": {"github_trending": {"max_items": 3}}, + } + with patch("llm.call_llm", new=AsyncMock(side_effect=RuntimeError("boom"))): + md, err = await summarize_github_trending([{"full_name": "o/r"}], config) + assert md == "" + assert "boom" in err + + +@pytest.mark.asyncio +async def test_select_ai_related_hn_parses_id_array(tmp_path): + prompt_path = tmp_path / "select.md" + prompt_path.write_text("k={k} candidates={candidates_json}", encoding="utf-8") + config = { + "model": "x", + "baseUrl": "http://x", + "apiKeyName": "DEEPSEEK_API_KEY", + "prompts": {"section_hackernews_select": str(prompt_path)}, + } + with patch("llm.call_llm", new=AsyncMock(return_value='["111", "222"]')): + ids, err = await select_ai_related_hn( + [{"id": "111"}, {"id": "222"}, {"id": "333"}], k=2, config=config + ) + assert ids == ["111", "222"] + assert err is None + + +@pytest.mark.asyncio +async def test_select_ai_related_hn_empty_array(tmp_path): + prompt_path = tmp_path / "select.md" + prompt_path.write_text("{k}{candidates_json}", encoding="utf-8") + config = { + "model": "x", + "baseUrl": "http://x", + "apiKeyName": "DEEPSEEK_API_KEY", + "prompts": {"section_hackernews_select": str(prompt_path)}, + } + with patch("llm.call_llm", new=AsyncMock(return_value="[]")): + ids, err = await select_ai_related_hn([{"id": "1"}], k=1, config=config) + assert ids == [] + assert err is None + + +@pytest.mark.asyncio +async def test_summarize_hackernews_happy(tmp_path): + prompt_path = tmp_path / "hn.md" + prompt_path.write_text("{stories_json}", encoding="utf-8") + config = { + "model": "x", + "baseUrl": "http://x", + "apiKeyName": "DEEPSEEK_API_KEY", + "prompts": {"section_hackernews": str(prompt_path)}, + } + with patch("llm.call_llm", new=AsyncMock(return_value="## HN summary")): + md, err = await summarize_hackernews( + [{"id": "1", "title": "t", "link_content": "x", "top_comments": []}], config + ) + assert md == "## HN summary" + assert err is None diff --git a/ai-daily-main/tests/pytest/test_main.py b/ai-daily-main/tests/pytest/test_main.py new file mode 100644 index 0000000..4f4baf1 --- /dev/null +++ b/ai-daily-main/tests/pytest/test_main.py @@ -0,0 +1,285 @@ +"""主程序逻辑测试""" + +import json +import pytest +import sys +from datetime import datetime, timezone, timedelta, date +from pathlib import Path +from unittest.mock import patch, MagicMock, AsyncMock + +sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src")) + +from main import ( + now_local, + parse_time_to_local, + calculate_push_times, + collect_entries_for_push, + main as run_main, +) + + +class TestNowLocal: + """测试获取本地时间""" + + def test_now_local_with_config(self, sample_config): + result = now_local(sample_config) + assert isinstance(result, datetime) + assert result.tzinfo is not None + + def test_now_local_without_config(self): + result = now_local() + assert isinstance(result, datetime) + + +class TestParseTimeToLocal: + """测试时间解析""" + + def test_parse_iso_format(self, sample_config): + result = parse_time_to_local("2024-01-15T10:30:00+00:00", sample_config) + assert result is not None + assert result.year == 2024 + assert result.month == 1 + assert result.day == 15 + + def test_parse_with_z_suffix(self, sample_config): + result = parse_time_to_local("2024-01-15T10:30:00Z", sample_config) + assert result is not None + assert result.year == 2024 + + def test_parse_invalid_format(self, sample_config): + result = parse_time_to_local("not-a-date", sample_config) + assert result is None + + def test_parse_none(self, sample_config): + result = parse_time_to_local("", sample_config) + assert result is None + + +class TestCalculatePushTimes: + """测试推送时间计算""" + + def test_calculate_single_cron(self, sample_config): + times = calculate_push_times(["30 8 * * *"], config=sample_config) + assert len(times) == 1 + assert times[0].hour == 8 + assert times[0].minute == 30 + + def test_calculate_multiple_crons(self, sample_config): + times = calculate_push_times(["0 8 * * *", "0 17 * * *"], config=sample_config) + assert len(times) == 2 + hours = [t.hour for t in times] + assert 8 in hours + assert 17 in hours + + def test_calculate_with_offset(self, sample_config): + times = calculate_push_times(["0 8 * * *"], offset_days=1, config=sample_config) + assert len(times) == 1 + expected_date = (datetime.now(timezone.utc) + timedelta(days=1)).date() + assert times[0].date() == expected_date + + def test_calculate_invalid_cron(self, sample_config): + times = calculate_push_times(["invalid cron"], config=sample_config) + assert times == [] + + +class TestCollectEntriesForPush: + """测试收集推送条目""" + + def test_collect_no_files(self, temp_dir): + to_push, context = collect_entries_for_push( + last_push_time=None, context_days=2, min_score=60, data_dir=str(temp_dir) + ) + assert to_push == [] + assert context == [] + + def test_collect_with_low_score(self, temp_dir): + now = datetime.now(timezone.utc) + + data = { + "meta": {"date": now.date().isoformat()}, + "entries": [ + { + "title": "Low Score", + "link": "https://example.com/1", + "score": 30, + "fetched_at": now.isoformat(), + } + ], + } + + fetch_file = temp_dir / f"fetch-{now.date().isoformat()}.json" + with open(fetch_file, "w") as f: + json.dump(data, f) + + to_push, context = collect_entries_for_push( + last_push_time=None, context_days=2, min_score=60, data_dir=str(temp_dir) + ) + + assert len(to_push) == 0 + + def test_collect_with_high_score(self, temp_dir): + now = datetime.now(timezone.utc) + + data = { + "meta": {"date": now.date().isoformat()}, + "entries": [ + { + "title": "High Score", + "link": "https://example.com/1", + "score": 85, + "fetched_at": now.isoformat(), + } + ], + } + + fetch_file = temp_dir / f"fetch-{now.date().isoformat()}.json" + with open(fetch_file, "w") as f: + json.dump(data, f) + + to_push, context = collect_entries_for_push( + last_push_time=None, context_days=2, min_score=60, data_dir=str(temp_dir) + ) + + assert len(to_push) == 1 + assert to_push[0]["score"] == 85 + + def test_collect_with_last_push_time(self, temp_dir): + now = datetime.now(timezone.utc) + last_push = now - timedelta(hours=2) + + data = { + "meta": {"date": now.date().isoformat()}, + "entries": [ + { + "title": "New Entry", + "link": "https://example.com/1", + "score": 80, + "fetched_at": now.isoformat(), + }, + { + "title": "Old Entry", + "link": "https://example.com/2", + "score": 80, + "fetched_at": last_push.isoformat(), + }, + ], + } + + fetch_file = temp_dir / f"fetch-{now.date().isoformat()}.json" + with open(fetch_file, "w") as f: + json.dump(data, f) + + to_push, context = collect_entries_for_push( + last_push_time=last_push, + context_days=2, + min_score=60, + data_dir=str(temp_dir), + ) + + assert len(to_push) == 1 + assert to_push[0]["title"] == "New Entry" + + def test_collect_context_limit(self, temp_dir): + now = datetime.now(timezone.utc) + + entries = [ + { + "title": f"Entry{i}", + "link": f"https://example.com/{i}", + "score": 50 + i, + "fetched_at": now.isoformat(), + } + for i in range(60) + ] + + data = {"meta": {"date": now.date().isoformat()}, "entries": entries} + + fetch_file = temp_dir / f"fetch-{now.date().isoformat()}.json" + with open(fetch_file, "w") as f: + json.dump(data, f) + + to_push, context = collect_entries_for_push( + last_push_time=None, context_days=2, min_score=60, data_dir=str(temp_dir) + ) + + assert len(context) <= 50 + + def test_collect_multi_day(self, temp_dir): + from src.config import get_timezone + + tz = get_timezone() + today = datetime.now(tz) + yesterday = today - timedelta(days=1) + + today_data = { + "meta": {"date": today.date().isoformat()}, + "entries": [ + { + "title": "Today Entry", + "link": "https://example.com/1", + "score": 80, + "fetched_at": today.isoformat(), + } + ], + } + + yesterday_data = { + "meta": {"date": yesterday.date().isoformat()}, + "entries": [ + { + "title": "Yesterday Entry", + "link": "https://example.com/2", + "score": 75, + "fetched_at": yesterday.isoformat(), + } + ], + } + + (temp_dir / f"fetch-{today.date().isoformat()}.json").write_text( + json.dumps(today_data) + ) + (temp_dir / f"fetch-{yesterday.date().isoformat()}.json").write_text( + json.dumps(yesterday_data) + ) + + to_push, context = collect_entries_for_push( + last_push_time=None, context_days=2, min_score=60, data_dir=str(temp_dir) + ) + + assert len(to_push) >= 1 + + +class TestMainStartup: + """测试主程序启动流程""" + + @pytest.mark.asyncio + async def test_main_checks_llm_before_starting_loops(self, sample_config): + with patch("main.load_config", return_value=sample_config), patch( + "main.check_llm_available", new_callable=AsyncMock + ) as mock_check, patch( + "main.fetch_loop", new_callable=AsyncMock + ) as mock_fetch_loop, patch( + "main.push_loop", new_callable=AsyncMock + ) as mock_push_loop: + await run_main() + + mock_check.assert_awaited_once_with(sample_config["llm"]) + mock_fetch_loop.assert_awaited_once_with(sample_config) + mock_push_loop.assert_awaited_once_with(sample_config) + + @pytest.mark.asyncio + async def test_main_exits_when_llm_health_check_fails(self, sample_config): + with patch("main.load_config", return_value=sample_config), patch( + "main.check_llm_available", new_callable=AsyncMock + ) as mock_check, patch( + "main.fetch_loop", new_callable=AsyncMock + ) as mock_fetch_loop, patch( + "main.push_loop", new_callable=AsyncMock + ) as mock_push_loop: + mock_check.side_effect = RuntimeError("health failed") + + await run_main() + + mock_check.assert_awaited_once_with(sample_config["llm"]) + mock_fetch_loop.assert_not_called() + mock_push_loop.assert_not_called() diff --git a/ai-daily-main/tests/pytest/test_main_morning_push.py b/ai-daily-main/tests/pytest/test_main_morning_push.py new file mode 100644 index 0000000..ef72b76 --- /dev/null +++ b/ai-daily-main/tests/pytest/test_main_morning_push.py @@ -0,0 +1,123 @@ +"""测试早报四模块编排:gather + insights 串行 + sentinel 拼装 + 失败隔离""" + +import sys +from pathlib import Path +from unittest.mock import patch, AsyncMock + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from src.main import _run_morning_push + + +def _insights_meta() -> dict: + return { + "title": "📰 AI Daily 每日精选 | 2026-05-22", + "lead": "lead text", + "highlights": ["a"], + "profile": "morning", + "date": "2026-05-22", + "excerpt": "", + "seotitle": "", + "seodescription": "", + } + + +@pytest.mark.asyncio +async def test_assembles_all_four_sections(sample_config): + sample_config["filter"]["push_context_days"] = 5 + + sent = {} + + async def fake_send(content, push_cfg, title=None, metadata=None): + sent["content"] = content + sent["metadata"] = metadata + + saved = {} + + def fake_save( + filepath, + content, + source_count, + total_entries, + profile="default", + metadata=None, + ): + saved["profile"] = profile + saved["content"] = content + saved["metadata"] = metadata + + with patch( + "src.main.run_rss_section", new=AsyncMock(return_value=("R", None, None)) + ), patch( + "src.main.run_github_section", new=AsyncMock(return_value=("G", None)) + ), patch( + "src.main.run_hackernews_section", new=AsyncMock(return_value=("H", None)) + ), patch( + "src.main.run_insights_section", + new=AsyncMock(return_value=("I", _insights_meta(), None)), + ), patch( + "src.main.send_to_platforms", new=AsyncMock(side_effect=fake_send) + ), patch( + "src.main.save_push_file", side_effect=fake_save + ): + await _run_morning_push(sample_config) + + assert "SECTION:rss" in sent["content"] + assert "SECTION:github" in sent["content"] + assert "SECTION:hackernews" in sent["content"] + assert "SECTION:insights" in sent["content"] + assert sent["metadata"]["profile"] == "morning" + assert saved["profile"] == "morning" + assert saved["metadata"]["lead"] == "lead text" + + +@pytest.mark.asyncio +async def test_rss_failure_raises_to_caller(sample_config): + sample_config["filter"]["push_context_days"] = 5 + + with patch( + "src.main.run_rss_section", + new=AsyncMock(return_value=("", None, "compose_digest 失败")), + ), patch( + "src.main.run_github_section", new=AsyncMock(return_value=("G", None)) + ), patch( + "src.main.run_hackernews_section", new=AsyncMock(return_value=("H", None)) + ), patch( + "src.main.notify_llm_errors", new=AsyncMock() + ): + with pytest.raises(RuntimeError): + await _run_morning_push(sample_config) + + +@pytest.mark.asyncio +async def test_section_failure_degrades_to_omission(sample_config): + sample_config["filter"]["push_context_days"] = 5 + + sent = {} + + async def fake_send(content, push_cfg, title=None, metadata=None): + sent["content"] = content + + with patch( + "src.main.run_rss_section", new=AsyncMock(return_value=("R", None, None)) + ), patch( + "src.main.run_github_section", new=AsyncMock(return_value=("", "gh down")) + ), patch( + "src.main.run_hackernews_section", new=AsyncMock(return_value=("H", None)) + ), patch( + "src.main.run_insights_section", + new=AsyncMock(return_value=("I", _insights_meta(), None)), + ), patch( + "src.main.notify_llm_errors", new=AsyncMock() + ), patch( + "src.main.send_to_platforms", new=AsyncMock(side_effect=fake_send) + ), patch( + "src.main.save_push_file" + ): + await _run_morning_push(sample_config) + + assert "SECTION:rss" in sent["content"] + assert "SECTION:github" not in sent["content"] + assert "SECTION:hackernews" in sent["content"] diff --git a/ai-daily-main/tests/pytest/test_main_run_push_job.py b/ai-daily-main/tests/pytest/test_main_run_push_job.py new file mode 100644 index 0000000..51cdc97 --- /dev/null +++ b/ai-daily-main/tests/pytest/test_main_run_push_job.py @@ -0,0 +1,43 @@ +"""测试 run_push_job 的早报/默认路径分发""" + +import sys +from pathlib import Path +from unittest.mock import patch, AsyncMock + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from src.main import run_push_job + + +@pytest.mark.asyncio +async def test_default_path_when_not_morning(sample_config): + sample_config["schedule"]["push_cron"] = ["0 8 * * *", "0 17 * * *"] + sample_config["filter"]["push_context_days"] = 5 + + with patch("src.main.is_morning_push", return_value=False), patch( + "src.main._run_default_push", new=AsyncMock(return_value=None) + ) as default_path, patch( + "src.main._run_morning_push", new=AsyncMock(return_value=None) + ) as morning_path: + await run_push_job(sample_config) + + default_path.assert_awaited_once() + morning_path.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_morning_path_when_morning(sample_config): + sample_config["schedule"]["push_cron"] = ["0 8 * * *", "0 17 * * *"] + sample_config["filter"]["push_context_days"] = 5 + + with patch("src.main.is_morning_push", return_value=True), patch( + "src.main._run_default_push", new=AsyncMock(return_value=None) + ) as default_path, patch( + "src.main._run_morning_push", new=AsyncMock(return_value=None) + ) as morning_path: + await run_push_job(sample_config) + + morning_path.assert_awaited_once() + default_path.assert_not_awaited() diff --git a/ai-daily-main/tests/pytest/test_morning_detection.py b/ai-daily-main/tests/pytest/test_morning_detection.py new file mode 100644 index 0000000..1807053 --- /dev/null +++ b/ai-daily-main/tests/pytest/test_morning_detection.py @@ -0,0 +1,60 @@ +"""测试早报判定:push_cron 最近最早匹配""" + +import sys +from datetime import datetime, timedelta, timezone +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from src.main import is_morning_push + + +TZ = timezone(timedelta(hours=8)) + + +def _cfg(push_cron): + return {"schedule": {"push_cron": push_cron, "timezone_hours": 8}} + + +def test_returns_false_when_push_cron_empty(): + assert is_morning_push(datetime(2026, 5, 17, 8, 0, tzinfo=TZ), {"schedule": {}}) is False + assert is_morning_push(datetime(2026, 5, 17, 8, 0, tzinfo=TZ), _cfg([])) is False + + +def test_single_cron_always_morning(): + """push_cron 只有一项时,任何触发(含手动 / 非整点)都视为早报""" + cfg = _cfg(["0 8 * * *"]) + assert is_morning_push(datetime(2026, 5, 17, 8, 0, tzinfo=TZ), cfg) is True + assert is_morning_push(datetime(2026, 5, 17, 12, 30, tzinfo=TZ), cfg) is True + assert is_morning_push(datetime(2026, 5, 17, 23, 59, tzinfo=TZ), cfg) is True + + +def test_earliest_cron_match_is_morning(): + """触发时刻离最早 cron 最近 → 早报""" + cfg = _cfg(["0 8 * * *", "0 17 * * *"]) + assert is_morning_push(datetime(2026, 5, 17, 8, 0, tzinfo=TZ), cfg) is True + assert is_morning_push(datetime(2026, 5, 17, 8, 30, tzinfo=TZ), cfg) is True + assert is_morning_push(datetime(2026, 5, 17, 6, 0, tzinfo=TZ), cfg) is True + + +def test_later_cron_match_not_morning(): + """触发时刻离非最早 cron 最近 → 默认""" + cfg = _cfg(["0 8 * * *", "0 17 * * *"]) + assert is_morning_push(datetime(2026, 5, 17, 17, 0, tzinfo=TZ), cfg) is False + assert is_morning_push(datetime(2026, 5, 17, 16, 30, tzinfo=TZ), cfg) is False + assert is_morning_push(datetime(2026, 5, 17, 22, 0, tzinfo=TZ), cfg) is False + + +def test_drift_tolerance_via_closest_match(): + """无显式容差,但「最近匹配」自动吸附小幅漂移 (08:01 仍归 08:00)""" + cfg = _cfg(["0 8 * * *", "0 17 * * *"]) + assert is_morning_push(datetime(2026, 5, 17, 8, 1, tzinfo=TZ), cfg) is True + assert is_morning_push(datetime(2026, 5, 17, 17, 1, tzinfo=TZ), cfg) is False + + +def test_three_crons_only_earliest_is_morning(): + """三条 cron 时,只有最早那条对应的触发是早报""" + cfg = _cfg(["0 8 * * *", "0 12 * * *", "0 20 * * *"]) + assert is_morning_push(datetime(2026, 5, 17, 8, 0, tzinfo=TZ), cfg) is True + assert is_morning_push(datetime(2026, 5, 17, 12, 0, tzinfo=TZ), cfg) is False + assert is_morning_push(datetime(2026, 5, 17, 20, 0, tzinfo=TZ), cfg) is False diff --git a/ai-daily-main/tests/pytest/test_processor.py b/ai-daily-main/tests/pytest/test_processor.py new file mode 100644 index 0000000..5b7e45d --- /dev/null +++ b/ai-daily-main/tests/pytest/test_processor.py @@ -0,0 +1,87 @@ +"""内容处理模块测试""" + +import pytest +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src")) + +from processor import html_to_markdown + + +class TestHtmlToMarkdown: + """测试HTML转Markdown""" + + def test_convert_basic_html(self): + html = "

Hello World

" + result = html_to_markdown(html) + assert "Hello World" in result + + def test_convert_with_links(self): + html = 'Click here' + result = html_to_markdown(html) + assert "[Click here](https://example.com)" in result + + def test_convert_with_images(self): + html = 'Image' + result = html_to_markdown(html) + assert "![Image](https://example.com/image.png)" in result + + def test_convert_with_headings(self): + html = "

Title

Subtitle

" + result = html_to_markdown(html) + assert "# Title" in result + assert "## Subtitle" in result + + def test_convert_with_lists(self): + html = "
  • Item 1
  • Item 2
" + result = html_to_markdown(html) + assert "Item 1" in result + assert "Item 2" in result + + def test_convert_with_strong_emphasis(self): + html = "Bold and italic" + result = html_to_markdown(html) + assert "**Bold**" in result + assert "*italic*" in result + + def test_relative_link_conversion(self): + html = 'Read more' + result = html_to_markdown(html, base_url="https://example.com/blog") + assert "https://example.com/article/123" in result + + def test_relative_image_conversion(self): + html = '' + result = html_to_markdown(html, base_url="https://example.com") + assert "https://example.com/images/logo.png" in result + + def test_absolute_link_unchanged(self): + html = 'Link' + result = html_to_markdown(html, base_url="https://example.com") + assert "https://other.com/page" in result + + def test_remove_xgo_ing_link(self): + html = "

Content

[⚡ Powered by xgo.ing](https://xgo.ing)

" + result = html_to_markdown(html) + assert "xgo.ing" not in result + assert "Content" in result + + def test_remove_xgo_ing_link_with_slash(self): + html = "

Content

[⚡ Powered by xgo.ing](https://xgo.ing/)

" + result = html_to_markdown(html) + assert "xgo.ing" not in result + + def test_clean_extra_newlines(self): + html = "

Line 1

\n\n\n\n

Line 2

" + result = html_to_markdown(html) + assert "\n\n\n\n" not in result + + def test_empty_html(self): + result = html_to_markdown("") + assert result.strip() == "" + + def test_html_with_nbsp(self): + html = "

Hello World

" + result = html_to_markdown(html) + assert "Hello" in result + assert "World" in result diff --git a/ai-daily-main/tests/pytest/test_push.py b/ai-daily-main/tests/pytest/test_push.py new file mode 100644 index 0000000..6fffdf0 --- /dev/null +++ b/ai-daily-main/tests/pytest/test_push.py @@ -0,0 +1,209 @@ +"""推送模块测试""" + +import os +import pytest +import sys +from pathlib import Path +from unittest.mock import patch, AsyncMock, MagicMock + +sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src")) + +from push.discord import DiscordPlatform +from push.feishu import FeishuPlatform +from push import create_platform + + +class TestDiscordPlatform: + """测试Discord推送""" + + def test_validate_config_valid(self): + config = { + "enabled": True, + "apiKeyName": "DISCORD_WEBHOOK_URL", + } + with patch.dict( + os.environ, + {"DISCORD_WEBHOOK_URL": "https://discord.com/api/webhooks/123456/abcdef"}, + ): + platform = DiscordPlatform(config) + assert platform.validate_config(config) is True + + def test_validate_config_disabled(self): + config = { + "enabled": False, + "apiKeyName": "DISCORD_WEBHOOK_URL", + } + with patch.dict( + os.environ, + {"DISCORD_WEBHOOK_URL": "https://discord.com/api/webhooks/123456/abcdef"}, + ): + platform = DiscordPlatform(config) + assert platform.validate_config(config) is False + + def test_validate_config_missing_webhook(self): + config = {"enabled": True, "apiKeyName": "DISCORD_WEBHOOK_URL"} + with patch.dict(os.environ, {"DISCORD_WEBHOOK_URL": ""}): + platform = DiscordPlatform(config) + assert platform.validate_config(config) is False + + def test_validate_config_invalid_url(self): + config = {"enabled": True, "apiKeyName": "DISCORD_WEBHOOK_URL"} + with patch.dict(os.environ, {"DISCORD_WEBHOOK_URL": "not-a-valid-url"}): + platform = DiscordPlatform(config) + assert platform.validate_config(config) is False + + def test_validate_config_wrong_domain(self): + config = {"enabled": True, "apiKeyName": "DISCORD_WEBHOOK_URL"} + with patch.dict( + os.environ, {"DISCORD_WEBHOOK_URL": "https://example.com/webhook"} + ): + platform = DiscordPlatform(config) + assert platform.validate_config(config) is False + + def test_split_content_short(self): + with patch.dict(os.environ, {"DISCORD_WEBHOOK_URL": "https://test.com"}): + config = {"apiKeyName": "DISCORD_WEBHOOK_URL"} + platform = DiscordPlatform(config) + short_content = "Hello" + chunks = platform._split_content(short_content, limit=2000) + assert len(chunks) == 1 + assert chunks[0] == "Hello" + + def test_split_content_long_message(self): + with patch.dict(os.environ, {"DISCORD_WEBHOOK_URL": "https://test.com"}): + config = {"apiKeyName": "DISCORD_WEBHOOK_URL"} + platform = DiscordPlatform(config) + long_content = "A\n" * 2500 + chunks = platform._split_content(long_content, limit=2000) + assert len(chunks) > 1 + assert all(len(c) <= 2000 for c in chunks) + + def test_split_content_exact_boundary(self): + with patch.dict(os.environ, {"DISCORD_WEBHOOK_URL": "https://test.com"}): + config = {"apiKeyName": "DISCORD_WEBHOOK_URL"} + platform = DiscordPlatform(config) + content = "A" * 2000 + chunks = platform._split_content(content, limit=2000) + assert len(chunks) == 1 + + def test_split_content_unicode(self): + with patch.dict(os.environ, {"DISCORD_WEBHOOK_URL": "https://test.com"}): + config = {"apiKeyName": "DISCORD_WEBHOOK_URL"} + platform = DiscordPlatform(config) + content = "你好" * 500 + chunks = platform._split_content(content, limit=100) + assert len(chunks) >= 1 + + @pytest.mark.asyncio + async def test_send_success(self): + with patch.dict( + os.environ, + {"DISCORD_WEBHOOK_URL": "https://discord.com/api/webhooks/test/abc"}, + ): + config = {"apiKeyName": "DISCORD_WEBHOOK_URL"} + platform = DiscordPlatform(config) + + with patch.object(platform, "send", new_callable=AsyncMock) as mock_send: + mock_send.return_value = True + + result = await mock_send("Test message") + + assert result is True + + @pytest.mark.asyncio + async def test_send_failure(self): + with patch.dict( + os.environ, + {"DISCORD_WEBHOOK_URL": "https://discord.com/api/webhooks/test/abc"}, + ): + config = {"apiKeyName": "DISCORD_WEBHOOK_URL"} + platform = DiscordPlatform(config) + + with patch.object(platform, "send", new_callable=AsyncMock) as mock_send: + mock_send.return_value = False + + result = await mock_send("Test message") + + assert result is False + + +class TestFeishuPlatform: + """测试飞书推送""" + + def test_validate_config_valid(self): + config = { + "enabled": True, + "apiKeyName": "FEISHU_WEBHOOK_URL", + } + with patch.dict(os.environ, {"FEISHU_WEBHOOK_URL": "https://open.feishu.cn/open-apis/bot/v2/hook/test"}): + platform = FeishuPlatform(config) + assert platform.validate_config(config) is True + + def test_validate_config_disabled(self): + config = { + "enabled": False, + "apiKeyName": "FEISHU_WEBHOOK_URL", + } + with patch.dict(os.environ, {"FEISHU_WEBHOOK_URL": "https://open.feishu.cn/open-apis/bot/v2/hook/test"}): + platform = FeishuPlatform(config) + assert platform.validate_config(config) is False + + def test_validate_config_missing_key(self): + config = {"enabled": True, "apiKeyName": "FEISHU_WEBHOOK_URL"} + with patch.dict(os.environ, {"FEISHU_WEBHOOK_URL": ""}): + platform = FeishuPlatform(config) + assert platform.validate_config(config) is False + + def test_validate_config_any_non_empty_webhook(self): + config = {"enabled": True, "apiKeyName": "FEISHU_WEBHOOK_URL"} + with patch.dict(os.environ, {"FEISHU_WEBHOOK_URL": "https://open.feishu.cn/open-apis/bot/v2/hook/test"}): + platform = FeishuPlatform(config) + assert platform.validate_config(config) is True + + +class TestPushFactory: + """测试平台工厂""" + + def test_create_enabled_platform(self): + config = { + "enabled": True, + "apiKeyName": "DISCORD_WEBHOOK_URL", + } + with patch.dict( + os.environ, + {"DISCORD_WEBHOOK_URL": "https://discord.com/api/webhooks/123/abc"}, + ): + platform = create_platform("discord", config) + assert platform is not None + + def test_create_disabled_platform_returns_none(self): + config = {"enabled": False, "apiKeyName": "DISCORD_WEBHOOK_URL"} + platform = create_platform("discord", config) + assert platform is None + + def test_create_unknown_platform_raises(self): + with pytest.raises(ValueError): + create_platform("unknown", {}) + + def test_create_feishu_platform(self): + config = { + "enabled": True, + "apiKeyName": "FEISHU_WEBHOOK_URL", + } + with patch.dict(os.environ, {"FEISHU_WEBHOOK_URL": "https://open.feishu.cn/open-apis/bot/v2/hook/test"}): + platform = create_platform("feishu", config) + assert platform is not None + assert isinstance(platform, FeishuPlatform) + + def test_create_discord_platform(self): + config = { + "enabled": True, + "apiKeyName": "DISCORD_WEBHOOK_URL", + } + with patch.dict( + os.environ, + {"DISCORD_WEBHOOK_URL": "https://discord.com/api/webhooks/test/abc"}, + ): + platform = create_platform("discord", config) + assert platform is not None + assert isinstance(platform, DiscordPlatform) diff --git a/ai-daily-main/tests/pytest/test_sections_github_enricher.py b/ai-daily-main/tests/pytest/test_sections_github_enricher.py new file mode 100644 index 0000000..9f470bf --- /dev/null +++ b/ai-daily-main/tests/pytest/test_sections_github_enricher.py @@ -0,0 +1,107 @@ +"""测试 GitHub REST API enrich 字段映射、archived 过滤、token 鉴权头""" + +import base64 +import sys +from pathlib import Path +from unittest.mock import patch, AsyncMock, MagicMock + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from src.sections.github.repo_enricher import enrich_repo, _auth_headers + + +def test_auth_headers_with_token(monkeypatch): + monkeypatch.setenv("GITHUB_TOKEN", "ghp_secret") + headers = _auth_headers(token_env="GITHUB_TOKEN") + assert headers["Authorization"] == "Bearer ghp_secret" + assert headers["Accept"] == "application/vnd.github+json" + + +def test_auth_headers_without_token(monkeypatch): + monkeypatch.delenv("GITHUB_TOKEN", raising=False) + headers = _auth_headers(token_env="GITHUB_TOKEN") + assert "Authorization" not in headers + assert headers["Accept"] == "application/vnd.github+json" + + +@pytest.mark.asyncio +async def test_enrich_repo_merges_metadata_and_readme(): + readme_body = "# Title\n\nProject description here." + readme_b64 = base64.b64encode(readme_body.encode("utf-8")).decode("ascii") + + metadata_payload = { + "description": "real desc", + "topics": ["llm", "rag"], + "license": {"spdx_id": "MIT"}, + "pushed_at": "2026-05-16T10:00:00Z", + "archived": False, + } + readme_payload = {"content": readme_b64, "encoding": "base64"} + + async def fake_get_json(session, url, **kwargs): + if url.endswith("/readme"): + return readme_payload + return metadata_payload + + base = { + "url": "https://github.com/o/r", + "full_name": "o/r", + "description": "from trending", + "language": "Python", + "stars_today": 100, + "stars_total": 5000, + } + + with patch( + "src.sections.github.repo_enricher._get_json", new=AsyncMock(side_effect=fake_get_json) + ): + enriched = await enrich_repo( + session=MagicMock(), repo=base, token_env="GITHUB_TOKEN", readme_max_chars=200 + ) + + assert enriched["topics"] == ["llm", "rag"] + assert enriched["license"] == "MIT" + assert enriched["pushed_at"] == "2026-05-16T10:00:00Z" + assert "Project description" in enriched["readme_excerpt"] + assert enriched["stars_today"] == 100 # trending 已有字段保留 + + +@pytest.mark.asyncio +async def test_enrich_repo_returns_none_when_archived(): + metadata_payload = {"archived": True, "topics": [], "pushed_at": "x"} + + async def fake_get_json(session, url, **kwargs): + if url.endswith("/readme"): + return {"content": ""} + return metadata_payload + + base = {"url": "https://github.com/o/r", "full_name": "o/r"} + with patch( + "src.sections.github.repo_enricher._get_json", new=AsyncMock(side_effect=fake_get_json) + ): + result = await enrich_repo( + session=MagicMock(), repo=base, token_env="GITHUB_TOKEN", readme_max_chars=200 + ) + assert result is None + + +@pytest.mark.asyncio +async def test_enrich_repo_truncates_readme(): + readme_body = "x" * 5000 + readme_b64 = base64.b64encode(readme_body.encode("utf-8")).decode("ascii") + + async def fake_get_json(session, url, **kwargs): + if url.endswith("/readme"): + return {"content": readme_b64, "encoding": "base64"} + return {"archived": False, "topics": [], "pushed_at": "p"} + + base = {"url": "https://github.com/o/r", "full_name": "o/r"} + with patch( + "src.sections.github.repo_enricher._get_json", new=AsyncMock(side_effect=fake_get_json) + ): + enriched = await enrich_repo( + session=MagicMock(), repo=base, token_env="GITHUB_TOKEN", readme_max_chars=100 + ) + assert len(enriched["readme_excerpt"]) == 100 diff --git a/ai-daily-main/tests/pytest/test_sections_github_scraper.py b/ai-daily-main/tests/pytest/test_sections_github_scraper.py new file mode 100644 index 0000000..783dbc8 --- /dev/null +++ b/ai-daily-main/tests/pytest/test_sections_github_scraper.py @@ -0,0 +1,40 @@ +"""测试 GitHub trending HTML 解析""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from src.sections.github.trending_scraper import parse_trending_html + + +def test_parse_trending_html_returns_repo_dicts(): + fixture = ( + Path(__file__).parent / "fixtures" / "github_trending.html" + ).read_text(encoding="utf-8") + + repos = parse_trending_html(fixture) + + assert len(repos) > 0 + first = repos[0] + assert first["url"].startswith("https://github.com/") + assert "/" in first["full_name"] + assert isinstance(first["stars_today"], int) + assert isinstance(first["stars_total"], int) + # description / language 可为空字符串但必须是 str + assert isinstance(first["description"], str) + assert isinstance(first["language"], str) + + +def test_parse_trending_html_dedupes_by_url(): + fixture = ( + Path(__file__).parent / "fixtures" / "github_trending.html" + ).read_text(encoding="utf-8") + repos = parse_trending_html(fixture) + urls = [r["url"] for r in repos] + assert len(urls) == len(set(urls)) + + +def test_parse_trending_html_empty_input(): + assert parse_trending_html("") == [] + assert parse_trending_html("no repos") == [] diff --git a/ai-daily-main/tests/pytest/test_sections_github_section.py b/ai-daily-main/tests/pytest/test_sections_github_section.py new file mode 100644 index 0000000..1776080 --- /dev/null +++ b/ai-daily-main/tests/pytest/test_sections_github_section.py @@ -0,0 +1,145 @@ +"""测试 GitHub 板块编排:抓取 → history 过滤 → enrich → LLM 总结""" + +import sys +from pathlib import Path +from unittest.mock import patch, AsyncMock + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from src.sections.github.section import run_github_section + +# Stub summarize_github_trending until Task 11 provides it +import src.llm as _llm +if not hasattr(_llm, "summarize_github_trending"): + async def _stub(*a, **k): + return "", None + _llm.summarize_github_trending = _stub + + +def _cfg(history_file: str, max_deep_dive: int = 10) -> dict: + return { + "filter": {"keep_days": 7}, + "sections": { + "github_trending": { + "enabled": True, + "max_items": 3, + "max_deep_dive": max_deep_dive, + "readme_max_chars": 3000, + "history_file": history_file, + "request_timeout": 10, + "tokenName": "GITHUB_TOKEN", + } + }, + "llm": { + "model": "x", + "baseUrl": "http://x", + "apiKeyName": "DEEPSEEK_API_KEY", + "prompts": {"section_github": "prompts/section_github.md"}, + }, + } + + +@pytest.mark.asyncio +async def test_disabled_returns_empty(tmp_path): + cfg = _cfg(str(tmp_path / "h.json")) + cfg["sections"]["github_trending"]["enabled"] = False + md, err = await run_github_section(cfg, now=None) + assert md == "" + assert err is None + + +@pytest.mark.asyncio +async def test_no_candidates_after_history_returns_empty(tmp_path): + history_path = tmp_path / "h.json" + # 预置 history,使得今日 scrape 出来的 repo 都已存在 + history_path.write_text( + '{"repos": {"https://github.com/a/b": "2026-05-16"}, "updated_at": "x"}', + encoding="utf-8", + ) + cfg = _cfg(str(history_path)) + + with patch( + "src.sections.github.section.fetch_trending_page", new=AsyncMock(return_value="") + ), patch( + "src.sections.github.section.parse_trending_html", + return_value=[{"url": "https://github.com/a/b", "full_name": "a/b"}], + ): + md, err = await run_github_section(cfg, now=None) + + assert md == "" + assert err is None + + +@pytest.mark.asyncio +async def test_happy_path_enriches_and_summarizes(tmp_path): + history_path = tmp_path / "h.json" + cfg = _cfg(str(history_path), max_deep_dive=10) + + repos = [ + { + "url": "https://github.com/o1/r1", + "full_name": "o1/r1", + "description": "d1", + "language": "Python", + "stars_today": 100, + "stars_total": 1000, + } + ] + enriched = [{**repos[0], "topics": ["llm"], "license": "MIT", "pushed_at": "p", "readme_excerpt": "rm"}] + + with patch( + "src.sections.github.section.fetch_trending_page", new=AsyncMock(return_value="") + ), patch( + "src.sections.github.section.parse_trending_html", return_value=repos + ), patch( + "src.sections.github.section.enrich_repos", + new=AsyncMock(return_value=(enriched, [])), + ), patch( + "src.llm.summarize_github_trending", + new=AsyncMock(return_value=("## GH section md", None)), + ): + md, err = await run_github_section(cfg, now=None) + + assert md == "## GH section md" + assert err is None + import json as _j + saved = _j.loads(history_path.read_text(encoding="utf-8")) + assert "https://github.com/o1/r1" in saved["repos"] + + +@pytest.mark.asyncio +async def test_truncates_candidates_to_max_deep_dive(tmp_path): + cfg = _cfg(str(tmp_path / "h.json"), max_deep_dive=2) + repos = [ + {"url": f"https://github.com/o/r{i}", "full_name": f"o/r{i}"} for i in range(5) + ] + captured = {} + + async def fake_enrich(candidates, **kwargs): + captured["count"] = len(candidates) + return [], [] + + with patch( + "src.sections.github.section.fetch_trending_page", new=AsyncMock(return_value="") + ), patch( + "src.sections.github.section.parse_trending_html", return_value=repos + ), patch( + "src.sections.github.section.enrich_repos", new=AsyncMock(side_effect=fake_enrich) + ): + await run_github_section(cfg, now=None) + + assert captured["count"] == 2 + + +@pytest.mark.asyncio +async def test_scrape_failure_returns_error(tmp_path): + cfg = _cfg(str(tmp_path / "h.json")) + with patch( + "src.sections.github.section.fetch_trending_page", + new=AsyncMock(side_effect=RuntimeError("HTTP 500")), + ): + md, err = await run_github_section(cfg, now=None) + assert md == "" + assert "HTTP 500" in err diff --git a/ai-daily-main/tests/pytest/test_sections_hackernews_enricher.py b/ai-daily-main/tests/pytest/test_sections_hackernews_enricher.py new file mode 100644 index 0000000..fdb2a94 --- /dev/null +++ b/ai-daily-main/tests/pytest/test_sections_hackernews_enricher.py @@ -0,0 +1,246 @@ +"""测试 HN enrich(Algolia 评论树 + 外链正文)""" + +import sys +from pathlib import Path +from unittest.mock import patch, AsyncMock, MagicMock + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from src.sections.hackernews.item_enricher import enrich_story + + +def _kwargs(**overrides): + base = dict( + top_comments=3, + top_l2_per_l1=2, + comment_max_chars=500, + comments_total_chars=60000, + link_content_max_chars=3000, + algolia_base="https://hn.algolia.com/api/v1", + timeout=10, + ) + base.update(overrides) + return base + + +@pytest.mark.asyncio +async def test_enrich_external_link_story_returns_tree(): + story = { + "id": "111", + "title": "T", + "url": "https://example.com/post", + "site": "example.com", + "points": 100, + "comments": 5, + "comments_url": "https://news.ycombinator.com/item?id=111", + } + algolia_payload = { + "text": None, + "children": [ + { + "text": "

comment one

", + "children": [ + {"text": "

reply 1a

"}, + {"text": "

reply 1b

"}, + {"text": "

reply 1c (should be dropped)

"}, + ], + }, + {"text": "

comment two

", "children": []}, + {"text": "

comment three

"}, + {"text": "

comment four (over top_comments cap)

"}, + ], + } + + async def fake_algolia(session, item_id, **kw): + return algolia_payload + + async def fake_external(session, url, **kw): + return "link body" + + with patch( + "src.sections.hackernews.item_enricher._fetch_algolia_item", + new=AsyncMock(side_effect=fake_algolia), + ), patch( + "src.sections.hackernews.item_enricher._fetch_external_markdown", + new=AsyncMock(side_effect=fake_external), + ): + enriched = await enrich_story( + session=MagicMock(), story=story, **_kwargs() + ) + + tree = enriched["top_comments"] + assert len(tree) == 3 + assert "comment one" in tree[0]["l1"] + assert len(tree[0]["replies"]) == 2 + assert "reply 1a" in tree[0]["replies"][0] + assert "reply 1b" in tree[0]["replies"][1] + assert tree[1]["replies"] == [] + assert tree[2]["replies"] == [] + assert "link body" in enriched["link_content"] + + +@pytest.mark.asyncio +async def test_enrich_show_hn_uses_root_text_no_external_fetch(): + story = { + "id": "222", + "title": "Show HN: T", + "url": "https://news.ycombinator.com/item?id=222", + "site": "", + "points": 200, + "comments": 10, + "comments_url": "https://news.ycombinator.com/item?id=222", + } + algolia_payload = { + "text": "

post body text

", + "children": [{"text": "

c1

"}], + } + link_calls = [] + + async def fake_algolia(session, item_id, **kw): + return algolia_payload + + async def fake_external(session, url, **kw): + link_calls.append(url) + return "should not be called" + + with patch( + "src.sections.hackernews.item_enricher._fetch_algolia_item", + new=AsyncMock(side_effect=fake_algolia), + ), patch( + "src.sections.hackernews.item_enricher._fetch_external_markdown", + new=AsyncMock(side_effect=fake_external), + ): + enriched = await enrich_story( + session=MagicMock(), story=story, **_kwargs() + ) + + assert link_calls == [] + assert "post body text" in enriched["link_content"] + assert enriched["top_comments"][0]["l1"].startswith("c1") or "c1" in enriched["top_comments"][0]["l1"] + + +@pytest.mark.asyncio +async def test_enrich_truncates_comments_and_link(): + story = { + "id": "333", + "title": "T", + "url": "https://example.com/a", + "site": "example.com", + "points": 100, + "comments": 2, + "comments_url": "x", + } + long_comment = "

" + ("y" * 2000) + "

" + long_reply = "

" + ("z" * 2000) + "

" + long_link = "z" * 5000 + + async def fake_algolia(session, item_id, **kw): + return { + "text": None, + "children": [ + {"text": long_comment, "children": [{"text": long_reply}]} + ], + } + + async def fake_external(session, url, **kw): + return long_link + + with patch( + "src.sections.hackernews.item_enricher._fetch_algolia_item", + new=AsyncMock(side_effect=fake_algolia), + ), patch( + "src.sections.hackernews.item_enricher._fetch_external_markdown", + new=AsyncMock(side_effect=fake_external), + ): + enriched = await enrich_story( + session=MagicMock(), + story=story, + **_kwargs(comment_max_chars=100, link_content_max_chars=200), + ) + + assert len(enriched["top_comments"][0]["l1"]) <= 100 + assert len(enriched["top_comments"][0]["replies"][0]) <= 100 + assert len(enriched["link_content"]) <= 200 + + +@pytest.mark.asyncio +async def test_enrich_failure_returns_partial(): + story = { + "id": "444", + "title": "T", + "url": "https://example.com/x", + "site": "example.com", + "points": 100, + "comments": 2, + "comments_url": "x", + } + + async def fake_algolia(session, item_id, **kw): + raise RuntimeError("algolia down") + + async def fake_external(session, url, **kw): + return "ok" + + with patch( + "src.sections.hackernews.item_enricher._fetch_algolia_item", + new=AsyncMock(side_effect=fake_algolia), + ), patch( + "src.sections.hackernews.item_enricher._fetch_external_markdown", + new=AsyncMock(side_effect=fake_external), + ): + enriched = await enrich_story( + session=MagicMock(), story=story, **_kwargs() + ) + + assert enriched["top_comments"] == [] + assert "ok" in enriched["link_content"] + + +@pytest.mark.asyncio +async def test_enrich_total_budget_stops_early(): + """累计字符达 comments_total_chars 立即停止,后续 L1 / L2 都不再加入""" + story = { + "id": "555", + "title": "T", + "url": "https://example.com/q", + "site": "example.com", + "points": 100, + "comments": 5, + "comments_url": "x", + } + big = "

" + ("a" * 500) + "

" # markdown 约 500 chars + + async def fake_algolia(session, item_id, **kw): + return { + "text": None, + "children": [{"text": big, "children": [{"text": big}, {"text": big}]} for _ in range(10)], + } + + async def fake_external(session, url, **kw): + return "x" + + with patch( + "src.sections.hackernews.item_enricher._fetch_algolia_item", + new=AsyncMock(side_effect=fake_algolia), + ), patch( + "src.sections.hackernews.item_enricher._fetch_external_markdown", + new=AsyncMock(side_effect=fake_external), + ): + enriched = await enrich_story( + session=MagicMock(), + story=story, + **_kwargs( + top_comments=10, + top_l2_per_l1=2, + comment_max_chars=500, + comments_total_chars=1500, + ), + ) + + tree = enriched["top_comments"] + total = sum(len(n["l1"]) + sum(len(r) for r in n["replies"]) for n in tree) + # 累计应该在 1500 附近停下(允许多收一条到 ~2000),不应该收全 10*3=30 条 + assert total <= 2000 + assert len(tree) < 10 diff --git a/ai-daily-main/tests/pytest/test_sections_hackernews_scraper.py b/ai-daily-main/tests/pytest/test_sections_hackernews_scraper.py new file mode 100644 index 0000000..b391be3 --- /dev/null +++ b/ai-daily-main/tests/pytest/test_sections_hackernews_scraper.py @@ -0,0 +1,59 @@ +"""测试 HN 首页 HTML 解析""" + +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from src.sections.hackernews.frontpage_scraper import parse_frontpage_html + + +def test_parse_frontpage_returns_stories(): + fixture = (Path(__file__).parent / "fixtures" / "hn_frontpage.html").read_text( + encoding="utf-8" + ) + + stories = parse_frontpage_html(fixture) + assert len(stories) >= 25 + s = stories[0] + print(json.dumps(stories[:5], indent=4, ensure_ascii=False)) + + assert s["id"] + assert s["title"] + assert s["url"] + assert isinstance(s["points"], int) + assert isinstance(s["comments"], int) + assert s["comments_url"].startswith("https://news.ycombinator.com/item?id=") + + +def test_parse_frontpage_detects_show_hn_internal_url(): + html = """ + + + + + + + +
+ + Ask HN: what's new? + +
+ + 50 points + by alice + 2 hours ago + | 5 comments + +
+ """ + stories = parse_frontpage_html(html) + assert len(stories) == 1 + s = stories[0] + assert s["id"] == "111" + assert s["url"].startswith("https://news.ycombinator.com/item?id=") + assert s["site"] == "" + assert s["points"] == 50 + assert s["comments"] == 5 diff --git a/ai-daily-main/tests/pytest/test_sections_hackernews_section.py b/ai-daily-main/tests/pytest/test_sections_hackernews_section.py new file mode 100644 index 0000000..3576026 --- /dev/null +++ b/ai-daily-main/tests/pytest/test_sections_hackernews_section.py @@ -0,0 +1,111 @@ +"""测试 HN 板块编排:scrape → select → enrich → LLM""" + +import sys +from pathlib import Path +from unittest.mock import patch, AsyncMock + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +# Stub HN LLM functions until Task 16 lands them +import src.llm as _llm +if not hasattr(_llm, "select_ai_related_hn"): + async def _stub_select(*a, **k): + return [], None + _llm.select_ai_related_hn = _stub_select +if not hasattr(_llm, "summarize_hackernews"): + async def _stub_summarize(*a, **k): + return "", None + _llm.summarize_hackernews = _stub_summarize + +from src.sections.hackernews.section import run_hackernews_section + + +def _cfg() -> dict: + return { + "filter": {"keep_days": 7}, + "sections": { + "hackernews": { + "enabled": True, + "select_k": 1, + "top_comments": 20, + "comment_max_chars": 500, + "link_content_max_chars": 3000, + "request_timeout": 10, + "algolia_base": "https://hn.algolia.com/api/v1", + } + }, + "llm": { + "model": "x", + "baseUrl": "http://x", + "apiKeyName": "DEEPSEEK_API_KEY", + "prompts": { + "section_hackernews_select": "prompts/section_hackernews_select.md", + "section_hackernews": "prompts/section_hackernews.md", + }, + }, + } + + +@pytest.mark.asyncio +async def test_disabled_returns_empty(): + cfg = _cfg() + cfg["sections"]["hackernews"]["enabled"] = False + md, err = await run_hackernews_section(cfg, now=None) + assert md == "" + assert err is None + + +@pytest.mark.asyncio +async def test_select_empty_returns_silent(): + cfg = _cfg() + with patch( + "src.sections.hackernews.section.fetch_frontpage", new=AsyncMock(return_value="") + ), patch( + "src.sections.hackernews.section.parse_frontpage_html", + return_value=[{"id": "1", "title": "x"}], + ), patch( + "src.llm.select_ai_related_hn", + new=AsyncMock(return_value=([], None)), + ): + md, err = await run_hackernews_section(cfg, now=None) + assert md == "" + assert err is None + + +@pytest.mark.asyncio +async def test_happy_path(): + cfg = _cfg() + front = [{"id": "1", "title": "AI thing", "url": "https://e.com/a", "site": "e.com", "points": 100, "comments": 5, "comments_url": "x"}] + enriched = [{**front[0], "link_content": "body", "top_comments": ["c1"]}] + + with patch( + "src.sections.hackernews.section.fetch_frontpage", new=AsyncMock(return_value="") + ), patch( + "src.sections.hackernews.section.parse_frontpage_html", return_value=front + ), patch( + "src.llm.select_ai_related_hn", + new=AsyncMock(return_value=(["1"], None)), + ), patch( + "src.sections.hackernews.section.enrich_stories", + new=AsyncMock(return_value=(enriched, [])), + ), patch( + "src.llm.summarize_hackernews", + new=AsyncMock(return_value=("## HN md", None)), + ): + md, err = await run_hackernews_section(cfg, now=None) + assert md == "## HN md" + assert err is None + + +@pytest.mark.asyncio +async def test_scrape_failure_returns_error(): + cfg = _cfg() + with patch( + "src.sections.hackernews.section.fetch_frontpage", + new=AsyncMock(side_effect=RuntimeError("net")), + ): + md, err = await run_hackernews_section(cfg, now=None) + assert md == "" + assert "net" in err diff --git a/ai-daily-main/tests/pytest/test_sections_insights.py b/ai-daily-main/tests/pytest/test_sections_insights.py new file mode 100644 index 0000000..46bbbc8 --- /dev/null +++ b/ai-daily-main/tests/pytest/test_sections_insights.py @@ -0,0 +1,66 @@ +"""测试 insights 模块""" + +import sys +from pathlib import Path +from unittest.mock import patch, AsyncMock + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +# Stub Task 17's LLM function until it lands +import src.llm as _llm +if not hasattr(_llm, "generate_trend_insights"): + async def _stub(*a, **k): + return "", None + _llm.generate_trend_insights = _stub + +from src.sections.insights.section import run_insights_section + + +def _cfg() -> dict: + return { + "filter": {"push_context_days": 5}, + "sections": {"insights": {"enabled": True}}, + "llm": { + "model": "x", + "baseUrl": "http://x", + "apiKeyName": "DEEPSEEK_API_KEY", + "prompts": {"insights": "prompts/insights.md"}, + }, + } + + +@pytest.mark.asyncio +async def test_disabled_returns_empty(): + cfg = _cfg() + cfg["sections"]["insights"]["enabled"] = False + md, meta, err = await run_insights_section("rss", "gh", "hn", cfg, now=None) + assert md == "" + assert meta is None + assert err is None + + +@pytest.mark.asyncio +async def test_marks_empty_sections_for_llm(): + cfg = _cfg() + captured = {} + + async def fake_gen(sections, config): + captured["sections"] = sections + return "insights md", None + + with patch( + "src.llm.generate_trend_insights", + new=AsyncMock(side_effect=fake_gen), + ): + md, meta, err = await run_insights_section("", "gh md", "", cfg, now=None) + + assert md == "insights md" + assert err is None + # metadata 由 parse_insights_with_metadata 注入默认标题/profile + assert meta["profile"] == "morning" + assert "📰 AI Daily 每日精选" in meta["title"] + assert captured["sections"]["rss"] == "(本次无内容)" + assert captured["sections"]["github"] == "gh md" + assert captured["sections"]["hackernews"] == "(本次无内容)" diff --git a/ai-daily-main/tests/pytest/test_sections_rss.py b/ai-daily-main/tests/pytest/test_sections_rss.py new file mode 100644 index 0000000..9cf0cad --- /dev/null +++ b/ai-daily-main/tests/pytest/test_sections_rss.py @@ -0,0 +1,76 @@ +"""src.main.collect_entries_for_push patched at source (lazy import in section)""" + +import sys +from pathlib import Path +from unittest.mock import patch, AsyncMock + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from src.sections.rss.section import run_rss_section + + +@pytest.mark.asyncio +async def test_returns_markdown_when_entries_present(sample_config, tmp_path): + digest_raw = ( + "---\n" + 'title: "🌙 AI Daily 晚报 | 测试"\n' + 'lead: "今日测试导读"\n' + "highlights:\n - 重点1\n" + "---\n\n" + "### 1️⃣ digest body" + ) + with patch( + "src.main.collect_entries_for_push", + return_value=([{"link": "x", "title": "t", "score": 80}], []), + ), patch( + "src.sections.rss.section.compose_digest", + new=AsyncMock(return_value=digest_raw), + ), patch( + "src.sections.rss.section.load_recent_push_content", return_value="" + ), patch( + "src.sections.rss.section.get_last_push_file", return_value=None + ): + md, meta, err = await run_rss_section(sample_config, now=None) + + assert md == "### 1️⃣ digest body" + assert err is None + assert meta["title"] == "🌙 AI Daily 晚报 | 测试" + assert meta["lead"] == "今日测试导读" + assert meta["highlights"] == ["重点1"] + assert meta["profile"] == "default" + + +@pytest.mark.asyncio +async def test_returns_empty_when_no_entries(sample_config): + with patch( + "src.main.collect_entries_for_push", return_value=([], []) + ), patch( + "src.sections.rss.section.get_last_push_file", return_value=None + ): + md, meta, err = await run_rss_section(sample_config, now=None) + + assert md == "" + assert meta is None + assert err is None + + +@pytest.mark.asyncio +async def test_returns_error_on_compose_failure(sample_config): + with patch( + "src.main.collect_entries_for_push", + return_value=([{"link": "x"}], []), + ), patch( + "src.sections.rss.section.compose_digest", + new=AsyncMock(side_effect=RuntimeError("LLM down")), + ), patch( + "src.sections.rss.section.load_recent_push_content", return_value="" + ), patch( + "src.sections.rss.section.get_last_push_file", return_value=None + ): + md, meta, err = await run_rss_section(sample_config, now=None) + + assert md == "" + assert meta is None + assert "LLM down" in err diff --git a/ai-daily-main/tests/pytest/test_storage.py b/ai-daily-main/tests/pytest/test_storage.py new file mode 100644 index 0000000..9677ddb --- /dev/null +++ b/ai-daily-main/tests/pytest/test_storage.py @@ -0,0 +1,257 @@ +"""存储模块测试""" + +import json +import pytest +import sys +from datetime import datetime, date, timedelta, timezone +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src")) + +from storage import ( + get_fetch_file, + get_push_file, + get_last_push_file, + extract_push_time, + read_entries, + read_fetch_data, + save_fetch_file, + append_entries, + format_entry, + json_to_md, + save_push_file, + load_existing_links, + cleanup_old_files, +) + + +class TestGetFetchFile: + """测试获取fetch文件路径""" + + def test_get_fetch_file_default(self): + result = get_fetch_file() + assert "fetch-" in result + assert result.endswith(".json") + + def test_get_fetch_file_specific_date(self): + result = get_fetch_file(date(2024, 1, 15)) + assert "fetch-2024-01-15.json" in result + + +class TestGetPushFile: + """测试获取push文件路径""" + + def test_get_push_file_default(self): + result = get_push_file() + assert "push-" in result + assert result.endswith(".md") + + def test_get_push_file_specific_time(self): + dt = datetime(2024, 1, 15, 8, 30, 0) + result = get_push_file(dt) + assert "push-2024-01-15-08-30-00.md" in result + + +class TestExtractPushTime: + """测试从文件名提取时间""" + + def test_extract_valid_time(self): + result = extract_push_time("news-data/push-2024-01-15-08-30-00.md") + assert result is not None + assert result.year == 2024 + assert result.month == 1 + assert result.day == 15 + + def test_extract_invalid_filename(self): + result = extract_push_time("invalid.md") + assert result is None + + +class TestGetLastPushFile: + """测试获取最新push文件""" + + def test_get_last_push_file_empty(self, temp_dir): + result = get_last_push_file(str(temp_dir)) + assert result is None + + def test_get_last_push_file_exists(self, temp_dir): + (temp_dir / "push-2024-01-14-10-00-00.md").touch() + (temp_dir / "push-2024-01-15-10-00-00.md").touch() + + result = get_last_push_file(str(temp_dir)) + assert "2024-01-15" in result + + +class TestReadWriteEntries: + """测试读写条目""" + + def test_read_entries(self, sample_fetch_json): + entries = read_entries(sample_fetch_json) + assert len(entries) == 3 + assert entries[0]["title"] == "Article 1" + + def test_read_entries_missing_file(self): + entries = read_entries("nonexistent.json") + assert entries == [] + + def test_read_fetch_data(self, sample_fetch_json): + data = read_fetch_data(sample_fetch_json) + assert "meta" in data + assert "entries" in data + assert len(data["entries"]) == 3 + + +class TestSaveFetchFile: + """测试保存fetch文件""" + + def test_save_fetch_file(self, temp_dir): + filepath = str(temp_dir / "test.json") + meta = {"date": "2024-01-15"} + entries = [{"title": "Test", "link": "https://example.com", "score": 80}] + + save_fetch_file(filepath, meta, entries) + + data = read_fetch_data(filepath) + assert data["meta"]["date"] == "2024-01-15" + assert len(data["entries"]) == 1 + + +class TestAppendEntries: + """测试追加条目""" + + def test_append_new_entries(self, temp_dir): + filepath = str(temp_dir / "test.json") + meta = {"date": "2024-01-15"} + + entries1 = [{"title": "Entry1", "link": "https://example.com/1", "score": 80}] + count1 = append_entries(filepath, entries1, meta) + assert count1 == 1 + + entries2 = [{"title": "Entry2", "link": "https://example.com/2", "score": 70}] + count2 = append_entries(filepath, entries2, meta) + assert count2 == 1 + + def test_append_duplicate_entries(self, temp_dir): + filepath = str(temp_dir / "test.json") + meta = {"date": "2024-01-15"} + + entries = [{"title": "Entry1", "link": "https://example.com/1", "score": 80}] + append_entries(filepath, entries, meta) + + all_entries = read_entries(filepath) + assert len(all_entries) == 1 + + count = append_entries(filepath, entries, meta) + all_entries_after = read_entries(filepath) + assert len(all_entries_after) == 1 + + def test_append_to_existing_file(self, temp_dir): + filepath = str(temp_dir / "test.json") + + data = { + "meta": {"date": "2024-01-15"}, + "entries": [{"title": "Old", "link": "https://old.com", "score": 60}], + } + with open(filepath, "w") as f: + json.dump(data, f) + + new_entries = [{"title": "New", "link": "https://new.com", "score": 70}] + count = append_entries(filepath, new_entries) + + entries = read_entries(filepath) + assert len(entries) == 2 + + +class TestFormatEntry: + """测试格式化条目""" + + def test_format_entry_basic(self, sample_entry): + result = format_entry(sample_entry) + assert "## Test Article Title" in result + assert "source: Test Source" in result + assert "score: 85" in result + + def test_format_entry_with_tags(self, sample_entry): + result = format_entry(sample_entry) + assert "AI" in result + assert "Tech" in result + + +class TestJsonToMd: + """测试JSON转Markdown""" + + def test_json_to_md_basic(self, sample_fetch_json): + data = read_fetch_data(sample_fetch_json) + result = json_to_md(data) + + assert "Article 1" in result + assert "Article 2" in result + assert "Article 3" in result + + def test_json_to_md_empty(self): + data = {"meta": {}, "entries": []} + result = json_to_md(data) + assert result == "" + + +class TestSavePushFile: + """测试保存推送文件""" + + def test_save_push_file(self, temp_dir): + filepath = str(temp_dir / "push-test.md") + content = "# Test Push\n\nContent here" + + save_push_file(filepath, content, 5, 10) + + with open(filepath, "r") as f: + content = f.read() + + assert "pushDate:" in content + assert "sourceCount: 5" in content + assert "totalEntries: 10" in content + assert "# Test Push" in content + + +class TestLoadExistingLinks: + """测试加载已有链接""" + + def test_load_existing_links_json(self, sample_fetch_json): + links = load_existing_links(sample_fetch_json) + assert len(links) == 3 + assert "https://example.com/1" in links + + def test_load_existing_links_missing(self): + links = load_existing_links("nonexistent.json") + assert links == set() + + def test_load_existing_links_empty_string(self): + links = load_existing_links("") + assert links == set() + + +class TestCleanupOldFiles: + """测试清理旧文件""" + + def test_cleanup_old_files(self, temp_dir): + old_date = (datetime.now() - timedelta(days=10)).date() + new_date = (datetime.now() - timedelta(days=1)).date() + + (temp_dir / f"fetch-{old_date}.json").touch() + (temp_dir / f"fetch-{new_date}.json").touch() + + cleanup_old_files(days=7, data_dir=str(temp_dir)) + + assert not (temp_dir / f"fetch-{old_date}.json").exists() + assert (temp_dir / f"fetch-{new_date}.json").exists() + + def test_cleanup_push_files(self, temp_dir): + old_time = datetime.now() - timedelta(days=10) + new_time = datetime.now() - timedelta(days=1) + + (temp_dir / f"push-{old_time.strftime('%Y-%m-%d-%H-%M-%S')}.md").touch() + (temp_dir / f"push-{new_time.strftime('%Y-%m-%d-%H-%M-%S')}.md").touch() + + cleanup_old_files(days=7, data_dir=str(temp_dir)) + + files = list(temp_dir.glob("push-*.md")) + assert len(files) == 1 diff --git a/ai-daily-main/tests/pytest/test_storage_sections.py b/ai-daily-main/tests/pytest/test_storage_sections.py new file mode 100644 index 0000000..b07a832 --- /dev/null +++ b/ai-daily-main/tests/pytest/test_storage_sections.py @@ -0,0 +1,163 @@ +"""测试新增的 sentinel 切片与 section-aware 读取""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src")) + +from storage import extract_section + + +class TestExtractSection: + def test_extract_section_with_sentinel(self): + md = ( + "intro\n" + "\n" + "RSS body\n" + "\n" + "\n" + "\n" + "GH body\n" + "\n" + ) + assert extract_section(md, "rss").strip() == "RSS body" + assert extract_section(md, "github").strip() == "GH body" + assert extract_section(md, "hackernews") == "" + + def test_extract_section_legacy_file_rss(self): + legacy = "# AI Daily\n### 1️⃣ foo\n### 2️⃣ bar\n" + assert extract_section(legacy, "rss") == legacy + + def test_extract_section_legacy_file_non_rss(self): + legacy = "# AI Daily\n### 1️⃣ foo\n" + assert extract_section(legacy, "github") == "" + assert extract_section(legacy, "hackernews") == "" + assert extract_section(legacy, "insights") == "" + + def test_extract_section_missing_end_marker(self): + broken = "\ncontent only\n" + assert extract_section(broken, "rss") == "" + + +from datetime import date, datetime, timedelta +from storage import save_push_file + + +from storage import TrendingHistory, load_trending_history + + +class TestTrendingHistory: + def test_load_missing_file(self, tmp_path): + path = tmp_path / "trending.json" + h = load_trending_history(str(path)) + assert h.repos == {} + + def test_touch_then_save_then_reload(self, tmp_path): + path = tmp_path / "trending.json" + h = load_trending_history(str(path)) + today = date(2026, 5, 17) + h.touch("https://github.com/a/b", today) + h.touch("https://github.com/c/d", today) + h.save() + + h2 = load_trending_history(str(path)) + assert h2.repos == { + "https://github.com/a/b": "2026-05-17", + "https://github.com/c/d": "2026-05-17", + } + + def test_contains_returns_membership(self, tmp_path): + h = load_trending_history(str(tmp_path / "x.json")) + h.touch("https://github.com/a/b", date(2026, 5, 17)) + assert "https://github.com/a/b" in h + assert "https://github.com/x/y" not in h + + def test_cleanup_removes_expired_entries(self, tmp_path): + path = tmp_path / "trending.json" + path.write_text( + '{"repos": {' + '"https://github.com/old/repo": "2026-05-01", ' + '"https://github.com/new/repo": "2026-05-15"' + '}, "updated_at": "2026-05-15T00:00:00+08:00"}', + encoding="utf-8", + ) + h = load_trending_history(str(path)) + h.cleanup(today=date(2026, 5, 17), keep_days=7) + assert "https://github.com/old/repo" not in h + assert "https://github.com/new/repo" in h + + def test_cleanup_keeps_today_inclusive(self, tmp_path): + h = load_trending_history(str(tmp_path / "x.json")) + h.touch("https://github.com/a/b", date(2026, 5, 10)) + # 2026-05-10 + 7 days = 2026-05-17 (last_seen 2026-05-10 仍在 keep 区间) + h.cleanup(today=date(2026, 5, 17), keep_days=7) + assert "https://github.com/a/b" in h + # 再过 1 天就出区间 + h.cleanup(today=date(2026, 5, 18), keep_days=7) + assert "https://github.com/a/b" not in h + + +class TestSavePushFileProfile: + def test_default_profile_when_not_specified(self, tmp_path): + f = tmp_path / "push-x.md" + save_push_file(str(f), "body content", source_count=1, total_entries=1) + text = f.read_text(encoding="utf-8") + assert 'profile: "default"' in text + assert "body content" in text + + def test_morning_profile(self, tmp_path): + f = tmp_path / "push-x.md" + save_push_file( + str(f), "body", source_count=2, total_entries=3, profile="morning" + ) + text = f.read_text(encoding="utf-8") + assert 'profile: "morning"' in text + + +from storage import cleanup_old_files +import json as _j + + +class TestCleanupOldFilesTrendingHistory: + def test_prunes_trending_history_entries_not_file(self, tmp_path): + path = tmp_path / "trending-history.json" + old_date = (datetime.now().date() - timedelta(days=30)).isoformat() + fresh_date = datetime.now().date().isoformat() + path.write_text( + '{"repos": {' + f'"https://github.com/a/b": "{old_date}", ' + f'"https://github.com/c/d": "{fresh_date}"' + '}, "updated_at": "..."}', + encoding="utf-8", + ) + cleanup_old_files(days=7, data_dir=str(tmp_path)) + # 文件应保留 + assert path.exists() + # 过期条目应被剪枝 + data = _j.loads(path.read_text(encoding="utf-8")) + assert "https://github.com/a/b" not in data["repos"] + assert "https://github.com/c/d" in data["repos"] + + +from storage import assemble_with_sentinels + + +class TestAssembleWithSentinels: + def test_assembles_all_sections_in_order(self): + out = assemble_with_sentinels( + {"rss": "R", "github": "G", "hackernews": "H", "insights": "I"} + ) + assert out.index("SECTION:rss") < out.index("SECTION:github") + assert out.index("SECTION:github") < out.index("SECTION:hackernews") + assert out.index("SECTION:hackernews") < out.index("SECTION:insights") + assert "\nR\n" in out + + def test_omits_empty_sections(self): + out = assemble_with_sentinels({"rss": "R", "github": "", "hackernews": "H", "insights": ""}) + assert "SECTION:github" not in out + assert "SECTION:insights" not in out + assert "SECTION:rss" in out + assert "SECTION:hackernews" in out + + def test_returns_empty_when_all_empty(self): + assert assemble_with_sentinels({"rss": "", "github": "", "hackernews": "", "insights": ""}) == "" diff --git a/ai-daily-main/tests/pytest/test_timezone.py b/ai-daily-main/tests/pytest/test_timezone.py new file mode 100644 index 0000000..dc3d62a --- /dev/null +++ b/ai-daily-main/tests/pytest/test_timezone.py @@ -0,0 +1,101 @@ +"""时区处理测试""" + +import pytest +import sys +from datetime import datetime, timezone, timedelta +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src")) + +from config import get_timezone + + +class TestTimezoneBasics: + """测试时区基础功能""" + + def test_get_timezone_positive_hours(self): + config = {"schedule": {"timezone_hours": 8}} + tz = get_timezone(config) + offset = tz.utcoffset(datetime.now()) + assert offset.total_seconds() == 8 * 3600 + + def test_get_timezone_negative_hours(self): + config = {"schedule": {"timezone_hours": -5}} + tz = get_timezone(config) + offset = tz.utcoffset(datetime.now()) + assert offset.total_seconds() == -5 * 3600 + + def test_get_timezone_zero_hours(self): + config = {"schedule": {"timezone_hours": 0}} + tz = get_timezone(config) + offset = tz.utcoffset(datetime.now()) + assert offset.total_seconds() == 0 + + def test_get_timezone_missing_schedule(self): + config = {} + tz = get_timezone(config) + assert tz is not None + + def test_get_timezone_none_config(self): + tz = get_timezone(None) + assert tz is not None + + +class TestTimezoneConversions: + """测试时区转换""" + + def test_utc_to_local(self): + config = {"schedule": {"timezone_hours": 8}} + tz = get_timezone(config) + + utc_time = datetime(2024, 1, 15, 10, 0, 0, tzinfo=timezone.utc) + local_time = utc_time.astimezone(tz) + + assert local_time.hour == 18 + + def test_cross_day_conversion(self): + config = {"schedule": {"timezone_hours": 8}} + tz = get_timezone(config) + + utc_time = datetime(2024, 1, 15, 20, 0, 0, tzinfo=timezone.utc) + local_time = utc_time.astimezone(tz) + + assert local_time.day == 16 + + def test_negative_timezone(self): + config = {"schedule": {"timezone_hours": -5}} + tz = get_timezone(config) + + utc_time = datetime(2024, 1, 15, 10, 0, 0, tzinfo=timezone.utc) + local_time = utc_time.astimezone(tz) + + assert local_time.hour == 5 + + +class TestTimezoneAwareDatetime: + """测试带时区的datetime操作""" + + def test_now_in_config_timezone(self): + config = {"schedule": {"timezone_hours": 8}} + tz = get_timezone(config) + + now_local = datetime.now(tz) + assert now_local.tzinfo == tz + + def test_timezone_aware_comparison(self): + config = {"schedule": {"timezone_hours": 8}} + tz = get_timezone(config) + + dt1 = datetime(2024, 1, 15, 10, 0, 0, tzinfo=timezone.utc) + dt2 = datetime(2024, 1, 15, 18, 0, 0, tzinfo=tz) + + assert dt1 == dt2 + + def test_naive_to_aware(self): + config = {"schedule": {"timezone_hours": 8}} + tz = get_timezone(config) + + naive = datetime(2024, 1, 15, 10, 0, 0) + aware = naive.replace(tzinfo=tz) + + assert aware.tzinfo == tz diff --git a/ai-daily-main/tests/resend_notify.py b/ai-daily-main/tests/resend_notify.py new file mode 100644 index 0000000..d45b965 --- /dev/null +++ b/ai-daily-main/tests/resend_notify.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""重新推送即时消息（notify 文件） + +Usage: + python tests/resend_notify.py news-data/notify-2026-05-21.md + python tests/resend_notify.py news-data/notify-2026-05-21.md --index 0 # 推送第一个块 +""" + +import argparse +import asyncio +import sys +from pathlib import Path + +from dotenv import load_dotenv + +load_dotenv() + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from src.config import load_config +from src.markdown_utils import parse_frontmatter +from src.push import send_to_platforms + + +def parse_notify_file(filepath: str): + """解析 notify 文件，返回所有推送块列表 + + Returns: + List[Dict]: [{"metadata": {...}, "content": "..."}, ...] + """ + with open(filepath, "r", encoding="utf-8") as f: + content = f.read() + + blocks = [] + for block in content.split("------"): + block = block.strip() + if not block: + continue + + metadata, body = parse_frontmatter(block) + if not metadata: + continue + + blocks.append({"metadata": metadata, "content": body}) + + return blocks + + +async def resend_notify(filepath: str, config: dict, index: int = -1): + """重新推送 notify 文件中的即时消息 + + Args: + filepath: notify 文件路径 + config: 配置字典 + index: 推送第几个块（-1 表示最新的一个） + + Returns: + bool: 推送是否成功 + """ + if not Path(filepath).exists(): + print(f"❌ 文件不存在: {filepath}") + return False + + blocks = parse_notify_file(filepath) + if not blocks: + print("❌ 文件中没有有效的推送块") + return False + + print(f"\n📋 文件中共有 {len(blocks)} 个推送块") + + # 选择要推送的块 + if index == -1: + block = blocks[-1] + print(f" 使用最新的一个（第 {len(blocks)} 个）") + elif 0 <= index < len(blocks): + block = blocks[index] + print(f" 使用第 {index + 1} 个") + else: + print(f"❌ 索引超出范围: {index} (有效范围: 0-{len(blocks)-1})") + return False + + metadata = block["metadata"] + content = block["content"] + # 拼接推送标题，与 main.py 的即时推送保持一致 + raw_title = metadata.get("title", "") + title = "🚨 AI Daily 快讯 | " + raw_title if raw_title else "🚨 AI Daily 快讯" + + print(f"\n📤 准备推送:") + print(f" 标题: {title}") + print(f" 时间: {metadata.get('pushTime', 'N/A')}") + + # 推送 + try: + await send_to_platforms(content, config["push"], title=title, metadata=metadata) + print("\n✅ 推送成功!") + return True + except Exception as e: + print(f"\n❌ 推送失败: {e}") + import traceback + traceback.print_exc() + return False + + +async def main(): + parser = argparse.ArgumentParser(description="重新推送即时消息（notify 文件）") + parser.add_argument("filepath", help="notify 文件路径") + parser.add_argument("--index", type=int, default=-1, help="推送第几个块（-1=最新，默认）") + args = parser.parse_args() + + print("=" * 60) + print("🚨 即时消息推送工具") + print("=" * 60) + + # 加载配置 + try: + config = load_config() + print("✅ 配置加载成功") + except Exception as e: + print(f"❌ 加载配置失败: {e}") + return 1 + + # 显示推送平台配置状态 + print("\n📋 推送平台配置:") + import os + for platform_name, platform_conf in config.get("push", {}).items(): + enabled = platform_conf.get("enabled", False) + api_key_name = platform_conf.get("apiKeyName", "") + has_key = bool(os.environ.get(api_key_name, "")) + status = "✅" if (enabled and has_key) else "⚠️" + print(f" {status} {platform_name}: enabled={enabled}, has_key={has_key}") + + # 推送 + success = await resend_notify(args.filepath, config, args.index) + return 0 if success else 1 + + +if __name__ == "__main__": + try: + sys.exit(asyncio.run(main())) + except KeyboardInterrupt: + print("\n\n👋 已取消") + sys.exit(130) diff --git a/ai-daily-main/tests/resend_push.py b/ai-daily-main/tests/resend_push.py new file mode 100644 index 0000000..e188661 --- /dev/null +++ b/ai-daily-main/tests/resend_push.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""重新推送已保存的 push 文件 + +Usage: + python tests/resend_push.py news-data/push-2026-05-21-08-00-00.md +""" + +import argparse +import asyncio +import sys +from pathlib import Path + +from dotenv import load_dotenv + +load_dotenv() + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from src.config import load_config +from src.markdown_utils import parse_frontmatter +from src.push import send_to_platforms + + +async def resend_push_file(filepath: str, config: dict): + """读取 push 文件，解析 metadata 并重新推送 + + Args: + filepath: push 文件路径 + config: 配置字典 + + Returns: + bool: 推送是否成功 + """ + if not Path(filepath).exists(): + print(f"❌ 文件不存在: {filepath}") + return False + + # 读取文件内容 + with open(filepath, "r", encoding="utf-8") as f: + content = f.read() + + # 解析 frontmatter（复用 markdown_utils） + metadata, body = parse_frontmatter(content) + if not metadata: + print("❌ 文件格式错误：无法解析 YAML frontmatter") + return False + + # 拼接推送标题，与 main.py 的 push 任务保持一致 + raw_title = metadata.get("title", "") + title = "📰 AI Daily 每日精选 | " + raw_title if raw_title else "📰 AI Daily 每日精选" + + print(f"\n📤 准备推送文件: {Path(filepath).name}") + print(f" 标题: {title}") + print(f" Profile: {metadata.get('profile', 'N/A')}") + print(f" 日期: {metadata.get('date') or metadata.get('pushDate', 'N/A')}") + + # 推送到所有平台 + try: + await send_to_platforms(body, config["push"], title=title, metadata=metadata) + print("\n✅ 推送成功!") + return True + except Exception as e: + print(f"\n❌ 推送失败: {e}") + import traceback + + traceback.print_exc() + return False + + +async def main(): + parser = argparse.ArgumentParser(description="重新推送已保存的 push 文件") + parser.add_argument("filepath", help="push 文件路径") + args = parser.parse_args() + + print("=" * 60) + print("📤 重新推送工具") + print("=" * 60) + + # 加载配置 + try: + config = load_config() + print("✅ 配置加载成功") + except Exception as e: + print(f"❌ 加载配置失败: {e}") + return 1 + + # 显示推送平台配置状态 + print("\n📋 推送平台配置:") + import os + for platform_name, platform_conf in config.get("push", {}).items(): + enabled = platform_conf.get("enabled", False) + api_key_name = platform_conf.get("apiKeyName", "") + has_key = bool(os.environ.get(api_key_name, "")) + + status = "✅" if (enabled and has_key) else "⚠️" + print(f" {status} {platform_name}: enabled={enabled}, has_key={has_key} ({api_key_name})") + + # 推送文件 + success = await resend_push_file(args.filepath, config) + return 0 if success else 1 + + +if __name__ == "__main__": + try: + sys.exit(asyncio.run(main())) + except KeyboardInterrupt: + print("\n\n👋 已取消") + sys.exit(130) diff --git a/ai-daily-main/tests/run_llm_test.py b/ai-daily-main/tests/run_llm_test.py new file mode 100644 index 0000000..0ffeaed --- /dev/null +++ b/ai-daily-main/tests/run_llm_test.py @@ -0,0 +1,383 @@ +#!/usr/bin/env python3 +"""测试LLM评分和推送功能 - 独立运行脚本 + +Usage: + # 先激活虚拟环境 + source ../.venv/bin/activate + + # 测试评分 + python tests/run_llm_test.py --score + + # 测试即时推送 + python tests/run_llm_test.py --immediate-push --push +""" + +import argparse +import asyncio +import json +import os +import sys +from datetime import datetime, timedelta, timezone +from pathlib import Path + +# 检查是否在虚拟环境中 +if not hasattr(sys, "real_prefix") and not ( + hasattr(sys, "base_prefix") and sys.base_prefix != sys.prefix +): + print("⚠️ 建议先激活虚拟环境: source .venv/bin/activate") + print("") + +# 加载 .env 文件 +from dotenv import load_dotenv + +load_dotenv() + +# 添加项目根目录到路径 +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from src.config import get_timezone, load_config +from src.llm import compose_digest, generate_immediate_push, score_batch +from src.push import send_to_platforms +from src.storage import read_fetch_data, save_fetch_file + + +def parse_args(): + """解析命令行参数""" + parser = argparse.ArgumentParser(description="测试LLM评分和推送") + parser.add_argument( + "--input", + "-i", + type=str, + default="tests/news-data/fetch-{date}.json", + help="输入文件路径，支持{date}占位符 (默认: tests/news-data/fetch-{date}.json)", + ) + parser.add_argument( + "--date", + "-d", + type=str, + default=datetime.now(get_timezone()).strftime("%Y-%m-%d"), + help="日期，格式YYYY-MM-DD (默认: 今天)", + ) + parser.add_argument( + "--limit", "-l", type=int, default=0, help="测试的消息数量 (默认: 0表示全部)" + ) + + # 测试模式选择 + parser.add_argument("--score", action="store_true", help="测试评分") + parser.add_argument("--immediate-push", action="store_true", help="测试即时推送") + parser.add_argument("--digest", action="store_true", help="测试汇总推送") + parser.add_argument("--push", action="store_true", help="推送到Discord") + parser.add_argument("--all", action="store_true", help="运行所有测试") + + return parser.parse_args() + + +def should_run(args, mode: str) -> bool: + """判断是否运行某个模式""" + # 如果没有任何特定模式指定，默认运行评分 + if args.all: + return True + + # 检查是否指定了任何模式 + any_mode = args.score or args.immediate_push or args.digest + + if mode == "score": + return args.score or not any_mode # 默认运行评分 + elif mode == "immediate_push": + return args.immediate_push + elif mode == "digest": + return args.digest + return False + + +async def run_llm_test(): + """主函数""" + args = parse_args() + + print("=" * 60) + print("🤖 LLM测试脚本") + print("=" * 60) + + # 构建输入文件路径 + input_path = args.input.format(date=args.date) + print(f"\n📂 输入文件: {input_path}") + + # 读取数据 + if not Path(input_path).exists(): + print(f"❌ 文件不存在: {input_path}") + print("\n💡 提示: 先运行 fetch_news.py 获取新闻数据") + print(" python tests/fetch_news.py --hours 1") + return False + + print(input_path) + data = read_fetch_data(input_path) + entries = data.get("entries", []) + meta = data.get("meta", {}) + + print(f" ✓ 共 {len(entries)} 条") + + if not entries: + print("❌ 没有条目可测试") + return False + + # 限制测试数量 (0表示全部) + if args.limit > 0: + test_entries = entries[: args.limit] + print(f" 测试前 {len(test_entries)} 条") + else: + test_entries = entries + print(f" 测试全部 {len(test_entries)} 条") + + # 显示待评分条目 + print(f"\n📄 测试条目:") + for i, e in enumerate(test_entries[:5], 1): + print(f" [{i}] {e.get('title', 'N/A')[:45]}...") + print(f" 来源: {e.get('source', 'N/A')}") + + # 加载配置 + print("\n⚙️ 加载配置...") + config = load_config() + llm_config = config["llm"] + + print(f" ✓ 提供商: {llm_config.get('provider', 'openai')}") + print(f" ✓ 模型: {llm_config.get('model', 'N/A')}") + print(f" ✓ BaseURL: {llm_config.get('baseUrl', 'N/A')}") + + # 检查API key + api_key_name = llm_config.get("apiKeyName", "OPENAI_API_KEY") + api_key = os.environ.get(api_key_name) + if not api_key: + print(f"\n❌ 未设置环境变量: {api_key_name}") + return False + + print(f" ✓ API Key: {api_key[:10]}...") + + # 检查是否启用推送 + push_enabled = args.push and config.get("push") + if push_enabled: + print("\n🔌 推送已启用 (将推送到所有已配置的平台)") + + # ========== 测试评分 ========== + if should_run(args, "score"): + print("\n" + "-" * 60) + print("🎯 测试: 评分 (score_batch)") + print("-" * 60) + + try: + scored, score_errors = await score_batch(test_entries, llm_config) + if score_errors: + print("\n⚠️ 评分存在异常:") + for error in score_errors: + print(f" - {error}") + print("\n✅ 评分完成!") + + # 显示评分结果 + print("\n📊 评分结果:") + for i, e in enumerate(scored[:5], 1): + print(f"\n [{i}] {e['title'][:40]}...") + print(f" 评分: {e.get('score', 'N/A')}/100") + print(f" 标签: {e.get('tags', [])}") + print(f" 摘要: {e.get('summary', 'N/A')[:60]}...") + + # 保存评分结果到JSON文件 + print(f"\n💾 保存评分结果到: {input_path}") + + # 构建link到评分的映射 + score_map = {e.get("link"): e for e in scored if e.get("link")} + + # 更新所有entries的评分 + all_entries = data.get("entries", []) + for i, entry in enumerate(all_entries): + link = entry.get("link") + if link in score_map: + all_entries[i] = score_map[link] + + save_fetch_file(input_path, meta, all_entries) + print(f" ✅ 已保存 {len(scored)} 条评分结果") + + # 更新test_entries为评分后的数据 + test_entries = scored + + except Exception as e: + print(f"\n❌ 评分失败: {e}") + import traceback + + traceback.print_exc() + return False + + # ========== 测试即时推送 ========== + if should_run(args, "immediate_push"): + print("\n" + "-" * 60) + print("🔥 测试: 即时推送 (generate_immediate_push)") + print("-" * 60) + + # 筛选高分条目 (>=80分)用于推送 + hot_entries = [e for e in test_entries if e.get("score", 0) >= 90] + if not hot_entries: + hot_entries = test_entries[-3:-1] # 如果没有高分，取前2条 + + print(f"\n使用 {len(hot_entries)} 条高分消息生成推送...") + + # 加载近期推送上下文用于测试 + context_days = config.get("filter", {}).get("context_days", 3) + from src.llm import parse_immediate_push_with_metadata + from src.storage import ( + get_notify_file, + load_recent_notify_content, + load_recent_push_content, + save_notify_file, + ) + + recent_notify = load_recent_notify_content(context_days) + recent_push = load_recent_push_content(context_days) + recent_context = ( + f"=== 近期即时推送 ===\n{recent_notify}\n\n" + f"=== 近期汇总推送 ===\n{recent_push}" + ) + + try: + # 传入上下文参数 + push_content, immediate_push_error = await generate_immediate_push( + hot_entries, llm_config, recent_push_context=recent_context + ) + timestamp = datetime.now(get_timezone()).strftime("%Y-%m-%d") + content_without_title, metadata = parse_immediate_push_with_metadata( + push_content, f"🚨 AI Daily 快讯 | {timestamp}" + ) + metadata["pushTime"] = datetime.now(get_timezone()).isoformat() + push_content = content_without_title + + if immediate_push_error: + print(f"\n⚠️ 即时推送生成异常: {immediate_push_error}") + push_content = "" + print(f"\n✅ 推送内容生成完成!") + print(f"\n📤 推送内容预览:") + print("-" * 40) + print( + push_content[:500] + "..." if len(push_content) > 500 else push_content + ) + print("-" * 40) + + # 检查是否有实际内容需要推送 + no_content_marker = config.get("filter", {}).get( + "no_content_marker", "[NO_NEW_CONTENT]" + ) + if no_content_marker in push_content: + print(f"\nℹ️ 无新内容需要推送 (LLM判定为重复内容)") + else: + # 推送到所有启用的平台 + if push_enabled: + print("\n📤 推送消息...") + await send_to_platforms( + push_content, + config["push"], + title="🚨 AI Daily 快讯 | " + metadata["title"], + metadata=metadata, + ) + print(" ✅ 推送成功!") + + # 保存到 notify 文件 + notify_file = get_notify_file() + save_notify_file(notify_file, push_content, metadata) + print(f"\n💾 已保存即时推送到 {notify_file}") + + except Exception as e: + print(f"\n❌ 即时推送生成失败: {e}") + import traceback + + traceback.print_exc() + + # ========== 测试汇总推送 ========== + if should_run(args, "digest"): + print("\n" + "-" * 60) + print("📰 测试: 汇总推送 (compose_digest)") + print("-" * 60) + + # 构建上下文（从 fetch 文件读取的历史数据） + context = test_entries[:10] # 使用前10条作为模拟上下文 + + print(f"\n使用 {len(test_entries)} 条消息生成汇总...") + + # 加载近期推送上下文 + push_context_days = config.get( + "filter", + ).get("push_context_days", 5) + from src.storage import get_push_file, load_recent_push_content, save_push_file + + recent_push_context_str = load_recent_push_content(push_context_days) + + try: + raw_digest = await compose_digest( + test_entries, + context, + llm_config, + recent_push_context=recent_push_context_str, + ) + from src.llm import parse_digest_with_metadata + + date_str = datetime.now(get_timezone()).strftime("%Y-%m-%d") + digest_content, metadata = parse_digest_with_metadata(raw_digest, date_str) + metadata["pushTime"] = datetime.now(get_timezone()).isoformat() + + print(f"\n✅ 汇总内容生成完成!") + print(f" 标题: {metadata['title']}") + print(f" 导读: {metadata.get('lead', '')[:60]}") + print(f" 重点: {metadata.get('highlights', [])}") + print(f"\n📰 汇总内容预览:") + print("-" * 40) + print( + digest_content[:500] + "..." + if len(digest_content) > 500 + else digest_content + ) + print("-" * 40) + + # 推送到所有启用的平台 + if push_enabled: + print("\n📤 推送消息...") + await send_to_platforms( + digest_content, + config["push"], + title="📰 AI Daily 每日精选 | " + metadata["title"], + metadata=metadata, + ) + print(" ✅ 推送成功!") + + # 保存到 push 文件 + push_file = get_push_file() + save_push_file( + push_file, + digest_content, + len(test_entries), + len(test_entries), + profile="default", + metadata=metadata, + ) + print(f"\n💾 已保存汇总到 {push_file}") + + except Exception as e: + print(f"\n❌ 汇总推送生成失败: {e}") + import traceback + + traceback.print_exc() + + print("\n" + "=" * 60) + print("✅ LLM测试完成!") + print("=" * 60) + + return True + + +if __name__ == "__main__": + try: + success = asyncio.run(run_llm_test()) + sys.exit(0 if success else 1) + except KeyboardInterrupt: + print("\n\n👋 已取消") + sys.exit(130) + except Exception as e: + print(f"\n❌ 错误: {e}") + import traceback + + traceback.print_exc() + sys.exit(1) diff --git a/ai-daily-main/tests/run_morning_push.py b/ai-daily-main/tests/run_morning_push.py new file mode 100644 index 0000000..1397593 --- /dev/null +++ b/ai-daily-main/tests/run_morning_push.py @@ -0,0 +1,30 @@ +"""模拟一次完整早报推送(强制 is_morning=True,但不发送到推送渠道)""" + +import asyncio +import os +import sys +from unittest.mock import AsyncMock, patch + +from dotenv import load_dotenv + +load_dotenv() +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from src.config import load_config +from src.main import _run_morning_push + + +async def main(): + config = load_config() + async def fake_send(content, push_cfg): + print("\n" + "=" * 60) + print("📤 假推送内容(实际不会发送)") + print("=" * 60) + print(content) + + with patch("src.main.send_to_platforms", new=AsyncMock(side_effect=fake_send)): + await _run_morning_push(config) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/ai-daily-main/tests/test_cleanup_old_files.py b/ai-daily-main/tests/test_cleanup_old_files.py new file mode 100644 index 0000000..580364c --- /dev/null +++ b/ai-daily-main/tests/test_cleanup_old_files.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""测试 cleanup_old_files 函数 + +功能： +1. 在 tests/cleanup_test_data 文件夹内创建不同日期的测试文件 +2. 运行 cleanup_old_files +3. 显示清理结果 +""" + +import os +import sys +from datetime import date, datetime, timedelta +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from src.storage import cleanup_old_files + + +def create_test_files(): + """创建测试文件""" + test_dir = Path("tests/cleanup_test_data") + test_dir.mkdir(parents=True, exist_ok=True) + + today = date.today() + created_files = [] + + # 创建不同日期的文件 + file_specs = [ + # (文件名模式, 日期偏移, 文件类型) + ("fetch", -10, "json"), # 10天前 - 应该删除 + ("fetch", -8, "json"), # 8天前 - 应该删除 + ("fetch", -7, "json"), # 7天前 - 应该删除 + ("fetch", -6, "json"), # 6天前 - 应该保留 + ("fetch", -3, "json"), # 3天前 - 应该保留 + ("fetch", -1, "json"), # 1天前 - 应该保留 + ("fetch", 0, "json"), # 今天 - 应该保留 + ("push", -10, "md"), # 10天前 - 应该删除 + ("push", -7, "md"), # 7天前 - 应该删除 + ("push", -5, "md"), # 5天前 - 应该保留 + ("push", -2, "md"), # 2天前 - 应该保留 + ("push", 0, "md"), # 今天 - 应该保留 + ("notify", -9, "md"), # 9天前 - 应该删除 + ("notify", -6, "md"), # 6天前 - 应该保留 + ("notify", -1, "md"), # 1天前 - 应该保留 + ("notify", 0, "md"), # 今天 - 应该保留 + ] + + print(f"\n📅 今天是: {today}") + print(f" cutoff: {today - timedelta(days=7)} (7天前)") + print( + f" 将删除 < {today - timedelta(days=6)} (< {(today - timedelta(days=6)).strftime('%m-%d')}) 的文件" + ) + print( + f" 保留 >= {today - timedelta(days=6)} (>= {(today - timedelta(days=6)).strftime('%m-%d')}) 的文件" + ) + + print(f"\n📂 创建测试文件到: {test_dir}") + print("-" * 50) + + for prefix, offset, ext in file_specs: + file_date = today + timedelta(days=offset) + + if prefix == "push": + # push 文件带时间戳 + filename = f"push-{file_date.isoformat()}-08-00-00.{ext}" + else: + filename = f"{prefix}-{file_date.isoformat()}.{ext}" + + filepath = test_dir / filename + + # 创建文件并写入内容 + with open(filepath, "w") as f: + f.write(f"测试文件 - {filename}") + + status = "🗑️ 将删除" if offset <= -7 else "✅ 将保留" + print(f" {status}: {filename}") + created_files.append(filename) + + print("-" * 50) + print(f"✅ 创建了 {len(created_files)} 个测试文件") + + return test_dir + + +def list_files_after_cleanup(test_dir: Path): + """显示清理后的文件""" + print(f"\n📂 清理后的文件列表:") + print("-" * 50) + + files = sorted(test_dir.glob("*")) + if not files: + print(" (空目录)") + else: + for f in files: + print(f" ✅ {f.name}") + + print("-" * 50) + print(f" 共 {len(files)} 个文件") + + +def main(): + """主函数""" + print("=" * 60) + print("🧪 cleanup_old_files 测试") + print("=" * 60) + + # 1. 创建测试文件 + test_dir = create_test_files() + + # 2. 列出清理前的文件 + print(f"\n📂 清理前的文件列表:") + print("-" * 50) + for f in sorted(test_dir.glob("*")): + print(f" {f.name}") + print("-" * 50) + print(f" 共 {len(list(test_dir.glob('*')))} 个文件") + + # 3. 运行清理 + print("\n🚀 运行 cleanup_old_files...") + cleanup_old_files(days=7, data_dir=str(test_dir)) + + # 4. 列出清理后的文件 + list_files_after_cleanup(test_dir) + + print("\n" + "=" * 60) + print("🎉 测试完成!") + print("=" * 60) + + # 自动清理测试目录 + import shutil + + shutil.rmtree(test_dir) + print(f"✅ 已删除测试目录: {test_dir}") + + +if __name__ == "__main__": + main() diff --git a/ai-daily-main/tests/test_fetch_lookback.py b/ai-daily-main/tests/test_fetch_lookback.py new file mode 100644 index 0000000..e49c17a --- /dev/null +++ b/ai-daily-main/tests/test_fetch_lookback.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +"""测试 fetch_lookback_minutes 功能脚本 + +测试内容： +1. cutoff 时间计算是否正确 +2. load_existing_links 阈值逻辑 +3. 跨天边界的去重逻辑 +""" + +import os +import sys +from datetime import datetime, timedelta, timezone +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from dotenv import load_dotenv + +load_dotenv() + +from src.config import get_timezone, load_config +from src.storage import get_fetch_file, load_existing_links, read_entries + + +def test_cutoff_calculation(): + """测试 cutoff 时间计算""" + print("\n" + "=" * 60) + print("📊 Test 1: cutoff 时间计算") + print("=" * 60) + + config = load_config() + interval = config["schedule"]["fetch_interval_minutes"] + lookback = config["schedule"].get("fetch_lookback_minutes", 120) + + # 确保 lookback >= interval + lookback = max(lookback, interval) + threshold = lookback + interval + + print(f"\n配置值:") + print(f" fetch_interval_minutes: {interval}") + print( + f" fetch_lookback_minutes: {config['schedule'].get('fetch_lookback_minutes', 120)}" + ) + print(f" 修正后的 lookback: {lookback}") + print(f" threshold (lookback + interval): {threshold}") + + # 模拟计算 cutoff + now_utc = datetime.now(timezone.utc) + cutoff = now_utc - timedelta(minutes=lookback) + + print(f"\n计算结果:") + print(f" 当前 UTC 时间: {now_utc.strftime('%Y-%m-%d %H:%M:%S')}") + print(f" cutoff 时间: {cutoff.strftime('%Y-%m-%d %H:%M:%S')}") + print(f" 过去 {lookback} 分钟") + + assert lookback >= interval, f"lookback ({lookback}) 应该 >= interval ({interval})" + print("\n✅ Test 1 通过: cutoff 计算正确") + + +def test_threshold_logic(): + """测试 load_existing_links 阈值逻辑""" + print("\n" + "=" * 60) + print("📊 Test 2: load_existing_links 阈值逻辑") + print("=" * 60) + + config = load_config() + interval = config["schedule"]["fetch_interval_minutes"] + lookback = config["schedule"].get("fetch_lookback_minutes", 120) + lookback = max(lookback, interval) + threshold = lookback + interval + + tz = get_timezone(config) + + # 测试不同时间点 + test_cases = [ + ("02:00", threshold, True, "凌晨2点应该需要昨天"), + ("02:29", threshold, True, "02:29 应该需要昨天"), + ("02:30", threshold, False, "02:30 开始只需要当天"), + ("12:00", threshold, False, "中午12点只需要当天"), + ("23:59", threshold, False, "23:59 只需要当天"), + ] + + print(f"\n阈值: {threshold} 分钟 (即 {threshold // 60}小时 {threshold % 60}分钟)") + print(f"\n测试结果:") + + all_passed = True + for time_str, thresh, expected_need_yesterday, desc in test_cases: + hour, minute = map(int, time_str.split(":")) + current_minutes = hour * 60 + minute + need_yesterday = current_minutes < thresh + + status = "✅" if need_yesterday == expected_need_yesterday else "❌" + print( + f" {status} {time_str}: 需要昨天={need_yesterday} (预期: {expected_need_yesterday}) - {desc}" + ) + + if need_yesterday != expected_need_yesterday: + all_passed = False + + if all_passed: + print("\n✅ Test 2 通过: 阈值逻辑正确") + else: + print("\n❌ Test 2 失败: 阈值逻辑有问题") + + +def test_load_existing_links_files(): + """测试实际加载文件功能""" + print("\n" + "=" * 60) + print("📊 Test 3: 实际加载文件测试") + print("=" * 60) + + config = load_config() + interval = config["schedule"]["fetch_interval_minutes"] + lookback = config["schedule"].get("fetch_lookback_minutes", 120) + lookback = max(lookback, interval) + threshold = lookback + interval + + tz = get_timezone(config) + now = datetime.now(tz) + current_minutes = now.hour * 60 + now.minute + need_yesterday = current_minutes < threshold + + print(f"\n当前时间: {now.strftime('%Y-%m-%d %H:%M:%S')}") + print(f"当前分钟数: {current_minutes}") + print(f"阈值: {threshold}") + print(f"需要加载昨天: {need_yesterday}") + + # 测试当天文件 + today_file = get_fetch_file() + print(f"\n当天文件: {today_file}") + print(f"文件存在: {Path(today_file).exists()}") + + # 测试 load_existing_links 函数 + existing = load_existing_links(today_file, threshold) + print(f"加载到的链接数: {len(existing)}") + + if need_yesterday: + yesterday = (now - timedelta(days=1)).date() + yesterday_file = get_fetch_file(yesterday) + print(f"\n昨天文件: {yesterday_file}") + print(f"文件存在: {Path(yesterday_file).exists()}") + + if Path(yesterday_file).exists(): + yesterday_entries = read_entries(yesterday_file) + print(f"昨天文件条目数: {len(yesterday_entries)}") + + print("\n✅ Test 3 完成: 文件加载功能正常") + + +def test_mock_time(): + """模拟不同时间测试阈值逻辑""" + print("\n" + "=" * 60) + print("📊 Test 4: 模拟时间测试") + print("=" * 60) + + config = load_config() + interval = config["schedule"]["fetch_interval_minutes"] + lookback = config["schedule"].get("fetch_lookback_minutes", 120) + lookback = max(lookback, interval) + threshold = lookback + interval + + print(f"\n配置: interval={interval}, lookback={lookback}, threshold={threshold}") + + test_times = [ + (0, 0), # 00:00 + (2, 20), # 02:20 + (2, 30), # 02:30 + (3, 0), # 03:00 + (8, 0), # 08:00 + (12, 0), # 12:00 + (23, 59), # 23:59 + ] + + print("\n模拟时间测试:") + for hour, minute in test_times: + current_minutes = hour * 60 + minute + need_yesterday = current_minutes < threshold + + time_str = f"{hour:02d}:{minute:02d}" + status = "🔴 需要昨天" if need_yesterday else "🟢 只需当天" + print(f" {time_str} ({current_minutes:4d}分钟): {status}") + + print("\n✅ Test 4 完成") + + +def main(): + """主函数""" + print("\n" + "=" * 60) + print("🧪 fetch_lookback_minutes 功能测试") + print("=" * 60) + + try: + test_cutoff_calculation() + test_threshold_logic() + test_load_existing_links_files() + test_mock_time() + + print("\n" + "=" * 60) + print("🎉 所有测试完成!") + print("=" * 60) + return 0 + + except Exception as e: + print(f"\n❌ 测试失败: {e}") + import traceback + + traceback.print_exc() + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ai-daily-main/tests/test_insights_section.py b/ai-daily-main/tests/test_insights_section.py new file mode 100644 index 0000000..a1a8fa4 --- /dev/null +++ b/ai-daily-main/tests/test_insights_section.py @@ -0,0 +1,93 @@ +"""测试 insights section - 从早报文件解析三个板块并生成洞察""" + +import asyncio +import re +import sys +from datetime import datetime +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from dotenv import load_dotenv + +from src.config import get_timezone, load_config +from src.sections.insights.section import run_insights_section + +load_dotenv() + + +def parse_sections(content: str) -> dict: + """从早报内容中解析出各个板块""" + sections = {} + pattern = r"\n(.*?)\n" + + for match in re.finditer(pattern, content, re.DOTALL): + section_name = match.group(1) + section_content = match.group(2) + sections[section_name] = section_content + + return sections + + +async def main(): + if len(sys.argv) < 2: + print("用法: python test_insights_section.py <早报文件路径>") + print( + "示例: python test_insights_section.py news-data/push-2026-05-25-08-00-00.md" + ) + sys.exit(1) + + filepath = sys.argv[1] + + # 读取早报文件 + with open(filepath, "r", encoding="utf-8") as f: + content = f.read() + + # 解析板块 + sections = parse_sections(content) + + print(f"📄 解析文件: {filepath}") + print(f"📋 找到板块: {list(sections.keys())}") + print() + + rss_md = sections.get("rss", "") + gh_md = sections.get("github", "") + hn_md = sections.get("hackernews", "") + + if not rss_md and not gh_md and not hn_md: + print("❌ 未找到任何板块内容") + sys.exit(1) + + print(f"RSS 板块: {len(rss_md)} 字符") + print(f"GitHub 板块: {len(gh_md)} 字符") + print(f"HackerNews 板块: {len(hn_md)} 字符") + print() + + # 加载配置 + config = load_config() + now = datetime.now(get_timezone(config)) + + # 调用 insights section + print("🤖 生成洞察中...") + insights_md, metadata, error = await run_insights_section( + rss_md, gh_md, hn_md, config, now + ) + + if error: + print(f"❌ 错误: {error}") + sys.exit(1) + + print("\n" + "=" * 60) + print("📊 Insights 板块结果:") + print("=" * 60) + print(insights_md) + print("=" * 60) + + if metadata: + print("\n📋 Metadata:") + for key, value in metadata.items(): + print(f" {key}: {value}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/ai-daily-main/tests/test_push_loop.py b/ai-daily-main/tests/test_push_loop.py new file mode 100644 index 0000000..85c44a4 --- /dev/null +++ b/ai-daily-main/tests/test_push_loop.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""测试 push_loop 时间逻辑 - 直接调用 main.py""" +import asyncio +import sys +from datetime import datetime, timedelta +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from src.config import get_timezone, load_config + +# 记录调用时间 +call_times = [] + +async def mock_run_push_job(config): + """模拟 push job""" + now = datetime.now(get_timezone(config)) + call_times.append(now) + print(f"\n{'='*40}") + print(f"📤 Mock Push Job 被调用 | {now.strftime('%Y-%m-%d %H:%M:%S')}") + print(f"{'='*40}\n") + +async def main(): + print("🧪 测试 push_loop 时间逻辑") + print("="*50) + + # 加载配置 + config = load_config() + tz = get_timezone(config) + + # 动态设置 cron：约 30 秒后 和 60 秒后触发 + # 标准 cron 是5字段（分 时 日 月 周），通过计算实现秒级等待 + now = datetime.now(tz) + + # 计算等待时间： + # 第一次：约 30 秒后（下一分钟，等待 = 60 - 当前秒数） + # 第二次：约 60 秒后（下两分钟，等待 = 120 - 当前秒数） + sec_to_wait_1 = 60 - now.second # 到下一分钟的剩余秒数 + sec_to_wait_2 = sec_to_wait_1 + 60 # 再加一分钟 + + min_1 = (now.minute + 1) % 60 + min_2 = (now.minute + 2) % 60 + + cron1 = f"{min_1} {now.hour} * * *" + cron2 = f"{min_2} {now.hour} * * *" + + config['schedule']['push_cron'] = [cron1, cron2] + + print(f"\n当前时间: {now.strftime('%H:%M:%S')}") + print(f"测试配置:") + print(f" - 第1次推送: {cron1} (约 {sec_to_wait_1}s 后)") + print(f" - 第2次推送: {cron2} (约 {sec_to_wait_2}s 后)") + print() + + # 使用 patch mock run_push_job，设置超时 3 分钟 + from src import main as main_module + + test_task = None + push_task = None + + async def run_test(): + with patch.object(main_module, 'run_push_job', mock_run_push_job): + await main_module.push_loop(config) + + async def timeout_guard(): + await asyncio.sleep(180) # 3分钟超时 + print("\n⏱️ 测试超时") + if push_task: + push_task.cancel() + + try: + # 同时运行 push_loop 和超时守卫 + push_task = asyncio.create_task(run_test()) + timeout_task = asyncio.create_task(timeout_guard()) + + # 等待 push_task 完成或超时 + while push_task and not push_task.done(): + if len(call_times) >= 2: + push_task.cancel() + break + await asyncio.sleep(0.1) + + timeout_task.cancel() + + except asyncio.CancelledError: + pass + + # 验证结果 + print("\n" + "="*50) + print("📊 测试结果验证") + print("="*50) + + if len(call_times) >= 2: + print(f"✅ 成功调用 {len(call_times)} 次") + for i, t in enumerate(call_times, 1): + print(f" 第{i}次: {t.strftime('%H:%M:%S')}") + + interval = (call_times[1] - call_times[0]).total_seconds() + print(f"\n实际间隔: {interval:.1f} 秒") + if 55 <= interval <= 65: + print("✅ 间隔正确 (约60秒)") + else: + print(f"⚠️ 间隔异常 (期望 ~60秒)") + else: + print(f"❌ 只调用了 {len(call_times)} 次，期望 2 次") + +if __name__ == "__main__": + try: + asyncio.run(main()) + except KeyboardInterrupt: + print("\n\n👋 测试已取消") + sys.exit(130) diff --git a/ai-daily-main/uv.lock b/ai-daily-main/uv.lock new file mode 100644 index 0000000..c1ebe07 --- /dev/null +++ b/ai-daily-main/uv.lock @@ -0,0 +1,889 @@ +version = 1 +revision = 3 +requires-python = ">=3.12" + +[[package]] +name = "aiohappyeyeballs" +version = "2.6.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/26/30/f84a107a9c4331c14b2b586036f40965c128aa4fee4dda5d3d51cb14ad54/aiohappyeyeballs-2.6.1.tar.gz", hash = "sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558", size = 22760, upload-time = "2025-03-12T01:42:48.764Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0f/15/5bf3b99495fb160b63f95972b81750f18f7f4e02ad051373b669d17d44f2/aiohappyeyeballs-2.6.1-py3-none-any.whl", hash = "sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8", size = 15265, upload-time = "2025-03-12T01:42:47.083Z" }, +] + +[[package]] +name = "aiohttp" +version = "3.13.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohappyeyeballs" }, + { name = "aiosignal" }, + { name = "attrs" }, + { name = "frozenlist" }, + { name = "multidict" }, + { name = "propcache" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/77/9a/152096d4808df8e4268befa55fba462f440f14beab85e8ad9bf990516918/aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1", size = 7858271, upload-time = "2026-03-31T22:01:03.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/6f/353954c29e7dcce7cf00280a02c75f30e133c00793c7a2ed3776d7b2f426/aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9", size = 748876, upload-time = "2026-03-31T21:57:36.319Z" }, + { url = "https://files.pythonhosted.org/packages/f5/1b/428a7c64687b3b2e9cd293186695affc0e1e54a445d0361743b231f11066/aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416", size = 499557, upload-time = "2026-03-31T21:57:38.236Z" }, + { url = "https://files.pythonhosted.org/packages/29/47/7be41556bfbb6917069d6a6634bb7dd5e163ba445b783a90d40f5ac7e3a7/aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2", size = 500258, upload-time = "2026-03-31T21:57:39.923Z" }, + { url = "https://files.pythonhosted.org/packages/67/84/c9ecc5828cb0b3695856c07c0a6817a99d51e2473400f705275a2b3d9239/aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4", size = 1749199, upload-time = "2026-03-31T21:57:41.938Z" }, + { url = "https://files.pythonhosted.org/packages/f0/d3/3c6d610e66b495657622edb6ae7c7fd31b2e9086b4ec50b47897ad6042a9/aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9", size = 1721013, upload-time = "2026-03-31T21:57:43.904Z" }, + { url = "https://files.pythonhosted.org/packages/49/a0/24409c12217456df0bae7babe3b014e460b0b38a8e60753d6cb339f6556d/aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5", size = 1781501, upload-time = "2026-03-31T21:57:46.285Z" }, + { url = "https://files.pythonhosted.org/packages/98/9d/b65ec649adc5bccc008b0957a9a9c691070aeac4e41cea18559fef49958b/aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e", size = 1878981, upload-time = "2026-03-31T21:57:48.734Z" }, + { url = "https://files.pythonhosted.org/packages/57/d8/8d44036d7eb7b6a8ec4c5494ea0c8c8b94fbc0ed3991c1a7adf230df03bf/aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1", size = 1767934, upload-time = "2026-03-31T21:57:51.171Z" }, + { url = "https://files.pythonhosted.org/packages/31/04/d3f8211f273356f158e3464e9e45484d3fb8c4ce5eb2f6fe9405c3273983/aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286", size = 1566671, upload-time = "2026-03-31T21:57:53.326Z" }, + { url = "https://files.pythonhosted.org/packages/41/db/073e4ebe00b78e2dfcacff734291651729a62953b48933d765dc513bf798/aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9", size = 1705219, upload-time = "2026-03-31T21:57:55.385Z" }, + { url = "https://files.pythonhosted.org/packages/48/45/7dfba71a2f9fd97b15c95c06819de7eb38113d2cdb6319669195a7d64270/aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88", size = 1743049, upload-time = "2026-03-31T21:57:57.341Z" }, + { url = "https://files.pythonhosted.org/packages/18/71/901db0061e0f717d226386a7f471bb59b19566f2cae5f0d93874b017271f/aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3", size = 1749557, upload-time = "2026-03-31T21:57:59.626Z" }, + { url = "https://files.pythonhosted.org/packages/08/d5/41eebd16066e59cd43728fe74bce953d7402f2b4ddfdfef2c0e9f17ca274/aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b", size = 1558931, upload-time = "2026-03-31T21:58:01.972Z" }, + { url = "https://files.pythonhosted.org/packages/30/e6/4a799798bf05740e66c3a1161079bda7a3dd8e22ca392481d7a7f9af82a6/aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe", size = 1774125, upload-time = "2026-03-31T21:58:04.007Z" }, + { url = "https://files.pythonhosted.org/packages/84/63/7749337c90f92bc2cb18f9560d67aa6258c7060d1397d21529b8004fcf6f/aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14", size = 1732427, upload-time = "2026-03-31T21:58:06.337Z" }, + { url = "https://files.pythonhosted.org/packages/98/de/cf2f44ff98d307e72fb97d5f5bbae3bfcb442f0ea9790c0bf5c5c2331404/aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3", size = 433534, upload-time = "2026-03-31T21:58:08.712Z" }, + { url = "https://files.pythonhosted.org/packages/aa/ca/eadf6f9c8fa5e31d40993e3db153fb5ed0b11008ad5d9de98a95045bed84/aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1", size = 460446, upload-time = "2026-03-31T21:58:10.945Z" }, + { url = "https://files.pythonhosted.org/packages/78/e9/d76bf503005709e390122d34e15256b88f7008e246c4bdbe915cd4f1adce/aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61", size = 742930, upload-time = "2026-03-31T21:58:13.155Z" }, + { url = "https://files.pythonhosted.org/packages/57/00/4b7b70223deaebd9bb85984d01a764b0d7bd6526fcdc73cca83bcbe7243e/aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832", size = 496927, upload-time = "2026-03-31T21:58:15.073Z" }, + { url = "https://files.pythonhosted.org/packages/9c/f5/0fb20fb49f8efdcdce6cd8127604ad2c503e754a8f139f5e02b01626523f/aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9", size = 497141, upload-time = "2026-03-31T21:58:17.009Z" }, + { url = "https://files.pythonhosted.org/packages/3b/86/b7c870053e36a94e8951b803cb5b909bfbc9b90ca941527f5fcafbf6b0fa/aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090", size = 1732476, upload-time = "2026-03-31T21:58:18.925Z" }, + { url = "https://files.pythonhosted.org/packages/b5/e5/4e161f84f98d80c03a238671b4136e6530453d65262867d989bbe78244d0/aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b", size = 1706507, upload-time = "2026-03-31T21:58:21.094Z" }, + { url = "https://files.pythonhosted.org/packages/d4/56/ea11a9f01518bd5a2a2fcee869d248c4b8a0cfa0bb13401574fa31adf4d4/aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a", size = 1773465, upload-time = "2026-03-31T21:58:23.159Z" }, + { url = "https://files.pythonhosted.org/packages/eb/40/333ca27fb74b0383f17c90570c748f7582501507307350a79d9f9f3c6eb1/aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8", size = 1873523, upload-time = "2026-03-31T21:58:25.59Z" }, + { url = "https://files.pythonhosted.org/packages/f0/d2/e2f77eef1acb7111405433c707dc735e63f67a56e176e72e9e7a2cd3f493/aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665", size = 1754113, upload-time = "2026-03-31T21:58:27.624Z" }, + { url = "https://files.pythonhosted.org/packages/fb/56/3f653d7f53c89669301ec9e42c95233e2a0c0a6dd051269e6e678db4fdb0/aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540", size = 1562351, upload-time = "2026-03-31T21:58:29.918Z" }, + { url = "https://files.pythonhosted.org/packages/ec/a6/9b3e91eb8ae791cce4ee736da02211c85c6f835f1bdfac0594a8a3b7018c/aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb", size = 1693205, upload-time = "2026-03-31T21:58:32.214Z" }, + { url = "https://files.pythonhosted.org/packages/98/fc/bfb437a99a2fcebd6b6eaec609571954de2ed424f01c352f4b5504371dd3/aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46", size = 1730618, upload-time = "2026-03-31T21:58:34.728Z" }, + { url = "https://files.pythonhosted.org/packages/e4/b6/c8534862126191a034f68153194c389addc285a0f1347d85096d349bbc15/aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8", size = 1745185, upload-time = "2026-03-31T21:58:36.909Z" }, + { url = "https://files.pythonhosted.org/packages/0b/93/4ca8ee2ef5236e2707e0fd5fecb10ce214aee1ff4ab307af9c558bda3b37/aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d", size = 1557311, upload-time = "2026-03-31T21:58:39.38Z" }, + { url = "https://files.pythonhosted.org/packages/57/ae/76177b15f18c5f5d094f19901d284025db28eccc5ae374d1d254181d33f4/aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6", size = 1773147, upload-time = "2026-03-31T21:58:41.476Z" }, + { url = "https://files.pythonhosted.org/packages/01/a4/62f05a0a98d88af59d93b7fcac564e5f18f513cb7471696ac286db970d6a/aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c", size = 1730356, upload-time = "2026-03-31T21:58:44.049Z" }, + { url = "https://files.pythonhosted.org/packages/e4/85/fc8601f59dfa8c9523808281f2da571f8b4699685f9809a228adcc90838d/aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc", size = 432637, upload-time = "2026-03-31T21:58:46.167Z" }, + { url = "https://files.pythonhosted.org/packages/c0/1b/ac685a8882896acf0f6b31d689e3792199cfe7aba37969fa91da63a7fa27/aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83", size = 458896, upload-time = "2026-03-31T21:58:48.119Z" }, + { url = "https://files.pythonhosted.org/packages/5d/ce/46572759afc859e867a5bc8ec3487315869013f59281ce61764f76d879de/aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c", size = 745721, upload-time = "2026-03-31T21:58:50.229Z" }, + { url = "https://files.pythonhosted.org/packages/13/fe/8a2efd7626dbe6049b2ef8ace18ffda8a4dfcbe1bcff3ac30c0c7575c20b/aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be", size = 497663, upload-time = "2026-03-31T21:58:52.232Z" }, + { url = "https://files.pythonhosted.org/packages/9b/91/cc8cc78a111826c54743d88651e1687008133c37e5ee615fee9b57990fac/aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25", size = 499094, upload-time = "2026-03-31T21:58:54.566Z" }, + { url = "https://files.pythonhosted.org/packages/0a/33/a8362cb15cf16a3af7e86ed11962d5cd7d59b449202dc576cdc731310bde/aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56", size = 1726701, upload-time = "2026-03-31T21:58:56.864Z" }, + { url = "https://files.pythonhosted.org/packages/45/0c/c091ac5c3a17114bd76cbf85d674650969ddf93387876cf67f754204bd77/aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2", size = 1683360, upload-time = "2026-03-31T21:58:59.072Z" }, + { url = "https://files.pythonhosted.org/packages/23/73/bcee1c2b79bc275e964d1446c55c54441a461938e70267c86afaae6fba27/aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a", size = 1773023, upload-time = "2026-03-31T21:59:01.776Z" }, + { url = "https://files.pythonhosted.org/packages/c7/ef/720e639df03004fee2d869f771799d8c23046dec47d5b81e396c7cda583a/aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be", size = 1853795, upload-time = "2026-03-31T21:59:04.568Z" }, + { url = "https://files.pythonhosted.org/packages/bd/c9/989f4034fb46841208de7aeeac2c6d8300745ab4f28c42f629ba77c2d916/aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b", size = 1730405, upload-time = "2026-03-31T21:59:07.221Z" }, + { url = "https://files.pythonhosted.org/packages/ce/75/ee1fd286ca7dc599d824b5651dad7b3be7ff8d9a7e7b3fe9820d9180f7db/aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94", size = 1558082, upload-time = "2026-03-31T21:59:09.484Z" }, + { url = "https://files.pythonhosted.org/packages/c3/20/1e9e6650dfc436340116b7aa89ff8cb2bbdf0abc11dfaceaad8f74273a10/aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d", size = 1692346, upload-time = "2026-03-31T21:59:12.068Z" }, + { url = "https://files.pythonhosted.org/packages/d8/40/8ebc6658d48ea630ac7903912fe0dd4e262f0e16825aa4c833c56c9f1f56/aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7", size = 1698891, upload-time = "2026-03-31T21:59:14.552Z" }, + { url = "https://files.pythonhosted.org/packages/d8/78/ea0ae5ec8ba7a5c10bdd6e318f1ba5e76fcde17db8275188772afc7917a4/aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772", size = 1742113, upload-time = "2026-03-31T21:59:17.068Z" }, + { url = "https://files.pythonhosted.org/packages/8a/66/9d308ed71e3f2491be1acb8769d96c6f0c47d92099f3bc9119cada27b357/aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5", size = 1553088, upload-time = "2026-03-31T21:59:19.541Z" }, + { url = "https://files.pythonhosted.org/packages/da/a6/6cc25ed8dfc6e00c90f5c6d126a98e2cf28957ad06fa1036bd34b6f24a2c/aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1", size = 1757976, upload-time = "2026-03-31T21:59:22.311Z" }, + { url = "https://files.pythonhosted.org/packages/c1/2b/cce5b0ffe0de99c83e5e36d8f828e4161e415660a9f3e58339d07cce3006/aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b", size = 1712444, upload-time = "2026-03-31T21:59:24.635Z" }, + { url = "https://files.pythonhosted.org/packages/6c/cf/9e1795b4160c58d29421eafd1a69c6ce351e2f7c8d3c6b7e4ca44aea1a5b/aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3", size = 438128, upload-time = "2026-03-31T21:59:27.291Z" }, + { url = "https://files.pythonhosted.org/packages/22/4d/eaedff67fc805aeba4ba746aec891b4b24cebb1a7d078084b6300f79d063/aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162", size = 464029, upload-time = "2026-03-31T21:59:29.429Z" }, + { url = "https://files.pythonhosted.org/packages/79/11/c27d9332ee20d68dd164dc12a6ecdef2e2e35ecc97ed6cf0d2442844624b/aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a", size = 778758, upload-time = "2026-03-31T21:59:31.547Z" }, + { url = "https://files.pythonhosted.org/packages/04/fb/377aead2e0a3ba5f09b7624f702a964bdf4f08b5b6728a9799830c80041e/aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254", size = 512883, upload-time = "2026-03-31T21:59:34.098Z" }, + { url = "https://files.pythonhosted.org/packages/bb/a6/aa109a33671f7a5d3bd78b46da9d852797c5e665bfda7d6b373f56bff2ec/aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36", size = 516668, upload-time = "2026-03-31T21:59:36.497Z" }, + { url = "https://files.pythonhosted.org/packages/79/b3/ca078f9f2fa9563c36fb8ef89053ea2bb146d6f792c5104574d49d8acb63/aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f", size = 1883461, upload-time = "2026-03-31T21:59:38.723Z" }, + { url = "https://files.pythonhosted.org/packages/b7/e3/a7ad633ca1ca497b852233a3cce6906a56c3225fb6d9217b5e5e60b7419d/aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800", size = 1747661, upload-time = "2026-03-31T21:59:41.187Z" }, + { url = "https://files.pythonhosted.org/packages/33/b9/cd6fe579bed34a906d3d783fe60f2fa297ef55b27bb4538438ee49d4dc41/aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf", size = 1863800, upload-time = "2026-03-31T21:59:43.84Z" }, + { url = "https://files.pythonhosted.org/packages/c0/3f/2c1e2f5144cefa889c8afd5cf431994c32f3b29da9961698ff4e3811b79a/aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b", size = 1958382, upload-time = "2026-03-31T21:59:46.187Z" }, + { url = "https://files.pythonhosted.org/packages/66/1d/f31ec3f1013723b3babe3609e7f119c2c2fb6ef33da90061a705ef3e1bc8/aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a", size = 1803724, upload-time = "2026-03-31T21:59:48.656Z" }, + { url = "https://files.pythonhosted.org/packages/0e/b4/57712dfc6f1542f067daa81eb61da282fab3e6f1966fca25db06c4fc62d5/aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8", size = 1640027, upload-time = "2026-03-31T21:59:51.284Z" }, + { url = "https://files.pythonhosted.org/packages/25/3c/734c878fb43ec083d8e31bf029daae1beafeae582d1b35da234739e82ee7/aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be", size = 1806644, upload-time = "2026-03-31T21:59:53.753Z" }, + { url = "https://files.pythonhosted.org/packages/20/a5/f671e5cbec1c21d044ff3078223f949748f3a7f86b14e34a365d74a5d21f/aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b", size = 1791630, upload-time = "2026-03-31T21:59:56.239Z" }, + { url = "https://files.pythonhosted.org/packages/0b/63/fb8d0ad63a0b8a99be97deac8c04dacf0785721c158bdf23d679a87aa99e/aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6", size = 1809403, upload-time = "2026-03-31T21:59:59.103Z" }, + { url = "https://files.pythonhosted.org/packages/59/0c/bfed7f30662fcf12206481c2aac57dedee43fe1c49275e85b3a1e1742294/aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037", size = 1634924, upload-time = "2026-03-31T22:00:02.116Z" }, + { url = "https://files.pythonhosted.org/packages/17/d6/fd518d668a09fd5a3319ae5e984d4d80b9a4b3df4e21c52f02251ef5a32e/aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500", size = 1836119, upload-time = "2026-03-31T22:00:04.756Z" }, + { url = "https://files.pythonhosted.org/packages/78/b7/15fb7a9d52e112a25b621c67b69c167805cb1f2ab8f1708a5c490d1b52fe/aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9", size = 1772072, upload-time = "2026-03-31T22:00:07.494Z" }, + { url = "https://files.pythonhosted.org/packages/7e/df/57ba7f0c4a553fc2bd8b6321df236870ec6fd64a2a473a8a13d4f733214e/aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8", size = 471819, upload-time = "2026-03-31T22:00:10.277Z" }, + { url = "https://files.pythonhosted.org/packages/62/29/2f8418269e46454a26171bfdd6a055d74febf32234e474930f2f60a17145/aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9", size = 505441, upload-time = "2026-03-31T22:00:12.791Z" }, +] + +[[package]] +name = "aiosignal" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "frozenlist" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007, upload-time = "2025-07-03T22:54:43.528Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, +] + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + +[[package]] +name = "beautifulsoup4" +version = "4.14.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "soupsieve" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c3/b0/1c6a16426d389813b48d95e26898aff79abbde42ad353958ad95cc8c9b21/beautifulsoup4-4.14.3.tar.gz", hash = "sha256:6292b1c5186d356bba669ef9f7f051757099565ad9ada5dd630bd9de5fa7fb86", size = 627737, upload-time = "2025-11-30T15:08:26.084Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1a/39/47f9197bdd44df24d67ac8893641e16f386c984a0619ef2ee4c51fbbc019/beautifulsoup4-4.14.3-py3-none-any.whl", hash = "sha256:0918bfe44902e6ad8d57732ba310582e98da931428d231a5ecb9e7c703a735bb", size = 107721, upload-time = "2025-11-30T15:08:24.087Z" }, +] + +[[package]] +name = "certifi" +version = "2026.4.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/25/ee/6caf7a40c36a1220410afe15a1cc64993a1f864871f698c0f93acb72842a/certifi-2026.4.22.tar.gz", hash = "sha256:8d455352a37b71bf76a79caa83a3d6c25afee4a385d632127b6afb3963f1c580", size = 137077, upload-time = "2026-04-22T11:26:11.191Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl", hash = "sha256:3cb2210c8f88ba2318d29b0388d1023c8492ff72ecdde4ebdaddbb13a31b1c4a", size = 135707, upload-time = "2026-04-22T11:26:09.372Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/a1/67fe25fac3c7642725500a3f6cfe5821ad557c3abb11c9d20d12c7008d3e/charset_normalizer-3.4.7.tar.gz", hash = "sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5", size = 144271, upload-time = "2026-04-02T09:28:39.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/eb/4fc8d0a7110eb5fc9cc161723a34a8a6c200ce3b4fbf681bc86feee22308/charset_normalizer-3.4.7-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46", size = 311328, upload-time = "2026-04-02T09:26:24.331Z" }, + { url = "https://files.pythonhosted.org/packages/f8/e3/0fadc706008ac9d7b9b5be6dc767c05f9d3e5df51744ce4cc9605de7b9f4/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2", size = 208061, upload-time = "2026-04-02T09:26:25.568Z" }, + { url = "https://files.pythonhosted.org/packages/42/f0/3dd1045c47f4a4604df85ec18ad093912ae1344ac706993aff91d38773a2/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b", size = 229031, upload-time = "2026-04-02T09:26:26.865Z" }, + { url = "https://files.pythonhosted.org/packages/dc/67/675a46eb016118a2fbde5a277a5d15f4f69d5f3f5f338e5ee2f8948fcf43/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a", size = 225239, upload-time = "2026-04-02T09:26:28.044Z" }, + { url = "https://files.pythonhosted.org/packages/4b/f8/d0118a2f5f23b02cd166fa385c60f9b0d4f9194f574e2b31cef350ad7223/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116", size = 216589, upload-time = "2026-04-02T09:26:29.239Z" }, + { url = "https://files.pythonhosted.org/packages/b1/f1/6d2b0b261b6c4ceef0fcb0d17a01cc5bc53586c2d4796fa04b5c540bc13d/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb", size = 202733, upload-time = "2026-04-02T09:26:30.5Z" }, + { url = "https://files.pythonhosted.org/packages/6f/c0/7b1f943f7e87cc3db9626ba17807d042c38645f0a1d4415c7a14afb5591f/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1", size = 212652, upload-time = "2026-04-02T09:26:31.709Z" }, + { url = "https://files.pythonhosted.org/packages/38/dd/5a9ab159fe45c6e72079398f277b7d2b523e7f716acc489726115a910097/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15", size = 211229, upload-time = "2026-04-02T09:26:33.282Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ff/531a1cad5ca855d1c1a8b69cb71abfd6d85c0291580146fda7c82857caa1/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5", size = 203552, upload-time = "2026-04-02T09:26:34.845Z" }, + { url = "https://files.pythonhosted.org/packages/c1/4c/a5fb52d528a8ca41f7598cb619409ece30a169fbdf9cdce592e53b46c3a6/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d", size = 230806, upload-time = "2026-04-02T09:26:36.152Z" }, + { url = "https://files.pythonhosted.org/packages/59/7a/071feed8124111a32b316b33ae4de83d36923039ef8cf48120266844285b/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7", size = 212316, upload-time = "2026-04-02T09:26:37.672Z" }, + { url = "https://files.pythonhosted.org/packages/fd/35/f7dba3994312d7ba508e041eaac39a36b120f32d4c8662b8814dab876431/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464", size = 227274, upload-time = "2026-04-02T09:26:38.93Z" }, + { url = "https://files.pythonhosted.org/packages/8a/2d/a572df5c9204ab7688ec1edc895a73ebded3b023bb07364710b05dd1c9be/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49", size = 218468, upload-time = "2026-04-02T09:26:40.17Z" }, + { url = "https://files.pythonhosted.org/packages/86/eb/890922a8b03a568ca2f336c36585a4713c55d4d67bf0f0c78924be6315ca/charset_normalizer-3.4.7-cp312-cp312-win32.whl", hash = "sha256:2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c", size = 148460, upload-time = "2026-04-02T09:26:41.416Z" }, + { url = "https://files.pythonhosted.org/packages/35/d9/0e7dffa06c5ab081f75b1b786f0aefc88365825dfcd0ac544bdb7b2b6853/charset_normalizer-3.4.7-cp312-cp312-win_amd64.whl", hash = "sha256:5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6", size = 159330, upload-time = "2026-04-02T09:26:42.554Z" }, + { url = "https://files.pythonhosted.org/packages/9e/5d/481bcc2a7c88ea6b0878c299547843b2521ccbc40980cb406267088bc701/charset_normalizer-3.4.7-cp312-cp312-win_arm64.whl", hash = "sha256:56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d", size = 147828, upload-time = "2026-04-02T09:26:44.075Z" }, + { url = "https://files.pythonhosted.org/packages/c1/3b/66777e39d3ae1ddc77ee606be4ec6d8cbd4c801f65e5a1b6f2b11b8346dd/charset_normalizer-3.4.7-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f496c9c3cc02230093d8330875c4c3cdfc3b73612a5fd921c65d39cbcef08063", size = 309627, upload-time = "2026-04-02T09:26:45.198Z" }, + { url = "https://files.pythonhosted.org/packages/2e/4e/b7f84e617b4854ade48a1b7915c8ccfadeba444d2a18c291f696e37f0d3b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ea948db76d31190bf08bd371623927ee1339d5f2a0b4b1b4a4439a65298703c", size = 207008, upload-time = "2026-04-02T09:26:46.824Z" }, + { url = "https://files.pythonhosted.org/packages/c4/bb/ec73c0257c9e11b268f018f068f5d00aa0ef8c8b09f7753ebd5f2880e248/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a277ab8928b9f299723bc1a2dabb1265911b1a76341f90a510368ca44ad9ab66", size = 228303, upload-time = "2026-04-02T09:26:48.397Z" }, + { url = "https://files.pythonhosted.org/packages/85/fb/32d1f5033484494619f701e719429c69b766bfc4dbc61aa9e9c8c166528b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3bec022aec2c514d9cf199522a802bd007cd588ab17ab2525f20f9c34d067c18", size = 224282, upload-time = "2026-04-02T09:26:49.684Z" }, + { url = "https://files.pythonhosted.org/packages/fa/07/330e3a0dda4c404d6da83b327270906e9654a24f6c546dc886a0eb0ffb23/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e044c39e41b92c845bc815e5ae4230804e8e7bc29e399b0437d64222d92809dd", size = 215595, upload-time = "2026-04-02T09:26:50.915Z" }, + { url = "https://files.pythonhosted.org/packages/e3/7c/fc890655786e423f02556e0216d4b8c6bcb6bdfa890160dc66bf52dee468/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:f495a1652cf3fbab2eb0639776dad966c2fb874d79d87ca07f9d5f059b8bd215", size = 201986, upload-time = "2026-04-02T09:26:52.197Z" }, + { url = "https://files.pythonhosted.org/packages/d8/97/bfb18b3db2aed3b90cf54dc292ad79fdd5ad65c4eae454099475cbeadd0d/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e712b419df8ba5e42b226c510472b37bd57b38e897d3eca5e8cfd410a29fa859", size = 211711, upload-time = "2026-04-02T09:26:53.49Z" }, + { url = "https://files.pythonhosted.org/packages/6f/a5/a581c13798546a7fd557c82614a5c65a13df2157e9ad6373166d2a3e645d/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7804338df6fcc08105c7745f1502ba68d900f45fd770d5bdd5288ddccb8a42d8", size = 210036, upload-time = "2026-04-02T09:26:54.975Z" }, + { url = "https://files.pythonhosted.org/packages/8c/bf/b3ab5bcb478e4193d517644b0fb2bf5497fbceeaa7a1bc0f4d5b50953861/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:481551899c856c704d58119b5025793fa6730adda3571971af568f66d2424bb5", size = 202998, upload-time = "2026-04-02T09:26:56.303Z" }, + { url = "https://files.pythonhosted.org/packages/e7/4e/23efd79b65d314fa320ec6017b4b5834d5c12a58ba4610aa353af2e2f577/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f59099f9b66f0d7145115e6f80dd8b1d847176df89b234a5a6b3f00437aa0832", size = 230056, upload-time = "2026-04-02T09:26:57.554Z" }, + { url = "https://files.pythonhosted.org/packages/b9/9f/1e1941bc3f0e01df116e68dc37a55c4d249df5e6fa77f008841aef68264f/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:f59ad4c0e8f6bba240a9bb85504faa1ab438237199d4cce5f622761507b8f6a6", size = 211537, upload-time = "2026-04-02T09:26:58.843Z" }, + { url = "https://files.pythonhosted.org/packages/80/0f/088cbb3020d44428964a6c97fe1edfb1b9550396bf6d278330281e8b709c/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3dedcc22d73ec993f42055eff4fcfed9318d1eeb9a6606c55892a26964964e48", size = 226176, upload-time = "2026-04-02T09:27:00.437Z" }, + { url = "https://files.pythonhosted.org/packages/6a/9f/130394f9bbe06f4f63e22641d32fc9b202b7e251c9aef4db044324dac493/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64f02c6841d7d83f832cd97ccf8eb8a906d06eb95d5276069175c696b024b60a", size = 217723, upload-time = "2026-04-02T09:27:02.021Z" }, + { url = "https://files.pythonhosted.org/packages/73/55/c469897448a06e49f8fa03f6caae97074fde823f432a98f979cc42b90e69/charset_normalizer-3.4.7-cp313-cp313-win32.whl", hash = "sha256:4042d5c8f957e15221d423ba781e85d553722fc4113f523f2feb7b188cc34c5e", size = 148085, upload-time = "2026-04-02T09:27:03.192Z" }, + { url = "https://files.pythonhosted.org/packages/5d/78/1b74c5bbb3f99b77a1715c91b3e0b5bdb6fe302d95ace4f5b1bec37b0167/charset_normalizer-3.4.7-cp313-cp313-win_amd64.whl", hash = "sha256:3946fa46a0cf3e4c8cb1cc52f56bb536310d34f25f01ca9b6c16afa767dab110", size = 158819, upload-time = "2026-04-02T09:27:04.454Z" }, + { url = "https://files.pythonhosted.org/packages/68/86/46bd42279d323deb8687c4a5a811fd548cb7d1de10cf6535d099877a9a9f/charset_normalizer-3.4.7-cp313-cp313-win_arm64.whl", hash = "sha256:80d04837f55fc81da168b98de4f4b797ef007fc8a79ab71c6ec9bc4dd662b15b", size = 147915, upload-time = "2026-04-02T09:27:05.971Z" }, + { url = "https://files.pythonhosted.org/packages/97/c8/c67cb8c70e19ef1960b97b22ed2a1567711de46c4ddf19799923adc836c2/charset_normalizer-3.4.7-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c36c333c39be2dbca264d7803333c896ab8fa7d4d6f0ab7edb7dfd7aea6e98c0", size = 309234, upload-time = "2026-04-02T09:27:07.194Z" }, + { url = "https://files.pythonhosted.org/packages/99/85/c091fdee33f20de70d6c8b522743b6f831a2f1cd3ff86de4c6a827c48a76/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c2aed2e5e41f24ea8ef1590b8e848a79b56f3a5564a65ceec43c9d692dc7d8a", size = 208042, upload-time = "2026-04-02T09:27:08.749Z" }, + { url = "https://files.pythonhosted.org/packages/87/1c/ab2ce611b984d2fd5d86a5a8a19c1ae26acac6bad967da4967562c75114d/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:54523e136b8948060c0fa0bc7b1b50c32c186f2fceee897a495406bb6e311d2b", size = 228706, upload-time = "2026-04-02T09:27:09.951Z" }, + { url = "https://files.pythonhosted.org/packages/a8/29/2b1d2cb00bf085f59d29eb773ce58ec2d325430f8c216804a0a5cd83cbca/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:715479b9a2802ecac752a3b0efa2b0b60285cf962ee38414211abdfccc233b41", size = 224727, upload-time = "2026-04-02T09:27:11.175Z" }, + { url = "https://files.pythonhosted.org/packages/47/5c/032c2d5a07fe4d4855fea851209cca2b6f03ebeb6d4e3afdb3358386a684/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bd6c2a1c7573c64738d716488d2cdd3c00e340e4835707d8fdb8dc1a66ef164e", size = 215882, upload-time = "2026-04-02T09:27:12.446Z" }, + { url = "https://files.pythonhosted.org/packages/2c/c2/356065d5a8b78ed04499cae5f339f091946a6a74f91e03476c33f0ab7100/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:c45e9440fb78f8ddabcf714b68f936737a121355bf59f3907f4e17721b9d1aae", size = 200860, upload-time = "2026-04-02T09:27:13.721Z" }, + { url = "https://files.pythonhosted.org/packages/0c/cd/a32a84217ced5039f53b29f460962abb2d4420def55afabe45b1c3c7483d/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3534e7dcbdcf757da6b85a0bbf5b6868786d5982dd959b065e65481644817a18", size = 211564, upload-time = "2026-04-02T09:27:15.272Z" }, + { url = "https://files.pythonhosted.org/packages/44/86/58e6f13ce26cc3b8f4a36b94a0f22ae2f00a72534520f4ae6857c4b81f89/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e8ac484bf18ce6975760921bb6148041faa8fef0547200386ea0b52b5d27bf7b", size = 211276, upload-time = "2026-04-02T09:27:16.834Z" }, + { url = "https://files.pythonhosted.org/packages/8f/fe/d17c32dc72e17e155e06883efa84514ca375f8a528ba2546bee73fc4df81/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a5fe03b42827c13cdccd08e6c0247b6a6d4b5e3cdc53fd1749f5896adcdc2356", size = 201238, upload-time = "2026-04-02T09:27:18.229Z" }, + { url = "https://files.pythonhosted.org/packages/6a/29/f33daa50b06525a237451cdb6c69da366c381a3dadcd833fa5676bc468b3/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2d6eb928e13016cea4f1f21d1e10c1cebd5a421bc57ddf5b1142ae3f86824fab", size = 230189, upload-time = "2026-04-02T09:27:19.445Z" }, + { url = "https://files.pythonhosted.org/packages/b6/6e/52c84015394a6a0bdcd435210a7e944c5f94ea1055f5cc5d56c5fe368e7b/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e74327fb75de8986940def6e8dee4f127cc9752bee7355bb323cc5b2659b6d46", size = 211352, upload-time = "2026-04-02T09:27:20.79Z" }, + { url = "https://files.pythonhosted.org/packages/8c/d7/4353be581b373033fb9198bf1da3cf8f09c1082561e8e922aa7b39bf9fe8/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:d6038d37043bced98a66e68d3aa2b6a35505dc01328cd65217cefe82f25def44", size = 227024, upload-time = "2026-04-02T09:27:22.063Z" }, + { url = "https://files.pythonhosted.org/packages/30/45/99d18aa925bd1740098ccd3060e238e21115fffbfdcb8f3ece837d0ace6c/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7579e913a5339fb8fa133f6bbcfd8e6749696206cf05acdbdca71a1b436d8e72", size = 217869, upload-time = "2026-04-02T09:27:23.486Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/5ee478aa53f4bb7996482153d4bfe1b89e0f087f0ab6b294fcf92d595873/charset_normalizer-3.4.7-cp314-cp314-win32.whl", hash = "sha256:5b77459df20e08151cd6f8b9ef8ef1f961ef73d85c21a555c7eed5b79410ec10", size = 148541, upload-time = "2026-04-02T09:27:25.146Z" }, + { url = "https://files.pythonhosted.org/packages/48/77/72dcb0921b2ce86420b2d79d454c7022bf5be40202a2a07906b9f2a35c97/charset_normalizer-3.4.7-cp314-cp314-win_amd64.whl", hash = "sha256:92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f", size = 159634, upload-time = "2026-04-02T09:27:26.642Z" }, + { url = "https://files.pythonhosted.org/packages/c6/a3/c2369911cd72f02386e4e340770f6e158c7980267da16af8f668217abaa0/charset_normalizer-3.4.7-cp314-cp314-win_arm64.whl", hash = "sha256:67f6279d125ca0046a7fd386d01b311c6363844deac3e5b069b514ba3e63c246", size = 148384, upload-time = "2026-04-02T09:27:28.271Z" }, + { url = "https://files.pythonhosted.org/packages/94/09/7e8a7f73d24dba1f0035fbbf014d2c36828fc1bf9c88f84093e57d315935/charset_normalizer-3.4.7-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:effc3f449787117233702311a1b7d8f59cba9ced946ba727bdc329ec69028e24", size = 330133, upload-time = "2026-04-02T09:27:29.474Z" }, + { url = "https://files.pythonhosted.org/packages/8d/da/96975ddb11f8e977f706f45cddd8540fd8242f71ecdb5d18a80723dcf62c/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79", size = 216257, upload-time = "2026-04-02T09:27:30.793Z" }, + { url = "https://files.pythonhosted.org/packages/e5/e8/1d63bf8ef2d388e95c64b2098f45f84758f6d102a087552da1485912637b/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:733784b6d6def852c814bce5f318d25da2ee65dd4839a0718641c696e09a2960", size = 234851, upload-time = "2026-04-02T09:27:32.44Z" }, + { url = "https://files.pythonhosted.org/packages/9b/40/e5ff04233e70da2681fa43969ad6f66ca5611d7e669be0246c4c7aaf6dc8/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a89c23ef8d2c6b27fd200a42aa4ac72786e7c60d40efdc76e6011260b6e949c4", size = 233393, upload-time = "2026-04-02T09:27:34.03Z" }, + { url = "https://files.pythonhosted.org/packages/be/c1/06c6c49d5a5450f76899992f1ee40b41d076aee9279b49cf9974d2f313d5/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c114670c45346afedc0d947faf3c7f701051d2518b943679c8ff88befe14f8e", size = 223251, upload-time = "2026-04-02T09:27:35.369Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9f/f2ff16fb050946169e3e1f82134d107e5d4ae72647ec8a1b1446c148480f/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:a180c5e59792af262bf263b21a3c49353f25945d8d9f70628e73de370d55e1e1", size = 206609, upload-time = "2026-04-02T09:27:36.661Z" }, + { url = "https://files.pythonhosted.org/packages/69/d5/a527c0cd8d64d2eab7459784fb4169a0ac76e5a6fc5237337982fd61347e/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3c9a494bc5ec77d43cea229c4f6db1e4d8fe7e1bbffa8b6f0f0032430ff8ab44", size = 220014, upload-time = "2026-04-02T09:27:38.019Z" }, + { url = "https://files.pythonhosted.org/packages/7e/80/8a7b8104a3e203074dc9aa2c613d4b726c0e136bad1cc734594b02867972/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8d828b6667a32a728a1ad1d93957cdf37489c57b97ae6c4de2860fa749b8fc1e", size = 218979, upload-time = "2026-04-02T09:27:39.37Z" }, + { url = "https://files.pythonhosted.org/packages/02/9a/b759b503d507f375b2b5c153e4d2ee0a75aa215b7f2489cf314f4541f2c0/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cf1493cd8607bec4d8a7b9b004e699fcf8f9103a9284cc94962cb73d20f9d4a3", size = 209238, upload-time = "2026-04-02T09:27:40.722Z" }, + { url = "https://files.pythonhosted.org/packages/c2/4e/0f3f5d47b86bdb79256e7290b26ac847a2832d9a4033f7eb2cd4bcf4bb5b/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0c96c3b819b5c3e9e165495db84d41914d6894d55181d2d108cc1a69bfc9cce0", size = 236110, upload-time = "2026-04-02T09:27:42.33Z" }, + { url = "https://files.pythonhosted.org/packages/96/23/bce28734eb3ed2c91dcf93abeb8a5cf393a7b2749725030bb630e554fdd8/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:752a45dc4a6934060b3b0dab47e04edc3326575f82be64bc4fc293914566503e", size = 219824, upload-time = "2026-04-02T09:27:43.924Z" }, + { url = "https://files.pythonhosted.org/packages/2c/6f/6e897c6984cc4d41af319b077f2f600fc8214eb2fe2d6bcb79141b882400/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:8778f0c7a52e56f75d12dae53ae320fae900a8b9b4164b981b9c5ce059cd1fcb", size = 233103, upload-time = "2026-04-02T09:27:45.348Z" }, + { url = "https://files.pythonhosted.org/packages/76/22/ef7bd0fe480a0ae9b656189ec00744b60933f68b4f42a7bb06589f6f576a/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ce3412fbe1e31eb81ea42f4169ed94861c56e643189e1e75f0041f3fe7020abe", size = 225194, upload-time = "2026-04-02T09:27:46.706Z" }, + { url = "https://files.pythonhosted.org/packages/c5/a7/0e0ab3e0b5bc1219bd80a6a0d4d72ca74d9250cb2382b7c699c147e06017/charset_normalizer-3.4.7-cp314-cp314t-win32.whl", hash = "sha256:c03a41a8784091e67a39648f70c5f97b5b6a37f216896d44d2cdcb82615339a0", size = 159827, upload-time = "2026-04-02T09:27:48.053Z" }, + { url = "https://files.pythonhosted.org/packages/7a/1d/29d32e0fb40864b1f878c7f5a0b343ae676c6e2b271a2d55cc3a152391da/charset_normalizer-3.4.7-cp314-cp314t-win_amd64.whl", hash = "sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c", size = 174168, upload-time = "2026-04-02T09:27:49.795Z" }, + { url = "https://files.pythonhosted.org/packages/de/32/d92444ad05c7a6e41fb2036749777c163baf7a0301a040cb672d6b2b1ae9/charset_normalizer-3.4.7-cp314-cp314t-win_arm64.whl", hash = "sha256:c35abb8bfff0185efac5878da64c45dafd2b37fb0383add1be155a763c1f083d", size = 153018, upload-time = "2026-04-02T09:27:51.116Z" }, + { url = "https://files.pythonhosted.org/packages/db/8f/61959034484a4a7c527811f4721e75d02d653a35afb0b6054474d8185d4c/charset_normalizer-3.4.7-py3-none-any.whl", hash = "sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d", size = 61958, upload-time = "2026-04-02T09:28:37.794Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "croniter" +version = "6.2.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/de/5832661ed55107b8a09af3f0a2e71e0957226a59eb1dcf0a445cce6daf20/croniter-6.2.2.tar.gz", hash = "sha256:ba60832a5ec8e12e51b8691c3309a113d1cf6526bdf1a48150ce8ec7a532d0ab", size = 113762, upload-time = "2026-03-15T08:43:48.112Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/39/783980e78cb92c2d7bdb1fc7dbc86e94ccc6d58224d76a7f1f51b6c51e30/croniter-6.2.2-py3-none-any.whl", hash = "sha256:a5d17b1060974d36251ea4faf388233eca8acf0d09cbd92d35f4c4ac8f279960", size = 45422, upload-time = "2026-03-15T08:43:46.626Z" }, +] + +[[package]] +name = "daily-news" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "aiohttp" }, + { name = "beautifulsoup4" }, + { name = "croniter" }, + { name = "feedparser" }, + { name = "markdownify" }, + { name = "pytest" }, + { name = "pytest-asyncio" }, + { name = "pytest-mock" }, + { name = "python-dotenv" }, + { name = "pyyaml" }, + { name = "requests" }, +] + +[package.metadata] +requires-dist = [ + { name = "aiohttp", specifier = ">=3.9.0" }, + { name = "beautifulsoup4", specifier = ">=4.12.0" }, + { name = "croniter", specifier = ">=6.0.0" }, + { name = "feedparser", specifier = ">=6.0.0" }, + { name = "markdownify", specifier = ">=0.11.0" }, + { name = "pytest", specifier = ">=7.0.0" }, + { name = "pytest-asyncio", specifier = ">=0.21.0" }, + { name = "pytest-mock", specifier = ">=3.10.0" }, + { name = "python-dotenv", specifier = ">=1.0.0" }, + { name = "pyyaml", specifier = ">=6.0" }, + { name = "requests", specifier = ">=2.34.2" }, +] + +[[package]] +name = "feedparser" +version = "6.0.12" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sgmllib3k" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dc/79/db7edb5e77d6dfbc54d7d9df72828be4318275b2e580549ff45a962f6461/feedparser-6.0.12.tar.gz", hash = "sha256:64f76ce90ae3e8ef5d1ede0f8d3b50ce26bcce71dd8ae5e82b1cd2d4a5f94228", size = 286579, upload-time = "2025-09-10T13:33:59.486Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/eb/c96d64137e29ae17d83ad2552470bafe3a7a915e85434d9942077d7fd011/feedparser-6.0.12-py3-none-any.whl", hash = "sha256:6bbff10f5a52662c00a2e3f86a38928c37c48f77b3c511aedcd51de933549324", size = 81480, upload-time = "2025-09-10T13:33:58.022Z" }, +] + +[[package]] +name = "frozenlist" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2d/f5/c831fac6cc817d26fd54c7eaccd04ef7e0288806943f7cc5bbf69f3ac1f0/frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad", size = 45875, upload-time = "2025-10-06T05:38:17.865Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/29/948b9aa87e75820a38650af445d2ef2b6b8a6fab1a23b6bb9e4ef0be2d59/frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1", size = 87782, upload-time = "2025-10-06T05:36:06.649Z" }, + { url = "https://files.pythonhosted.org/packages/64/80/4f6e318ee2a7c0750ed724fa33a4bdf1eacdc5a39a7a24e818a773cd91af/frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b", size = 50594, upload-time = "2025-10-06T05:36:07.69Z" }, + { url = "https://files.pythonhosted.org/packages/2b/94/5c8a2b50a496b11dd519f4a24cb5496cf125681dd99e94c604ccdea9419a/frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4", size = 50448, upload-time = "2025-10-06T05:36:08.78Z" }, + { url = "https://files.pythonhosted.org/packages/6a/bd/d91c5e39f490a49df14320f4e8c80161cfcce09f1e2cde1edd16a551abb3/frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383", size = 242411, upload-time = "2025-10-06T05:36:09.801Z" }, + { url = "https://files.pythonhosted.org/packages/8f/83/f61505a05109ef3293dfb1ff594d13d64a2324ac3482be2cedc2be818256/frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4", size = 243014, upload-time = "2025-10-06T05:36:11.394Z" }, + { url = "https://files.pythonhosted.org/packages/d8/cb/cb6c7b0f7d4023ddda30cf56b8b17494eb3a79e3fda666bf735f63118b35/frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8", size = 234909, upload-time = "2025-10-06T05:36:12.598Z" }, + { url = "https://files.pythonhosted.org/packages/31/c5/cd7a1f3b8b34af009fb17d4123c5a778b44ae2804e3ad6b86204255f9ec5/frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b", size = 250049, upload-time = "2025-10-06T05:36:14.065Z" }, + { url = "https://files.pythonhosted.org/packages/c0/01/2f95d3b416c584a1e7f0e1d6d31998c4a795f7544069ee2e0962a4b60740/frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52", size = 256485, upload-time = "2025-10-06T05:36:15.39Z" }, + { url = "https://files.pythonhosted.org/packages/ce/03/024bf7720b3abaebcff6d0793d73c154237b85bdf67b7ed55e5e9596dc9a/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29", size = 237619, upload-time = "2025-10-06T05:36:16.558Z" }, + { url = "https://files.pythonhosted.org/packages/69/fa/f8abdfe7d76b731f5d8bd217827cf6764d4f1d9763407e42717b4bed50a0/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3", size = 250320, upload-time = "2025-10-06T05:36:17.821Z" }, + { url = "https://files.pythonhosted.org/packages/f5/3c/b051329f718b463b22613e269ad72138cc256c540f78a6de89452803a47d/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143", size = 246820, upload-time = "2025-10-06T05:36:19.046Z" }, + { url = "https://files.pythonhosted.org/packages/0f/ae/58282e8f98e444b3f4dd42448ff36fa38bef29e40d40f330b22e7108f565/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608", size = 250518, upload-time = "2025-10-06T05:36:20.763Z" }, + { url = "https://files.pythonhosted.org/packages/8f/96/007e5944694d66123183845a106547a15944fbbb7154788cbf7272789536/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa", size = 239096, upload-time = "2025-10-06T05:36:22.129Z" }, + { url = "https://files.pythonhosted.org/packages/66/bb/852b9d6db2fa40be96f29c0d1205c306288f0684df8fd26ca1951d461a56/frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf", size = 39985, upload-time = "2025-10-06T05:36:23.661Z" }, + { url = "https://files.pythonhosted.org/packages/b8/af/38e51a553dd66eb064cdf193841f16f077585d4d28394c2fa6235cb41765/frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746", size = 44591, upload-time = "2025-10-06T05:36:24.958Z" }, + { url = "https://files.pythonhosted.org/packages/a7/06/1dc65480ab147339fecc70797e9c2f69d9cea9cf38934ce08df070fdb9cb/frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd", size = 40102, upload-time = "2025-10-06T05:36:26.333Z" }, + { url = "https://files.pythonhosted.org/packages/2d/40/0832c31a37d60f60ed79e9dfb5a92e1e2af4f40a16a29abcc7992af9edff/frozenlist-1.8.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a", size = 85717, upload-time = "2025-10-06T05:36:27.341Z" }, + { url = "https://files.pythonhosted.org/packages/30/ba/b0b3de23f40bc55a7057bd38434e25c34fa48e17f20ee273bbde5e0650f3/frozenlist-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7", size = 49651, upload-time = "2025-10-06T05:36:28.855Z" }, + { url = "https://files.pythonhosted.org/packages/0c/ab/6e5080ee374f875296c4243c381bbdef97a9ac39c6e3ce1d5f7d42cb78d6/frozenlist-1.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40", size = 49417, upload-time = "2025-10-06T05:36:29.877Z" }, + { url = "https://files.pythonhosted.org/packages/d5/4e/e4691508f9477ce67da2015d8c00acd751e6287739123113a9fca6f1604e/frozenlist-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027", size = 234391, upload-time = "2025-10-06T05:36:31.301Z" }, + { url = "https://files.pythonhosted.org/packages/40/76/c202df58e3acdf12969a7895fd6f3bc016c642e6726aa63bd3025e0fc71c/frozenlist-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822", size = 233048, upload-time = "2025-10-06T05:36:32.531Z" }, + { url = "https://files.pythonhosted.org/packages/f9/c0/8746afb90f17b73ca5979c7a3958116e105ff796e718575175319b5bb4ce/frozenlist-1.8.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121", size = 226549, upload-time = "2025-10-06T05:36:33.706Z" }, + { url = "https://files.pythonhosted.org/packages/7e/eb/4c7eefc718ff72f9b6c4893291abaae5fbc0c82226a32dcd8ef4f7a5dbef/frozenlist-1.8.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5", size = 239833, upload-time = "2025-10-06T05:36:34.947Z" }, + { url = "https://files.pythonhosted.org/packages/c2/4e/e5c02187cf704224f8b21bee886f3d713ca379535f16893233b9d672ea71/frozenlist-1.8.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e", size = 245363, upload-time = "2025-10-06T05:36:36.534Z" }, + { url = "https://files.pythonhosted.org/packages/1f/96/cb85ec608464472e82ad37a17f844889c36100eed57bea094518bf270692/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11", size = 229314, upload-time = "2025-10-06T05:36:38.582Z" }, + { url = "https://files.pythonhosted.org/packages/5d/6f/4ae69c550e4cee66b57887daeebe006fe985917c01d0fff9caab9883f6d0/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1", size = 243365, upload-time = "2025-10-06T05:36:40.152Z" }, + { url = "https://files.pythonhosted.org/packages/7a/58/afd56de246cf11780a40a2c28dc7cbabbf06337cc8ddb1c780a2d97e88d8/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1", size = 237763, upload-time = "2025-10-06T05:36:41.355Z" }, + { url = "https://files.pythonhosted.org/packages/cb/36/cdfaf6ed42e2644740d4a10452d8e97fa1c062e2a8006e4b09f1b5fd7d63/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8", size = 240110, upload-time = "2025-10-06T05:36:42.716Z" }, + { url = "https://files.pythonhosted.org/packages/03/a8/9ea226fbefad669f11b52e864c55f0bd57d3c8d7eb07e9f2e9a0b39502e1/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed", size = 233717, upload-time = "2025-10-06T05:36:44.251Z" }, + { url = "https://files.pythonhosted.org/packages/1e/0b/1b5531611e83ba7d13ccc9988967ea1b51186af64c42b7a7af465dcc9568/frozenlist-1.8.0-cp313-cp313-win32.whl", hash = "sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496", size = 39628, upload-time = "2025-10-06T05:36:45.423Z" }, + { url = "https://files.pythonhosted.org/packages/d8/cf/174c91dbc9cc49bc7b7aab74d8b734e974d1faa8f191c74af9b7e80848e6/frozenlist-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231", size = 43882, upload-time = "2025-10-06T05:36:46.796Z" }, + { url = "https://files.pythonhosted.org/packages/c1/17/502cd212cbfa96eb1388614fe39a3fc9ab87dbbe042b66f97acb57474834/frozenlist-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62", size = 39676, upload-time = "2025-10-06T05:36:47.8Z" }, + { url = "https://files.pythonhosted.org/packages/d2/5c/3bbfaa920dfab09e76946a5d2833a7cbdf7b9b4a91c714666ac4855b88b4/frozenlist-1.8.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94", size = 89235, upload-time = "2025-10-06T05:36:48.78Z" }, + { url = "https://files.pythonhosted.org/packages/d2/d6/f03961ef72166cec1687e84e8925838442b615bd0b8854b54923ce5b7b8a/frozenlist-1.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c", size = 50742, upload-time = "2025-10-06T05:36:49.837Z" }, + { url = "https://files.pythonhosted.org/packages/1e/bb/a6d12b7ba4c3337667d0e421f7181c82dda448ce4e7ad7ecd249a16fa806/frozenlist-1.8.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52", size = 51725, upload-time = "2025-10-06T05:36:50.851Z" }, + { url = "https://files.pythonhosted.org/packages/bc/71/d1fed0ffe2c2ccd70b43714c6cab0f4188f09f8a67a7914a6b46ee30f274/frozenlist-1.8.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51", size = 284533, upload-time = "2025-10-06T05:36:51.898Z" }, + { url = "https://files.pythonhosted.org/packages/c9/1f/fb1685a7b009d89f9bf78a42d94461bc06581f6e718c39344754a5d9bada/frozenlist-1.8.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65", size = 292506, upload-time = "2025-10-06T05:36:53.101Z" }, + { url = "https://files.pythonhosted.org/packages/e6/3b/b991fe1612703f7e0d05c0cf734c1b77aaf7c7d321df4572e8d36e7048c8/frozenlist-1.8.0-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82", size = 274161, upload-time = "2025-10-06T05:36:54.309Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ec/c5c618767bcdf66e88945ec0157d7f6c4a1322f1473392319b7a2501ded7/frozenlist-1.8.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714", size = 294676, upload-time = "2025-10-06T05:36:55.566Z" }, + { url = "https://files.pythonhosted.org/packages/7c/ce/3934758637d8f8a88d11f0585d6495ef54b2044ed6ec84492a91fa3b27aa/frozenlist-1.8.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d", size = 300638, upload-time = "2025-10-06T05:36:56.758Z" }, + { url = "https://files.pythonhosted.org/packages/fc/4f/a7e4d0d467298f42de4b41cbc7ddaf19d3cfeabaf9ff97c20c6c7ee409f9/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506", size = 283067, upload-time = "2025-10-06T05:36:57.965Z" }, + { url = "https://files.pythonhosted.org/packages/dc/48/c7b163063d55a83772b268e6d1affb960771b0e203b632cfe09522d67ea5/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51", size = 292101, upload-time = "2025-10-06T05:36:59.237Z" }, + { url = "https://files.pythonhosted.org/packages/9f/d0/2366d3c4ecdc2fd391e0afa6e11500bfba0ea772764d631bbf82f0136c9d/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e", size = 289901, upload-time = "2025-10-06T05:37:00.811Z" }, + { url = "https://files.pythonhosted.org/packages/b8/94/daff920e82c1b70e3618a2ac39fbc01ae3e2ff6124e80739ce5d71c9b920/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0", size = 289395, upload-time = "2025-10-06T05:37:02.115Z" }, + { url = "https://files.pythonhosted.org/packages/e3/20/bba307ab4235a09fdcd3cc5508dbabd17c4634a1af4b96e0f69bfe551ebd/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41", size = 283659, upload-time = "2025-10-06T05:37:03.711Z" }, + { url = "https://files.pythonhosted.org/packages/fd/00/04ca1c3a7a124b6de4f8a9a17cc2fcad138b4608e7a3fc5877804b8715d7/frozenlist-1.8.0-cp313-cp313t-win32.whl", hash = "sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b", size = 43492, upload-time = "2025-10-06T05:37:04.915Z" }, + { url = "https://files.pythonhosted.org/packages/59/5e/c69f733a86a94ab10f68e496dc6b7e8bc078ebb415281d5698313e3af3a1/frozenlist-1.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888", size = 48034, upload-time = "2025-10-06T05:37:06.343Z" }, + { url = "https://files.pythonhosted.org/packages/16/6c/be9d79775d8abe79b05fa6d23da99ad6e7763a1d080fbae7290b286093fd/frozenlist-1.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042", size = 41749, upload-time = "2025-10-06T05:37:07.431Z" }, + { url = "https://files.pythonhosted.org/packages/f1/c8/85da824b7e7b9b6e7f7705b2ecaf9591ba6f79c1177f324c2735e41d36a2/frozenlist-1.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0", size = 86127, upload-time = "2025-10-06T05:37:08.438Z" }, + { url = "https://files.pythonhosted.org/packages/8e/e8/a1185e236ec66c20afd72399522f142c3724c785789255202d27ae992818/frozenlist-1.8.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f", size = 49698, upload-time = "2025-10-06T05:37:09.48Z" }, + { url = "https://files.pythonhosted.org/packages/a1/93/72b1736d68f03fda5fdf0f2180fb6caaae3894f1b854d006ac61ecc727ee/frozenlist-1.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c", size = 49749, upload-time = "2025-10-06T05:37:10.569Z" }, + { url = "https://files.pythonhosted.org/packages/a7/b2/fabede9fafd976b991e9f1b9c8c873ed86f202889b864756f240ce6dd855/frozenlist-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2", size = 231298, upload-time = "2025-10-06T05:37:11.993Z" }, + { url = "https://files.pythonhosted.org/packages/3a/3b/d9b1e0b0eed36e70477ffb8360c49c85c8ca8ef9700a4e6711f39a6e8b45/frozenlist-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8", size = 232015, upload-time = "2025-10-06T05:37:13.194Z" }, + { url = "https://files.pythonhosted.org/packages/dc/94/be719d2766c1138148564a3960fc2c06eb688da592bdc25adcf856101be7/frozenlist-1.8.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686", size = 225038, upload-time = "2025-10-06T05:37:14.577Z" }, + { url = "https://files.pythonhosted.org/packages/e4/09/6712b6c5465f083f52f50cf74167b92d4ea2f50e46a9eea0523d658454ae/frozenlist-1.8.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e", size = 240130, upload-time = "2025-10-06T05:37:15.781Z" }, + { url = "https://files.pythonhosted.org/packages/f8/d4/cd065cdcf21550b54f3ce6a22e143ac9e4836ca42a0de1022da8498eac89/frozenlist-1.8.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a", size = 242845, upload-time = "2025-10-06T05:37:17.037Z" }, + { url = "https://files.pythonhosted.org/packages/62/c3/f57a5c8c70cd1ead3d5d5f776f89d33110b1addae0ab010ad774d9a44fb9/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128", size = 229131, upload-time = "2025-10-06T05:37:18.221Z" }, + { url = "https://files.pythonhosted.org/packages/6c/52/232476fe9cb64f0742f3fde2b7d26c1dac18b6d62071c74d4ded55e0ef94/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f", size = 240542, upload-time = "2025-10-06T05:37:19.771Z" }, + { url = "https://files.pythonhosted.org/packages/5f/85/07bf3f5d0fb5414aee5f47d33c6f5c77bfe49aac680bfece33d4fdf6a246/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7", size = 237308, upload-time = "2025-10-06T05:37:20.969Z" }, + { url = "https://files.pythonhosted.org/packages/11/99/ae3a33d5befd41ac0ca2cc7fd3aa707c9c324de2e89db0e0f45db9a64c26/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30", size = 238210, upload-time = "2025-10-06T05:37:22.252Z" }, + { url = "https://files.pythonhosted.org/packages/b2/60/b1d2da22f4970e7a155f0adde9b1435712ece01b3cd45ba63702aea33938/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7", size = 231972, upload-time = "2025-10-06T05:37:23.5Z" }, + { url = "https://files.pythonhosted.org/packages/3f/ab/945b2f32de889993b9c9133216c068b7fcf257d8595a0ac420ac8677cab0/frozenlist-1.8.0-cp314-cp314-win32.whl", hash = "sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806", size = 40536, upload-time = "2025-10-06T05:37:25.581Z" }, + { url = "https://files.pythonhosted.org/packages/59/ad/9caa9b9c836d9ad6f067157a531ac48b7d36499f5036d4141ce78c230b1b/frozenlist-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0", size = 44330, upload-time = "2025-10-06T05:37:26.928Z" }, + { url = "https://files.pythonhosted.org/packages/82/13/e6950121764f2676f43534c555249f57030150260aee9dcf7d64efda11dd/frozenlist-1.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b", size = 40627, upload-time = "2025-10-06T05:37:28.075Z" }, + { url = "https://files.pythonhosted.org/packages/c0/c7/43200656ecc4e02d3f8bc248df68256cd9572b3f0017f0a0c4e93440ae23/frozenlist-1.8.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d", size = 89238, upload-time = "2025-10-06T05:37:29.373Z" }, + { url = "https://files.pythonhosted.org/packages/d1/29/55c5f0689b9c0fb765055629f472c0de484dcaf0acee2f7707266ae3583c/frozenlist-1.8.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed", size = 50738, upload-time = "2025-10-06T05:37:30.792Z" }, + { url = "https://files.pythonhosted.org/packages/ba/7d/b7282a445956506fa11da8c2db7d276adcbf2b17d8bb8407a47685263f90/frozenlist-1.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930", size = 51739, upload-time = "2025-10-06T05:37:32.127Z" }, + { url = "https://files.pythonhosted.org/packages/62/1c/3d8622e60d0b767a5510d1d3cf21065b9db874696a51ea6d7a43180a259c/frozenlist-1.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c", size = 284186, upload-time = "2025-10-06T05:37:33.21Z" }, + { url = "https://files.pythonhosted.org/packages/2d/14/aa36d5f85a89679a85a1d44cd7a6657e0b1c75f61e7cad987b203d2daca8/frozenlist-1.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24", size = 292196, upload-time = "2025-10-06T05:37:36.107Z" }, + { url = "https://files.pythonhosted.org/packages/05/23/6bde59eb55abd407d34f77d39a5126fb7b4f109a3f611d3929f14b700c66/frozenlist-1.8.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37", size = 273830, upload-time = "2025-10-06T05:37:37.663Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3f/22cff331bfad7a8afa616289000ba793347fcd7bc275f3b28ecea2a27909/frozenlist-1.8.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a", size = 294289, upload-time = "2025-10-06T05:37:39.261Z" }, + { url = "https://files.pythonhosted.org/packages/a4/89/5b057c799de4838b6c69aa82b79705f2027615e01be996d2486a69ca99c4/frozenlist-1.8.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2", size = 300318, upload-time = "2025-10-06T05:37:43.213Z" }, + { url = "https://files.pythonhosted.org/packages/30/de/2c22ab3eb2a8af6d69dc799e48455813bab3690c760de58e1bf43b36da3e/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef", size = 282814, upload-time = "2025-10-06T05:37:45.337Z" }, + { url = "https://files.pythonhosted.org/packages/59/f7/970141a6a8dbd7f556d94977858cfb36fa9b66e0892c6dd780d2219d8cd8/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe", size = 291762, upload-time = "2025-10-06T05:37:46.657Z" }, + { url = "https://files.pythonhosted.org/packages/c1/15/ca1adae83a719f82df9116d66f5bb28bb95557b3951903d39135620ef157/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8", size = 289470, upload-time = "2025-10-06T05:37:47.946Z" }, + { url = "https://files.pythonhosted.org/packages/ac/83/dca6dc53bf657d371fbc88ddeb21b79891e747189c5de990b9dfff2ccba1/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a", size = 289042, upload-time = "2025-10-06T05:37:49.499Z" }, + { url = "https://files.pythonhosted.org/packages/96/52/abddd34ca99be142f354398700536c5bd315880ed0a213812bc491cff5e4/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e", size = 283148, upload-time = "2025-10-06T05:37:50.745Z" }, + { url = "https://files.pythonhosted.org/packages/af/d3/76bd4ed4317e7119c2b7f57c3f6934aba26d277acc6309f873341640e21f/frozenlist-1.8.0-cp314-cp314t-win32.whl", hash = "sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df", size = 44676, upload-time = "2025-10-06T05:37:52.222Z" }, + { url = "https://files.pythonhosted.org/packages/89/76/c615883b7b521ead2944bb3480398cbb07e12b7b4e4d073d3752eb721558/frozenlist-1.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd", size = 49451, upload-time = "2025-10-06T05:37:53.425Z" }, + { url = "https://files.pythonhosted.org/packages/e0/a3/5982da14e113d07b325230f95060e2169f5311b1017ea8af2a29b374c289/frozenlist-1.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79", size = 42507, upload-time = "2025-10-06T05:37:54.513Z" }, + { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" }, +] + +[[package]] +name = "idna" +version = "3.15" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/77/7b3966d0b9d1d31a36ddf1746926a11dface89a83409bf1483f0237aa758/idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc", size = 199245, upload-time = "2026-05-12T22:45:57.011Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", size = 72340, upload-time = "2026-05-12T22:45:55.733Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "markdownify" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3f/bc/c8c8eea5335341306b0fa7e1cb33c5e1c8d24ef70ddd684da65f41c49c92/markdownify-1.2.2.tar.gz", hash = "sha256:b274f1b5943180b031b699b199cbaeb1e2ac938b75851849a31fd0c3d6603d09", size = 18816, upload-time = "2025-11-16T19:21:18.565Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/ce/f1e3e9d959db134cedf06825fae8d5b294bd368aacdd0831a3975b7c4d55/markdownify-1.2.2-py3-none-any.whl", hash = "sha256:3f02d3cc52714084d6e589f70397b6fc9f2f3a8531481bf35e8cc39f975e186a", size = 15724, upload-time = "2025-11-16T19:21:17.622Z" }, +] + +[[package]] +name = "multidict" +version = "6.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1a/c2/c2d94cbe6ac1753f3fc980da97b3d930efe1da3af3c9f5125354436c073d/multidict-6.7.1.tar.gz", hash = "sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d", size = 102010, upload-time = "2026-01-26T02:46:45.979Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/9c/f20e0e2cf80e4b2e4b1c365bf5fe104ee633c751a724246262db8f1a0b13/multidict-6.7.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172", size = 76893, upload-time = "2026-01-26T02:43:52.754Z" }, + { url = "https://files.pythonhosted.org/packages/fe/cf/18ef143a81610136d3da8193da9d80bfe1cb548a1e2d1c775f26b23d024a/multidict-6.7.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd", size = 45456, upload-time = "2026-01-26T02:43:53.893Z" }, + { url = "https://files.pythonhosted.org/packages/a9/65/1caac9d4cd32e8433908683446eebc953e82d22b03d10d41a5f0fefe991b/multidict-6.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7", size = 43872, upload-time = "2026-01-26T02:43:55.041Z" }, + { url = "https://files.pythonhosted.org/packages/cf/3b/d6bd75dc4f3ff7c73766e04e705b00ed6dbbaccf670d9e05a12b006f5a21/multidict-6.7.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53", size = 251018, upload-time = "2026-01-26T02:43:56.198Z" }, + { url = "https://files.pythonhosted.org/packages/fd/80/c959c5933adedb9ac15152e4067c702a808ea183a8b64cf8f31af8ad3155/multidict-6.7.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75", size = 258883, upload-time = "2026-01-26T02:43:57.499Z" }, + { url = "https://files.pythonhosted.org/packages/86/85/7ed40adafea3d4f1c8b916e3b5cc3a8e07dfcdcb9cd72800f4ed3ca1b387/multidict-6.7.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b", size = 242413, upload-time = "2026-01-26T02:43:58.755Z" }, + { url = "https://files.pythonhosted.org/packages/d2/57/b8565ff533e48595503c785f8361ff9a4fde4d67de25c207cd0ba3befd03/multidict-6.7.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733", size = 268404, upload-time = "2026-01-26T02:44:00.216Z" }, + { url = "https://files.pythonhosted.org/packages/e0/50/9810c5c29350f7258180dfdcb2e52783a0632862eb334c4896ac717cebcb/multidict-6.7.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a", size = 269456, upload-time = "2026-01-26T02:44:02.202Z" }, + { url = "https://files.pythonhosted.org/packages/f3/8d/5e5be3ced1d12966fefb5c4ea3b2a5b480afcea36406559442c6e31d4a48/multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961", size = 256322, upload-time = "2026-01-26T02:44:03.56Z" }, + { url = "https://files.pythonhosted.org/packages/31/6e/d8a26d81ac166a5592782d208dd90dfdc0a7a218adaa52b45a672b46c122/multidict-6.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582", size = 253955, upload-time = "2026-01-26T02:44:04.845Z" }, + { url = "https://files.pythonhosted.org/packages/59/4c/7c672c8aad41534ba619bcd4ade7a0dc87ed6b8b5c06149b85d3dd03f0cd/multidict-6.7.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e", size = 251254, upload-time = "2026-01-26T02:44:06.133Z" }, + { url = "https://files.pythonhosted.org/packages/7b/bd/84c24de512cbafbdbc39439f74e967f19570ce7924e3007174a29c348916/multidict-6.7.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3", size = 252059, upload-time = "2026-01-26T02:44:07.518Z" }, + { url = "https://files.pythonhosted.org/packages/fa/ba/f5449385510825b73d01c2d4087bf6d2fccc20a2d42ac34df93191d3dd03/multidict-6.7.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6", size = 263588, upload-time = "2026-01-26T02:44:09.382Z" }, + { url = "https://files.pythonhosted.org/packages/d7/11/afc7c677f68f75c84a69fe37184f0f82fce13ce4b92f49f3db280b7e92b3/multidict-6.7.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a", size = 259642, upload-time = "2026-01-26T02:44:10.73Z" }, + { url = "https://files.pythonhosted.org/packages/2b/17/ebb9644da78c4ab36403739e0e6e0e30ebb135b9caf3440825001a0bddcb/multidict-6.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba", size = 251377, upload-time = "2026-01-26T02:44:12.042Z" }, + { url = "https://files.pythonhosted.org/packages/ca/a4/840f5b97339e27846c46307f2530a2805d9d537d8b8bd416af031cad7fa0/multidict-6.7.1-cp312-cp312-win32.whl", hash = "sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511", size = 41887, upload-time = "2026-01-26T02:44:14.245Z" }, + { url = "https://files.pythonhosted.org/packages/80/31/0b2517913687895f5904325c2069d6a3b78f66cc641a86a2baf75a05dcbb/multidict-6.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19", size = 46053, upload-time = "2026-01-26T02:44:15.371Z" }, + { url = "https://files.pythonhosted.org/packages/0c/5b/aba28e4ee4006ae4c7df8d327d31025d760ffa992ea23812a601d226e682/multidict-6.7.1-cp312-cp312-win_arm64.whl", hash = "sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf", size = 43307, upload-time = "2026-01-26T02:44:16.852Z" }, + { url = "https://files.pythonhosted.org/packages/f2/22/929c141d6c0dba87d3e1d38fbdf1ba8baba86b7776469f2bc2d3227a1e67/multidict-6.7.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2b41f5fed0ed563624f1c17630cb9941cf2309d4df00e494b551b5f3e3d67a23", size = 76174, upload-time = "2026-01-26T02:44:18.509Z" }, + { url = "https://files.pythonhosted.org/packages/c7/75/bc704ae15fee974f8fccd871305e254754167dce5f9e42d88a2def741a1d/multidict-6.7.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:84e61e3af5463c19b67ced91f6c634effb89ef8bfc5ca0267f954451ed4bb6a2", size = 45116, upload-time = "2026-01-26T02:44:19.745Z" }, + { url = "https://files.pythonhosted.org/packages/79/76/55cd7186f498ed080a18440c9013011eb548f77ae1b297206d030eb1180a/multidict-6.7.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:935434b9853c7c112eee7ac891bc4cb86455aa631269ae35442cb316790c1445", size = 43524, upload-time = "2026-01-26T02:44:21.571Z" }, + { url = "https://files.pythonhosted.org/packages/e9/3c/414842ef8d5a1628d68edee29ba0e5bcf235dbfb3ccd3ea303a7fe8c72ff/multidict-6.7.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:432feb25a1cb67fe82a9680b4d65fb542e4635cb3166cd9c01560651ad60f177", size = 249368, upload-time = "2026-01-26T02:44:22.803Z" }, + { url = "https://files.pythonhosted.org/packages/f6/32/befed7f74c458b4a525e60519fe8d87eef72bb1e99924fa2b0f9d97a221e/multidict-6.7.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e82d14e3c948952a1a85503817e038cba5905a3352de76b9a465075d072fba23", size = 256952, upload-time = "2026-01-26T02:44:24.306Z" }, + { url = "https://files.pythonhosted.org/packages/03/d6/c878a44ba877f366630c860fdf74bfb203c33778f12b6ac274936853c451/multidict-6.7.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4cfb48c6ea66c83bcaaf7e4dfa7ec1b6bbcf751b7db85a328902796dfde4c060", size = 240317, upload-time = "2026-01-26T02:44:25.772Z" }, + { url = "https://files.pythonhosted.org/packages/68/49/57421b4d7ad2e9e60e25922b08ceb37e077b90444bde6ead629095327a6f/multidict-6.7.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d540e51b7e8e170174555edecddbd5538105443754539193e3e1061864d444d", size = 267132, upload-time = "2026-01-26T02:44:27.648Z" }, + { url = "https://files.pythonhosted.org/packages/b7/fe/ec0edd52ddbcea2a2e89e174f0206444a61440b40f39704e64dc807a70bd/multidict-6.7.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:273d23f4b40f3dce4d6c8a821c741a86dec62cded82e1175ba3d99be128147ed", size = 268140, upload-time = "2026-01-26T02:44:29.588Z" }, + { url = "https://files.pythonhosted.org/packages/b0/73/6e1b01cbeb458807aa0831742232dbdd1fa92bfa33f52a3f176b4ff3dc11/multidict-6.7.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d624335fd4fa1c08a53f8b4be7676ebde19cd092b3895c421045ca87895b429", size = 254277, upload-time = "2026-01-26T02:44:30.902Z" }, + { url = "https://files.pythonhosted.org/packages/6a/b2/5fb8c124d7561a4974c342bc8c778b471ebbeb3cc17df696f034a7e9afe7/multidict-6.7.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:12fad252f8b267cc75b66e8fc51b3079604e8d43a75428ffe193cd9e2195dfd6", size = 252291, upload-time = "2026-01-26T02:44:32.31Z" }, + { url = "https://files.pythonhosted.org/packages/5a/96/51d4e4e06bcce92577fcd488e22600bd38e4fd59c20cb49434d054903bd2/multidict-6.7.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:03ede2a6ffbe8ef936b92cb4529f27f42be7f56afcdab5ab739cd5f27fb1cbf9", size = 250156, upload-time = "2026-01-26T02:44:33.734Z" }, + { url = "https://files.pythonhosted.org/packages/db/6b/420e173eec5fba721a50e2a9f89eda89d9c98fded1124f8d5c675f7a0c0f/multidict-6.7.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:90efbcf47dbe33dcf643a1e400d67d59abeac5db07dc3f27d6bdeae497a2198c", size = 249742, upload-time = "2026-01-26T02:44:35.222Z" }, + { url = "https://files.pythonhosted.org/packages/44/a3/ec5b5bd98f306bc2aa297b8c6f11a46714a56b1e6ef5ebda50a4f5d7c5fb/multidict-6.7.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c4b9bfc148f5a91be9244d6264c53035c8a0dcd2f51f1c3c6e30e30ebaa1c84", size = 262221, upload-time = "2026-01-26T02:44:36.604Z" }, + { url = "https://files.pythonhosted.org/packages/cd/f7/e8c0d0da0cd1e28d10e624604e1a36bcc3353aaebdfdc3a43c72bc683a12/multidict-6.7.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:401c5a650f3add2472d1d288c26deebc540f99e2fb83e9525007a74cd2116f1d", size = 258664, upload-time = "2026-01-26T02:44:38.008Z" }, + { url = "https://files.pythonhosted.org/packages/52/da/151a44e8016dd33feed44f730bd856a66257c1ee7aed4f44b649fb7edeb3/multidict-6.7.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:97891f3b1b3ffbded884e2916cacf3c6fc87b66bb0dde46f7357404750559f33", size = 249490, upload-time = "2026-01-26T02:44:39.386Z" }, + { url = "https://files.pythonhosted.org/packages/87/af/a3b86bf9630b732897f6fc3f4c4714b90aa4361983ccbdcd6c0339b21b0c/multidict-6.7.1-cp313-cp313-win32.whl", hash = "sha256:e1c5988359516095535c4301af38d8a8838534158f649c05dd1050222321bcb3", size = 41695, upload-time = "2026-01-26T02:44:41.318Z" }, + { url = "https://files.pythonhosted.org/packages/b2/35/e994121b0e90e46134673422dd564623f93304614f5d11886b1b3e06f503/multidict-6.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:960c83bf01a95b12b08fd54324a4eb1d5b52c88932b5cba5d6e712bb3ed12eb5", size = 45884, upload-time = "2026-01-26T02:44:42.488Z" }, + { url = "https://files.pythonhosted.org/packages/ca/61/42d3e5dbf661242a69c97ea363f2d7b46c567da8eadef8890022be6e2ab0/multidict-6.7.1-cp313-cp313-win_arm64.whl", hash = "sha256:563fe25c678aaba333d5399408f5ec3c383ca5b663e7f774dd179a520b8144df", size = 43122, upload-time = "2026-01-26T02:44:43.664Z" }, + { url = "https://files.pythonhosted.org/packages/6d/b3/e6b21c6c4f314bb956016b0b3ef2162590a529b84cb831c257519e7fde44/multidict-6.7.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:c76c4bec1538375dad9d452d246ca5368ad6e1c9039dadcf007ae59c70619ea1", size = 83175, upload-time = "2026-01-26T02:44:44.894Z" }, + { url = "https://files.pythonhosted.org/packages/fb/76/23ecd2abfe0957b234f6c960f4ade497f55f2c16aeb684d4ecdbf1c95791/multidict-6.7.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:57b46b24b5d5ebcc978da4ec23a819a9402b4228b8a90d9c656422b4bdd8a963", size = 48460, upload-time = "2026-01-26T02:44:46.106Z" }, + { url = "https://files.pythonhosted.org/packages/c4/57/a0ed92b23f3a042c36bc4227b72b97eca803f5f1801c1ab77c8a212d455e/multidict-6.7.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e954b24433c768ce78ab7929e84ccf3422e46deb45a4dc9f93438f8217fa2d34", size = 46930, upload-time = "2026-01-26T02:44:47.278Z" }, + { url = "https://files.pythonhosted.org/packages/b5/66/02ec7ace29162e447f6382c495dc95826bf931d3818799bbef11e8f7df1a/multidict-6.7.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3bd231490fa7217cc832528e1cd8752a96f0125ddd2b5749390f7c3ec8721b65", size = 242582, upload-time = "2026-01-26T02:44:48.604Z" }, + { url = "https://files.pythonhosted.org/packages/58/18/64f5a795e7677670e872673aca234162514696274597b3708b2c0d276cce/multidict-6.7.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:253282d70d67885a15c8a7716f3a73edf2d635793ceda8173b9ecc21f2fb8292", size = 250031, upload-time = "2026-01-26T02:44:50.544Z" }, + { url = "https://files.pythonhosted.org/packages/c8/ed/e192291dbbe51a8290c5686f482084d31bcd9d09af24f63358c3d42fd284/multidict-6.7.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0b4c48648d7649c9335cf1927a8b87fa692de3dcb15faa676c6a6f1f1aabda43", size = 228596, upload-time = "2026-01-26T02:44:51.951Z" }, + { url = "https://files.pythonhosted.org/packages/1e/7e/3562a15a60cf747397e7f2180b0a11dc0c38d9175a650e75fa1b4d325e15/multidict-6.7.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:98bc624954ec4d2c7cb074b8eefc2b5d0ce7d482e410df446414355d158fe4ca", size = 257492, upload-time = "2026-01-26T02:44:53.902Z" }, + { url = "https://files.pythonhosted.org/packages/24/02/7d0f9eae92b5249bb50ac1595b295f10e263dd0078ebb55115c31e0eaccd/multidict-6.7.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b99af4d9eec0b49927b4402bcbb58dea89d3e0db8806a4086117019939ad3dd", size = 255899, upload-time = "2026-01-26T02:44:55.316Z" }, + { url = "https://files.pythonhosted.org/packages/00/e3/9b60ed9e23e64c73a5cde95269ef1330678e9c6e34dd4eb6b431b85b5a10/multidict-6.7.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6aac4f16b472d5b7dc6f66a0d49dd57b0e0902090be16594dc9ebfd3d17c47e7", size = 247970, upload-time = "2026-01-26T02:44:56.783Z" }, + { url = "https://files.pythonhosted.org/packages/3e/06/538e58a63ed5cfb0bd4517e346b91da32fde409d839720f664e9a4ae4f9d/multidict-6.7.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:21f830fe223215dffd51f538e78c172ed7c7f60c9b96a2bf05c4848ad49921c3", size = 245060, upload-time = "2026-01-26T02:44:58.195Z" }, + { url = "https://files.pythonhosted.org/packages/b2/2f/d743a3045a97c895d401e9bd29aaa09b94f5cbdf1bd561609e5a6c431c70/multidict-6.7.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:f5dd81c45b05518b9aa4da4aa74e1c93d715efa234fd3e8a179df611cc85e5f4", size = 235888, upload-time = "2026-01-26T02:44:59.57Z" }, + { url = "https://files.pythonhosted.org/packages/38/83/5a325cac191ab28b63c52f14f1131f3b0a55ba3b9aa65a6d0bf2a9b921a0/multidict-6.7.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:eb304767bca2bb92fb9c5bd33cedc95baee5bb5f6c88e63706533a1c06ad08c8", size = 243554, upload-time = "2026-01-26T02:45:01.054Z" }, + { url = "https://files.pythonhosted.org/packages/20/1f/9d2327086bd15da2725ef6aae624208e2ef828ed99892b17f60c344e57ed/multidict-6.7.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:c9035dde0f916702850ef66460bc4239d89d08df4d02023a5926e7446724212c", size = 252341, upload-time = "2026-01-26T02:45:02.484Z" }, + { url = "https://files.pythonhosted.org/packages/e8/2c/2a1aa0280cf579d0f6eed8ee5211c4f1730bd7e06c636ba2ee6aafda302e/multidict-6.7.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:af959b9beeb66c822380f222f0e0a1889331597e81f1ded7f374f3ecb0fd6c52", size = 246391, upload-time = "2026-01-26T02:45:03.862Z" }, + { url = "https://files.pythonhosted.org/packages/e5/03/7ca022ffc36c5a3f6e03b179a5ceb829be9da5783e6fe395f347c0794680/multidict-6.7.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:41f2952231456154ee479651491e94118229844dd7226541788be783be2b5108", size = 243422, upload-time = "2026-01-26T02:45:05.296Z" }, + { url = "https://files.pythonhosted.org/packages/dc/1d/b31650eab6c5778aceed46ba735bd97f7c7d2f54b319fa916c0f96e7805b/multidict-6.7.1-cp313-cp313t-win32.whl", hash = "sha256:df9f19c28adcb40b6aae30bbaa1478c389efd50c28d541d76760199fc1037c32", size = 47770, upload-time = "2026-01-26T02:45:06.754Z" }, + { url = "https://files.pythonhosted.org/packages/ac/5b/2d2d1d522e51285bd61b1e20df8f47ae1a9d80839db0b24ea783b3832832/multidict-6.7.1-cp313-cp313t-win_amd64.whl", hash = "sha256:d54ecf9f301853f2c5e802da559604b3e95bb7a3b01a9c295c6ee591b9882de8", size = 53109, upload-time = "2026-01-26T02:45:08.044Z" }, + { url = "https://files.pythonhosted.org/packages/3d/a3/cc409ba012c83ca024a308516703cf339bdc4b696195644a7215a5164a24/multidict-6.7.1-cp313-cp313t-win_arm64.whl", hash = "sha256:5a37ca18e360377cfda1d62f5f382ff41f2b8c4ccb329ed974cc2e1643440118", size = 45573, upload-time = "2026-01-26T02:45:09.349Z" }, + { url = "https://files.pythonhosted.org/packages/91/cc/db74228a8be41884a567e88a62fd589a913708fcf180d029898c17a9a371/multidict-6.7.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8f333ec9c5eb1b7105e3b84b53141e66ca05a19a605368c55450b6ba208cb9ee", size = 75190, upload-time = "2026-01-26T02:45:10.651Z" }, + { url = "https://files.pythonhosted.org/packages/d5/22/492f2246bb5b534abd44804292e81eeaf835388901f0c574bac4eeec73c5/multidict-6.7.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a407f13c188f804c759fc6a9f88286a565c242a76b27626594c133b82883b5c2", size = 44486, upload-time = "2026-01-26T02:45:11.938Z" }, + { url = "https://files.pythonhosted.org/packages/f1/4f/733c48f270565d78b4544f2baddc2fb2a245e5a8640254b12c36ac7ac68e/multidict-6.7.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0e161ddf326db5577c3a4cc2d8648f81456e8a20d40415541587a71620d7a7d1", size = 43219, upload-time = "2026-01-26T02:45:14.346Z" }, + { url = "https://files.pythonhosted.org/packages/24/bb/2c0c2287963f4259c85e8bcbba9182ced8d7fca65c780c38e99e61629d11/multidict-6.7.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:1e3a8bb24342a8201d178c3b4984c26ba81a577c80d4d525727427460a50c22d", size = 245132, upload-time = "2026-01-26T02:45:15.712Z" }, + { url = "https://files.pythonhosted.org/packages/a7/f9/44d4b3064c65079d2467888794dea218d1601898ac50222ab8a9a8094460/multidict-6.7.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97231140a50f5d447d3164f994b86a0bed7cd016e2682f8650d6a9158e14fd31", size = 252420, upload-time = "2026-01-26T02:45:17.293Z" }, + { url = "https://files.pythonhosted.org/packages/8b/13/78f7275e73fa17b24c9a51b0bd9d73ba64bb32d0ed51b02a746eb876abe7/multidict-6.7.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6b10359683bd8806a200fd2909e7c8ca3a7b24ec1d8132e483d58e791d881048", size = 233510, upload-time = "2026-01-26T02:45:19.356Z" }, + { url = "https://files.pythonhosted.org/packages/4b/25/8167187f62ae3cbd52da7893f58cb036b47ea3fb67138787c76800158982/multidict-6.7.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:283ddac99f7ac25a4acadbf004cb5ae34480bbeb063520f70ce397b281859362", size = 264094, upload-time = "2026-01-26T02:45:20.834Z" }, + { url = "https://files.pythonhosted.org/packages/a1/e7/69a3a83b7b030cf283fb06ce074a05a02322359783424d7edf0f15fe5022/multidict-6.7.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:538cec1e18c067d0e6103aa9a74f9e832904c957adc260e61cd9d8cf0c3b3d37", size = 260786, upload-time = "2026-01-26T02:45:22.818Z" }, + { url = "https://files.pythonhosted.org/packages/fe/3b/8ec5074bcfc450fe84273713b4b0a0dd47c0249358f5d82eb8104ffe2520/multidict-6.7.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7eee46ccb30ff48a1e35bb818cc90846c6be2b68240e42a78599166722cea709", size = 248483, upload-time = "2026-01-26T02:45:24.368Z" }, + { url = "https://files.pythonhosted.org/packages/48/5a/d5a99e3acbca0e29c5d9cba8f92ceb15dce78bab963b308ae692981e3a5d/multidict-6.7.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa263a02f4f2dd2d11a7b1bb4362aa7cb1049f84a9235d31adf63f30143469a0", size = 248403, upload-time = "2026-01-26T02:45:25.982Z" }, + { url = "https://files.pythonhosted.org/packages/35/48/e58cd31f6c7d5102f2a4bf89f96b9cf7e00b6c6f3d04ecc44417c00a5a3c/multidict-6.7.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2e1425e2f99ec5bd36c15a01b690a1a2456209c5deed58f95469ffb46039ccbb", size = 240315, upload-time = "2026-01-26T02:45:27.487Z" }, + { url = "https://files.pythonhosted.org/packages/94/33/1cd210229559cb90b6786c30676bb0c58249ff42f942765f88793b41fdce/multidict-6.7.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:497394b3239fc6f0e13a78a3e1b61296e72bf1c5f94b4c4eb80b265c37a131cd", size = 245528, upload-time = "2026-01-26T02:45:28.991Z" }, + { url = "https://files.pythonhosted.org/packages/64/f2/6e1107d226278c876c783056b7db43d800bb64c6131cec9c8dfb6903698e/multidict-6.7.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:233b398c29d3f1b9676b4b6f75c518a06fcb2ea0b925119fb2c1bc35c05e1601", size = 258784, upload-time = "2026-01-26T02:45:30.503Z" }, + { url = "https://files.pythonhosted.org/packages/4d/c1/11f664f14d525e4a1b5327a82d4de61a1db604ab34c6603bb3c2cc63ad34/multidict-6.7.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:93b1818e4a6e0930454f0f2af7dfce69307ca03cdcfb3739bf4d91241967b6c1", size = 251980, upload-time = "2026-01-26T02:45:32.603Z" }, + { url = "https://files.pythonhosted.org/packages/e1/9f/75a9ac888121d0c5bbd4ecf4eead45668b1766f6baabfb3b7f66a410e231/multidict-6.7.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f33dc2a3abe9249ea5d8360f969ec7f4142e7ac45ee7014d8f8d5acddf178b7b", size = 243602, upload-time = "2026-01-26T02:45:34.043Z" }, + { url = "https://files.pythonhosted.org/packages/9a/e7/50bf7b004cc8525d80dbbbedfdc7aed3e4c323810890be4413e589074032/multidict-6.7.1-cp314-cp314-win32.whl", hash = "sha256:3ab8b9d8b75aef9df299595d5388b14530839f6422333357af1339443cff777d", size = 40930, upload-time = "2026-01-26T02:45:36.278Z" }, + { url = "https://files.pythonhosted.org/packages/e0/bf/52f25716bbe93745595800f36fb17b73711f14da59ed0bb2eba141bc9f0f/multidict-6.7.1-cp314-cp314-win_amd64.whl", hash = "sha256:5e01429a929600e7dab7b166062d9bb54a5eed752384c7384c968c2afab8f50f", size = 45074, upload-time = "2026-01-26T02:45:37.546Z" }, + { url = "https://files.pythonhosted.org/packages/97/ab/22803b03285fa3a525f48217963da3a65ae40f6a1b6f6cf2768879e208f9/multidict-6.7.1-cp314-cp314-win_arm64.whl", hash = "sha256:4885cb0e817aef5d00a2e8451d4665c1808378dc27c2705f1bf4ef8505c0d2e5", size = 42471, upload-time = "2026-01-26T02:45:38.889Z" }, + { url = "https://files.pythonhosted.org/packages/e0/6d/f9293baa6146ba9507e360ea0292b6422b016907c393e2f63fc40ab7b7b5/multidict-6.7.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:0458c978acd8e6ea53c81eefaddbbee9c6c5e591f41b3f5e8e194780fe026581", size = 82401, upload-time = "2026-01-26T02:45:40.254Z" }, + { url = "https://files.pythonhosted.org/packages/7a/68/53b5494738d83558d87c3c71a486504d8373421c3e0dbb6d0db48ad42ee0/multidict-6.7.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c0abd12629b0af3cf590982c0b413b1e7395cd4ec026f30986818ab95bfaa94a", size = 48143, upload-time = "2026-01-26T02:45:41.635Z" }, + { url = "https://files.pythonhosted.org/packages/37/e8/5284c53310dcdc99ce5d66563f6e5773531a9b9fe9ec7a615e9bc306b05f/multidict-6.7.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:14525a5f61d7d0c94b368a42cff4c9a4e7ba2d52e2672a7b23d84dc86fb02b0c", size = 46507, upload-time = "2026-01-26T02:45:42.99Z" }, + { url = "https://files.pythonhosted.org/packages/e4/fc/6800d0e5b3875568b4083ecf5f310dcf91d86d52573160834fb4bfcf5e4f/multidict-6.7.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17307b22c217b4cf05033dabefe68255a534d637c6c9b0cc8382718f87be4262", size = 239358, upload-time = "2026-01-26T02:45:44.376Z" }, + { url = "https://files.pythonhosted.org/packages/41/75/4ad0973179361cdf3a113905e6e088173198349131be2b390f9fa4da5fc6/multidict-6.7.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a7e590ff876a3eaf1c02a4dfe0724b6e69a9e9de6d8f556816f29c496046e59", size = 246884, upload-time = "2026-01-26T02:45:47.167Z" }, + { url = "https://files.pythonhosted.org/packages/c3/9c/095bb28b5da139bd41fb9a5d5caff412584f377914bd8787c2aa98717130/multidict-6.7.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5fa6a95dfee63893d80a34758cd0e0c118a30b8dcb46372bf75106c591b77889", size = 225878, upload-time = "2026-01-26T02:45:48.698Z" }, + { url = "https://files.pythonhosted.org/packages/07/d0/c0a72000243756e8f5a277b6b514fa005f2c73d481b7d9e47cd4568aa2e4/multidict-6.7.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a0543217a6a017692aa6ae5cc39adb75e587af0f3a82288b1492eb73dd6cc2a4", size = 253542, upload-time = "2026-01-26T02:45:50.164Z" }, + { url = "https://files.pythonhosted.org/packages/c0/6b/f69da15289e384ecf2a68837ec8b5ad8c33e973aa18b266f50fe55f24b8c/multidict-6.7.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f99fe611c312b3c1c0ace793f92464d8cd263cc3b26b5721950d977b006b6c4d", size = 252403, upload-time = "2026-01-26T02:45:51.779Z" }, + { url = "https://files.pythonhosted.org/packages/a2/76/b9669547afa5a1a25cd93eaca91c0da1c095b06b6d2d8ec25b713588d3a1/multidict-6.7.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9004d8386d133b7e6135679424c91b0b854d2d164af6ea3f289f8f2761064609", size = 244889, upload-time = "2026-01-26T02:45:53.27Z" }, + { url = "https://files.pythonhosted.org/packages/7e/a9/a50d2669e506dad33cfc45b5d574a205587b7b8a5f426f2fbb2e90882588/multidict-6.7.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e628ef0e6859ffd8273c69412a2465c4be4a9517d07261b33334b5ec6f3c7489", size = 241982, upload-time = "2026-01-26T02:45:54.919Z" }, + { url = "https://files.pythonhosted.org/packages/c5/bb/1609558ad8b456b4827d3c5a5b775c93b87878fd3117ed3db3423dfbce1b/multidict-6.7.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:841189848ba629c3552035a6a7f5bf3b02eb304e9fea7492ca220a8eda6b0e5c", size = 232415, upload-time = "2026-01-26T02:45:56.981Z" }, + { url = "https://files.pythonhosted.org/packages/d8/59/6f61039d2aa9261871e03ab9dc058a550d240f25859b05b67fd70f80d4b3/multidict-6.7.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:ce1bbd7d780bb5a0da032e095c951f7014d6b0a205f8318308140f1a6aba159e", size = 240337, upload-time = "2026-01-26T02:45:58.698Z" }, + { url = "https://files.pythonhosted.org/packages/a1/29/fdc6a43c203890dc2ae9249971ecd0c41deaedfe00d25cb6564b2edd99eb/multidict-6.7.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b26684587228afed0d50cf804cc71062cc9c1cdf55051c4c6345d372947b268c", size = 248788, upload-time = "2026-01-26T02:46:00.862Z" }, + { url = "https://files.pythonhosted.org/packages/a9/14/a153a06101323e4cf086ecee3faadba52ff71633d471f9685c42e3736163/multidict-6.7.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9f9af11306994335398293f9958071019e3ab95e9a707dc1383a35613f6abcb9", size = 242842, upload-time = "2026-01-26T02:46:02.824Z" }, + { url = "https://files.pythonhosted.org/packages/41/5f/604ae839e64a4a6efc80db94465348d3b328ee955e37acb24badbcd24d83/multidict-6.7.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b4938326284c4f1224178a560987b6cf8b4d38458b113d9b8c1db1a836e640a2", size = 240237, upload-time = "2026-01-26T02:46:05.898Z" }, + { url = "https://files.pythonhosted.org/packages/5f/60/c3a5187bf66f6fb546ff4ab8fb5a077cbdd832d7b1908d4365c7f74a1917/multidict-6.7.1-cp314-cp314t-win32.whl", hash = "sha256:98655c737850c064a65e006a3df7c997cd3b220be4ec8fe26215760b9697d4d7", size = 48008, upload-time = "2026-01-26T02:46:07.468Z" }, + { url = "https://files.pythonhosted.org/packages/0c/f7/addf1087b860ac60e6f382240f64fb99f8bfb532bb06f7c542b83c29ca61/multidict-6.7.1-cp314-cp314t-win_amd64.whl", hash = "sha256:497bde6223c212ba11d462853cfa4f0ae6ef97465033e7dc9940cdb3ab5b48e5", size = 53542, upload-time = "2026-01-26T02:46:08.809Z" }, + { url = "https://files.pythonhosted.org/packages/4c/81/4629d0aa32302ef7b2ec65c75a728cc5ff4fa410c50096174c1632e70b3e/multidict-6.7.1-cp314-cp314t-win_arm64.whl", hash = "sha256:2bbd113e0d4af5db41d5ebfe9ccaff89de2120578164f86a5d17d5a576d1e5b2", size = 44719, upload-time = "2026-01-26T02:46:11.146Z" }, + { url = "https://files.pythonhosted.org/packages/81/08/7036c080d7117f28a4af526d794aab6a84463126db031b007717c1a6676e/multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56", size = 12319, upload-time = "2026-01-26T02:46:44.004Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "propcache" +version = "0.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/44/c87281c333769159c50594f22610f77398a47ccbfbbf23074e744e86f87c/propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427", size = 50208, upload-time = "2026-05-08T21:02:12.199Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/cb/e27bc2b2737a0bb49962b275efa051e8f1c35a936df7d5139b6b658b7dc9/propcache-0.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:806719138ecd720339a12410fb9614ac9b2b2d3a5fdf8235d56981c36f4039ba", size = 95887, upload-time = "2026-05-08T21:00:11.277Z" }, + { url = "https://files.pythonhosted.org/packages/e6/13/b8ae04c59392f8d11c6cd9fb4011d1dc7c86b81225c770280300e259ffe1/propcache-0.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:db2b80ea58eab4f86b2beec3cc8b39e8ff9276ac20e96b7cce43c8ae84cd6b5a", size = 54654, upload-time = "2026-05-08T21:00:12.604Z" }, + { url = "https://files.pythonhosted.org/packages/2c/7d/49777a3e20b55863d4794384a38acd460c04157b0a00f8602b0d508b8431/propcache-0.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e5cbfac9f61484f7e9f3597775500cd3ebe8274e9b050c38f9525c77c97520bf", size = 55190, upload-time = "2026-05-08T21:00:13.935Z" }, + { url = "https://files.pythonhosted.org/packages/44/c7/085d0cd63062e84044e3f05797749c3f8e3938ff3aeb0eb2f69d43fafc91/propcache-0.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbc581d2814337da56222fab8dc5f161cd798a434e49bac27930aaef798e144", size = 59995, upload-time = "2026-05-08T21:00:15.526Z" }, + { url = "https://files.pythonhosted.org/packages/9c/42/32cf8e3009e92b2645cf1e944f701e8ea4e924dffde1ee26db860bcbf7e4/propcache-0.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:857187f381f88c8e2fa2fe56ab94879d011b883d5a2ee5a1b60a8cd2a06846d9", size = 63422, upload-time = "2026-05-08T21:00:16.824Z" }, + { url = "https://files.pythonhosted.org/packages/9e/1b/f112433f99fc979431b87a39ef169e3f8df070d99a72792c56d6937ac48b/propcache-0.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:178b4a2cdaac1818e2bf1c5a99b94383fa73ea5382e032a48dec07dc5668dc42", size = 64342, upload-time = "2026-05-08T21:00:18.362Z" }, + { url = "https://files.pythonhosted.org/packages/14/15/5574111ae50dd6e879456888c0eadd4c5a869959775854e18e18a6b345f3/propcache-0.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476", size = 61639, upload-time = "2026-05-08T21:00:19.692Z" }, + { url = "https://files.pythonhosted.org/packages/cc/da/4d775080b1490c0ae604acda868bd71aabe3a89ed16f2aa4339eb8a283e7/propcache-0.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5671d09a36b06d0fd4a3da0fccbcae360e9b1570924171a15e9e0997f0249fba", size = 61588, upload-time = "2026-05-08T21:00:21.155Z" }, + { url = "https://files.pythonhosted.org/packages/04/ac/f076982cbe2195ee9cf32de5a1e46951d9fb399fc207f390562dd0fd8fb2/propcache-0.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:80168e2ebe4d3ec6599d10ad8f520304ae1cad9b6c5a95372aef1b66b7bfb53a", size = 60029, upload-time = "2026-05-08T21:00:22.713Z" }, + { url = "https://files.pythonhosted.org/packages/70/60/189be62e0dd898dce3b331e1b8c7a543cd3a405ac0c81fe8ee8a9d5d77e1/propcache-0.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:45f11346f884bc47444f6e6647131055844134c3175b629f84952e2b5cd62b64", size = 56774, upload-time = "2026-05-08T21:00:24.001Z" }, + { url = "https://files.pythonhosted.org/packages/ea/9e/93377b9c7939c1ffae98f878dee955efadfd638078bc86dbc21f9d52f651/propcache-0.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e778ebd44ef4f66ed60a0416b06b489687db264a9c0b3620362f26489492913", size = 63532, upload-time = "2026-05-08T21:00:25.545Z" }, + { url = "https://files.pythonhosted.org/packages/14/f9/590ef6cfb9b8028d516d287812ece32bb0bc5f11fbb9c8bf6b2e6313fec8/propcache-0.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:c0cb9ed24c8964e172768d455a38254c2dd8a552905729ce006cad3d3dda59b1", size = 61592, upload-time = "2026-05-08T21:00:27.186Z" }, + { url = "https://files.pythonhosted.org/packages/b4/5e/70958b3034c297a630bba2f17ca7abc2d5f39a803ad7e370ab79d1ecd022/propcache-0.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1d1ad32d9d4355e2be65574fd0bfd3677e7066b009cd5b9b2dee8aa6a6393b33", size = 64788, upload-time = "2026-05-08T21:00:28.8Z" }, + { url = "https://files.pythonhosted.org/packages/12/fd/77fe5936d8c3086ca9048f7f415f122ed82e53884a9ec193646b42deef06/propcache-0.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a", size = 62514, upload-time = "2026-05-08T21:00:30.098Z" }, + { url = "https://files.pythonhosted.org/packages/cf/74/66bd798b5b3be70aa1b391f5cc9d6a0a5532d7fd3b19ec0b213e72e6ad9d/propcache-0.5.2-cp312-cp312-win32.whl", hash = "sha256:8c7972d8f193740d9175f0998ab38717e6cd322d5935c5b0fef8c0d323fd9031", size = 39018, upload-time = "2026-05-08T21:00:31.622Z" }, + { url = "https://files.pythonhosted.org/packages/61/7c/5c0d34aa3024694d6dcb9271cdbdd08c4e47c1c0ad95ec7e7bc74cdea145/propcache-0.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:d9ee8826a7d47863a08ac44e1a5f611a462eefc3a194b492da242128bec75b42", size = 42322, upload-time = "2026-05-08T21:00:32.918Z" }, + { url = "https://files.pythonhosted.org/packages/4d/91/875812f1a3feb20ceba818ef39fbe4d92f1081e04ac815c822496d0d038b/propcache-0.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:2800a4a8ead6b28cccd1ec54b59346f0def7922ee1c7598e8499c733cfbb7c84", size = 38172, upload-time = "2026-05-08T21:00:35.124Z" }, + { url = "https://files.pythonhosted.org/packages/c5/09/f049e45385503fe67db75a6b6186a7b9f0c3930366dc960522c312a825b1/propcache-0.5.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:099aaf4b4d1a02265b92a977edf00b5c4f63b3b17ac6de39b0d637c9cac0188a", size = 94457, upload-time = "2026-05-08T21:00:36.355Z" }, + { url = "https://files.pythonhosted.org/packages/6b/65/83d1d05655baf63113731bd5a1008435e14f8d1e5a06cbe4ec5b23ad7a31/propcache-0.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:68ce1c44c7a813a7f71ea04315a8c7b330b63db99d059a797a4651bb6f69f117", size = 53835, upload-time = "2026-05-08T21:00:38.072Z" }, + { url = "https://files.pythonhosted.org/packages/a9/12/a6ba6482bb5ea3260c000c9b20881c95fa11c6b30173715668259f844ed7/propcache-0.5.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fc299c129490f55f254cd90be0deca4764e36e9a7c08b4aa588479a3bbed3098", size = 54545, upload-time = "2026-05-08T21:00:39.319Z" }, + { url = "https://files.pythonhosted.org/packages/a9/19/7fa086f5764c59ec8a8e157cd93aa8497acc00aba9dcdec56bfffb32602d/propcache-0.5.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6ae2198be502c10f09b2516e7b5d019816924bc3183a43ce792a7bd6625e6f4", size = 59886, upload-time = "2026-05-08T21:00:40.621Z" }, + { url = "https://files.pythonhosted.org/packages/a1/e4/5d7663dc8235956c8f5281698a3af1d351d8820341ddd890f59d9a9127f2/propcache-0.5.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6041d31504dc1779d700e1edcfb08eea334b357620b06681a4eabb57a74e574e", size = 63261, upload-time = "2026-05-08T21:00:41.775Z" }, + { url = "https://files.pythonhosted.org/packages/4a/4a/15a03adee24d6350da4292caeac44c34c033d2afe5e87eb370f38854560f/propcache-0.5.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7eabc04151c78a9f4d5bbb5f1faf571e4defeb4b585e0fe95b60ff2dbe4d3d7", size = 64184, upload-time = "2026-05-08T21:00:43.018Z" }, + { url = "https://files.pythonhosted.org/packages/8b/c6/979176efdaa3d239e36d503d5af63a0a773b36662ed8f52e5b6a6d9fd40e/propcache-0.5.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4db0ba63d693afd40d249bd93f842b5f144f8fcbb83de05660373bcf30517b1d", size = 61534, upload-time = "2026-05-08T21:00:44.507Z" }, + { url = "https://files.pythonhosted.org/packages/c8/22/63e8cd1bae4c2d2be6493b6b7d10566ddafad88137cfbc99964a1119853c/propcache-0.5.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dbcf7675229b35d31abb6547d8ebc8c27a830ac3f9a794edff6254873ec7c0a", size = 61500, upload-time = "2026-05-08T21:00:45.796Z" }, + { url = "https://files.pythonhosted.org/packages/60/5a/28e5d9acbac1cc9ccb67045e8c1b943aa8d79fdf39c93bd73cacd68008ea/propcache-0.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d310c013aad2c72f1c3f2f8dd3279d460a858c551f97aeb8c63e4693cca7b4d2", size = 59994, upload-time = "2026-05-08T21:00:47.093Z" }, + { url = "https://files.pythonhosted.org/packages/f3/40/db650677f554a95b9c01a7c9d93d629e93a15562f5deb4573c9ee136fed2/propcache-0.5.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:06187263ddad280d05b4d8a8b3bb7d164cbebd469236544a42e6d9b28ac6a4fa", size = 56884, upload-time = "2026-05-08T21:00:48.376Z" }, + { url = "https://files.pythonhosted.org/packages/80/45/70b39b89516ff8b96bf732fa6fded8cef20f293cb1508690101c3c07ec51/propcache-0.5.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3115559b8effafd63b142ea5ed53d63a16ea6469cbc63dce4ee194b42db5d853", size = 63464, upload-time = "2026-05-08T21:00:49.954Z" }, + { url = "https://files.pythonhosted.org/packages/f9/e2/fa59d3a89eac5534293124af4f1d0d0ada091ce4a0ab4610ce03fd2bdd8d/propcache-0.5.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c60462af8e6dc30c35407c7237ea908d777b22862bbee27bc4699c0d8bcdc45a", size = 61588, upload-time = "2026-05-08T21:00:51.281Z" }, + { url = "https://files.pythonhosted.org/packages/0b/97/efb547a55c4bc7381cfb202d6a2239ac621045277bc1ea5dfd3a7f0516c0/propcache-0.5.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:40314bca9ac559716fe374094fc81c11dcc34b64fd6c585360f5775690505704", size = 64667, upload-time = "2026-05-08T21:00:52.602Z" }, + { url = "https://files.pythonhosted.org/packages/92/56/f5c7d9b4b7595d5127da38974d791b2153f3d1eae6c674af3583ace92ad3/propcache-0.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cfa21e036ce1e1db2be04ba3b85d2df1bb1702fa01932d984c5464c665228ff4", size = 62463, upload-time = "2026-05-08T21:00:54.303Z" }, + { url = "https://files.pythonhosted.org/packages/bd/3b/484a3a65fc9f9f60c41dcd17b428bace5389544e2c680994534a20755066/propcache-0.5.2-cp313-cp313-win32.whl", hash = "sha256:f156a3529f38063b6dbaf356e15602a7f95f8055b1295a438433a6386f10463d", size = 38621, upload-time = "2026-05-08T21:00:55.808Z" }, + { url = "https://files.pythonhosted.org/packages/1c/fd/3f0f10dba4dabad3bf53102be007abf55481067952bde0fdddff439e7c61/propcache-0.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:dfed59d0a5aeb01e242e66ff0300bc4a265a7c05f612d30016f0b60b1017d757", size = 41649, upload-time = "2026-05-08T21:00:57.061Z" }, + { url = "https://files.pythonhosted.org/packages/90/ec/6ce619cc32bb500a482f811f9cd509368b4e58e638d13f2c68f370d6b475/propcache-0.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:ba338430e87ceb9c8f0cf754de38a9860560261e56c00376debd628698a7364f", size = 37636, upload-time = "2026-05-08T21:00:58.646Z" }, + { url = "https://files.pythonhosted.org/packages/1b/82/c1d268bbbf2ef981c5bf0fbbe746db617c66e3bcefe431a1aa8943fbe23a/propcache-0.5.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a592f5f3da71c8691c788c13cb6734b6d17663d2e1cb8caddf0673d01ef8847d", size = 98872, upload-time = "2026-05-08T21:00:59.889Z" }, + { url = "https://files.pythonhosted.org/packages/f4/d4/52c871e73e864e6b34c0e2d58ac1ec5ccd149497ddc7ad2137ae98323a35/propcache-0.5.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6a997d0489e9668a384fcfd5061b857aa5361de73191cac204d04b889cfbbafa", size = 56257, upload-time = "2026-05-08T21:01:01.195Z" }, + { url = "https://files.pythonhosted.org/packages/67/f0/9b90ca2a210b3d09bcfcd96ecd0f55545c091535abce2a45de2775cfd357/propcache-0.5.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:10734b5484ea113152ee25a91dccedf81631791805d2c9ccb054958e51842c94", size = 56696, upload-time = "2026-05-08T21:01:02.941Z" }, + { url = "https://files.pythonhosted.org/packages/9d/0e/6e9d4ba07c8e56e21ddec1e75f12148142b21ca83a51871babce095334f4/propcache-0.5.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cafca7e56c12bb02ae16d283742bef25a61122e9dab2b5b3f2ccbe589ce32164", size = 62378, upload-time = "2026-05-08T21:01:04.475Z" }, + { url = "https://files.pythonhosted.org/packages/65/19/c10badaa463dde8a27ce884f8ee2ec37e6035b7c9f5ff0c8f74f06f08dac/propcache-0.5.2-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f064f8d2b59177878b7615df1735cd8fe3462ed6be8c7b217d17a276489c2b7f", size = 65283, upload-time = "2026-05-08T21:01:05.959Z" }, + { url = "https://files.pythonhosted.org/packages/b0/b6/93bea99ca80e19cef6512a8580e5b7857bbe09422d9daa7fd4ef5723306c/propcache-0.5.2-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f78abfa8dfc32376fd1aacf597b2f2fbbe0ea751419aee718af5d4f82537ef8c", size = 66616, upload-time = "2026-05-08T21:01:07.228Z" }, + { url = "https://files.pythonhosted.org/packages/83/e4/5c7462e50625f051f37fb38b8224f7639f667184bbd34424ec83819bb1b7/propcache-0.5.2-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7467da8a9822bf1a55336f877340c5bcbd3c482afc43a99771169f74a26dedc", size = 63773, upload-time = "2026-05-08T21:01:08.514Z" }, + { url = "https://files.pythonhosted.org/packages/ca/b6/99238894047b13c823be25027e736626cd414a52a5e30d2c3347c2733529/propcache-0.5.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a6ddc6ac9e25de626c1f129c1b467d7ecd33ce2237d3fd0c4e429feef0a7ee1f", size = 63664, upload-time = "2026-05-08T21:01:09.874Z" }, + { url = "https://files.pythonhosted.org/packages/85/1e/a3a1a63116a2b8edb415a8bb9a6f0c34bd03830b1e18e8ce2904e1dc1cf4/propcache-0.5.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2f22cbbac9e26a8e864c0985ff1268d5d939d53d9d9411a9824279097e03a2cb", size = 62643, upload-time = "2026-05-08T21:01:11.132Z" }, + { url = "https://files.pythonhosted.org/packages/e4/03/893cf147de2fc6543c5eaa07ad833170e7e2a2385725bbebe8c0503723bb/propcache-0.5.2-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:fc76378c62a0f04d0cd82fbb1a2cd2d7e28fcb40d5873f28a6c44e388aaa2751", size = 59595, upload-time = "2026-05-08T21:01:12.387Z" }, + { url = "https://files.pythonhosted.org/packages/86/3b/04c1a2e12c57766568ba75ba72b3bf2042818d4c1425fab6fc07155c7cff/propcache-0.5.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:acd2c8edba48e31e58a363b8cf4e5c7db3b04b3f9e371f601df30d9b0d244836", size = 65711, upload-time = "2026-05-08T21:01:13.676Z" }, + { url = "https://files.pythonhosted.org/packages/1c/34/80f8d0099f8d6bacc4de1624c85672681c8cd1149ca2da0e38fd120b817f/propcache-0.5.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:452b5065457eb9991ec5eb38ff41d6cd4c991c9ac7c531c4d5849ae473a9a13f", size = 64247, upload-time = "2026-05-08T21:01:14.936Z" }, + { url = "https://files.pythonhosted.org/packages/f3/1a/8b08f3a5f1037e9e370c55883ceeeee0f6dd0416fb2d2d67b8bfc91f2a79/propcache-0.5.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:3430bb2bfe1331885c427745a751e774ee679fd4344f80b97bf879815fe8fa55", size = 67102, upload-time = "2026-05-08T21:01:16.281Z" }, + { url = "https://files.pythonhosted.org/packages/34/68/8bdb7bb7756d76e005490649d10e4a8369e610c74d619f71e1aedf889e9c/propcache-0.5.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cef6cea3922890dd6c9654971001fa797b526c16ab5e1e46c05fd6f877be7568", size = 64964, upload-time = "2026-05-08T21:01:17.57Z" }, + { url = "https://files.pythonhosted.org/packages/0a/aa/50fb0b5d3968b61a510926ff8b8465f1d6e976b3ab74496d7a4b9fc42515/propcache-0.5.2-cp313-cp313t-win32.whl", hash = "sha256:72d61e16dd78228b58c5d47be830ff3da7e5f139abdf0aef9d86cde1c5cf2191", size = 42546, upload-time = "2026-05-08T21:01:18.946Z" }, + { url = "https://files.pythonhosted.org/packages/ae/4c/0ddbae64321bd4a95bcbfc19307238016b5b1fee645c84626c8d539e5b74/propcache-0.5.2-cp313-cp313t-win_amd64.whl", hash = "sha256:0958834041a0166d343b8d2cedcd8bcbaeb4fdbe0cf08320c5379f143c3be6e7", size = 46330, upload-time = "2026-05-08T21:01:20.162Z" }, + { url = "https://files.pythonhosted.org/packages/00/d9/9cddc8efb78d8af264c5ec9f6d10b62f57c515feda8d321595f56010fb23/propcache-0.5.2-cp313-cp313t-win_arm64.whl", hash = "sha256:6de8bd93ddde9b992cf2b2e0d796d501a19026b5b9fd87356d7d0779531a8d96", size = 40521, upload-time = "2026-05-08T21:01:21.399Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ea/23ee535d90ce8bcc465a3028eb3cc0ce3bd1005f4bb27710b30587de798d/propcache-0.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:46088abff4cba581dea21ae0467a480526cb25aa5f3c269e909f800328bc3999", size = 94662, upload-time = "2026-05-08T21:01:22.683Z" }, + { url = "https://files.pythonhosted.org/packages/b5/06/c5a52f419b5d8972f8d46a7577476090d8e3263ff589ce40b5ca4968d5be/propcache-0.5.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fc88b26f08d634f7bc819a7852e5214f5802641ab8d9fd5326892292eee1993e", size = 53928, upload-time = "2026-05-08T21:01:23.986Z" }, + { url = "https://files.pythonhosted.org/packages/63/b1/4260d67d6bd85e58a66b72d54ce15d5de789b6f3870cc6bedf8ff9667401/propcache-0.5.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:97797ebb098e670a2f92dd66f32897e30d7615b14e7f59711de23e30a9072539", size = 54650, upload-time = "2026-05-08T21:01:25.305Z" }, + { url = "https://files.pythonhosted.org/packages/70/06/2f46c318e3307cd7a6a7481def374ce838c0fe20084b39dd54b0879d0e99/propcache-0.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba57fffe4ac99c5d30076161b5866336d97600769bad35cc68f7774b15298a4e", size = 59912, upload-time = "2026-05-08T21:01:26.545Z" }, + { url = "https://files.pythonhosted.org/packages/4c/29/fe1aebec2ce57ab985a9c382bded1124431f85078113aa222c5d278430d4/propcache-0.5.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:583c19759d9eec1e5b69e2fbef36a7d9c326041be9746cb822d335c8cedc2979", size = 63300, upload-time = "2026-05-08T21:01:27.937Z" }, + { url = "https://files.pythonhosted.org/packages/b4/18/2334b26768b6c82be8c69e83671b767d5ef426aa09b0cba6c2ea47816774/propcache-0.5.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d0326e2e5e1f3163fa306c834e48e8d490e5fae607a097a40c0648109b47ba80", size = 64208, upload-time = "2026-05-08T21:01:29.484Z" }, + { url = "https://files.pythonhosted.org/packages/2b/76/7f1bfd6afff4c5e38e36a3c6d68eb5f4b7311ea80baf693db78d95b603c4/propcache-0.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e00820e192c8dbebcafb383ebbf99030895f09905e7a0eb2e0340a0bcc2bc825", size = 61633, upload-time = "2026-05-08T21:01:31.068Z" }, + { url = "https://files.pythonhosted.org/packages/c4/46/b3ff8aba2b4953a3e50de2cf72f1b5748b8eca93b15f3dc2c84339084c09/propcache-0.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c66afea89b1e43725731d2004732a046fe6fe955d51f952c3e95a7314a284a39", size = 61724, upload-time = "2026-05-08T21:01:32.374Z" }, + { url = "https://files.pythonhosted.org/packages/c5/01/814cfcafbcff954f94c01cf30e097ddc88a076b5440fbcf4570753437d40/propcache-0.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d4dc37dec6c6cdad0b57881a5658fd14fbf53e333b1a86cf86559f190e1d9ec4", size = 60069, upload-time = "2026-05-08T21:01:33.67Z" }, + { url = "https://files.pythonhosted.org/packages/da/68/5c6f7622d510cc666a300687e06fd060c1a43361c0c9b20d284f06d8096a/propcache-0.5.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5570dbcc97571c15f68068e529c92715a12f8d54030e272d264b377e22bd17a5", size = 57099, upload-time = "2026-05-08T21:01:34.915Z" }, + { url = "https://files.pythonhosted.org/packages/55/27/9cb0b4c679124085327957d42521c99dba04c88c90c3e55a6f0b633ebccc/propcache-0.5.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f814362777a9f841adddb200ecdf8f5cb1e5a3c4b7a86378edbd6ccb26edd702", size = 63391, upload-time = "2026-05-08T21:01:36.231Z" }, + { url = "https://files.pythonhosted.org/packages/f0/9d/7258aaa5bdf60fc6f27591eef6fe52768cb0beda7140be477c8b12c9794a/propcache-0.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:196913dea116aeb5a2ba95af4ddcb7ea85559ae07d8eee8751688310d09168c3", size = 61626, upload-time = "2026-05-08T21:01:37.545Z" }, + { url = "https://files.pythonhosted.org/packages/8e/0d/41c602003e8a9b16fe1e7eadf62c7bfba9d5474370b24200bf48b315f45f/propcache-0.5.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6e7b8719005dd1175be4ab1cd25e9b98659a5e0347331506ec6760d2773a7fb5", size = 64781, upload-time = "2026-05-08T21:01:38.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f3/38e66b1856e9bd079deea015bc4a55f7767c0e4db2f7dcf69e7e680ba4ce/propcache-0.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:51f96d685ab16e88cab128cd37a52c5da540809c8b879fa047731bfcb4ad35a4", size = 62570, upload-time = "2026-05-08T21:01:40.415Z" }, + { url = "https://files.pythonhosted.org/packages/95/ca/bbfe9b910ce57dde8bb4876b4520fc02a4e89497c10de26be936758a3aaa/propcache-0.5.2-cp314-cp314-win32.whl", hash = "sha256:cc6fc3cc62e8501d3ed62894425040d2728ecddb1ed072737a5c70bd537aa9f0", size = 39436, upload-time = "2026-05-08T21:01:41.654Z" }, + { url = "https://files.pythonhosted.org/packages/61/d2/45c9defbaa1ea297035d9d4cce9e8f80daafbf19319c6007f157c6256ea9/propcache-0.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:81e3a30b0bb60caa22033dd0f8a3618d1d67356212514f62c57db75cb0ef410c", size = 42373, upload-time = "2026-05-08T21:01:43.041Z" }, + { url = "https://files.pythonhosted.org/packages/44/68/9ea5103f41d5217d7d6ec24db90018e23aebec070c3f9a6e54d12b841fd8/propcache-0.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:0d2c9bf8528f135dbb805ce027567e09164f7efa51a2be07458a2c0420f292d0", size = 38554, upload-time = "2026-05-08T21:01:44.336Z" }, + { url = "https://files.pythonhosted.org/packages/8a/81/fadf555f42d3b762eea8a53950b0489fdc0aa9da5f8ed9e10ce0a4e01b48/propcache-0.5.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:4bc8ff1feffc6a61c7002ffe84634c41b822e104990ae009f44a0834430070bb", size = 99395, upload-time = "2026-05-08T21:01:45.883Z" }, + { url = "https://files.pythonhosted.org/packages/f5/c9/c61e134a686949cf7971af3a390148b1156f7be81c73bc0cd12c873e2d48/propcache-0.5.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:79aa3ff0a9b566633b642fa9caf7e21ed1c13d6feca718187873f199e1514078", size = 56653, upload-time = "2026-05-08T21:01:47.307Z" }, + { url = "https://files.pythonhosted.org/packages/cb/73/daf935ea7048ddd7ec8eec5345b4a40b619d2d178b3c0a0900796bc3c794/propcache-0.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1b31822f4474c4036bae62de9402710051d431a606d6a0f907fec79935a071aa", size = 56914, upload-time = "2026-05-08T21:01:48.573Z" }, + { url = "https://files.pythonhosted.org/packages/79/9f/aba959b435ea18617edd7cf0a7ad0b9c574b8fc7e3d2cd55fb59cb255d33/propcache-0.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13fef48778b5a2a756523fdb781326b028ca75e32858b04f2cdd19f394564917", size = 62567, upload-time = "2026-05-08T21:01:49.903Z" }, + { url = "https://files.pythonhosted.org/packages/6c/a1/859942de9a791ff42f6141736f5b37749b8f53e65edfa49638c67dd67e6a/propcache-0.5.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8b73ab70f1a3351fbc71f663b3e645af6dd0329100c353081cf69c37433fc6fe", size = 65542, upload-time = "2026-05-08T21:01:51.204Z" }, + { url = "https://files.pythonhosted.org/packages/b5/61/315bc0fd6c0fc7f80a528b8afd209e5fc4a875ea79571b91b8f50f442907/propcache-0.5.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5538d2c13d93e4698af7e092b57bc7298fd35d1d58e656ae18f23ee0d0378e03", size = 66845, upload-time = "2026-05-08T21:01:52.539Z" }, + { url = "https://files.pythonhosted.org/packages/47/f7/9f8122e3132e8e354ac41975ef8f1099be7d5a16bc7ae562734e993665c0/propcache-0.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd645f03898405cabe694fb8bc35241e3a9c332ec85627584fe3de201452b335", size = 63985, upload-time = "2026-05-08T21:01:53.847Z" }, + { url = "https://files.pythonhosted.org/packages/c8/54/c317819ec157cbf6f35df9df9657a6f82daf34d5faf15948b2f639c2192e/propcache-0.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a473b3440261e0c60706e732b2ed2f517857344fc21bf48fdfe211e2d98eb285", size = 63999, upload-time = "2026-05-08T21:01:55.179Z" }, + { url = "https://files.pythonhosted.org/packages/5a/56/387e3f7dfce0a9233df41fb888aa1c30222cb4bbbf09537c02dd9bd85fe2/propcache-0.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7afa37062e6650640e932e4cc9297d81f9f42d9944029cc386b8247dea4da837", size = 62779, upload-time = "2026-05-08T21:01:57.489Z" }, + { url = "https://files.pythonhosted.org/packages/a1/9c/596784cb5824ed61ee960d3f8655a3f0993e107c6e98ab6c818b7fb92ccb/propcache-0.5.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:8a90efd5777e996e42d568db9ac740b944d691e565cbfd31b2f7832f9184b2b8", size = 59796, upload-time = "2026-05-08T21:01:58.736Z" }, + { url = "https://files.pythonhosted.org/packages/c2/3d/1a6cfa1726a48542c1e8784a0761421476a5b68e09b7f36bf95eb954aaba/propcache-0.5.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:f19bb891234d72535764d703bfed1153cc34f4214d5bd7150aee1eec9e8f4366", size = 66023, upload-time = "2026-05-08T21:02:00.228Z" }, + { url = "https://files.pythonhosted.org/packages/e4/0e/05fd6990369477076e4e280bcb970de760fddf0161a46e988bc95f7940ec/propcache-0.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:32775082acd2d807ee3db715c7770d38767b817870acfa08c29e057f3c4d5b56", size = 64448, upload-time = "2026-05-08T21:02:01.888Z" }, + { url = "https://files.pythonhosted.org/packages/cd/86/5f8da315a4309c62c10c0b2516b17492d5d3bbe1bb862b96604db67e2a37/propcache-0.5.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9282fb1a3bccd038da9f768b927b24a0c753e466c086b7c4f3c6982851eefb2d", size = 67329, upload-time = "2026-05-08T21:02:03.484Z" }, + { url = "https://files.pythonhosted.org/packages/da/d3/3368efe79ab21f0cdf86ef49895811c9cc933131d4cde1f28a624e22e712/propcache-0.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cc49723e2f60d6b32a0f0b08a3fd6d13203c07f1cd9566cfce0f12a917c967a2", size = 65172, upload-time = "2026-05-08T21:02:04.745Z" }, + { url = "https://files.pythonhosted.org/packages/d5/07/127e8b0bacfb325396196f9d976a22453049b89b9b2b08477cc3145faa44/propcache-0.5.2-cp314-cp314t-win32.whl", hash = "sha256:2d7aa89ebca5acc98cba9d1472d976e394782f587bad6661003602a619fd1821", size = 43813, upload-time = "2026-05-08T21:02:06.025Z" }, + { url = "https://files.pythonhosted.org/packages/88/fb/46dad6c0ae49ed230ab1b16c890c2b6314e2403e6c412976f4a72d64a527/propcache-0.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d447bb0b3054be5818458fbb171208b1d9ff11eba14e18ca18b90cbb45767370", size = 47764, upload-time = "2026-05-08T21:02:07.353Z" }, + { url = "https://files.pythonhosted.org/packages/e7/c4/a47d0a63aa309d10d59ede6e9d4cff03a344a79d1f0f4cd0cd74997b53e0/propcache-0.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fe67a3d11cd9b4efabfa45c3d00ffba2b26811442a73a581a94b67c2b5faccf6", size = 41140, upload-time = "2026-05-08T21:02:09.065Z" }, + { url = "https://files.pythonhosted.org/packages/3a/ed/1cdcab6ba3d6ab7feca11fc14f0eeea80755bb53ef4e892079f31b10a25f/propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe", size = 14036, upload-time = "2026-05-08T21:02:10.673Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165, upload-time = "2026-04-07T17:16:18.027Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" }, +] + +[[package]] +name = "pytest-asyncio" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/90/2c/8af215c0f776415f3590cac4f9086ccefd6fd463befeae41cd4d3f193e5a/pytest_asyncio-1.3.0.tar.gz", hash = "sha256:d7f52f36d231b80ee124cd216ffb19369aa168fc10095013c6b014a34d3ee9e5", size = 50087, upload-time = "2025-11-10T16:07:47.256Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl", hash = "sha256:611e26147c7f77640e6d0a92a38ed17c3e9848063698d5c93d5aa7aa11cebff5", size = 15075, upload-time = "2025-11-10T16:07:45.537Z" }, +] + +[[package]] +name = "pytest-mock" +version = "3.15.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/68/14/eb014d26be205d38ad5ad20d9a80f7d201472e08167f0bb4361e251084a9/pytest_mock-3.15.1.tar.gz", hash = "sha256:1849a238f6f396da19762269de72cb1814ab44416fa73a8686deac10b0d87a0f", size = 34036, upload-time = "2025-09-16T16:37:27.081Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/cc/06253936f4a7fa2e0f48dfe6d851d9c56df896a9ab09ac019d70b760619c/pytest_mock-3.15.1-py3-none-any.whl", hash = "sha256:0a25e2eb88fe5168d535041d09a4529a188176ae608a6d249ee65abc0949630d", size = 10095, upload-time = "2025-09-16T16:37:25.734Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "python-dotenv" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + +[[package]] +name = "sgmllib3k" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9e/bd/3704a8c3e0942d711c1299ebf7b9091930adae6675d7c8f476a7ce48653c/sgmllib3k-1.0.0.tar.gz", hash = "sha256:7868fb1c8bfa764c1ac563d3cf369c381d1325d36124933a726f29fcdaa812e9", size = 5750, upload-time = "2010-08-24T14:33:52.445Z" } + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "soupsieve" +version = "2.8.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7b/ae/2d9c981590ed9999a0d91755b47fc74f74de286b0f5cee14c9269041e6c4/soupsieve-2.8.3.tar.gz", hash = "sha256:3267f1eeea4251fb42728b6dfb746edc9acaffc4a45b27e19450b676586e8349", size = 118627, upload-time = "2026-01-20T04:27:02.457Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/46/2c/1462b1d0a634697ae9e55b3cecdcb64788e8b7d63f54d923fcd0bb140aed/soupsieve-2.8.3-py3-none-any.whl", hash = "sha256:ed64f2ba4eebeab06cc4962affce381647455978ffc1e36bb79a545b91f45a95", size = 37016, upload-time = "2026-01-20T04:27:01.012Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "urllib3" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, +] + +[[package]] +name = "yarl" +version = "1.23.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "multidict" }, + { name = "propcache" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/23/6e/beb1beec874a72f23815c1434518bfc4ed2175065173fb138c3705f658d4/yarl-1.23.0.tar.gz", hash = "sha256:53b1ea6ca88ebd4420379c330aea57e258408dd0df9af0992e5de2078dc9f5d5", size = 194676, upload-time = "2026-03-01T22:07:53.373Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/8a/94615bc31022f711add374097ad4144d569e95ff3c38d39215d07ac153a0/yarl-1.23.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1932b6b8bba8d0160a9d1078aae5838a66039e8832d41d2992daa9a3a08f7860", size = 124737, upload-time = "2026-03-01T22:05:12.897Z" }, + { url = "https://files.pythonhosted.org/packages/e3/6f/c6554045d59d64052698add01226bc867b52fe4a12373415d7991fdca95d/yarl-1.23.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:411225bae281f114067578891bc75534cfb3d92a3b4dfef7a6ca78ba354e6069", size = 87029, upload-time = "2026-03-01T22:05:14.376Z" }, + { url = "https://files.pythonhosted.org/packages/19/2a/725ecc166d53438bc88f76822ed4b1e3b10756e790bafd7b523fe97c322d/yarl-1.23.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:13a563739ae600a631c36ce096615fe307f131344588b0bc0daec108cdb47b25", size = 86310, upload-time = "2026-03-01T22:05:15.71Z" }, + { url = "https://files.pythonhosted.org/packages/99/30/58260ed98e6ff7f90ba84442c1ddd758c9170d70327394a6227b310cd60f/yarl-1.23.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cbf44c5cb4a7633d078788e1b56387e3d3cf2b8139a3be38040b22d6c3221c8", size = 97587, upload-time = "2026-03-01T22:05:17.384Z" }, + { url = "https://files.pythonhosted.org/packages/76/0a/8b08aac08b50682e65759f7f8dde98ae8168f72487e7357a5d684c581ef9/yarl-1.23.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:53ad387048f6f09a8969631e4de3f1bf70c50e93545d64af4f751b2498755072", size = 92528, upload-time = "2026-03-01T22:05:18.804Z" }, + { url = "https://files.pythonhosted.org/packages/52/07/0b7179101fe5f8385ec6c6bb5d0cb9f76bd9fb4a769591ab6fb5cdbfc69a/yarl-1.23.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4a59ba56f340334766f3a4442e0efd0af895fae9e2b204741ef885c446b3a1a8", size = 105339, upload-time = "2026-03-01T22:05:20.235Z" }, + { url = "https://files.pythonhosted.org/packages/d3/8a/36d82869ab5ec829ca8574dfcb92b51286fcfb1e9c7a73659616362dc880/yarl-1.23.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:803a3c3ce4acc62eaf01eaca1208dcf0783025ef27572c3336502b9c232005e7", size = 105061, upload-time = "2026-03-01T22:05:22.268Z" }, + { url = "https://files.pythonhosted.org/packages/66/3e/868e5c3364b6cee19ff3e1a122194fa4ce51def02c61023970442162859e/yarl-1.23.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a3d2bff8f37f8d0f96c7ec554d16945050d54462d6e95414babaa18bfafc7f51", size = 100132, upload-time = "2026-03-01T22:05:23.638Z" }, + { url = "https://files.pythonhosted.org/packages/cf/26/9c89acf82f08a52cb52d6d39454f8d18af15f9d386a23795389d1d423823/yarl-1.23.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c75eb09e8d55bceb4367e83496ff8ef2bc7ea6960efb38e978e8073ea59ecb67", size = 99289, upload-time = "2026-03-01T22:05:25.749Z" }, + { url = "https://files.pythonhosted.org/packages/6f/54/5b0db00d2cb056922356104468019c0a132e89c8d3ab67d8ede9f4483d2a/yarl-1.23.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:877b0738624280e34c55680d6054a307aa94f7d52fa0e3034a9cc6e790871da7", size = 96950, upload-time = "2026-03-01T22:05:27.318Z" }, + { url = "https://files.pythonhosted.org/packages/f6/40/10fa93811fd439341fad7e0718a86aca0de9548023bbb403668d6555acab/yarl-1.23.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:b5405bb8f0e783a988172993cfc627e4d9d00432d6bbac65a923041edacf997d", size = 93960, upload-time = "2026-03-01T22:05:28.738Z" }, + { url = "https://files.pythonhosted.org/packages/bc/d2/8ae2e6cd77d0805f4526e30ec43b6f9a3dfc542d401ac4990d178e4bf0cf/yarl-1.23.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c3a3598a832590c5a3ce56ab5576361b5688c12cb1d39429cf5dba30b510760", size = 104703, upload-time = "2026-03-01T22:05:30.438Z" }, + { url = "https://files.pythonhosted.org/packages/2f/0c/b3ceacf82c3fe21183ce35fa2acf5320af003d52bc1fcf5915077681142e/yarl-1.23.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:8419ebd326430d1cbb7efb5292330a2cf39114e82df5cc3d83c9a0d5ebeaf2f2", size = 98325, upload-time = "2026-03-01T22:05:31.835Z" }, + { url = "https://files.pythonhosted.org/packages/9d/e0/12900edd28bdab91a69bd2554b85ad7b151f64e8b521fe16f9ad2f56477a/yarl-1.23.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:be61f6fff406ca40e3b1d84716fde398fc08bc63dd96d15f3a14230a0973ed86", size = 105067, upload-time = "2026-03-01T22:05:33.358Z" }, + { url = "https://files.pythonhosted.org/packages/15/61/74bb1182cf79c9bbe4eb6b1f14a57a22d7a0be5e9cedf8e2d5c2086474c3/yarl-1.23.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3ceb13c5c858d01321b5d9bb65e4cf37a92169ea470b70fec6f236b2c9dd7e34", size = 100285, upload-time = "2026-03-01T22:05:35.4Z" }, + { url = "https://files.pythonhosted.org/packages/69/7f/cd5ef733f2550de6241bd8bd8c3febc78158b9d75f197d9c7baa113436af/yarl-1.23.0-cp312-cp312-win32.whl", hash = "sha256:fffc45637bcd6538de8b85f51e3df3223e4ad89bccbfca0481c08c7fc8b7ed7d", size = 82359, upload-time = "2026-03-01T22:05:36.811Z" }, + { url = "https://files.pythonhosted.org/packages/f5/be/25216a49daeeb7af2bec0db22d5e7df08ed1d7c9f65d78b14f3b74fd72fc/yarl-1.23.0-cp312-cp312-win_amd64.whl", hash = "sha256:f69f57305656a4852f2a7203efc661d8c042e6cc67f7acd97d8667fb448a426e", size = 87674, upload-time = "2026-03-01T22:05:38.171Z" }, + { url = "https://files.pythonhosted.org/packages/d2/35/aeab955d6c425b227d5b7247eafb24f2653fedc32f95373a001af5dfeb9e/yarl-1.23.0-cp312-cp312-win_arm64.whl", hash = "sha256:6e87a6e8735b44816e7db0b2fbc9686932df473c826b0d9743148432e10bb9b9", size = 81879, upload-time = "2026-03-01T22:05:40.006Z" }, + { url = "https://files.pythonhosted.org/packages/9a/4b/a0a6e5d0ee8a2f3a373ddef8a4097d74ac901ac363eea1440464ccbe0898/yarl-1.23.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:16c6994ac35c3e74fb0ae93323bf8b9c2a9088d55946109489667c510a7d010e", size = 123796, upload-time = "2026-03-01T22:05:41.412Z" }, + { url = "https://files.pythonhosted.org/packages/67/b6/8925d68af039b835ae876db5838e82e76ec87b9782ecc97e192b809c4831/yarl-1.23.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4a42e651629dafb64fd5b0286a3580613702b5809ad3f24934ea87595804f2c5", size = 86547, upload-time = "2026-03-01T22:05:42.841Z" }, + { url = "https://files.pythonhosted.org/packages/ae/50/06d511cc4b8e0360d3c94af051a768e84b755c5eb031b12adaaab6dec6e5/yarl-1.23.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7c6b9461a2a8b47c65eef63bb1c76a4f1c119618ffa99ea79bc5bb1e46c5821b", size = 85854, upload-time = "2026-03-01T22:05:44.85Z" }, + { url = "https://files.pythonhosted.org/packages/c4/f4/4e30b250927ffdab4db70da08b9b8d2194d7c7b400167b8fbeca1e4701ca/yarl-1.23.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2569b67d616eab450d262ca7cb9f9e19d2f718c70a8b88712859359d0ab17035", size = 98351, upload-time = "2026-03-01T22:05:46.836Z" }, + { url = "https://files.pythonhosted.org/packages/86/fc/4118c5671ea948208bdb1492d8b76bdf1453d3e73df051f939f563e7dcc5/yarl-1.23.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e9d9a4d06d3481eab79803beb4d9bd6f6a8e781ec078ac70d7ef2dcc29d1bea5", size = 92711, upload-time = "2026-03-01T22:05:48.316Z" }, + { url = "https://files.pythonhosted.org/packages/56/11/1ed91d42bd9e73c13dc9e7eb0dd92298d75e7ac4dd7f046ad0c472e231cd/yarl-1.23.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f514f6474e04179d3d33175ed3f3e31434d3130d42ec153540d5b157deefd735", size = 106014, upload-time = "2026-03-01T22:05:50.028Z" }, + { url = "https://files.pythonhosted.org/packages/ce/c9/74e44e056a23fbc33aca71779ef450ca648a5bc472bdad7a82339918f818/yarl-1.23.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fda207c815b253e34f7e1909840fd14299567b1c0eb4908f8c2ce01a41265401", size = 105557, upload-time = "2026-03-01T22:05:51.416Z" }, + { url = "https://files.pythonhosted.org/packages/66/fe/b1e10b08d287f518994f1e2ff9b6d26f0adeecd8dd7d533b01bab29a3eda/yarl-1.23.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:34b6cf500e61c90f305094911f9acc9c86da1a05a7a3f5be9f68817043f486e4", size = 101559, upload-time = "2026-03-01T22:05:52.872Z" }, + { url = "https://files.pythonhosted.org/packages/72/59/c5b8d94b14e3d3c2a9c20cb100119fd534ab5a14b93673ab4cc4a4141ea5/yarl-1.23.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d7504f2b476d21653e4d143f44a175f7f751cd41233525312696c76aa3dbb23f", size = 100502, upload-time = "2026-03-01T22:05:54.954Z" }, + { url = "https://files.pythonhosted.org/packages/77/4f/96976cb54cbfc5c9fd73ed4c51804f92f209481d1fb190981c0f8a07a1d7/yarl-1.23.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:578110dd426f0d209d1509244e6d4a3f1a3e9077655d98c5f22583d63252a08a", size = 98027, upload-time = "2026-03-01T22:05:56.409Z" }, + { url = "https://files.pythonhosted.org/packages/63/6e/904c4f476471afdbad6b7e5b70362fb5810e35cd7466529a97322b6f5556/yarl-1.23.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:609d3614d78d74ebe35f54953c5bbd2ac647a7ddb9c30a5d877580f5e86b22f2", size = 95369, upload-time = "2026-03-01T22:05:58.141Z" }, + { url = "https://files.pythonhosted.org/packages/9d/40/acfcdb3b5f9d68ef499e39e04d25e141fe90661f9d54114556cf83be8353/yarl-1.23.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4966242ec68afc74c122f8459abd597afd7d8a60dc93d695c1334c5fd25f762f", size = 105565, upload-time = "2026-03-01T22:06:00.286Z" }, + { url = "https://files.pythonhosted.org/packages/5e/c6/31e28f3a6ba2869c43d124f37ea5260cac9c9281df803c354b31f4dd1f3c/yarl-1.23.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e0fd068364a6759bc794459f0a735ab151d11304346332489c7972bacbe9e72b", size = 99813, upload-time = "2026-03-01T22:06:01.712Z" }, + { url = "https://files.pythonhosted.org/packages/08/1f/6f65f59e72d54aa467119b63fc0b0b1762eff0232db1f4720cd89e2f4a17/yarl-1.23.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:39004f0ad156da43e86aa71f44e033de68a44e5a31fc53507b36dd253970054a", size = 105632, upload-time = "2026-03-01T22:06:03.188Z" }, + { url = "https://files.pythonhosted.org/packages/a3/c4/18b178a69935f9e7a338127d5b77d868fdc0f0e49becd286d51b3a18c61d/yarl-1.23.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e5723c01a56c5028c807c701aa66722916d2747ad737a046853f6c46f4875543", size = 101895, upload-time = "2026-03-01T22:06:04.651Z" }, + { url = "https://files.pythonhosted.org/packages/8f/54/f5b870b5505663911dba950a8e4776a0dbd51c9c54c0ae88e823e4b874a0/yarl-1.23.0-cp313-cp313-win32.whl", hash = "sha256:1b6b572edd95b4fa8df75de10b04bc81acc87c1c7d16bcdd2035b09d30acc957", size = 82356, upload-time = "2026-03-01T22:06:06.04Z" }, + { url = "https://files.pythonhosted.org/packages/7a/84/266e8da36879c6edcd37b02b547e2d9ecdfea776be49598e75696e3316e1/yarl-1.23.0-cp313-cp313-win_amd64.whl", hash = "sha256:baaf55442359053c7d62f6f8413a62adba3205119bcb6f49594894d8be47e5e3", size = 87515, upload-time = "2026-03-01T22:06:08.107Z" }, + { url = "https://files.pythonhosted.org/packages/00/fd/7e1c66efad35e1649114fa13f17485f62881ad58edeeb7f49f8c5e748bf9/yarl-1.23.0-cp313-cp313-win_arm64.whl", hash = "sha256:fb4948814a2a98e3912505f09c9e7493b1506226afb1f881825368d6fb776ee3", size = 81785, upload-time = "2026-03-01T22:06:10.181Z" }, + { url = "https://files.pythonhosted.org/packages/9c/fc/119dd07004f17ea43bb91e3ece6587759edd7519d6b086d16bfbd3319982/yarl-1.23.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:aecfed0b41aa72b7881712c65cf764e39ce2ec352324f5e0837c7048d9e6daaa", size = 130719, upload-time = "2026-03-01T22:06:11.708Z" }, + { url = "https://files.pythonhosted.org/packages/e6/0d/9f2348502fbb3af409e8f47730282cd6bc80dec6630c1e06374d882d6eb2/yarl-1.23.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:a41bcf68efd19073376eb8cf948b8d9be0af26256403e512bb18f3966f1f9120", size = 89690, upload-time = "2026-03-01T22:06:13.429Z" }, + { url = "https://files.pythonhosted.org/packages/50/93/e88f3c80971b42cfc83f50a51b9d165a1dbf154b97005f2994a79f212a07/yarl-1.23.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:cde9a2ecd91668bcb7f077c4966d8ceddb60af01b52e6e3e2680e4cf00ad1a59", size = 89851, upload-time = "2026-03-01T22:06:15.53Z" }, + { url = "https://files.pythonhosted.org/packages/1c/07/61c9dd8ba8f86473263b4036f70fb594c09e99c0d9737a799dfd8bc85651/yarl-1.23.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5023346c4ee7992febc0068e7593de5fa2bf611848c08404b35ebbb76b1b0512", size = 95874, upload-time = "2026-03-01T22:06:17.553Z" }, + { url = "https://files.pythonhosted.org/packages/9e/e9/f9ff8ceefba599eac6abddcfb0b3bee9b9e636e96dbf54342a8577252379/yarl-1.23.0-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d1009abedb49ae95b136a8904a3f71b342f849ffeced2d3747bf29caeda218c4", size = 88710, upload-time = "2026-03-01T22:06:19.004Z" }, + { url = "https://files.pythonhosted.org/packages/eb/78/0231bfcc5d4c8eec220bc2f9ef82cb4566192ea867a7c5b4148f44f6cbcd/yarl-1.23.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a8d00f29b42f534cc8aa3931cfe773b13b23e561e10d2b26f27a8d309b0e82a1", size = 101033, upload-time = "2026-03-01T22:06:21.203Z" }, + { url = "https://files.pythonhosted.org/packages/cd/9b/30ea5239a61786f18fd25797151a17fbb3be176977187a48d541b5447dd4/yarl-1.23.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:95451e6ce06c3e104556d73b559f5da6c34a069b6b62946d3ad66afcd51642ea", size = 100817, upload-time = "2026-03-01T22:06:22.738Z" }, + { url = "https://files.pythonhosted.org/packages/62/e2/a4980481071791bc83bce2b7a1a1f7adcabfa366007518b4b845e92eeee3/yarl-1.23.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:531ef597132086b6cf96faa7c6c1dcd0361dd5f1694e5cc30375907b9b7d3ea9", size = 97482, upload-time = "2026-03-01T22:06:24.21Z" }, + { url = "https://files.pythonhosted.org/packages/e5/1e/304a00cf5f6100414c4b5a01fc7ff9ee724b62158a08df2f8170dfc72a2d/yarl-1.23.0-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:88f9fb0116fbfcefcab70f85cf4b74a2b6ce5d199c41345296f49d974ddb4123", size = 95949, upload-time = "2026-03-01T22:06:25.697Z" }, + { url = "https://files.pythonhosted.org/packages/68/03/093f4055ed4cae649ac53bca3d180bd37102e9e11d048588e9ab0c0108d0/yarl-1.23.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:e7b0460976dc75cb87ad9cc1f9899a4b97751e7d4e77ab840fc9b6d377b8fd24", size = 95839, upload-time = "2026-03-01T22:06:27.309Z" }, + { url = "https://files.pythonhosted.org/packages/b9/28/4c75ebb108f322aa8f917ae10a8ffa4f07cae10a8a627b64e578617df6a0/yarl-1.23.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:115136c4a426f9da976187d238e84139ff6b51a20839aa6e3720cd1026d768de", size = 90696, upload-time = "2026-03-01T22:06:29.048Z" }, + { url = "https://files.pythonhosted.org/packages/23/9c/42c2e2dd91c1a570402f51bdf066bfdb1241c2240ba001967bad778e77b7/yarl-1.23.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:ead11956716a940c1abc816b7df3fa2b84d06eaed8832ca32f5c5e058c65506b", size = 100865, upload-time = "2026-03-01T22:06:30.525Z" }, + { url = "https://files.pythonhosted.org/packages/74/05/1bcd60a8a0a914d462c305137246b6f9d167628d73568505fce3f1cb2e65/yarl-1.23.0-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:fe8f8f5e70e6dbdfca9882cd9deaac058729bcf323cf7a58660901e55c9c94f6", size = 96234, upload-time = "2026-03-01T22:06:32.692Z" }, + { url = "https://files.pythonhosted.org/packages/90/b2/f52381aac396d6778ce516b7bc149c79e65bfc068b5de2857ab69eeea3b7/yarl-1.23.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:a0e317df055958a0c1e79e5d2aa5a5eaa4a6d05a20d4b0c9c3f48918139c9fc6", size = 100295, upload-time = "2026-03-01T22:06:34.268Z" }, + { url = "https://files.pythonhosted.org/packages/e5/e8/638bae5bbf1113a659b2435d8895474598afe38b4a837103764f603aba56/yarl-1.23.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6f0fd84de0c957b2d280143522c4f91a73aada1923caee763e24a2b3fda9f8a5", size = 97784, upload-time = "2026-03-01T22:06:35.864Z" }, + { url = "https://files.pythonhosted.org/packages/80/25/a3892b46182c586c202629fc2159aa13975d3741d52ebd7347fd501d48d5/yarl-1.23.0-cp313-cp313t-win32.whl", hash = "sha256:93a784271881035ab4406a172edb0faecb6e7d00f4b53dc2f55919d6c9688595", size = 88313, upload-time = "2026-03-01T22:06:37.39Z" }, + { url = "https://files.pythonhosted.org/packages/43/68/8c5b36aa5178900b37387937bc2c2fe0e9505537f713495472dcf6f6fccc/yarl-1.23.0-cp313-cp313t-win_amd64.whl", hash = "sha256:dd00607bffbf30250fe108065f07453ec124dbf223420f57f5e749b04295e090", size = 94932, upload-time = "2026-03-01T22:06:39.579Z" }, + { url = "https://files.pythonhosted.org/packages/c6/cc/d79ba8292f51f81f4dc533a8ccfb9fc6992cabf0998ed3245de7589dc07c/yarl-1.23.0-cp313-cp313t-win_arm64.whl", hash = "sha256:ac09d42f48f80c9ee1635b2fcaa819496a44502737660d3c0f2ade7526d29144", size = 84786, upload-time = "2026-03-01T22:06:41.988Z" }, + { url = "https://files.pythonhosted.org/packages/90/98/b85a038d65d1b92c3903ab89444f48d3cee490a883477b716d7a24b1a78c/yarl-1.23.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:21d1b7305a71a15b4794b5ff22e8eef96ff4a6d7f9657155e5aa419444b28912", size = 124455, upload-time = "2026-03-01T22:06:43.615Z" }, + { url = "https://files.pythonhosted.org/packages/39/54/bc2b45559f86543d163b6e294417a107bb87557609007c007ad889afec18/yarl-1.23.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:85610b4f27f69984932a7abbe52703688de3724d9f72bceb1cca667deff27474", size = 86752, upload-time = "2026-03-01T22:06:45.425Z" }, + { url = "https://files.pythonhosted.org/packages/24/f9/e8242b68362bffe6fb536c8db5076861466fc780f0f1b479fc4ffbebb128/yarl-1.23.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23f371bd662cf44a7630d4d113101eafc0cfa7518a2760d20760b26021454719", size = 86291, upload-time = "2026-03-01T22:06:46.974Z" }, + { url = "https://files.pythonhosted.org/packages/ea/d8/d1cb2378c81dd729e98c716582b1ccb08357e8488e4c24714658cc6630e8/yarl-1.23.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c4a80f77dc1acaaa61f0934176fccca7096d9b1ff08c8ba9cddf5ae034a24319", size = 99026, upload-time = "2026-03-01T22:06:48.459Z" }, + { url = "https://files.pythonhosted.org/packages/0a/ff/7196790538f31debe3341283b5b0707e7feb947620fc5e8236ef28d44f72/yarl-1.23.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:bd654fad46d8d9e823afbb4f87c79160b5a374ed1ff5bde24e542e6ba8f41434", size = 92355, upload-time = "2026-03-01T22:06:50.306Z" }, + { url = "https://files.pythonhosted.org/packages/c1/56/25d58c3eddde825890a5fe6aa1866228377354a3c39262235234ab5f616b/yarl-1.23.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:682bae25f0a0dd23a056739f23a134db9f52a63e2afd6bfb37ddc76292bbd723", size = 106417, upload-time = "2026-03-01T22:06:52.1Z" }, + { url = "https://files.pythonhosted.org/packages/51/8a/882c0e7bc8277eb895b31bce0138f51a1ba551fc2e1ec6753ffc1e7c1377/yarl-1.23.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a82836cab5f197a0514235aaf7ffccdc886ccdaa2324bc0aafdd4ae898103039", size = 106422, upload-time = "2026-03-01T22:06:54.424Z" }, + { url = "https://files.pythonhosted.org/packages/42/2b/fef67d616931055bf3d6764885990a3ac647d68734a2d6a9e1d13de437a2/yarl-1.23.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1c57676bdedc94cd3bc37724cf6f8cd2779f02f6aba48de45feca073e714fe52", size = 101915, upload-time = "2026-03-01T22:06:55.895Z" }, + { url = "https://files.pythonhosted.org/packages/18/6a/530e16aebce27c5937920f3431c628a29a4b6b430fab3fd1c117b26ff3f6/yarl-1.23.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c7f8dc16c498ff06497c015642333219871effba93e4a2e8604a06264aca5c5c", size = 100690, upload-time = "2026-03-01T22:06:58.21Z" }, + { url = "https://files.pythonhosted.org/packages/88/08/93749219179a45e27b036e03260fda05190b911de8e18225c294ac95bbc9/yarl-1.23.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:5ee586fb17ff8f90c91cf73c6108a434b02d69925f44f5f8e0d7f2f260607eae", size = 98750, upload-time = "2026-03-01T22:06:59.794Z" }, + { url = "https://files.pythonhosted.org/packages/d9/cf/ea424a004969f5d81a362110a6ac1496d79efdc6d50c2c4b2e3ea0fc2519/yarl-1.23.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:17235362f580149742739cc3828b80e24029d08cbb9c4bda0242c7b5bc610a8e", size = 94685, upload-time = "2026-03-01T22:07:01.375Z" }, + { url = "https://files.pythonhosted.org/packages/e2/b7/14341481fe568e2b0408bcf1484c652accafe06a0ade9387b5d3fd9df446/yarl-1.23.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:0793e2bd0cf14234983bbb371591e6bea9e876ddf6896cdcc93450996b0b5c85", size = 106009, upload-time = "2026-03-01T22:07:03.151Z" }, + { url = "https://files.pythonhosted.org/packages/0a/e6/5c744a9b54f4e8007ad35bce96fbc9218338e84812d36f3390cea616881a/yarl-1.23.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:3650dc2480f94f7116c364096bc84b1d602f44224ef7d5c7208425915c0475dd", size = 100033, upload-time = "2026-03-01T22:07:04.701Z" }, + { url = "https://files.pythonhosted.org/packages/0c/23/e3bfc188d0b400f025bc49d99793d02c9abe15752138dcc27e4eaf0c4a9e/yarl-1.23.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f40e782d49630ad384db66d4d8b73ff4f1b8955dc12e26b09a3e3af064b3b9d6", size = 106483, upload-time = "2026-03-01T22:07:06.231Z" }, + { url = "https://files.pythonhosted.org/packages/72/42/f0505f949a90b3f8b7a363d6cbdf398f6e6c58946d85c6d3a3bc70595b26/yarl-1.23.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:94f8575fbdf81749008d980c17796097e645574a3b8c28ee313931068dad14fe", size = 102175, upload-time = "2026-03-01T22:07:08.4Z" }, + { url = "https://files.pythonhosted.org/packages/aa/65/b39290f1d892a9dd671d1c722014ca062a9c35d60885d57e5375db0404b5/yarl-1.23.0-cp314-cp314-win32.whl", hash = "sha256:c8aa34a5c864db1087d911a0b902d60d203ea3607d91f615acd3f3108ac32169", size = 83871, upload-time = "2026-03-01T22:07:09.968Z" }, + { url = "https://files.pythonhosted.org/packages/a9/5b/9b92f54c784c26e2a422e55a8d2607ab15b7ea3349e28359282f84f01d43/yarl-1.23.0-cp314-cp314-win_amd64.whl", hash = "sha256:63e92247f383c85ab00dd0091e8c3fa331a96e865459f5ee80353c70a4a42d70", size = 89093, upload-time = "2026-03-01T22:07:11.501Z" }, + { url = "https://files.pythonhosted.org/packages/e0/7d/8a84dc9381fd4412d5e7ff04926f9865f6372b4c2fd91e10092e65d29eb8/yarl-1.23.0-cp314-cp314-win_arm64.whl", hash = "sha256:70efd20be968c76ece7baa8dafe04c5be06abc57f754d6f36f3741f7aa7a208e", size = 83384, upload-time = "2026-03-01T22:07:13.069Z" }, + { url = "https://files.pythonhosted.org/packages/dd/8d/d2fad34b1c08aa161b74394183daa7d800141aaaee207317e82c790b418d/yarl-1.23.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:9a18d6f9359e45722c064c97464ec883eb0e0366d33eda61cb19a244bf222679", size = 131019, upload-time = "2026-03-01T22:07:14.903Z" }, + { url = "https://files.pythonhosted.org/packages/19/ff/33009a39d3ccf4b94d7d7880dfe17fb5816c5a4fe0096d9b56abceea9ac7/yarl-1.23.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:2803ed8b21ca47a43da80a6fd1ed3019d30061f7061daa35ac54f63933409412", size = 89894, upload-time = "2026-03-01T22:07:17.372Z" }, + { url = "https://files.pythonhosted.org/packages/0c/f1/dab7ac5e7306fb79c0190766a3c00b4cb8d09a1f390ded68c85a5934faf5/yarl-1.23.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:394906945aa8b19fc14a61cf69743a868bb8c465efe85eee687109cc540b98f4", size = 89979, upload-time = "2026-03-01T22:07:19.361Z" }, + { url = "https://files.pythonhosted.org/packages/aa/b1/08e95f3caee1fad6e65017b9f26c1d79877b502622d60e517de01e72f95d/yarl-1.23.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:71d006bee8397a4a89f469b8deb22469fe7508132d3c17fa6ed871e79832691c", size = 95943, upload-time = "2026-03-01T22:07:21.266Z" }, + { url = "https://files.pythonhosted.org/packages/c0/cc/6409f9018864a6aa186c61175b977131f373f1988e198e031236916e87e4/yarl-1.23.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:62694e275c93d54f7ccedcfef57d42761b2aad5234b6be1f3e3026cae4001cd4", size = 88786, upload-time = "2026-03-01T22:07:23.129Z" }, + { url = "https://files.pythonhosted.org/packages/76/40/cc22d1d7714b717fde2006fad2ced5efe5580606cb059ae42117542122f3/yarl-1.23.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a31de1613658308efdb21ada98cbc86a97c181aa050ba22a808120bb5be3ab94", size = 101307, upload-time = "2026-03-01T22:07:24.689Z" }, + { url = "https://files.pythonhosted.org/packages/8f/0d/476c38e85ddb4c6ec6b20b815bdd779aa386a013f3d8b85516feee55c8dc/yarl-1.23.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fb1e8b8d66c278b21d13b0a7ca22c41dd757a7c209c6b12c313e445c31dd3b28", size = 100904, upload-time = "2026-03-01T22:07:26.287Z" }, + { url = "https://files.pythonhosted.org/packages/72/32/0abe4a76d59adf2081dcb0397168553ece4616ada1c54d1c49d8936c74f8/yarl-1.23.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:50f9d8d531dfb767c565f348f33dd5139a6c43f5cbdf3f67da40d54241df93f6", size = 97728, upload-time = "2026-03-01T22:07:27.906Z" }, + { url = "https://files.pythonhosted.org/packages/b7/35/7b30f4810fba112f60f5a43237545867504e15b1c7647a785fbaf588fac2/yarl-1.23.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:575aa4405a656e61a540f4a80eaa5260f2a38fff7bfdc4b5f611840d76e9e277", size = 95964, upload-time = "2026-03-01T22:07:30.198Z" }, + { url = "https://files.pythonhosted.org/packages/2d/86/ed7a73ab85ef00e8bb70b0cb5421d8a2a625b81a333941a469a6f4022828/yarl-1.23.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:041b1a4cefacf65840b4e295c6985f334ba83c30607441ae3cf206a0eed1a2e4", size = 95882, upload-time = "2026-03-01T22:07:32.132Z" }, + { url = "https://files.pythonhosted.org/packages/19/90/d56967f61a29d8498efb7afb651e0b2b422a1e9b47b0ab5f4e40a19b699b/yarl-1.23.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:d38c1e8231722c4ce40d7593f28d92b5fc72f3e9774fe73d7e800ec32299f63a", size = 90797, upload-time = "2026-03-01T22:07:34.404Z" }, + { url = "https://files.pythonhosted.org/packages/72/00/8b8f76909259f56647adb1011d7ed8b321bcf97e464515c65016a47ecdf0/yarl-1.23.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:d53834e23c015ee83a99377db6e5e37d8484f333edb03bd15b4bc312cc7254fb", size = 101023, upload-time = "2026-03-01T22:07:35.953Z" }, + { url = "https://files.pythonhosted.org/packages/ac/e2/cab11b126fb7d440281b7df8e9ddbe4851e70a4dde47a202b6642586b8d9/yarl-1.23.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:2e27c8841126e017dd2a054a95771569e6070b9ee1b133366d8b31beb5018a41", size = 96227, upload-time = "2026-03-01T22:07:37.594Z" }, + { url = "https://files.pythonhosted.org/packages/c2/9b/2c893e16bfc50e6b2edf76c1a9eb6cb0c744346197e74c65e99ad8d634d0/yarl-1.23.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:76855800ac56f878847a09ce6dba727c93ca2d89c9e9d63002d26b916810b0a2", size = 100302, upload-time = "2026-03-01T22:07:39.334Z" }, + { url = "https://files.pythonhosted.org/packages/28/ec/5498c4e3a6d5f1003beb23405671c2eb9cdbf3067d1c80f15eeafe301010/yarl-1.23.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e09fd068c2e169a7070d83d3bde728a4d48de0549f975290be3c108c02e499b4", size = 98202, upload-time = "2026-03-01T22:07:41.717Z" }, + { url = "https://files.pythonhosted.org/packages/fe/c3/cd737e2d45e70717907f83e146f6949f20cc23cd4bf7b2688727763aa458/yarl-1.23.0-cp314-cp314t-win32.whl", hash = "sha256:73309162a6a571d4cbd3b6a1dcc703c7311843ae0d1578df6f09be4e98df38d4", size = 90558, upload-time = "2026-03-01T22:07:43.433Z" }, + { url = "https://files.pythonhosted.org/packages/e1/19/3774d162f6732d1cfb0b47b4140a942a35ca82bb19b6db1f80e9e7bdc8f8/yarl-1.23.0-cp314-cp314t-win_amd64.whl", hash = "sha256:4503053d296bc6e4cbd1fad61cf3b6e33b939886c4f249ba7c78b602214fabe2", size = 97610, upload-time = "2026-03-01T22:07:45.773Z" }, + { url = "https://files.pythonhosted.org/packages/51/47/3fa2286c3cb162c71cdb34c4224d5745a1ceceb391b2bd9b19b668a8d724/yarl-1.23.0-cp314-cp314t-win_arm64.whl", hash = "sha256:44bb7bef4ea409384e3f8bc36c063d77ea1b8d4a5b2706956c0d6695f07dcc25", size = 86041, upload-time = "2026-03-01T22:07:49.026Z" }, + { url = "https://files.pythonhosted.org/packages/69/68/c8739671f5699c7dc470580a4f821ef37c32c4cb0b047ce223a7f115757f/yarl-1.23.0-py3-none-any.whl", hash = "sha256:a2df6afe50dea8ae15fa34c9f824a3ee958d785fd5d089063d960bae1daa0a3f", size = 48288, upload-time = "2026-03-01T22:07:51.388Z" }, +] diff --git a/config/.env.example b/config/.env.example new file mode 100644 index 0000000..3cb59e7 --- /dev/null +++ b/config/.env.example @@ -0,0 +1,17 @@ +# AI 配置 +OPENAI_API_KEY=your_api_key_here +OPENAI_API_BASE=https://api.openai.com/v1 +OPENAI_MODEL=gpt-4o-mini +WEBZINE_MODEL=deepseek-v3.2 + +# 系统配置 +TIME_ZONE=Asia/Shanghai +LOG_LEVEL=INFO + +# RSS代理（微信公众号转RSS服务，默认使用 werss.yynnice.top） +RSS_PROXY_BASE=https://werss.yynnice.top + +# 推送配置 +WECHAT_TARGET=your_wechat_target_here +WECHAT_ACCOUNT=your_wechat_account_here +FEISHU_WEBHOOK=your_feishu_webhook_here diff --git a/config/rss_feeds.txt b/config/rss_feeds.txt new file mode 100644 index 0000000..bd8a1c0 --- /dev/null +++ b/config/rss_feeds.txt @@ -0,0 +1,20 @@ +# 军事科技RSS订阅源列表 +# 格式：<源名称>| +# 以#开头的行是注释，会自动忽略 +# 空行也会自动跳过 + +# 英文军事源 +Defense One|https://www.defenseone.com/rss/all/ +#Seapower|https://seapowermagazine.org/feed/ +The War Zone|https://www.twz.com/feed + +# 中文微信公众号源 +NEWUAS|https://werss.yynnice.top/feed/MP_WXS_3246130840.rss +国防科技要闻|https://werss.yynnice.top/feed/MP_WXS_3921202880.rss +战略前沿技术|https://werss.yynnice.top/feed/MP_WXS_3271903347.rss +#无人机邦|https://werss.yynnice.top/feed/MP_WXS_3957361285.rss +#浮空飞行器|https://werss.yynnice.top/feed/MP_WXS_3291643707.rss +#海鹰资讯|https://werss.yynnice.top/feed/MP_WXS_3091381580.rss +#渊亭防务|https://werss.yynnice.top/feed/MP_WXS_3865628826.rss +#电波之矛|https://werss.yynnice.top/feed/MP_WXS_3004998095.rss +#龙牙的一座山|https://werss.yynnice.top/feed/MP_WXS_3517933009.rss diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..53dc2d6 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,50 @@ +version: '3.8' + +services: + military-digest: + build: . + container_name: military-digest-v3 + volumes: + - ./output:/app/output + - ./logs:/app/logs + - ./config:/app/config:ro + environment: + - TZ=Asia/Shanghai + - OPENAI_API_KEY=${OPENAI_API_KEY} + - OPENAI_API_BASE=${OPENAI_API_BASE} + - OPENAI_MODEL=${OPENAI_MODEL} + - WEBZINE_MODEL=${WEBZINE_MODEL} + - RSS_PROXY_BASE=${RSS_PROXY_BASE} + - WECHAT_ACCOUNT=${WECHAT_ACCOUNT} + - WECHAT_TARGET=${WECHAT_TARGET} + - FEISHU_WEBHOOK=${FEISHU_WEBHOOK} + restart: "no" + profiles: + - manual + + # 每日早8点定时执行 + military-digest-cron: + build: . + container_name: military-digest-cron + volumes: + - ./output:/app/output + - ./logs:/app/logs + - ./config:/app/config:ro + environment: + - TZ=Asia/Shanghai + - OPENAI_API_KEY=${OPENAI_API_KEY} + - OPENAI_API_BASE=${OPENAI_API_BASE} + - OPENAI_MODEL=${OPENAI_MODEL} + - WEBZINE_MODEL=${WEBZINE_MODEL} + - RSS_PROXY_BASE=${RSS_PROXY_BASE} + - WECHAT_ACCOUNT=${WECHAT_ACCOUNT} + - WECHAT_TARGET=${WECHAT_TARGET} + - FEISHU_WEBHOOK=${FEISHU_WEBHOOK} + restart: unless-stopped + entrypoint: | + bash -c " + echo '0 8 * * * cd /app && python scripts/military_daily_report_v3.py >> /app/logs/cron.log 2>&1' > /etc/crontabs/root + crond -f -l 2 + " + profiles: + - cron diff --git a/my-daily/.dockerignore b/my-daily/.dockerignore new file mode 100644 index 0000000..c1e372d --- /dev/null +++ b/my-daily/.dockerignore @@ -0,0 +1,56 @@ +# 配置文件（通过 volume 挂载，不打包进镜像） +.env +config.json + +# Git +.git/ +.gitignore + +# Python +__pycache__/ +*.pyc +*.pyo +.venv/ +venv/ + +# 运行时数据（通过 volume 挂载） +data/ +logs/ +docker/ + +# 文档（仅排除根目录下的） +/CONFIG.md +/CHANGELOG.md +/PROGRESS.md +/README.md +/QUICKSTART.md +/docs/ +# 其他散落的 md 文件 +/20260529_*.md +/1.md +/01*.md +/02*.md + +# 测试脚本 +scripts/test_*.py +scripts/debug_*.py +scripts/diagnose_*.py +scripts/audit_*.py +scripts/demo_*.py +scripts/gen_test_*.py +scripts/verify_*.py +scripts/analyze_*.py +scripts/sort_opml.py + +# 系统部署（仅 Linux/Windows 原生使用） +systemd/ +scripts/*.sh +scripts/*.bat +scripts/install.sh +scripts/uninstall*.sh +scripts/status.sh +scripts/setup_*.sh + +# 临时文件 +*.txt +!requirements.txt \ No newline at end of file diff --git a/my-daily/.env.example b/my-daily/.env.example new file mode 100644 index 0000000..b16aa9f --- /dev/null +++ b/my-daily/.env.example @@ -0,0 +1,28 @@ +# AI 配置（OpenAI 兼容接口，支持 deepseek / 火山引擎 等） +OPENAI_API_KEY=your_api_key_here +OPENAI_API_BASE=https://api.openai.com/v1 +OPENAI_MODEL=gpt-4o-mini + +# 推送配置 +FEISHU_WEBHOOK=your_feishu_webhook_here + +# 飞书图片推送配置（如需直接发送图片需配置以下两项） +FEISHU_APP_ID=your_feishu_app_id_here +FEISHU_APP_SECRET=your_feishu_app_secret_here + +# 图床上传配置 +IMAGE_HOSTING_PASSWORD=your_image_hosting_password_here + +# ── 定时计划配置（取消注释即生效，优先级最高）── +# FETCH_INTERVAL_MINUTES=30 # RSS 拉取间隔（分钟） +# FETCH_LOOKBACK_MINUTES=60 # 回溯时间窗口（分钟） +# PUSH_CRON="0 8 * * *" # 推送定时计划（cron 表达式，多个用逗号分隔） +# TIMEZONE_HOURS=8 # 时区（相对于 UTC 的小时数） + +# ── 评价管线参数覆盖（取消注释即生效，优先级最高）── +# HEAT_THRESHOLD=30 # 热度阈值固定值（0-100） +# HOT_THRESHOLD=90 # 即时推送分数线 +# CLUSTER_MERGE_THRESHOLD=4 # 聚类合并阈值 +# PRECISE_BLEND_RATIO=0.4 # LLM 精评融合权重（精评分占比） +# MEMORY_WINDOW_HOURS=24 # 簇记忆去重时间窗口 +# CROSS_DAY_MIN_OVERLAP=0.05 # 跨日关联实体重叠阈值 diff --git a/my-daily/.gitignore b/my-daily/.gitignore new file mode 100644 index 0000000..57aa321 --- /dev/null +++ b/my-daily/.gitignore @@ -0,0 +1,9 @@ +.env +config.json +data/ +output/ +__pycache__/ +*.pyc +.venv/ +venv/ +*.pickle diff --git a/my-daily/001.txt b/my-daily/001.txt new file mode 100644 index 0000000..bccbac0 --- /dev/null +++ b/my-daily/001.txt @@ -0,0 +1,1443 @@ +================================================== +🔄 Fetch Job | 2026-07-01 06:35:13 +================================================== + 📂 共 131 个订阅源 | 时间窗口: 60 分钟 + 📡 The War Zone: 0 条 (总 33 条) + 📡 Defense One: 0 条 (总 25 条) + 📡 Seapower: 0 条 (总 10 条) + 📡 90号茶室: 0 条 (总 30 条) + 📡 安全圈: 0 条 (总 30 条) + 📡 NEWUAS: 0 条 (总 30 条) + 📡 兵路文化: 0 条 (总 11 条) + 📡 白马 V 视角: 0 条 (总 30 条) + 📡 补三刀: 0 条 (总 24 条) + 📡 半导体行业观察: 0 条 (总 30 条) + 📡 艾瑞咨询: 0 条 (总 30 条) + 📡 北大纵横: 0 条 (总 30 条) + 📡 安全内参: 0 条 (总 30 条) + 📡 IPP评论: 0 条 (总 30 条) + 📡 参考消息: 0 条 (总 30 条) + 📡 补壹刀: 0 条 (总 30 条) + 📡 参考消息智库: 0 条 (总 30 条) + 📡 纯科学: 0 条 (总 28 条) + 📡 大浪淘沙: 0 条 (总 30 条) + 📡 从心不逾矩的防务菌: 0 条 (总 30 条) + 📡 大柳树防务: 0 条 (总 30 条) + 📡 电波之矛: 0 条 (总 30 条) + 📡 大湾区评论: 0 条 (总 30 条) + 📡 动态大参考: 0 条 (总 30 条) + 📡 东南亚问题研究: 0 条 (总 30 条) + 📡 防务快讯: 0 条 (总 30 条) + 📡 防务指南: 0 条 (总 16 条) + 📡 浮空飞行器: 0 条 (总 30 条) + 📡 复旦大学中国研究院: 0 条 (总 30 条) + 📡 阜成门六号院: 0 条 (总 25 条) + 📡 凤凰网军事频道: 0 条 (总 30 条) + 📡 观察者网: 0 条 (总 30 条) + 📡 光明军事: 0 条 (总 30 条) + 📡 国观智库: 0 条 (总 30 条) + 📡 国防科技要闻: 0 条 (总 30 条) + 📡 国防时报排头兵: 0 条 (总 30 条) + 📡 国科环宇: 0 条 (总 18 条) + 📡 国防工业出版社: 0 条 (总 30 条) + 📡 国际法务: 0 条 (总 30 条) + 📡 海洋防务前沿: 0 条 (总 30 条) + 📡 航小宇: 0 条 (总 30 条) + 📡 环时枢密院十号: 0 条 (总 30 条) + 📡 黑天鹅商业情报站: 0 条 (总 30 条) + 📡 环太国际战略: 0 条 (总 23 条) + 📡 海鹰资讯: 0 条 (总 30 条) + 📡 华山穹剑: 0 条 (总 30 条) + 📡 华语智库: 0 条 (总 30 条) + 📡 混沌巡洋舰: 0 条 (总 30 条) + 📡 胡锡进观察: 0 条 (总 30 条) + 📡 解放军报: 0 条 (总 30 条) + 📡 今日台湾: 0 条 (总 30 条) + 📡 精确打击洞见: 0 条 (总 30 条) + 📡 军事高科技在线: 0 条 (总 30 条) + 📡 军事文摘: 0 条 (总 30 条) + 📡 军民融合观察: 0 条 (总 30 条) + 📡 军鹰动态: 0 条 (总 30 条) + 📡 科技导报: 0 条 (总 30 条) + 📡 跨境前沿: 0 条 (总 30 条) + 📡 空天大视野: 0 条 (总 30 条) + 📡 军武次位面: 0 条 (总 30 条) + 📡 空天界: 0 条 (总 30 条) + 📡 六爷阿旦: 2 条 (总 30 条) + 📡 砺剑: 0 条 (总 30 条) + 📡 李光满说: 0 条 (总 30 条) + 📡 蓝血研究: 0 条 (总 30 条) + 📡 瞭望智库: 0 条 (总 30 条) + 📡 龙牙的一座山: 0 条 (总 30 条) + 📡 米尔观天下: 0 条 (总 30 条) + 📡 迷彩虎观察: 0 条 (总 30 条) + 📡 美国驻华大使馆: 0 条 (总 30 条) + 📡 梅特涅的信徒: 0 条 (总 30 条) + 📡 明叔杂谈: 0 条 (总 30 条) + 📡 澎湃新闻: 0 条 (总 30 条) + 📡 前沿深度解码: 0 条 (总 30 条) + 📡 秦安战略: 0 条 (总 30 条) + 📡 破圈了: 0 条 (总 30 条) + 📡 奇安网情局: 0 条 (总 30 条) + 📡 欧亚新观察: 0 条 (总 30 条) + 📡 盘古智库: 0 条 (总 30 条) + 📡 全球技术地图: 0 条 (总 30 条) + 📡 牛弹琴: 0 条 (总 30 条) + 📡 人大重阳: 0 条 (总 30 条) + 📡 人民日报: 0 条 (总 30 条) + 📡 南方周末: 0 条 (总 30 条) + 📡 三剑客: 0 条 (总 30 条) + 📡 上海国际问题研究院: 0 条 (总 30 条) + 📡 盛唐如松: 0 条 (总 30 条) + 📡 施展世界: 0 条 (总 24 条) + 📡 书剑杂谈: 0 条 (总 30 条) + 📡 戎评: 0 条 (总 30 条) + 📡 搜狐科技: 0 条 (总 30 条) + 📡 数字孪生战场: 0 条 (总 30 条) + 📡 望穹科技: 0 条 (总 30 条) + 📡 外军防务研究前沿: 0 条 (总 30 条) + 📡 网信前沿观察: 0 条 (总 30 条) + 📡 时政国关分析: 0 条 (总 30 条) + 📡 太空与网络: 0 条 (总 30 条) + 📡 沃德舆情观察: 0 条 (总 30 条) + 📡 无人机邦: 0 条 (总 30 条) + 📡 新华国际头条: 0 条 (总 30 条) + 📡 侠客岛: 0 条 (总 30 条) + 📡 无人争锋: 0 条 (总 30 条) + 📡 无人机反制: 0 条 (总 30 条) + 📡 乌鸦校尉: 0 条 (总 30 条) + 📡 喜乔智研: 0 条 (总 30 条) + 📡 央广军事: 0 条 (总 30 条) + 📡 一个坏土豆: 0 条 (总 30 条) + 📡 亚太安全与海洋研究: 0 条 (总 19 条) + 📡 渊亭防务: 0 条 (总 30 条) + 📡 一号哨位: 0 条 (总 30 条) + 📡 战略纵横家: 0 条 (总 30 条) + 📡 舆情文摘: 0 条 (总 30 条) + 📡 占豪: 0 条 (总 30 条) + 📡 泽平宏观: 0 条 (总 30 条) + 📡 远望智库: 0 条 (总 30 条) + 📡 职场老校尉: 0 条 (总 10 条) + 📡 中国国防报: 0 条 (总 30 条) + 📡 战略前沿技术: 0 条 (总 30 条) + 📡 知远战略与防务研究所: 0 条 (总 30 条) + 📡 智强战略咨询: 0 条 (总 30 条) + 📡 中国舰船: 0 条 (总 30 条) + 📡 占知智库: 0 条 (总 30 条) + 📡 正和岛: 0 条 (总 30 条) + 📡 智源社区: 0 条 (总 30 条) + 📡 中国现代国际关系研究院: 0 条 (总 30 条) + 📡 中国南海研究院: 0 条 (总 25 条) + 📡 中国周边安全研究中心: 0 条 (总 30 条) + 📡 中国指挥与控制学会: 0 条 (总 30 条) + 📡 中国舆论场: 0 条 (总 30 条) + 📡 装备参考: 0 条 (总 30 条) + 📡 最黑科技: 0 条 (总 30 条) + 📥 抓取到 2 条原始消息 + 🆕 新消息 0 条 | 缓存命中: 2 | 已有链接: 11 +06:36:11 [INFO] +================================================================ + 📊 运行监控报告 +================================================================ + +── ⏱ 阶段耗时 ── + RSS抓取　　　　　 57.9s + 去重　　　　　　　　 0.0s + ──────────────────── + 合计　　　　　　　　 57.9s / 总运行 58s + +── 📡 文章统计 ── + 总计: 2 缓存命中: 2 本次处理: 0 + 缓存命中率: 100% + +── 🤖 API调用统计 ── + 无API调用（全部命中缓存） + +── 📰 源抓取统计 ── + 源总数: 131 成功: 131 失败: 0 + +================================================================ + ⏰ 下次抓取: 29.0分钟后 + +================================================== +🔄 Fetch Job | 2026-07-01 07:05:13 +================================================== + 📂 共 131 个订阅源 | 时间窗口: 60 分钟 + 📡 The War Zone: 0 条 (总 33 条) + 📡 Defense One: 0 条 (总 25 条) + 📡 Seapower: 0 条 (总 10 条) + 📡 NEWUAS: 0 条 (总 30 条) + 📡 兵路文化: 0 条 (总 11 条) + 📡 安全圈: 0 条 (总 30 条) + 📡 白马 V 视角: 0 条 (总 30 条) + 📡 补三刀: 0 条 (总 24 条) + 📡 半导体行业观察: 0 条 (总 30 条) + 📡 90号茶室: 0 条 (总 30 条) + 📡 艾瑞咨询: 0 条 (总 30 条) + 📡 北大纵横: 0 条 (总 30 条) + 📡 补壹刀: 0 条 (总 30 条) + 📡 安全内参: 0 条 (总 30 条) + 📡 参考消息: 0 条 (总 30 条) + 📡 参考消息智库: 0 条 (总 30 条) + 📡 IPP评论: 0 条 (总 30 条) + 📡 纯科学: 0 条 (总 28 条) + 📡 大柳树防务: 0 条 (总 30 条) + 📡 大浪淘沙: 0 条 (总 30 条) + 📡 从心不逾矩的防务菌: 0 条 (总 30 条) + 📡 大湾区评论: 0 条 (总 30 条) + 📡 东南亚问题研究: 0 条 (总 30 条) + 📡 动态大参考: 0 条 (总 30 条) + 📡 电波之矛: 0 条 (总 30 条) + 📡 阜成门六号院: 0 条 (总 25 条) + 📡 防务快讯: 0 条 (总 30 条) + 📡 复旦大学中国研究院: 0 条 (总 30 条) + 📡 浮空飞行器: 0 条 (总 30 条) + 📡 防务指南: 0 条 (总 16 条) + 📡 观察者网: 0 条 (总 30 条) + 📡 国防科技要闻: 0 条 (总 30 条) + 📡 光明军事: 0 条 (总 30 条) + 📡 国观智库: 0 条 (总 30 条) + 📡 国防工业出版社: 0 条 (总 30 条) + 📡 国防时报排头兵: 0 条 (总 30 条) + 📡 国科环宇: 0 条 (总 18 条) + 📡 航小宇: 0 条 (总 30 条) + 📡 凤凰网军事频道: 0 条 (总 30 条) + 📡 环时枢密院十号: 0 条 (总 30 条) + 📡 国际法务: 0 条 (总 30 条) + 📡 海洋防务前沿: 0 条 (总 30 条) + 📡 华语智库: 0 条 (总 30 条) + 📡 黑天鹅商业情报站: 0 条 (总 30 条) + 📡 海鹰资讯: 0 条 (总 30 条) + 📡 华山穹剑: 0 条 (总 30 条) + 📡 环太国际战略: 0 条 (总 23 条) + 📡 胡锡进观察: 0 条 (总 30 条) + 📡 混沌巡洋舰: 0 条 (总 30 条) + 📡 解放军报: 0 条 (总 30 条) + 📡 精确打击洞见: 0 条 (总 30 条) + 📡 军民融合观察: 0 条 (总 30 条) + 📡 今日台湾: 0 条 (总 30 条) + 📡 军事文摘: 0 条 (总 30 条) + 📡 军事高科技在线: 0 条 (总 30 条) + 📡 跨境前沿: 0 条 (总 30 条) + 📡 军鹰动态: 0 条 (总 30 条) + 📡 科技导报: 0 条 (总 30 条) + 📡 空天界: 0 条 (总 30 条) + 📡 李光满说: 0 条 (总 30 条) + 📡 空天大视野: 0 条 (总 30 条) + 📡 六爷阿旦: 2 条 (总 30 条) + 📡 砺剑: 0 条 (总 30 条) + 📡 瞭望智库: 0 条 (总 30 条) + 📡 军武次位面: 0 条 (总 30 条) + 📡 龙牙的一座山: 0 条 (总 30 条) + 📡 蓝血研究: 0 条 (总 30 条) + 📡 迷彩虎观察: 0 条 (总 30 条) + 📡 美国驻华大使馆: 0 条 (总 30 条) + 📡 明叔杂谈: 0 条 (总 30 条) + 📡 米尔观天下: 0 条 (总 30 条) + 📡 梅特涅的信徒: 0 条 (总 30 条) + 📡 秦安战略: 0 条 (总 30 条) + 📡 澎湃新闻: 0 条 (总 30 条) + 📡 前沿深度解码: 0 条 (总 30 条) + 📡 欧亚新观察: 0 条 (总 30 条) + 📡 破圈了: 0 条 (总 30 条) + 📡 奇安网情局: 0 条 (总 30 条) + 📡 全球技术地图: 0 条 (总 30 条) + 📡 盘古智库: 0 条 (总 30 条) + 📡 牛弹琴: 0 条 (总 30 条) + 📡 人民日报: 0 条 (总 30 条) + 📡 三剑客: 0 条 (总 30 条) + 📡 人大重阳: 0 条 (总 30 条) + 📡 盛唐如松: 0 条 (总 30 条) + 📡 上海国际问题研究院: 0 条 (总 30 条) + 📡 南方周末: 0 条 (总 30 条) + 📡 戎评: 0 条 (总 30 条) + 📡 施展世界: 0 条 (总 24 条) + 📡 书剑杂谈: 0 条 (总 30 条) + 📡 时政国关分析: 0 条 (总 30 条) + 📡 数字孪生战场: 0 条 (总 30 条) + 📡 外军防务研究前沿: 0 条 (总 30 条) + 📡 望穹科技: 0 条 (总 30 条) + 📡 搜狐科技: 0 条 (总 30 条) + 📡 网信前沿观察: 0 条 (总 30 条) + 📡 太空与网络: 0 条 (总 30 条) + 📡 新华国际头条: 0 条 (总 30 条) + 📡 无人机邦: 0 条 (总 30 条) + 📡 侠客岛: 0 条 (总 30 条) + 📡 沃德舆情观察: 0 条 (总 30 条) + 📡 喜乔智研: 0 条 (总 30 条) + 📡 无人争锋: 0 条 (总 30 条) + 📡 无人机反制: 0 条 (总 30 条) + 📡 乌鸦校尉: 0 条 (总 30 条) + 📡 央广军事: 0 条 (总 30 条) + 📡 一个坏土豆: 0 条 (总 30 条) + 📡 亚太安全与海洋研究: 0 条 (总 19 条) + 📡 一号哨位: 0 条 (总 30 条) + 📡 舆情文摘: 0 条 (总 30 条) + 📡 渊亭防务: 0 条 (总 30 条) + 📡 战略纵横家: 0 条 (总 30 条) + 📡 占豪: 0 条 (总 30 条) + 📡 泽平宏观: 0 条 (总 30 条) + 📡 远望智库: 0 条 (总 30 条) + 📡 战略前沿技术: 0 条 (总 30 条) + 📡 职场老校尉: 0 条 (总 10 条) + 📡 智强战略咨询: 0 条 (总 30 条) + 📡 知远战略与防务研究所: 0 条 (总 30 条) + 📡 正和岛: 0 条 (总 30 条) + 📡 智源社区: 0 条 (总 30 条) + 📡 中国国防报: 0 条 (总 30 条) + 📡 中国舰船: 0 条 (总 30 条) + 📡 占知智库: 0 条 (总 30 条) + 📡 中国现代国际关系研究院: 0 条 (总 30 条) + 📡 中国南海研究院: 0 条 (总 25 条) + 📡 中国周边安全研究中心: 0 条 (总 30 条) + 📡 中国舆论场: 0 条 (总 30 条) + 📡 中国指挥与控制学会: 0 条 (总 30 条) + 📡 最黑科技: 0 条 (总 30 条) + 📡 装备参考: 0 条 (总 30 条) + 📥 抓取到 2 条原始消息 + 🆕 新消息 0 条 | 缓存命中: 2 | 已有链接: 11 +07:06:15 [INFO] +================================================================ + 📊 运行监控报告 +================================================================ + +── ⏱ 阶段耗时 ── + RSS抓取　　　　　 61.5s + 去重　　　　　　　　 0.0s + ──────────────────── + 合计　　　　　　　　 61.5s / 总运行 62s + +── 📡 文章统计 ── + 总计: 2 缓存命中: 2 本次处理: 0 + 缓存命中率: 100% + +── 🤖 API调用统计 ── + 无API调用（全部命中缓存） + +── 📰 源抓取统计 ── + 源总数: 131 成功: 131 失败: 0 + +================================================================ + ⏰ 下次抓取: 29.0分钟后 + +================================================== +🔄 Fetch Job | 2026-07-01 07:35:13 +================================================== + 📂 共 131 个订阅源 | 时间窗口: 60 分钟 + 📡 The War Zone: 0 条 (总 33 条) + 📡 Seapower: 0 条 (总 10 条) + 📡 Defense One: 0 条 (总 25 条) + 📡 兵路文化: 0 条 (总 11 条) + 📡 NEWUAS: 0 条 (总 30 条) + 📡 安全圈: 0 条 (总 30 条) + 📡 白马 V 视角: 0 条 (总 30 条) + 📡 半导体行业观察: 0 条 (总 30 条) + 📡 补三刀: 0 条 (总 24 条) + 📡 参考消息: 0 条 (总 30 条) + 📡 东南亚问题研究: 0 条 (总 30 条) + 📡 电波之矛: 0 条 (总 30 条) + 📡 补壹刀: 0 条 (总 30 条) + 📡 参考消息智库: 0 条 (总 30 条) + 📡 动态大参考: 0 条 (总 30 条) + 📡 大柳树防务: 0 条 (总 30 条) + 📡 大浪淘沙: 0 条 (总 30 条) + 📡 阜成门六号院: 0 条 (总 25 条) + 📡 纯科学: 0 条 (总 28 条) + 📡 防务快讯: 0 条 (总 30 条) + 📡 复旦大学中国研究院: 0 条 (总 30 条) + 📡 防务指南: 0 条 (总 16 条) + 📡 浮空飞行器: 0 条 (总 30 条) + 📡 观察者网: 0 条 (总 30 条) + 📡 国防科技要闻: 0 条 (总 30 条) + 📡 大湾区评论: 0 条 (总 30 条) + 📡 从心不逾矩的防务菌: 0 条 (总 30 条) + 📡 国观智库: 0 条 (总 30 条) + 📡 国科环宇: 0 条 (总 18 条) + 📡 国防工业出版社: 0 条 (总 30 条) + 📡 光明军事: 0 条 (总 30 条) + 📡 环时枢密院十号: 0 条 (总 30 条) + 📡 国防时报排头兵: 0 条 (总 30 条) + 📡 航小宇: 0 条 (总 30 条) + 📡 华语智库: 0 条 (总 30 条) + 📡 华山穹剑: 0 条 (总 30 条) + 📡 黑天鹅商业情报站: 0 条 (总 30 条) + 📡 海洋防务前沿: 0 条 (总 30 条) + 📡 国际法务: 0 条 (总 30 条) + 📡 环太国际战略: 0 条 (总 23 条) + 📡 胡锡进观察: 0 条 (总 30 条) + 📡 今日台湾: 0 条 (总 30 条) + 📡 精确打击洞见: 0 条 (总 30 条) + 📡 解放军报: 0 条 (总 30 条) + 📡 海鹰资讯: 0 条 (总 30 条) + 📡 凤凰网军事频道: 0 条 (总 30 条) + 📡 混沌巡洋舰: 0 条 (总 30 条) + 📡 军事文摘: 0 条 (总 30 条) + 📡 军事高科技在线: 0 条 (总 30 条) + 📡 跨境前沿: 0 条 (总 30 条) + 📡 军鹰动态: 0 条 (总 30 条) + 📡 李光满说: 0 条 (总 30 条) + 📡 科技导报: 0 条 (总 30 条) + 📡 六爷阿旦: 2 条 (总 30 条) + 📡 军民融合观察: 0 条 (总 30 条) + 📡 瞭望智库: 0 条 (总 30 条) + 📡 空天界: 0 条 (总 30 条) + 📡 空天大视野: 0 条 (总 30 条) + 📡 砺剑: 0 条 (总 30 条) + 📡 迷彩虎观察: 0 条 (总 30 条) + 📡 米尔观天下: 0 条 (总 30 条) + 📡 明叔杂谈: 0 条 (总 30 条) + 📡 龙牙的一座山: 0 条 (总 30 条) + 📡 美国驻华大使馆: 0 条 (总 30 条) + 📡 蓝血研究: 0 条 (总 30 条) + 📡 军武次位面: 0 条 (总 30 条) + 📡 澎湃新闻: 0 条 (总 30 条) + 📡 秦安战略: 0 条 (总 30 条) + 📡 破圈了: 0 条 (总 30 条) + 📡 前沿深度解码: 0 条 (总 30 条) + 📡 奇安网情局: 0 条 (总 30 条) + 📡 欧亚新观察: 0 条 (总 30 条) + 📡 盘古智库: 0 条 (总 30 条) + 📡 全球技术地图: 0 条 (总 30 条) + 📡 盛唐如松: 0 条 (总 30 条) + 📡 人民日报: 0 条 (总 30 条) + 📡 上海国际问题研究院: 0 条 (总 30 条) + 📡 三剑客: 0 条 (总 30 条) + 📡 牛弹琴: 0 条 (总 30 条) + 📡 梅特涅的信徒: 0 条 (总 30 条) + 📡 人大重阳: 0 条 (总 30 条) + 📡 戎评: 0 条 (总 30 条) + 📡 南方周末: 0 条 (总 30 条) + 📡 书剑杂谈: 0 条 (总 30 条) + 📡 外军防务研究前沿: 0 条 (总 30 条) + 📡 望穹科技: 0 条 (总 30 条) + 📡 无人机邦: 0 条 (总 30 条) + 📡 搜狐科技: 0 条 (总 30 条) + 📡 施展世界: 0 条 (总 24 条) + 📡 数字孪生战场: 0 条 (总 30 条) + 📡 沃德舆情观察: 0 条 (总 30 条) + 📡 时政国关分析: 0 条 (总 30 条) + 📡 喜乔智研: 0 条 (总 30 条) + 📡 新华国际头条: 0 条 (总 30 条) + 📡 网信前沿观察: 0 条 (总 30 条) + 📡 太空与网络: 0 条 (总 30 条) + 📡 侠客岛: 0 条 (总 30 条) + 📡 无人争锋: 0 条 (总 30 条) + 📡 无人机反制: 0 条 (总 30 条) + 📡 央广军事: 0 条 (总 30 条) + 📡 乌鸦校尉: 0 条 (总 30 条) + 📡 舆情文摘: 0 条 (总 30 条) + 📡 渊亭防务: 0 条 (总 30 条) + 📡 占豪: 0 条 (总 30 条) + 📡 战略纵横家: 0 条 (总 30 条) + 📡 泽平宏观: 0 条 (总 30 条) + 📡 职场老校尉: 0 条 (总 10 条) + 📡 一个坏土豆: 0 条 (总 30 条) + 📡 亚太安全与海洋研究: 0 条 (总 19 条) + 📡 一号哨位: 0 条 (总 30 条) + 📡 中国国防报: 0 条 (总 30 条) + 📡 知远战略与防务研究所: 0 条 (总 30 条) + 📡 智强战略咨询: 0 条 (总 30 条) + 📡 中国舰船: 0 条 (总 30 条) + 📡 占知智库: 0 条 (总 30 条) + 📡 远望智库: 0 条 (总 30 条) + 📡 中国南海研究院: 0 条 (总 25 条) + 📡 中国现代国际关系研究院: 0 条 (总 30 条) + 📡 战略前沿技术: 0 条 (总 30 条) + 📡 智源社区: 0 条 (总 30 条) + 📡 中国周边安全研究中心: 0 条 (总 30 条) + 📡 中国舆论场: 0 条 (总 30 条) + 📡 正和岛: 0 条 (总 30 条) + 📡 中国指挥与控制学会: 0 条 (总 30 条) + 📡 装备参考: 0 条 (总 30 条) + 📡 最黑科技: 0 条 (总 30 条) + ⚠️ 获取失败 90号茶室: + ⚠️ 获取失败 IPP评论: + ⚠️ 获取失败 艾瑞咨询: + ⚠️ 获取失败 安全内参: + ⚠️ 获取失败 北大纵横: + 📥 抓取到 2 条原始消息 + 🆕 新消息 0 条 | 缓存命中: 2 | 已有链接: 11 +07:37:18 [INFO] +================================================================ + 📊 运行监控报告 +================================================================ + +── ⏱ 阶段耗时 ── + RSS抓取　　　　　 125.0s + 去重　　　　　　　　 0.0s + ──────────────────── + 合计　　　　　　　　 125.0s / 总运行 125s + +── 📡 文章统计 ── + 总计: 2 缓存命中: 2 本次处理: 0 + 缓存命中率: 100% + +── 🤖 API调用统计 ── + 无API调用（全部命中缓存） + +── 📰 源抓取统计 ── + 源总数: 131 成功: 126 失败: 5 + 失败源: 90号茶室, IPP评论, 艾瑞咨询, 安全内参, 北大纵横 + +================================================================ + ⏰ 下次抓取: 27.9分钟后 + 📤 执行推送: 2026-07-01 08:00:01 + +================================================== +📤 Push Job | 2026-07-01 08:00:01 +================================================== +08:00:01 [INFO] [热度阈值] 均值=43.2 σ=5.3 ×1.5 原始=51.2 clamp=[35,80] → 最终=51.2 + 📝 簇2 注入前情: 五角大楼通过'War Force'项目招募两年制技术人才，提供参与政策制定和国家... + 🔄 簇5防退化重试(1/2): 违规1项 + 🔄 簇5防退化重试(2/2): 违规2项 + 📝 簇0 注入前情: 美国陆军在太平洋测试AI与无人艇协同后勤，验证自主航行与智能调度技术，以支持分布... + 🔄 簇0防退化重试(1/2): 违规1项 + ⚠️ 簇0洞察失败: 洞察 JSON 解析失败: Expecting ',' delimiter: line 13 column 530 (char 787) + 🔄 簇4防退化重试(1/2): 违规1项 + 🔄 簇4防退化重试(2/2): 违规2项 + 🔍 洞察生成完成 +08:02:05 [INFO] [热度阈值] 均值=43.2 σ=5.3 ×1.5 原始=51.2 clamp=[35,80] → 最终=51.2 +08:02:05 [INFO] [热度阈值] 均值=43.2 σ=5.3 ×1.5 原始=51.2 clamp=[35,80] → 最终=51.2 +08:02:05 [INFO] [热度阈值] 均值=43.2 σ=5.3 ×1.5 原始=51.2 clamp=[35,80] → 最终=51.2 + ✅ 已推送到 feishu + 💾 已保存热点速览到 data/cluster-20260701-080001.md + 📤 簇视图推送完成 +08:02:05 [INFO] ━━━ 网摘缓存策略启用 ━━━ +08:02:05 [INFO] ━━━ 网摘缓存策略启用 ━━━ +08:02:05 [INFO] ━━━ 网摘缓存策略启用 ━━━ +08:02:14 [INFO] 尝试1 结果: 正文✓(311字) | 价值点✓(43字) +08:02:14 [INFO] → 缓存正文 (尝试1, 311字) ✓ +08:02:14 [INFO] → 缓存价值点 (尝试1, 43字) ✓ +08:02:14 [INFO] 尝试1 双达标，直接返回 +08:02:14 [INFO] 尝试1 结果: 正文✓(324字) | 价值点✗(52字) +08:02:14 [INFO] → 缓存正文 (尝试1, 324字) ✓ +08:02:14 [INFO] 尝试2 开始前缓存: 正文已缓存(尝试1, 324字) +08:02:18 [INFO] 尝试1 结果: 正文✗(237字) | 价值点✗(59字) +08:02:25 [INFO] 尝试2 结果: 正文✓(277字) | 价值点✗(52字) +08:02:25 [INFO] 尝试3 开始前缓存: 正文已缓存(尝试1, 324字) +08:02:31 [INFO] 尝试3 结果: 正文✓(253字) | 价值点✓(43字) +08:02:31 [INFO] → 缓存价值点 (尝试3, 43字) ✓ +08:02:31 [INFO] 尝试3 双达标，直接返回 +08:02:37 [INFO] 尝试2 结果: 正文✗(379字) | 价值点✓(43字) +08:02:37 [INFO] → 缓存价值点 (尝试2, 43字) ✓ +08:02:37 [INFO] 尝试3 开始前缓存: 价值点已缓存(尝试2, 43字) +08:02:46 [INFO] 尝试3 结果: 正文✗(379字) | 价值点✓(43字) +08:02:46 [INFO] ─── 合并结果 ─── +08:02:46 [INFO] 价值点←尝试2(43字) | +08:02:46 [INFO] 最终: 正文379字(✗) | 价值点43字(✓达标) + 📝 网摘生成完成: 3 篇 +08:02:46 [INFO] [图片生成] 从JSON字符串生成单篇网摘图片，输出路径: data\images\20260701\webzine_1_20260701_080001.png +08:02:46 [INFO] [图片生成] 开始生成单篇网摘图片，输出路径: data\images\20260701\webzine_1_20260701_080001.png +08:02:47 [INFO] [图片生成] 网摘图片已保存: data\images\20260701\webzine_1_20260701_080001.png (命令数: 25, 高度: 1426px) + ├─ 单篇图片 1: webzine_1_20260701_080001.png +08:02:47 [INFO] [图片生成] 从JSON字符串生成单篇网摘图片，输出路径: data\images\20260701\webzine_2_20260701_080001.png +08:02:47 [INFO] [图片生成] 开始生成单篇网摘图片，输出路径: data\images\20260701\webzine_2_20260701_080001.png +08:02:47 [INFO] [图片生成] 网摘图片已保存: data\images\20260701\webzine_2_20260701_080001.png (命令数: 29, 高度: 1609px) + ├─ 单篇图片 2: webzine_2_20260701_080001.png +08:02:47 [INFO] [图片生成] 从JSON字符串生成单篇网摘图片，输出路径: data\images\20260701\webzine_3_20260701_080001.png +08:02:47 [INFO] [图片生成] 开始生成单篇网摘图片，输出路径: data\images\20260701\webzine_3_20260701_080001.png +08:02:47 [INFO] [图片生成] 网摘图片已保存: data\images\20260701\webzine_3_20260701_080001.png (命令数: 22, 高度: 1288px) + ├─ 单篇图片 3: webzine_3_20260701_080001.png +08:02:47 [INFO] [图片生成] 从JSON字符串列表生成合并网摘长图，共 3 篇，输出路径: data\images\20260701\webzine_combined_20260701_080001.png +08:02:47 [INFO] [图片生成] 开始生成合并网摘长图，共 3 篇，输出路径: data\images\20260701\webzine_combined_20260701_080001.png +08:02:47 [INFO] [图片生成] 合并长图已保存: data\images\20260701\webzine_combined_20260701_080001.png (命令数: 78, 高度: 4347px) + └─ 合并长图: webzine_combined_20260701_080001.png + 🖼️ 网摘图片生成完成: 4 张 + 📋 分类介绍: 2 个分类 + ✅ 已推送到 feishu + 💾 已保存每日精选到 data/digest-20260701-080001.md + 📤 单篇视图推送完成 + 🖼️ 推送合并网摘图片... + ✓ 图片上传成功: https://cloudimgs.yynnice.top/api/images/daily/webzine_combined_20260701_080001.png + 📝 图片链接消息已发送 + ✅ 图片已推送到 feishu + 📤 网摘图片推送完成 +08:02:51 [INFO] +================================================================ + 📊 运行监控报告 +================================================================ + +── ⏱ 阶段耗时 ── + 网摘生成　　　　　　 41.2s + 分类介绍　　　　　　 0.0s + ──────────────────── + 合计　　　　　　　　 41.2s / 总运行 170s + +── 📡 文章统计 ── + 总计: 0 缓存命中: 0 本次处理: 0 + +── 🤖 API调用统计 ── + 无API调用（全部命中缓存） + +── 📰 源抓取统计 ── + 源总数: 0 成功: 0 失败: 0 + +================================================================ + ⏰ 下次推送: 2026-07-02 08:00:00 (等待 1437.1 分钟) + +================================================== +🔄 Fetch Job | 2026-07-01 08:05:13 +================================================== + 📂 共 131 个订阅源 | 时间窗口: 60 分钟 + 📡 The War Zone: 0 条 (总 33 条) + 📡 Seapower: 0 条 (总 10 条) + 📡 Defense One: 1 条 (总 25 条) + 📡 艾瑞咨询: 0 条 (总 30 条) + 📡 NEWUAS: 0 条 (总 30 条) + 📡 安全圈: 0 条 (总 30 条) + 📡 兵路文化: 0 条 (总 11 条) + 📡 半导体行业观察: 0 条 (总 30 条) + 📡 补三刀: 0 条 (总 24 条) + 📡 90号茶室: 0 条 (总 30 条) + 📡 参考消息: 0 条 (总 30 条) + 📡 北大纵横: 0 条 (总 30 条) + 📡 参考消息智库: 0 条 (总 30 条) + 📡 补壹刀: 0 条 (总 30 条) + 📡 安全内参: 0 条 (总 30 条) + 📡 IPP评论: 0 条 (总 30 条) + 📡 大柳树防务: 0 条 (总 30 条) + 📡 大浪淘沙: 0 条 (总 30 条) + 📡 纯科学: 0 条 (总 28 条) + 📡 白马 V 视角: 0 条 (总 30 条) + 📡 从心不逾矩的防务菌: 0 条 (总 30 条) + 📡 大湾区评论: 0 条 (总 30 条) + 📡 东南亚问题研究: 0 条 (总 30 条) + 📡 电波之矛: 0 条 (总 30 条) + 📡 阜成门六号院: 0 条 (总 25 条) + 📡 复旦大学中国研究院: 0 条 (总 30 条) + 📡 防务快讯: 0 条 (总 30 条) + 📡 防务指南: 0 条 (总 16 条) + 📡 动态大参考: 0 条 (总 30 条) + 📡 浮空飞行器: 0 条 (总 30 条) + 📡 凤凰网军事频道: 0 条 (总 30 条) + 📡 国防工业出版社: 0 条 (总 30 条) + 📡 国科环宇: 0 条 (总 18 条) + 📡 国防科技要闻: 0 条 (总 30 条) + 📡 国观智库: 0 条 (总 30 条) + 📡 观察者网: 0 条 (总 30 条) + 📡 航小宇: 0 条 (总 30 条) + 📡 国防时报排头兵: 0 条 (总 30 条) + 📡 环时枢密院十号: 0 条 (总 30 条) + 📡 光明军事: 0 条 (总 30 条) + 📡 华山穹剑: 0 条 (总 30 条) + 📡 海洋防务前沿: 0 条 (总 30 条) + 📡 环太国际战略: 0 条 (总 23 条) + 📡 黑天鹅商业情报站: 0 条 (总 30 条) + 📡 国际法务: 0 条 (总 30 条) + 📡 华语智库: 0 条 (总 30 条) + 📡 海鹰资讯: 0 条 (总 30 条) + 📡 胡锡进观察: 0 条 (总 30 条) + 📡 混沌巡洋舰: 0 条 (总 30 条) + 📡 解放军报: 2 条 (总 30 条) + 📡 精确打击洞见: 0 条 (总 30 条) + 📡 今日台湾: 0 条 (总 30 条) + 📡 军事高科技在线: 0 条 (总 30 条) + 📡 军民融合观察: 0 条 (总 30 条) + 📡 跨境前沿: 0 条 (总 30 条) + 📡 军事文摘: 0 条 (总 30 条) + 📡 李光满说: 0 条 (总 30 条) + 📡 军武次位面: 0 条 (总 30 条) + 📡 砺剑: 0 条 (总 30 条) + 📡 瞭望智库: 0 条 (总 30 条) + 📡 科技导报: 0 条 (总 30 条) + 📡 军鹰动态: 0 条 (总 30 条) + 📡 空天大视野: 0 条 (总 30 条) + 📡 空天界: 0 条 (总 30 条) + 📡 六爷阿旦: 2 条 (总 30 条) + 📡 蓝血研究: 0 条 (总 30 条) + 📡 龙牙的一座山: 0 条 (总 30 条) + 📡 美国驻华大使馆: 0 条 (总 30 条) + 📡 迷彩虎观察: 0 条 (总 30 条) + 📡 明叔杂谈: 0 条 (总 30 条) + 📡 米尔观天下: 0 条 (总 30 条) + 📡 梅特涅的信徒: 0 条 (总 30 条) + 📡 澎湃新闻: 0 条 (总 30 条) + 📡 奇安网情局: 0 条 (总 30 条) + 📡 前沿深度解码: 0 条 (总 30 条) + 📡 破圈了: 0 条 (总 30 条) + 📡 秦安战略: 0 条 (总 30 条) + 📡 欧亚新观察: 0 条 (总 30 条) + 📡 盘古智库: 0 条 (总 30 条) + 📡 全球技术地图: 0 条 (总 30 条) + 📡 人大重阳: 0 条 (总 30 条) + 📡 三剑客: 0 条 (总 30 条) + 📡 盛唐如松: 0 条 (总 30 条) + 📡 牛弹琴: 0 条 (总 30 条) + 📡 上海国际问题研究院: 0 条 (总 30 条) + 📡 人民日报: 0 条 (总 30 条) + 📡 南方周末: 0 条 (总 30 条) + 📡 戎评: 0 条 (总 30 条) + 📡 施展世界: 0 条 (总 24 条) + 📡 时政国关分析: 0 条 (总 30 条) + 📡 书剑杂谈: 0 条 (总 30 条) + 📡 搜狐科技: 0 条 (总 30 条) + 📡 望穹科技: 0 条 (总 30 条) + 📡 数字孪生战场: 0 条 (总 30 条) + 📡 外军防务研究前沿: 0 条 (总 30 条) + 📡 无人机邦: 0 条 (总 30 条) + 📡 网信前沿观察: 0 条 (总 30 条) + 📡 太空与网络: 0 条 (总 30 条) + 📡 沃德舆情观察: 0 条 (总 30 条) + 📡 新华国际头条: 0 条 (总 30 条) + 📡 喜乔智研: 0 条 (总 30 条) + 📡 侠客岛: 0 条 (总 30 条) + 📡 无人争锋: 0 条 (总 30 条) + 📡 央广军事: 0 条 (总 30 条) + 📡 无人机反制: 0 条 (总 30 条) + 📡 乌鸦校尉: 0 条 (总 30 条) + 📡 舆情文摘: 0 条 (总 30 条) + 📡 渊亭防务: 0 条 (总 30 条) + 📡 一个坏土豆: 0 条 (总 30 条) + 📡 亚太安全与海洋研究: 0 条 (总 19 条) + 📡 泽平宏观: 0 条 (总 30 条) + 📡 占豪: 0 条 (总 30 条) + 📡 战略纵横家: 0 条 (总 30 条) + 📡 一号哨位: 0 条 (总 30 条) + 📡 远望智库: 0 条 (总 30 条) + 📡 战略前沿技术: 0 条 (总 30 条) + 📡 职场老校尉: 0 条 (总 10 条) + 📡 智强战略咨询: 0 条 (总 30 条) + 📡 知远战略与防务研究所: 0 条 (总 30 条) + 📡 占知智库: 0 条 (总 30 条) + 📡 正和岛: 0 条 (总 30 条) + 📡 中国国防报: 0 条 (总 30 条) + 📡 中国舰船: 0 条 (总 30 条) + 📡 智源社区: 0 条 (总 30 条) + 📡 中国南海研究院: 0 条 (总 25 条) + 📡 中国现代国际关系研究院: 0 条 (总 30 条) + 📡 中国舆论场: 0 条 (总 30 条) + 📡 装备参考: 0 条 (总 30 条) + 📡 中国周边安全研究中心: 0 条 (总 30 条) + 📡 中国指挥与控制学会: 0 条 (总 30 条) + 📡 最黑科技: 0 条 (总 30 条) + 📥 抓取到 5 条原始消息 + 🆕 新消息 3 条 | 缓存命中: 2 | 已有链接: 11 +08:06:24 [INFO] 中文文章：2 篇，待翻译标题的外文：1 篇 +08:06:24 [INFO] 开始并发翻译标题，并发数：6 +08:06:26 [INFO] 标题翻译完成，共处理 3 篇文章 +08:06:26 [INFO] 无需翻译正文的中文文章：2 篇，待翻译正文的外文：1 篇 +08:06:26 [INFO] 开始并发翻译正文，并发数：6 +08:06:28 [INFO] 正文翻译完成，共处理 3 篇文章 + 🤖 LLM评分中... + 📦 分成 1 个批次评分 (共 3 条) + 📊 评分分布: [85, 70, 70] + 📊 动态阈值: 79 | 合格条目: 1 + 🏷️ 实体提取完成: 1 条 + 🧩 聚类完成: 1 个簇 (多角度: 0, 最大簇: 1 篇) +08:06:40 [INFO] [热度阈值] 簇数=1<5, 保底阈值=40.0 + 🔥 簇热度: 0 个热点簇 (阈值=40.0) + 💾 已保存到 data/fetch-2026-07-01.json +08:06:40 [INFO] +================================================================ + 📊 运行监控报告 +================================================================ + +── ⏱ 阶段耗时 ── + RSS抓取　　　　　 71.1s + 去重　　　　　　　　 0.0s + 标题翻译　　　　　　 2.3s + 正文翻译　　　　　　 1.3s + AI评分分类　　　　 8.6s + 实体提取+角度分类　 3.9s + ──────────────────── + 合计　　　　　　　　 87.2s / 总运行 87s + +── 📡 文章统计 ── + 总计: 5 缓存命中: 2 本次处理: 3 + 缓存命中率: 40% + +── 🤖 API调用统计 ── + 无API调用（全部命中缓存） + +── 📰 源抓取统计 ── + 源总数: 131 成功: 131 失败: 0 + +================================================================ + ✅ Fetch Job 完成 | 新消息: 3 条 | 热点: 0 条 + ⏰ 下次抓取: 28.5分钟后 + +================================================== +🔄 Fetch Job | 2026-07-01 08:35:13 +================================================== + 📂 共 131 个订阅源 | 时间窗口: 60 分钟 + 📡 The War Zone: 0 条 (总 33 条) + 📡 Defense One: 1 条 (总 25 条) + 📡 Seapower: 0 条 (总 10 条) + 📡 NEWUAS: 0 条 (总 30 条) + 📡 兵路文化: 0 条 (总 11 条) + 📡 安全圈: 0 条 (总 30 条) + 📡 白马 V 视角: 0 条 (总 30 条) + 📡 补三刀: 0 条 (总 24 条) + 📡 半导体行业观察: 0 条 (总 30 条) + 📡 90号茶室: 0 条 (总 30 条) + 📡 艾瑞咨询: 1 条 (总 30 条) + 📡 补壹刀: 0 条 (总 30 条) + 📡 参考消息: 2 条 (总 30 条) + 📡 北大纵横: 0 条 (总 30 条) + 📡 参考消息智库: 0 条 (总 30 条) + 📡 安全内参: 0 条 (总 30 条) + 📡 大浪淘沙: 0 条 (总 30 条) + 📡 纯科学: 0 条 (总 28 条) + 📡 IPP评论: 0 条 (总 30 条) + 📡 从心不逾矩的防务菌: 0 条 (总 30 条) + 📡 电波之矛: 0 条 (总 30 条) + 📡 东南亚问题研究: 0 条 (总 30 条) + 📡 动态大参考: 0 条 (总 30 条) + 📡 防务快讯: 0 条 (总 30 条) + 📡 大柳树防务: 0 条 (总 30 条) + 📡 大湾区评论: 0 条 (总 30 条) + 📡 防务指南: 0 条 (总 16 条) + 📡 阜成门六号院: 0 条 (总 25 条) + 📡 复旦大学中国研究院: 0 条 (总 30 条) + 📡 观察者网: 0 条 (总 30 条) + 📡 浮空飞行器: 0 条 (总 30 条) + 📡 国防科技要闻: 0 条 (总 30 条) + 📡 国科环宇: 0 条 (总 18 条) + 📡 国观智库: 0 条 (总 30 条) + 📡 光明军事: 0 条 (总 30 条) + 📡 国防工业出版社: 0 条 (总 30 条) + 📡 国防时报排头兵: 0 条 (总 30 条) + 📡 航小宇: 0 条 (总 30 条) + 📡 环时枢密院十号: 0 条 (总 30 条) + 📡 华山穹剑: 0 条 (总 30 条) + 📡 黑天鹅商业情报站: 0 条 (总 30 条) + 📡 华语智库: 0 条 (总 30 条) + 📡 国际法务: 0 条 (总 30 条) + 📡 海洋防务前沿: 0 条 (总 30 条) + 📡 环太国际战略: 0 条 (总 23 条) + 📡 凤凰网军事频道: 0 条 (总 30 条) + 📡 海鹰资讯: 0 条 (总 30 条) + 📡 胡锡进观察: 0 条 (总 30 条) + 📡 解放军报: 2 条 (总 30 条) + 📡 混沌巡洋舰: 0 条 (总 30 条) + 📡 今日台湾: 0 条 (总 30 条) + 📡 精确打击洞见: 0 条 (总 30 条) + 📡 军事高科技在线: 0 条 (总 30 条) + 📡 军民融合观察: 0 条 (总 30 条) + 📡 军事文摘: 0 条 (总 30 条) + 📡 跨境前沿: 0 条 (总 30 条) + 📡 李光满说: 0 条 (总 30 条) + 📡 科技导报: 0 条 (总 30 条) + 📡 军鹰动态: 0 条 (总 30 条) + 📡 六爷阿旦: 2 条 (总 30 条) + 📡 瞭望智库: 0 条 (总 30 条) + 📡 砺剑: 0 条 (总 30 条) + 📡 空天界: 0 条 (总 30 条) + 📡 空天大视野: 0 条 (总 30 条) + 📡 龙牙的一座山: 0 条 (总 30 条) + 📡 蓝血研究: 0 条 (总 30 条) + 📡 军武次位面: 0 条 (总 30 条) + 📡 美国驻华大使馆: 0 条 (总 30 条) + 📡 米尔观天下: 0 条 (总 30 条) + 📡 明叔杂谈: 0 条 (总 30 条) + 📡 迷彩虎观察: 0 条 (总 30 条) + 📡 梅特涅的信徒: 0 条 (总 30 条) + 📡 秦安战略: 0 条 (总 30 条) + 📡 澎湃新闻: 1 条 (总 30 条) + 📡 前沿深度解码: 0 条 (总 30 条) + 📡 破圈了: 0 条 (总 30 条) + 📡 南方周末: 0 条 (总 30 条) + 📡 奇安网情局: 0 条 (总 30 条) + 📡 欧亚新观察: 0 条 (总 30 条) + 📡 牛弹琴: 0 条 (总 30 条) + 📡 盘古智库: 0 条 (总 30 条) + 📡 全球技术地图: 0 条 (总 30 条) + 📡 人民日报: 1 条 (总 30 条) + 📡 人大重阳: 0 条 (总 30 条) + 📡 三剑客: 0 条 (总 30 条) + 📡 戎评: 0 条 (总 30 条) + 📡 上海国际问题研究院: 0 条 (总 30 条) + 📡 盛唐如松: 0 条 (总 30 条) + 📡 书剑杂谈: 0 条 (总 30 条) + 📡 望穹科技: 0 条 (总 30 条) + 📡 外军防务研究前沿: 0 条 (总 30 条) + 📡 无人机邦: 0 条 (总 30 条) + 📡 搜狐科技: 0 条 (总 30 条) + 📡 数字孪生战场: 0 条 (总 30 条) + 📡 沃德舆情观察: 0 条 (总 30 条) + 📡 网信前沿观察: 0 条 (总 30 条) + 📡 太空与网络: 0 条 (总 30 条) + 📡 侠客岛: 0 条 (总 30 条) + 📡 喜乔智研: 0 条 (总 30 条) + 📡 施展世界: 0 条 (总 24 条) + 📡 新华国际头条: 0 条 (总 30 条) + 📡 时政国关分析: 1 条 (总 30 条) + 📡 无人机反制: 0 条 (总 30 条) + 📡 无人争锋: 0 条 (总 30 条) + 📡 乌鸦校尉: 0 条 (总 30 条) + 📡 央广军事: 2 条 (总 30 条) + 📡 亚太安全与海洋研究: 0 条 (总 19 条) + 📡 渊亭防务: 0 条 (总 30 条) + 📡 舆情文摘: 0 条 (总 30 条) + 📡 占豪: 0 条 (总 30 条) + 📡 泽平宏观: 0 条 (总 30 条) + 📡 一个坏土豆: 0 条 (总 30 条) + 📡 一号哨位: 0 条 (总 30 条) + 📡 远望智库: 0 条 (总 30 条) + 📡 战略纵横家: 2 条 (总 30 条) + 📡 职场老校尉: 0 条 (总 10 条) + 📡 智强战略咨询: 0 条 (总 30 条) + 📡 战略前沿技术: 0 条 (总 30 条) + 📡 中国国防报: 0 条 (总 30 条) + 📡 知远战略与防务研究所: 0 条 (总 30 条) + 📡 占知智库: 0 条 (总 30 条) + 📡 中国舰船: 0 条 (总 30 条) + 📡 智源社区: 0 条 (总 30 条) + 📡 中国现代国际关系研究院: 0 条 (总 30 条) + 📡 中国南海研究院: 0 条 (总 25 条) + 📡 正和岛: 0 条 (总 30 条) + 📡 中国舆论场: 0 条 (总 30 条) + 📡 中国周边安全研究中心: 0 条 (总 30 条) + 📡 中国指挥与控制学会: 0 条 (总 30 条) + 📡 装备参考: 0 条 (总 30 条) + 📡 最黑科技: 0 条 (总 30 条) + 📥 抓取到 15 条原始消息 + 🆕 新消息 10 条 | 缓存命中: 5 | 已有链接: 11 +08:36:27 [INFO] 中文文章：10 篇，待翻译标题的外文：0 篇 +08:36:27 [INFO] 无需翻译正文的中文文章：10 篇，待翻译正文的外文：0 篇 + 🤖 LLM评分中... + 📦 分成 3 个批次评分 (共 10 条) + 📊 评分进度: 2/3 批次完成 + 📊 评分进度: 3/3 批次完成 + 📊 评分分布: [88, 85, 85, 85, 75, 75, 70, 70, 65, 60] + 📊 动态阈值: 81 | 合格条目: 4 + 🏷️ 实体提取完成: 4 条 + 🧠 LLM消歧: 27 个未命中实体待判断 (模型: deepseek-v4-flash) + 🧩 聚类完成: 4 个簇 (多角度: 0, 最大簇: 1 篇) +08:37:04 [INFO] [热度阈值] 簇数=4<5, 保底阈值=40.0 + 🔥 簇热度: 0 个热点簇 (阈值=40.0) + 💾 已保存到 data/fetch-2026-07-01.json +08:37:04 [INFO] +================================================================ + 📊 运行监控报告 +================================================================ + +── ⏱ 阶段耗时 ── + RSS抓取　　　　　 74.1s + 去重　　　　　　　　 0.0s + 标题翻译　　　　　　 0.0s + 正文翻译　　　　　　 0.0s + AI评分分类　　　　 10.1s + 实体提取+角度分类　 27.0s + ──────────────────── + 合计　　　　　　　　 111.2s / 总运行 111s + +── 📡 文章统计 ── + 总计: 15 缓存命中: 5 本次处理: 10 + 缓存命中率: 33% + +── 🤖 API调用统计 ── + 无API调用（全部命中缓存） + +── 📰 源抓取统计 ── + 源总数: 131 成功: 131 失败: 0 + +================================================================ + ✅ Fetch Job 完成 | 新消息: 10 条 | 热点: 0 条 + ⏰ 下次抓取: 28.1分钟后 + +================================================== +🔄 Fetch Job | 2026-07-01 09:05:13 +================================================== + 📂 共 131 个订阅源 | 时间窗口: 60 分钟 + 📡 The War Zone: 0 条 (总 33 条) + 📡 Defense One: 0 条 (总 25 条) + 📡 Seapower: 0 条 (总 10 条) + 📡 兵路文化: 0 条 (总 11 条) + 📡 NEWUAS: 0 条 (总 30 条) + 📡 安全圈: 0 条 (总 30 条) + 📡 白马 V 视角: 0 条 (总 30 条) + 📡 半导体行业观察: 1 条 (总 30 条) + 📡 90号茶室: 0 条 (总 30 条) + 📡 艾瑞咨询: 0 条 (总 30 条) + 📡 北大纵横: 0 条 (总 30 条) + 📡 IPP评论: 0 条 (总 30 条) + 📡 补壹刀: 0 条 (总 30 条) + 📡 参考消息: 2 条 (总 30 条) + 📡 补三刀: 0 条 (总 25 条) + 📡 安全内参: 0 条 (总 30 条) + 📡 参考消息智库: 0 条 (总 30 条) + 📡 纯科学: 0 条 (总 28 条) + 📡 从心不逾矩的防务菌: 0 条 (总 30 条) + 📡 大柳树防务: 0 条 (总 30 条) + 📡 电波之矛: 0 条 (总 30 条) + 📡 大浪淘沙: 0 条 (总 30 条) + 📡 东南亚问题研究: 0 条 (总 30 条) + 📡 动态大参考: 0 条 (总 30 条) + 📡 阜成门六号院: 0 条 (总 25 条) + 📡 防务快讯: 0 条 (总 30 条) + 📡 防务指南: 0 条 (总 16 条) + 📡 大湾区评论: 0 条 (总 30 条) + 📡 复旦大学中国研究院: 0 条 (总 30 条) + 📡 浮空飞行器: 0 条 (总 30 条) + 📡 凤凰网军事频道: 0 条 (总 30 条) + 📡 观察者网: 0 条 (总 30 条) + 📡 国防科技要闻: 0 条 (总 30 条) + 📡 光明军事: 0 条 (总 30 条) + 📡 国观智库: 0 条 (总 30 条) + 📡 国防工业出版社: 0 条 (总 30 条) + 📡 国防时报排头兵: 0 条 (总 30 条) + 📡 国科环宇: 0 条 (总 18 条) + 📡 国际法务: 3 条 (总 30 条) + 📡 海洋防务前沿: 0 条 (总 30 条) + 📡 华语智库: 0 条 (总 30 条) + 📡 航小宇: 1 条 (总 30 条) + 📡 环时枢密院十号: 0 条 (总 30 条) + 📡 华山穹剑: 0 条 (总 30 条) + 📡 环太国际战略: 0 条 (总 23 条) + 📡 解放军报: 0 条 (总 30 条) + 📡 黑天鹅商业情报站: 0 条 (总 30 条) + 📡 胡锡进观察: 0 条 (总 30 条) + 📡 海鹰资讯: 0 条 (总 30 条) + 📡 混沌巡洋舰: 0 条 (总 30 条) + 📡 精确打击洞见: 0 条 (总 30 条) + 📡 今日台湾: 0 条 (总 30 条) + 📡 军事高科技在线: 0 条 (总 30 条) + 📡 军民融合观察: 0 条 (总 30 条) + 📡 军事文摘: 0 条 (总 30 条) + 📡 军鹰动态: 0 条 (总 30 条) + 📡 六爷阿旦: 0 条 (总 30 条) + 📡 李光满说: 0 条 (总 30 条) + 📡 跨境前沿: 0 条 (总 30 条) + 📡 砺剑: 0 条 (总 30 条) + 📡 科技导报: 0 条 (总 30 条) + 📡 瞭望智库: 1 条 (总 30 条) + 📡 空天大视野: 0 条 (总 30 条) + 📡 空天界: 0 条 (总 30 条) + 📡 龙牙的一座山: 0 条 (总 30 条) + 📡 迷彩虎观察: 1 条 (总 30 条) + 📡 美国驻华大使馆: 0 条 (总 30 条) + 📡 蓝血研究: 0 条 (总 30 条) + 📡 军武次位面: 0 条 (总 30 条) + 📡 米尔观天下: 1 条 (总 30 条) + 📡 梅特涅的信徒: 0 条 (总 30 条) + 📡 明叔杂谈: 0 条 (总 30 条) + 📡 秦安战略: 0 条 (总 30 条) + 📡 澎湃新闻: 0 条 (总 30 条) + 📡 前沿深度解码: 0 条 (总 30 条) + 📡 欧亚新观察: 0 条 (总 30 条) + 📡 破圈了: 0 条 (总 30 条) + 📡 奇安网情局: 0 条 (总 30 条) + 📡 全球技术地图: 0 条 (总 30 条) + 📡 盘古智库: 0 条 (总 30 条) + 📡 牛弹琴: 0 条 (总 30 条) + 📡 人大重阳: 0 条 (总 30 条) + 📡 南方周末: 0 条 (总 30 条) + 📡 人民日报: 0 条 (总 30 条) + 📡 三剑客: 0 条 (总 30 条) + 📡 上海国际问题研究院: 0 条 (总 30 条) + 📡 盛唐如松: 0 条 (总 30 条) + 📡 戎评: 0 条 (总 30 条) + 📡 施展世界: 0 条 (总 24 条) + 📡 书剑杂谈: 0 条 (总 30 条) + 📡 数字孪生战场: 0 条 (总 30 条) + 📡 搜狐科技: 0 条 (总 30 条) + 📡 外军防务研究前沿: 0 条 (总 30 条) + 📡 望穹科技: 0 条 (总 30 条) + 📡 时政国关分析: 0 条 (总 30 条) + 📡 太空与网络: 0 条 (总 30 条) + 📡 网信前沿观察: 0 条 (总 30 条) + 📡 沃德舆情观察: 0 条 (总 30 条) + 📡 无人机邦: 0 条 (总 30 条) + 📡 新华国际头条: 0 条 (总 30 条) + 📡 侠客岛: 0 条 (总 30 条) + 📡 无人机反制: 0 条 (总 30 条) + 📡 无人争锋: 0 条 (总 30 条) + 📡 喜乔智研: 0 条 (总 30 条) + 📡 乌鸦校尉: 0 条 (总 30 条) + 📡 央广军事: 0 条 (总 30 条) + 📡 渊亭防务: 0 条 (总 30 条) + 📡 舆情文摘: 0 条 (总 30 条) + 📡 一个坏土豆: 0 条 (总 30 条) + 📡 一号哨位: 0 条 (总 30 条) + 📡 泽平宏观: 0 条 (总 30 条) + 📡 亚太安全与海洋研究: 0 条 (总 19 条) + 📡 战略纵横家: 0 条 (总 30 条) + 📡 占豪: 0 条 (总 30 条) + 📡 远望智库: 0 条 (总 30 条) + 📡 职场老校尉: 0 条 (总 10 条) + 📡 战略前沿技术: 0 条 (总 30 条) + 📡 正和岛: 0 条 (总 30 条) + 📡 占知智库: 0 条 (总 30 条) + 📡 中国国防报: 0 条 (总 30 条) + 📡 智强战略咨询: 0 条 (总 30 条) + 📡 知远战略与防务研究所: 0 条 (总 30 条) + 📡 中国现代国际关系研究院: 0 条 (总 30 条) + 📡 中国舰船: 0 条 (总 30 条) + 📡 中国南海研究院: 0 条 (总 25 条) + 📡 智源社区: 0 条 (总 30 条) + 📡 中国舆论场: 0 条 (总 30 条) + 📡 中国指挥与控制学会: 0 条 (总 30 条) + 📡 最黑科技: 0 条 (总 30 条) + 📡 中国周边安全研究中心: 0 条 (总 30 条) + 📡 装备参考: 0 条 (总 30 条) + 📥 抓取到 10 条原始消息 + 🆕 新消息 8 条 | 缓存命中: 2 | 已有链接: 24 +09:06:25 [INFO] 中文文章：8 篇，待翻译标题的外文：0 篇 +09:06:25 [INFO] 无需翻译正文的中文文章：8 篇，待翻译正文的外文：0 篇 + 🤖 LLM评分中... + 📦 分成 3 个批次评分 (共 8 条) + 📊 评分进度: 2/3 批次完成 + 📊 评分进度: 3/3 批次完成 + 📊 评分分布: [85, 82, 70, 68, 65, 60, 55, 20] + 📊 动态阈值: 73 | 合格条目: 2 + 🏷️ 实体提取完成: 2 条 + 🧠 LLM消歧: 16 个未命中实体待判断 (模型: deepseek-v4-flash) + 🧩 聚类完成: 2 个簇 (多角度: 0, 最大簇: 1 篇) +09:06:45 [INFO] [热度阈值] 簇数=2<5, 保底阈值=40.0 + 🔥 簇热度: 0 个热点簇 (阈值=40.0) + 💾 已保存到 data/fetch-2026-07-01.json +09:06:45 [INFO] +================================================================ + 📊 运行监控报告 +================================================================ + +── ⏱ 阶段耗时 ── + RSS抓取　　　　　 71.7s + 去重　　　　　　　　 0.0s + 标题翻译　　　　　　 0.0s + 正文翻译　　　　　　 0.0s + AI评分分类　　　　 8.5s + 实体提取+角度分类　 11.4s + ──────────────────── + 合计　　　　　　　　 91.6s / 总运行 92s + +── 📡 文章统计 ── + 总计: 10 缓存命中: 2 本次处理: 8 + 缓存命中率: 20% + +── 🤖 API调用统计 ── + 无API调用（全部命中缓存） + +── 📰 源抓取统计 ── + 源总数: 131 成功: 131 失败: 0 + +================================================================ + ✅ Fetch Job 完成 | 新消息: 8 条 | 热点: 0 条 + ⏰ 下次抓取: 28.5分钟后 + +================================================== +🔄 Fetch Job | 2026-07-01 09:35:13 +================================================== + 📂 共 131 个订阅源 | 时间窗口: 60 分钟 + 📡 The War Zone: 0 条 (总 33 条) + 📡 Defense One: 0 条 (总 25 条) + 📡 Seapower: 0 条 (总 10 条) + 📡 NEWUAS: 0 条 (总 30 条) + 📡 兵路文化: 0 条 (总 11 条) + 📡 安全圈: 0 条 (总 30 条) + 📡 白马 V 视角: 0 条 (总 30 条) + 📡 补三刀: 0 条 (总 25 条) + 📡 半导体行业观察: 0 条 (总 30 条) + 📡 艾瑞咨询: 0 条 (总 30 条) + 📡 90号茶室: 0 条 (总 30 条) + 📡 北大纵横: 0 条 (总 30 条) + 📡 安全内参: 0 条 (总 30 条) + 📡 IPP评论: 0 条 (总 30 条) + 📡 补壹刀: 0 条 (总 30 条) + 📡 参考消息: 0 条 (总 30 条) + 📡 参考消息智库: 0 条 (总 30 条) + 📡 纯科学: 0 条 (总 28 条) + 📡 从心不逾矩的防务菌: 0 条 (总 30 条) + 📡 电波之矛: 0 条 (总 30 条) + 📡 大柳树防务: 0 条 (总 30 条) + 📡 动态大参考: 0 条 (总 30 条) + 📡 东南亚问题研究: 0 条 (总 30 条) + 📡 防务快讯: 0 条 (总 30 条) + 📡 大浪淘沙: 0 条 (总 30 条) + 📡 防务指南: 0 条 (总 16 条) + 📡 大湾区评论: 0 条 (总 30 条) + 📡 阜成门六号院: 0 条 (总 25 条) + 📡 复旦大学中国研究院: 0 条 (总 30 条) + 📡 浮空飞行器: 0 条 (总 30 条) + 📡 凤凰网军事频道: 0 条 (总 30 条) + 📡 观察者网: 0 条 (总 30 条) + 📡 国防科技要闻: 0 条 (总 30 条) + 📡 光明军事: 0 条 (总 30 条) + 📡 国防工业出版社: 0 条 (总 30 条) + 📡 国科环宇: 0 条 (总 18 条) + 📡 国观智库: 0 条 (总 30 条) + 📡 国防时报排头兵: 0 条 (总 30 条) + 📡 航小宇: 0 条 (总 30 条) + 📡 国际法务: 3 条 (总 30 条) + 📡 环时枢密院十号: 0 条 (总 30 条) + 📡 华山穹剑: 0 条 (总 30 条) + 📡 海鹰资讯: 0 条 (总 30 条) + 📡 黑天鹅商业情报站: 0 条 (总 30 条) + 📡 环太国际战略: 0 条 (总 23 条) + 📡 华语智库: 0 条 (总 30 条) + 📡 胡锡进观察: 0 条 (总 30 条) + 📡 混沌巡洋舰: 0 条 (总 30 条) + 📡 精确打击洞见: 0 条 (总 30 条) + 📡 今日台湾: 0 条 (总 30 条) + 📡 军事高科技在线: 0 条 (总 30 条) + 📡 军事文摘: 0 条 (总 30 条) + 📡 军民融合观察: 0 条 (总 30 条) + 📡 解放军报: 0 条 (总 30 条) + 📡 海洋防务前沿: 0 条 (总 30 条) + 📡 军鹰动态: 0 条 (总 30 条) + 📡 空天界: 0 条 (总 30 条) + 📡 空天大视野: 0 条 (总 30 条) + 📡 军武次位面: 0 条 (总 30 条) + 📡 跨境前沿: 0 条 (总 30 条) + 📡 科技导报: 0 条 (总 30 条) + 📡 砺剑: 0 条 (总 30 条) + 📡 蓝血研究: 0 条 (总 30 条) + 📡 李光满说: 0 条 (总 30 条) + 📡 瞭望智库: 1 条 (总 30 条) + 📡 六爷阿旦: 0 条 (总 30 条) + 📡 迷彩虎观察: 0 条 (总 30 条) + 📡 美国驻华大使馆: 0 条 (总 30 条) + 📡 龙牙的一座山: 0 条 (总 30 条) + 📡 梅特涅的信徒: 0 条 (总 30 条) + 📡 米尔观天下: 0 条 (总 30 条) + 📡 明叔杂谈: 0 条 (总 30 条) + 📡 澎湃新闻: 0 条 (总 30 条) + 📡 秦安战略: 0 条 (总 30 条) + 📡 破圈了: 0 条 (总 30 条) + 📡 欧亚新观察: 0 条 (总 30 条) + 📡 前沿深度解码: 0 条 (总 30 条) + 📡 盘古智库: 0 条 (总 30 条) + 📡 奇安网情局: 0 条 (总 30 条) + 📡 牛弹琴: 0 条 (总 30 条) + 📡 全球技术地图: 0 条 (总 30 条) + 📡 南方周末: 0 条 (总 30 条) + 📡 人民日报: 0 条 (总 30 条) + 📡 人大重阳: 0 条 (总 30 条) + 📡 戎评: 0 条 (总 30 条) + 📡 上海国际问题研究院: 0 条 (总 30 条) + 📡 三剑客: 0 条 (总 30 条) + 📡 盛唐如松: 0 条 (总 30 条) + 📡 书剑杂谈: 0 条 (总 30 条) + 📡 施展世界: 0 条 (总 24 条) + 📡 数字孪生战场: 0 条 (总 30 条) + 📡 外军防务研究前沿: 0 条 (总 30 条) + 📡 望穹科技: 0 条 (总 30 条) + 📡 搜狐科技: 0 条 (总 30 条) + 📡 时政国关分析: 0 条 (总 30 条) + 📡 太空与网络: 0 条 (总 30 条) + 📡 无人机邦: 0 条 (总 30 条) + 📡 沃德舆情观察: 0 条 (总 30 条) + 📡 无人机反制: 0 条 (总 30 条) + 📡 网信前沿观察: 0 条 (总 30 条) + 📡 侠客岛: 0 条 (总 30 条) + 📡 新华国际头条: 0 条 (总 30 条) + 📡 喜乔智研: 0 条 (总 30 条) + 📡 央广军事: 0 条 (总 30 条) + 📡 无人争锋: 0 条 (总 30 条) + 📡 亚太安全与海洋研究: 0 条 (总 19 条) + 📡 舆情文摘: 0 条 (总 30 条) + 📡 乌鸦校尉: 0 条 (总 30 条) + 📡 渊亭防务: 0 条 (总 30 条) + 📡 泽平宏观: 0 条 (总 30 条) + 📡 一个坏土豆: 0 条 (总 30 条) + 📡 一号哨位: 0 条 (总 30 条) + 📡 占豪: 0 条 (总 30 条) + 📡 远望智库: 0 条 (总 30 条) + 📡 职场老校尉: 0 条 (总 10 条) + 📡 战略前沿技术: 0 条 (总 30 条) + 📡 战略纵横家: 0 条 (总 30 条) + 📡 中国国防报: 0 条 (总 30 条) + 📡 正和岛: 0 条 (总 30 条) + 📡 知远战略与防务研究所: 0 条 (总 30 条) + 📡 占知智库: 0 条 (总 30 条) + 📡 智源社区: 0 条 (总 30 条) + 📡 中国舰船: 0 条 (总 30 条) + 📡 智强战略咨询: 0 条 (总 30 条) + 📡 中国现代国际关系研究院: 0 条 (总 30 条) + 📡 中国周边安全研究中心: 0 条 (总 30 条) + 📡 中国舆论场: 0 条 (总 30 条) + 📡 中国南海研究院: 0 条 (总 25 条) + 📡 中国指挥与控制学会: 0 条 (总 30 条) + 📡 装备参考: 0 条 (总 30 条) + 📡 最黑科技: 0 条 (总 30 条) + 📥 抓取到 4 条原始消息 + 🆕 新消息 0 条 | 缓存命中: 4 | 已有链接: 30 +09:36:16 [INFO] +================================================================ + 📊 运行监控报告 +================================================================ + +── ⏱ 阶段耗时 ── + RSS抓取　　　　　 62.7s + 去重　　　　　　　　 0.0s + ──────────────────── + 合计　　　　　　　　 62.7s / 总运行 63s + +── 📡 文章统计 ── + 总计: 4 缓存命中: 4 本次处理: 0 + 缓存命中率: 100% + +── 🤖 API调用统计 ── + 无API调用（全部命中缓存） + +── 📰 源抓取统计 ── + 源总数: 131 成功: 131 失败: 0 + +================================================================ + ⏰ 下次抓取: 29.0分钟后 + + + 🪖 军事科技每日资讯推送系统 - 全栈启动 +================================================== + 日志文件: D:\yinpeng\documents\code\military-digest-v3-full\my-daily\logs\20260701.log +✅ 数据管线后端已启动 (Fetch + Push 循环) +✅ Web 归档站点已启动 (http://0.0.0.0:8080) +================================================== +按 Ctrl+C 停止所有服务 +🪖 Web 归档站点 - 独立模式 + 地址: http://0.0.0.0:8080 + 数据目录: data/ + 按 Ctrl+C 停止 + * Serving Flask app 'web_archive.server' + * Debug mode: off +WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead. + * Running on all addresses (0.0.0.0) + * Running on http://127.0.0.1:8080 + * Running on http://192.168.1.40:8080 +Press CTRL+C to quit +🚀 军事科技每日资讯推送系统 +✅ 配置加载成功 +🔍 检查 LLM 接口可用性... + ✅ LLM 接口可用 +🔄 Fetch循环已启动 | 严格间隔: 30.0分钟 + +================================================== +🔄 Fetch Job | 2026-07-01 11:36:27 +================================================== + 📂 共 131 个订阅源 | 时间窗口: 60 分钟 +📤 Push循环已启动 | 定时: 0 8 * * * | 时区: UTC+08:00 + ⏰ 下次推送: 2026-07-02 08:00:00 (等待 1223.5 分钟) + 📡 Defense One: 0 条 (总 25 条) + 📡 The War Zone: 0 条 (总 33 条) + 📡 Seapower: 0 条 (总 10 条) + 📡 90号茶室: 0 条 (总 30 条) + 📡 NEWUAS: 0 条 (总 30 条) + 📡 兵路文化: 0 条 (总 11 条) + 📡 白马 V 视角: 0 条 (总 30 条) + 📡 半导体行业观察: 0 条 (总 30 条) + 📡 安全圈: 0 条 (总 30 条) + 📡 补三刀: 0 条 (总 25 条) + 📡 艾瑞咨询: 0 条 (总 30 条) + 📡 北大纵横: 0 条 (总 30 条) + 📡 参考消息: 0 条 (总 30 条) + 📡 安全内参: 0 条 (总 30 条) + 📡 参考消息智库: 0 条 (总 30 条) + 📡 补壹刀: 0 条 (总 30 条) + 📡 大浪淘沙: 0 条 (总 30 条) + 📡 电波之矛: 0 条 (总 30 条) + 📡 纯科学: 0 条 (总 28 条) + 📡 大柳树防务: 0 条 (总 30 条) + 📡 IPP评论: 0 条 (总 30 条) + 📡 东南亚问题研究: 0 条 (总 30 条) + 📡 动态大参考: 0 条 (总 30 条) + 📡 从心不逾矩的防务菌: 0 条 (总 30 条) + 📡 防务快讯: 0 条 (总 30 条) + 📡 大湾区评论: 0 条 (总 30 条) + 📡 防务指南: 0 条 (总 16 条) + 📡 阜成门六号院: 0 条 (总 25 条) + 📡 复旦大学中国研究院: 0 条 (总 30 条) + 📡 浮空飞行器: 0 条 (总 30 条) + 📡 国防科技要闻: 0 条 (总 30 条) + 📡 国观智库: 0 条 (总 30 条) + 📡 国科环宇: 0 条 (总 18 条) + 📡 观察者网: 0 条 (总 30 条) + 📡 国防工业出版社: 0 条 (总 30 条) + 📡 凤凰网军事频道: 0 条 (总 30 条) + 📡 国防时报排头兵: 0 条 (总 30 条) + 📡 光明军事: 0 条 (总 30 条) + 📡 航小宇: 0 条 (总 30 条) + 📡 国际法务: 0 条 (总 30 条) + 📡 环时枢密院十号: 0 条 (总 30 条) + 📡 黑天鹅商业情报站: 0 条 (总 30 条) + 📡 海洋防务前沿: 0 条 (总 30 条) + 📡 环太国际战略: 0 条 (总 23 条) + 📡 海鹰资讯: 0 条 (总 30 条) + 📡 华山穹剑: 0 条 (总 30 条) + 📡 华语智库: 0 条 (总 30 条) + 📡 军事文摘: 0 条 (总 30 条) + 📡 今日台湾: 0 条 (总 30 条) + 📡 解放军报: 0 条 (总 30 条) + 📡 胡锡进观察: 0 条 (总 30 条) + 📡 精确打击洞见: 0 条 (总 30 条) + 📡 军事高科技在线: 0 条 (总 30 条) + 📡 混沌巡洋舰: 0 条 (总 30 条) + 📡 军鹰动态: 0 条 (总 30 条) + 📡 军民融合观察: 0 条 (总 30 条) + 📡 科技导报: 0 条 (总 30 条) + 📡 跨境前沿: 0 条 (总 30 条) + 📡 空天大视野: 0 条 (总 30 条) + 📡 瞭望智库: 0 条 (总 30 条) + 📡 蓝血研究: 0 条 (总 30 条) + 📡 军武次位面: 0 条 (总 30 条) + 📡 李光满说: 0 条 (总 30 条) + 📡 空天界: 0 条 (总 30 条) + 📡 砺剑: 0 条 (总 30 条) + 📡 六爷阿旦: 0 条 (总 30 条) + 📡 龙牙的一座山: 0 条 (总 30 条) + 📡 明叔杂谈: 0 条 (总 30 条) + 📡 迷彩虎观察: 0 条 (总 30 条) + 📡 米尔观天下: 0 条 (总 30 条) + 📡 美国驻华大使馆: 0 条 (总 30 条) + 📡 澎湃新闻: 0 条 (总 30 条) + 📡 前沿深度解码: 0 条 (总 30 条) + 📡 秦安战略: 0 条 (总 30 条) + 📡 破圈了: 0 条 (总 30 条) + 📡 奇安网情局: 0 条 (总 30 条) + 📡 欧亚新观察: 0 条 (总 30 条) + 📡 梅特涅的信徒: 0 条 (总 30 条) + 📡 盘古智库: 0 条 (总 30 条) + 📡 牛弹琴: 0 条 (总 30 条) + 📡 全球技术地图: 0 条 (总 30 条) + 📡 人民日报: 0 条 (总 30 条) + 📡 人大重阳: 0 条 (总 30 条) + 📡 南方周末: 0 条 (总 30 条) + 📡 戎评: 0 条 (总 30 条) + 📡 上海国际问题研究院: 0 条 (总 30 条) + 📡 三剑客: 0 条 (总 30 条) + 📡 盛唐如松: 0 条 (总 30 条) + 📡 书剑杂谈: 0 条 (总 30 条) + 📡 数字孪生战场: 0 条 (总 30 条) + 📡 外军防务研究前沿: 0 条 (总 30 条) + 📡 望穹科技: 0 条 (总 30 条) + 📡 无人机邦: 0 条 (总 30 条) + 📡 搜狐科技: 0 条 (总 30 条) + 📡 沃德舆情观察: 0 条 (总 30 条) + 📡 施展世界: 0 条 (总 24 条) + 📡 网信前沿观察: 0 条 (总 30 条) + 📡 时政国关分析: 0 条 (总 30 条) + 📡 太空与网络: 0 条 (总 30 条) + 📡 新华国际头条: 0 条 (总 30 条) + 📡 乌鸦校尉: 0 条 (总 30 条) + 📡 侠客岛: 0 条 (总 30 条) + 📡 无人争锋: 0 条 (总 30 条) + 📡 无人机反制: 0 条 (总 30 条) + 📡 喜乔智研: 0 条 (总 30 条) + 📡 央广军事: 0 条 (总 30 条) + 📡 渊亭防务: 0 条 (总 30 条) + 📡 一号哨位: 0 条 (总 30 条) + 📡 舆情文摘: 0 条 (总 30 条) + 📡 亚太安全与海洋研究: 0 条 (总 19 条) + 📡 战略纵横家: 0 条 (总 30 条) + 📡 占豪: 0 条 (总 30 条) + 📡 泽平宏观: 0 条 (总 30 条) + 📡 一个坏土豆: 0 条 (总 30 条) + 📡 远望智库: 0 条 (总 30 条) + 📡 占知智库: 0 条 (总 30 条) + 📡 战略前沿技术: 0 条 (总 30 条) + 📡 职场老校尉: 0 条 (总 10 条) + 📡 中国国防报: 0 条 (总 30 条) + 📡 知远战略与防务研究所: 0 条 (总 30 条) + 📡 智强战略咨询: 0 条 (总 30 条) + 📡 智源社区: 0 条 (总 30 条) + 📡 正和岛: 0 条 (总 30 条) + 📡 中国舰船: 0 条 (总 30 条) + 📡 中国现代国际关系研究院: 0 条 (总 30 条) + 📡 中国舆论场: 0 条 (总 30 条) + 📡 中国周边安全研究中心: 0 条 (总 30 条) + 📡 中国南海研究院: 0 条 (总 25 条) + 📡 中国指挥与控制学会: 0 条 (总 30 条) + 📡 最黑科技: 0 条 (总 30 条) + 📡 装备参考: 0 条 (总 30 条) + 📥 抓取到 0 条原始消息 + ⏰ 下次抓取: 28.7分钟后 \ No newline at end of file diff --git a/my-daily/CHANGELOG.md b/my-daily/CHANGELOG.md new file mode 100644 index 0000000..b4957da --- /dev/null +++ b/my-daily/CHANGELOG.md @@ -0,0 +1,140 @@ +# 更新日志 + +## 2026-07-12 + +### 改进 +- **Fetch 管道重构**：[src/fetch_pipeline.py](./src/fetch_pipeline.py) 新增，将 `run_fetch_job()` 拆分为 8 个独立处理阶段函数（RSS抓取/去重/翻译/LLM评分/反馈修正/实体提取与聚类/热度计算/即时推送），采用管道模式串联，提升代码可维护性、可测试性和可扩展性。[main.py](./main.py) `run_fetch_job()` 从 305 行精简为 3 行，仅作为管道入口调用新模块 +- **快速上手指南**：[QUICKSTART.md](./QUICKSTART.md) 新增，包含环境准备、一键启动、常用命令和配置要点 +- **Web 归档解析器重构**：[web_archive/data_parser.py](./web_archive/data_parser.py) 引入 `BaseParser` / `WebzineParser` / `CategoryParser` 解析器模式，将 `_parse_webzine()` 和 `_parse_categories()` 重构为结构化解析器类，新增 `_safe_extract()` 安全正则提取和 `_split_sections()` 通用分段解析，提升代码可维护性和可测试性 + +### 修复 +- **RSS 标题过度截断**：[src/fetcher.py](./src/fetcher.py) `TITLE_MAX_CHARS` 从 40 提高到 120，修复英文军事标题（通常 50-100 字符）被截断导致 Web 归档页面显示不完整的问题 +- **start.py 子进程环境选择**：[start.py](./start.py) 简化 Python 解释器检测逻辑，固定优先使用 `py311_1` 环境，避免 `CONDA_PREFIX` 误判导致选错 base 环境 + +## 2026-07-01 + +### 新增 +- **一键启动日志系统**：[start.py](./start.py) 全面重写，从 `multiprocessing` 改为 `subprocess` 架构，新增 `Tee` 类实现终端输出与日志文件同步写入，日志文件按 `YYYYMMDD-HHmmSS.log` 命名存入 `logs/` 目录 +- **子进程实时输出捕获**：[start.py](./start.py) 通过 `stream_reader` 线程读取子进程管道输出，`[管线]` / `[Web]` 前缀区分来源，所有输出实时写入日志文件 + +### 修复 +- **聚类变量未初始化崩溃**：[main.py](./main.py) 修复 `run_fetch_job()` 中 `clusters` 变量在 `clustered_entries` 为空时未定义导致的 `cannot access local variable 'clusters'` 错误，新增 `clusters = []` 初始化 +- **Windows 终端 Emoji 编码错误**：[start.py](./start.py) 子进程环境变量添加 `PYTHONIOENCODING=utf-8` + `PYTHONUTF8=1`，修复 `print()` 输出 emoji 时 `UnicodeEncodeError: 'gbk' codec can't encode character` 错误 +- **子进程管道 GBK 解码错误**：[start.py](./start.py) `subprocess.Popen` 添加 `encoding='utf-8'`，修复 `UnicodeDecodeError: 'gbk' codec can't decode byte` 错误 +- **子进程输出缓冲延迟**：[start.py](./start.py) 环境变量添加 `PYTHONUNBUFFERED=1`，修复子进程输出在 Ctrl+C 退出时才批量写入日志的问题 + +## 2026-06-04 + +### 新增 +- **聚类引擎（3.6）**：[src/processors/cluster_engine.py](./src/processors/cluster_engine.py) 加权实体重叠 + 并查集细粒度聚类，equipment/event 权重=3（共享即合并），org/person 权重=2，location 权重=1，累计阈值≥4 合并，保留多角度报道 +- **聚类集成到 main.py**：[main.py](./main.py) 实体归一化后执行聚类，结果写入 `cluster_id` / `cluster_name` 字段并持久化到 fetch JSON +- **聚类验证脚本**：[scripts/test_cluster.py](./scripts/test_cluster.py) 模拟数据验证（16 篇 → 7 簇），[scripts/test_cluster_real.py](./scripts/test_cluster_real.py) 真实数据验证（134 篇 → 114 簇），[scripts/audit_clusters.py](./scripts/audit_clusters.py) 全量多篇簇人工核查 +- **簇记忆系统（3.7）**：[src/processors/cluster_memory.py](./src/processors/cluster_memory.py) SQLite 持久化已推送簇 ID 及时间戳，即时推送前过滤 24h 内已推送簇，避免同一热点反复推送。已集成到 main.py 热点推送流程 +- **簇热度计算（4.1）**：[src/processors/heat_calculator.py](./src/processors/heat_calculator.py) 四维热度评分：单篇最高分(40%) + 传播热度(30%) + 角度覆盖(20%) + 高优先加成(10%)，含时间衰减。已集成到 main.py，高热度簇中 ≥75 分文章自动提升为热点 +- **热点精评（4.4）**：[prompts/precise_heat.md](./prompts/precise_heat.md) + [src/llm.py](./src/llm.py) `precise_score_clusters()` Top 30 簇 LLM 四维精评（军事价值/时效/信息密度/传播），与启发式 6:4 融合，修正启发式对军事价值的判断偏差。阶段四 4/4 全部完成 +- **双视角洞察（5.2+5.3 合并）**：[prompts/cluster_insight.md](./prompts/cluster_insight.md) + [src/llm.py](./src/llm.py) `generate_cluster_insight()` 一次 LLM 调用同时输出精炼摘要(50-80字) + 结构化洞察(title/lead/highlights + 事件脉络/多角度分析/影响与展望)，使用 WEBZINE_MODEL pro 模型。已集成到 main.py 早报推送，新增 `🔍 热点洞察` 板块 +- **Prompt 防退化模块（5.4）**：[src/utils/prompt_checker.py](./src/utils/prompt_checker.py) 45 条禁止套话正则（空泛赞扬/模糊指代/空洞强化/乏味结论）+ 素材特异性检查，洞察生成后自动验证，违规则重试（最多 2 次） +- **摘要整合服务（5.5）**：[src/integrators/summary_integrator.py](./src/integrators/summary_integrator.py) 编排层：复用洞察 Top 5 的 summary 字段 + flash 模型补缺其余热簇，新增 `📋 今日速览` 板块。同时 5.6 洞察整合服务通过 `_generate_insights_section()` 一并完成 + +### 修复 +- **聚类泛化实体误合并**：[src/processors/cluster_engine.py](./src/processors/cluster_engine.py) 新增 `GENERIC_ENTITIES` 黑名单（无人机/导弹/雷达等23个类别词），泛化实体不触发直接合并，改走权重累加阈值判断。修复"沃尔玛无人机配送"与"俄乌冲突"被错误聚类的 bug +- **评分 JSON 截断**：[src/llm.py](./src/llm.py) `_parse_score_response()` 新增 `_try_fix_truncated_json()` 截断修复（补全括号/引号）；`call_llm()` 新增 `max_tokens=10240` 参数。经 6 个问题批次重跑验证，`max_tokens` 提升后 JSON 截断问题完全消失 +- **评分 URL 匹配**：[src/llm.py](./src/llm.py) `_reconcile_batch_results()` 新增 `_normalize_url()` 归一化匹配（去尾部斜杠、统一协议），修复 LLM 返回 URL 与输入不一致导致的匹配失败 +- **RSS 标题污染**：[src/fetcher.py](./src/fetcher.py) `_truncate_title()` → `_clean_title()`，三步清洗：按 `\n` 取首段（丢弃被 RSS 塞入标题的文章正文）→ 去首尾空白 → 超过 40 字截断 +- **内容连续换行**：[src/processor.py](./src/processor.py) `html_to_markdown()` 新增 `\n(\s*\n){2,}` → `\n\n` 正则，将含空格的连续空行（如 `\n \n \n`）压缩为单段落分隔 +- **标题兜底截断**：[src/llm.py](./src/llm.py) `_build_batch_prompt()` 新增 title `[:200]` 截断，防止异常标题绕过 fetcher 层清洗导致 Prompt 膨胀 + +### 改进 +- **评分进度日志**：[src/llm.py](./src/llm.py) `score_batch()` 新增每 10 批次进度输出 `📊 评分进度: N/401 批次完成`，替代原先的无进度静默等待 +- **评分异常诊断**：[scripts/diagnose_score_errors.py](./scripts/diagnose_score_errors.py) 独立重跑问题批次并捕获完整 LLM 响应（finish_reason / token 用量 / 响应尾部），用于定位根因 +- **簇热度详细日志**：[src/processors/heat_calculator.py](./src/processors/heat_calculator.py) `calculate_cluster_heat()` 对多篇簇输出四维分量日志（最高分×权重/传播/角度/优先），`calculate_hotspot_threshold()` 输出均值/σ/乘数/clamp 全链路 +- **簇记忆拦截日志**：[main.py](./main.py) 簇记忆过滤时逐条 `logger.info` 记录被拦截文章的簇 ID、簇名、评分、来源、标题，方便排查去重效果 +- **评价管线全面参数化**：评分/聚类/热度/记忆全部参数从硬编码迁移到 `.env > config.json > 默认值` 三级优先级。新增 `config.json` 的 `cluster`、`heat`（14 项）、`memory` 配置节，`.env` 新增 `HOT_THRESHOLD`/`CLUSTER_MERGE_THRESHOLD`/`PRECISE_BLEND_RATIO` 三个覆盖变量（共 5 个可选覆盖） +- **配置文档**：[CONFIG.md](./CONFIG.md) 环境变量与参数完整说明，含每个参数的默认值、作用环节、公式、调优建议 +- **评价管线文档**：[README.md](./README.md) 新增"评价管线详解"章节，完整描述从抓取到推送的 12 个步骤、每个步骤的配置项和计算公式 +- **早报架构重构**：[main.py](./main.py) Push 循环拆分为两波独立推送：🔥 热点速览（簇视图）→ 📰 军事科技每日精选（单篇视图），簇视图独立于 RSS 失败 +- **分类系统重设计**：[main.py](./main.py) `_categorize_entries()` 从 3 分类扩展为 5 分类（装备技术/地区安全/军事改革/产业观察/其他热点），多信号匹配（tags 子串 > 标题 > 角度），移除过于宽泛的兜底策略 +- **分类概览增强**：[main.py](./main.py) `_format_category_overview()` 每类显示 top 5（按 score 降序），含评分和可点击原文链接 +- **网摘 Prompt 改版**：[prompts/webzine.md](./prompts/webzine.md) 按参考消息风格重构：标题（主体+事件+态势）、价值点（35-45字）、正文（据 XX 报道起头，一段到底 280-320 字），JSON 输出含 title/value/body 三字段 +- **RSS Digest 移除**：[main.py](./main.py) 早报推送中移除 RSS Digest 板块，节省一次 LLM 调用；缺失的 `[持续跟踪]` 功能迁移至洞察 Prompt +- **洞察 [持续跟踪] 标记**：[prompts/cluster_insight.md](./prompts/cluster_insight.md) 跨日事件在事件脉络段首行标注 `[持续跟踪]`，区分前情提要（从素材早期报道提取）与最新突破 +- **跨日关联模块（6.3）**：[src/processors/cross_day.py](./src/processors/cross_day.py) 实体重叠（Jaccard）匹配今日簇与历史簇，输出热度/报道量变化趋势 + 趋势判定（上升/下降/持平/新晋），新增 `📈 跨日追踪` 板块 +- **网摘图片生成（6.6）**：[src/generators/image_renderer.py](./src/generators/image_renderer.py) 基于 PIL/Pillow 渲染网摘 JSON 为 PNG 图片，支持单篇 / 合并长图，自适应高度 + 动态文本换行；[src/generators/get_chinese_font.py](./src/generators/get_chinese_font.py) 中文字体管理（内置 NotoSansCJK → 系统字体 → 默认回退） +- **飞书图片推送**：[src/push/image_hosting.py](./src/push/image_hosting.py) 图床上传模块（公网 API 认证 → 文件上传 → URL 获取）+ [src/push/feishu.py](./src/push/feishu.py) 增强 `send_image()` 方法（图床上传 → Markdown 消息 → 飞书群） +- **图床配置**：`.env.example` 新增 `IMAGE_HOSTING_PASSWORD` 字段，支持从环境变量读取图床访问密码 + +### 修复 +- **图床 dir 参数编码问题**：`requests` 库将 `files={"dir": (None, "daily")}` 编码为 MIME 文件字段导致图床未识别目录，改为 `data={"dir": "daily"}` 分离发送后正常保存 +- **图床 URL 格式修正**：优先使用 `fullUrl` 完整字段，对相对路径 `url` 自动拼接域名前缀 + +### 改进 +- **早报推送流程扩展**：main.py 新增 `_generate_webzine_images()` 函数，在早报表象推送完成后自动生成并推送合并网摘图片到飞书群 +- **日志完善**：image_renderer.py 添加渲染命令执行、画布参数、字体加载等详细日志，便于排查渲染问题 +- **Docker 容器化部署**：[Dockerfile](./Dockerfile) 基于 Python 3.11-slim，内置 tini + Noto Sans CJK 中文字体；[docker-compose.yml](./docker-compose.yml) 三卷挂载（配置/数据/缓存），支持绿联云 NAS 部署 + +## 2026-06-01 + +### 新增 +- **OPML 订阅源支持**：改为从 `resources/rss_feeds.opml` 读取订阅源，支持分类分组（英文军事源 / 中文微信公众号源），共 137 个源。`config.json` 通过 `sources.base_opml` 字段引用 +- **`parse_opml()` / `merge_sources()`**：[src/config.py](./src/config.py) 新增 OPML 解析 + 源合并（base_opml + add - block - block_domains + xmlUrl 去重） +- **`test-fetch` 命令**：[main.py](./main.py) 新增测试抓取指令，支持 `--max N`（限定拉取源数量，默认 5）和 `--lookback N`（时间回溯窗口，分钟） +- **`--lookback` 参数**：[main.py](./main.py) `fetch` 和 `test-fetch` 命令均支持 `--lookback` 覆盖时间窗口，优先级：命令行 > config.json +- **评分后处理模块**：[src/scoring.py](./src/scoring.py) 实现硬约束（非目标域名 ≤79 / KOL 来源 ≤89）、时间衰减（半衰期可配，含最小年龄阈值）、动态阈值（均值 + 1.5×标准差，clamp [40, 90]） +- **网摘生成板块**：[src/sections/webzine/section.py](./src/sections/webzine/section.py) 参考消息风格网摘，支持字数校验重试（正文 280-320 字 / 价值点 35-45 字，最多 3 次）和并发生成 +- **提示词文件**：新增 [prompts/webzine.md](./prompts/webzine.md) 和 [prompts/category_overview.md](./prompts/category_overview.md) +- **多板块 Sentinel 支持**：[src/storage.py](./src/storage.py) `assemble_with_sentinels` 从仅支持 `rss` 扩展为 `rss → webzine → category_overview` 三板块 +- **早报 Webzine + 分类概览**：[main.py](./main.py) `_run_morning_push` 新增 TOP3 网摘生成和分类概览输出 + +### 修复 +- **`.env` 优先级**：[src/llm.py](./src/llm.py) `call_llm()` 和 [main.py](./main.py) `_llm_creds()` 中 `OPENAI_API_BASE` 和 `OPENAI_MODEL` 改为 `.env` 优先读取，config.json 作为回退 +- **Brotli 编码错误**：[src/fetcher.py](./src/fetcher.py) 请求头添加 `Accept-Encoding: gzip, deflate`，禁止服务器返回 br 压缩 +- **时间过滤 `break` → `continue`**：[src/fetcher.py](./src/fetcher.py) `_parse_feed_entries` 将 `break` 改为 `continue`，避免非严格倒序 RSS 导致漏掉新文章 +- **缓存 `published` 类型兼容**：[src/cache.py](./src/cache.py) `save_article()` 兼容 datetime / ISO 字符串 / None 三种 published 类型 +- **缓存 `id` 回退**：[src/cache.py](./src/cache.py) 当 `article["id"]` 不存在时自动使用 `link` 作为主键 +- **时间衰减过于激进**：[src/scoring.py](./src/scoring.py) 新增 `min_age_hours` 参数（默认 6h），文章发布不足该时间不衰减；衰减公式改为 `0.5^((age - min_age) / half_life)` + +### 改进 +- **每条源条目计数**：[src/fetcher.py](./src/fetcher.py) `fetch_single_feed_async` 输出 `📡 源名: N 条 (总 M 条)`，便于排查 +- **评分分布输出**：[main.py](./main.py) `run_fetch_job` 输出前 10 条评分分布 `📊 评分分布: [85, 72, 60, ...]` +- **时间窗口显示**：[main.py](./main.py) 输出增加 `时间窗口: N 分钟` +- **翻译管线接入**：[main.py](./main.py) `run_fetch_job` 集成了 `batch_translate_titles` 和 `batch_translate_contents` +- **Monitor 集成**：[main.py](./main.py) fetch/push 流程集成 `RunMonitor`，跟踪各阶段耗时 +- **ArticleCache 去重**：[main.py](./main.py) fetch 流程集成 SQLite 缓存去重 +- **config.json.example 更新**：新增 `translate`、`scoring`、`no_content_marker` 配置节，补充 webzine / category_overview prompt 路径 +- **translator 去回调化**：[src/translator.py](./src/translator.py) 移除 `call_ai_fn` 回调参数，直接使用 [src/llm.py](./src/llm.py) 的 `call_llm_sync` +- **即时推送阈值修正**：[config.json](./config.json) `hot_threshold` 从 85 改为 90，与 V3 开发计划一致 +- **源抓取成功率监控修复**：[src/fetcher.py](./src/fetcher.py) `fetch_all_feeds()` 回传源级成功/失败状态 → [main.py](./main.py) 接入 `monitor.record_source_result()` +- **部署脚本**：[scripts/setup_systemd.sh](./scripts/setup_systemd.sh) Linux systemd 一键安装（含卸载脚本）+ [scripts/setup_service.bat](./scripts/setup_service.bat) Windows 计划任务安装/管理 +- **源连通性检测脚本**：[scripts/check_sources.py](./scripts/check_sources.py) 逐源检测 HTTP 可达性 + RSS 可解析性 + 条目数/时效分布，输出与监控报表格式一致的汇总 +- **源成功率二次修复**：[src/fetcher.py](./src/fetcher.py) `fetch_single_feed_async()` 移除内部 try/except，异常向上传播至 `fetch_all_feeds` 的 `gather(return_exceptions=True)`，修复"源挂了但被计为成功"的误判 +- **实体提取 + 角度分类（阶段三第一轮）**：[prompts/extract_entities.md](./prompts/extract_entities.md) + [prompts/classify_angle.md](./prompts/classify_angle.md) + [src/processors/entity_extractor.py](./src/processors/entity_extractor.py)，5 类实体 + 6 类角度，实体和角度两个 LLM 调用并发执行，已集成到 fetch 流程的评分→缓存之间 +- **实体提取 Prompt 优化**：[prompts/extract_entities.md](./prompts/extract_entities.md) 新增政策/福利双列示例（军事/科技 + 政策/福利/机构），修复"军娃福利"无法提取实体（军队幼儿园、教育优待）的问题 +- **角度分类 Prompt 优化**：[prompts/classify_angle.md](./prompts/classify_angle.md) 新增广告→空角度规则、6 类反例表、决策树优先级，"战略分析"回退率从 90% 降至 20% +- **广告四层防御纵深**：[main.py](./main.py) 新增 `_is_ad_entry()` + `_has_any_entity()` 辅助函数，实体提取/热点推送/网摘Top3/分类概览全链路过滤，确保广告不进入下游 +- **缓存 tags 补存**：[src/cache.py](./src/cache.py) `save_article()` 修复 `ai_scores_json` 仅存 `scores` 漏存 `tags` 的问题，改为存储 `{score, tags, raw_scores}` +- **实体提取独立测试**：[scripts/test_entity_extract.py](./scripts/test_entity_extract.py) 从缓存读取最近文章直接测试实体提取+角度分类效果 +- **loop --max 参数**：[main.py](./main.py) `loop` 命令支持 `--max N`，仅拉取前 N 个源的长跑模式 +- **实体归一化（3.5）**：[src/processors/entity_normalizer.py](./src/processors/entity_normalizer.py) 双层归一化架构：第一层规则映射（200+ 别名，零 API 成本）+ 第二层 LLM 消歧（WEBZINE_MODEL=deepseek-v4-pro 判断未知别名），已集成到 fetch 流程实体提取→缓存之间 +- **LLM 消歧层**：[src/processors/entity_normalizer.py](./src/processors/entity_normalizer.py) 新增 `normalize_entries_async()` / `_llm_disambiguate_category()`，规则未命中的实体调用 WEBZINE_MODEL 判断是否可合并，批量并发执行 +- **LLM 消歧回流**：LLM 消歧发现的别名（霹雳-21/PL-21、DARPA、Strait of Hormuz 等 11 条）已回流至规则映射表，下次零成本命中 +- **映射表独立 JSON**：[src/processors/entity_aliases.json](./src/processors/entity_aliases.json) 别名数据从代码中抽离为独立 JSON 文件，新增别名无需修改代码 +- **Pickle 缓存**：[src/processors/entity_normalizer.py](./src/processors/entity_normalizer.py) JSON 未变时从 .pickle 缓存加载映射表 + 反向索引，跳过 JSON 解析；惰性加载，首次调用才触发 +- **映射表版本控制**：[src/processors/entity_aliases.json](./src/processors/entity_aliases.json) 新增 `_version` / `_updated` / `_changelog` 字段，通过 `get_alias_version()` 读取变更历史 +- **call_llm model_override**：[src/llm.py](./src/llm.py) `call_llm()` 新增 `model_override` 参数，支持按任务指定不同模型（如 WEBZINE_MODEL） + +--- + +## 初始版本 + +### 基础架构 +- 双循环架构：Fetch 循环（每 30 分钟）+ Push 循环（每日定时） +- RSS 异步抓取（aiohttp + feedparser） +- HTML → Markdown 转换（markdownify） +- 中英分离翻译管线（hanzi 占比 > 20% 判定 + ThreadPoolExecutor 并发） +- SQLite 三级缓存（article_cache + category_summary_cache + webzine_text） +- 11 阶段运行监控（RunMonitor） +- LLM 评分 / 摘要 / 即时推送（OpenAI 兼容接口） +- 飞书卡片消息 V2 推送（8000 字符自动分片） +- systemd 服务化部署（fetch.timer + push.timer） +- Sentinel 分段标记（单文件多板块管理） +- JSON 文件存储（fetch / push / notify） diff --git a/my-daily/CONFIG.md b/my-daily/CONFIG.md new file mode 100644 index 0000000..9c802ec --- /dev/null +++ b/my-daily/CONFIG.md @@ -0,0 +1,251 @@ +# 环境变量与参数配置说明 + +本文档说明项目中所有可配置参数的含义、默认值、作用环节和调整影响。 + +## 优先级规则 + +``` +.env 环境变量 > config.json > 源码硬编码默认值 + (最高优先级) (中等) (最低兜底) +``` + +--- + +## 一、环境变量（.env / .env.example） + +环境变量由 `dotenv` 在程序启动时加载到 `os.environ`，在对应模块初始化时读取覆盖。 + +### 1.1 AI 接口配置 + +| 变量 | 必填 | 示例 | 说明 | +|------|:---:|------|------| +| `OPENAI_API_KEY` | ✅ | `sk-xxx` | API 密钥 | +| `OPENAI_API_BASE` | ✅ | `https://api.deepseek.com` | API 端点 | +| `OPENAI_MODEL` | ✅ | `deepseek-v4-flash` | 默认模型（评分/摘要/推送/实体/角度/精评） | +| `WEBZINE_MODEL` | — | `deepseek-v4-pro` | 网摘/实体消歧专用模型（不填则复用 OPENAI_MODEL） | + +### 1.2 推送配置 + +| 变量 | 必填 | 示例 | 说明 | +|------|:---:|------|------| +| `FEISHU_WEBHOOK` | — | `https://open.feishu.cn/...` | 飞书机器人 Webhook，不填则跳过推送 | + +### 1.3 评价管线参数覆盖 + +以下变量**取消注释即生效**，覆盖 `config.json` 中的同名参数。 + +| 变量 | 默认 | 作用 | 对应步骤 | +|------|:---:|------|:---:| +| `HEAT_THRESHOLD` | — | **固定**热度阈值（0-100），替代动态计算 | 步骤 9 | +| `HOT_THRESHOLD` | 90 | 即时推送分数线 | 步骤 11 | +| `CLUSTER_MERGE_THRESHOLD` | 4 | 聚类合并所需累计权重 | 步骤 7 | +| `PRECISE_BLEND_RATIO` | 0.4 | LLM 精评分在融合热度中的占比 | 步骤 10 | +| `MEMORY_WINDOW_HOURS` | 24 | 簇记忆去重时间窗口（同簇几小时内不重复推送） | 步骤 12 | + +--- + +## 二、配置参数（config.json / config.json.example） + +### 2.1 filter — 过滤与推送阈值 + +| 参数 | 默认值 | 说明 | 影响 | +|------|:---:|------|------| +| `min_score` | 60 | 最低入库分数（低于此分的文章不入库） | 提高 → 减少无关文章入库 | +| `hot_threshold` | 90 | 即时推送分数线 | 提高 → 更少热点推送；降低 → 更多推送 | +| `context_days` | 2 | 推送上下文窗口（天） | 用于生成推送时参考近期内容 | +| `keep_days` | 7 | 数据文件保留天数 | 超过自动清理 | +| `no_content_marker` | `[NO_NEW_CONTENT]` | LLM 判定的"无新内容"标记文字 | — | +| `cluster_promotion_offset` | 15 | 簇提升分数线偏移 | 热点簇中 `score ≥ hot_threshold - this` 的文章自动提升为热点 | + +### 2.2 schedule — 调度 + +| 参数 | 默认值 | 说明 | +|------|:---:|------| +| `fetch_interval_minutes` | 30 | Fetch 循环间隔（分钟） | +| `fetch_lookback_minutes` | 60 | 抓取时间回溯窗口（分钟） | +| `push_cron` | `["0 8 * * *"]` | Push 定时（cron 表达式，默认每日 08:00） | +| `timezone_hours` | 8 | 时区偏移（+8 = 北京时间） | + +### 2.3 fetch — 抓取 + +| 参数 | 默认值 | 说明 | +|------|:---:|------| +| `max_workers` | 10 | RSS 并发抓取数 | +| `timeout` | 30 | 单源请求超时（秒） | + +### 2.4 translate — 翻译 + +| 参数 | 默认值 | 说明 | +|------|:---:|------| +| `max_concurrent` | 6 | 翻译并发数 | + +### 2.5 llm — LLM 调用 + +| 参数 | 默认值 | 说明 | +|------|:---:|------| +| `max_prompt_chars` | 10000 | 单批次 Prompt 字符数上限（超限自动拆分批次） | +| `max_concurrent_batches` | 3 | 评分并发批次数 | +| `prompts` | 见下文 | 各环节 Prompt 文件路径 | + +### 2.6 scoring — 单篇评分 + +| 参数 | 默认值 | 步骤 | 说明 | +|------|:---:|:---:|------| +| `non_target_max_score` | 79 | 2 | 非目标域名（社交媒体）文章评分上限 | +| `kol_max_score` | 89 | 2 | KOL 个人来源文章评分上限 | +| `half_life_hours` | 12 | 3 | 时间衰减半衰期（小时） | +| `min_age_hours` | 6 | 3 | 最小衰减年龄（发布不足此时不衰减） | +| `dynamic_threshold_multiplier` | 1.5 | 4 | 动态阈值的 σ 乘数 | +| `fallback_threshold` | 60 | 4 | 数据不足时的保底阈值 | + +**时间衰减公式**： +``` +有效年龄 = max(0, 当前时间 - 发布时间 - min_age_hours) +衰减因子 = 0.5 ^ (有效年龄 / half_life_hours) +最终分数 = 原分数 × 衰减因子 +``` + +**动态阈值公式**： +``` +阈值 = 均值 + dynamic_threshold_multiplier × σ +clamp[40, 90] +``` + +### 2.7 cluster — 聚类成簇 + +| 参数 | 默认值 | 说明 | 调优建议 | +|------|:---:|------|------| +| `merge_threshold` | 4 | 累计权重达到此值才合并两篇文章 | 提高 → 更严格、更小的簇；降低 → 更宽松、更大的簇 | +| `core_entity_weight` | 3 | 达到此权重的实体类别可触发直接合并 | — | +| `entity_weights.equipment` | 3 | 装备实体的权重 | 共享装备即直接合并（泛化黑名单内除外） | +| `entity_weights.event` | 3 | 事件实体的权重 | 共享事件即直接合并 | +| `entity_weights.org` | 2 | 机构实体的权重 | 需与其他实体累计 ≥ `merge_threshold` | +| `entity_weights.person` | 2 | 人物实体的权重 | 同上 | +| `entity_weights.location` | 1 | 地点实体的权重 | 同上 | +| `generic_entities` | 23 个词 | 不触发直接合并的泛化实体黑名单 | 如 `无人机`、`导弹`、`雷达` 等 | + +**聚类逻辑**： +- 共享 equipment/event 实体（不在黑名单中）→ **直接合并** +- 共享 org/person/location 或黑名单内的 equipment → **累计权重 ≥ merge_threshold 才合并** +- 通过并查集传递：A 与 B 合并、B 与 C 合并 → A-B-C 同簇 + +### 2.8 heat — 簇热度计算 + +#### 启发式热度公式（步骤 8） + +| 参数 | 默认值 | 说明 | +|------|:---:|------| +| `score_weight` | 0.4 | 单篇最高分权重（40%） | +| `propagation_weight` | 0.3 | 传播热度权重（30%） | +| `angle_weight` | 0.2 | 角度覆盖权重（20%） | +| `priority_weight` | 0.1 | 高优先级权重（10%） | +| `propagation_cap` | 30 | 传播热度得分上限 | +| `angle_cap` | 20 | 角度覆盖得分上限 | +| `priority_cap` | 10 | 高优先级加成上限 | +| `priority_per_90plus` | 5 | 每篇 ≥90 分文章的额外加分 | + +``` +热度 = min(100, + 最高分 × score_weight × 时间衰减 + + min(报道数 × log(来源多样性) × 3, propagation_cap) + + min(角度数/6 × angle_cap, angle_cap) + + min(≥90分文章数 × priority_per_90plus, priority_cap) +) +``` + +#### 热度阈值（步骤 9） + +| 参数 | 默认值 | 说明 | +|------|:---:|------| +| `threshold_clamp_min` | 35 | 热度阈值下限（动态阈值不低于此值） | +| `threshold_clamp_max` | 80 | 热度阈值上限 | +| `threshold_multiplier` | 1.5 | 动态阈值的 σ 乘数 | + +``` +热度阈值 = 均值 + threshold_multiplier × σ +clamp[threshold_clamp_min, threshold_clamp_max] +``` + +#### LLM 精评融合（步骤 10） + +| 参数 | 默认值 | 说明 | +|------|:---:|------| +| `precise_blend_ratio` | 0.4 | 精评分在融合热度中的占比（0-1） | +| `precise_top_n` | 30 | 送入 LLM 精评的簇数量 | + +``` +融合热度 = 启发式热度 × (1 - precise_blend_ratio) + 精评分 × precise_blend_ratio +``` + +### 2.9 memory — 簇记忆去重 + +| 参数 | 默认值 | 步骤 | 说明 | +|------|:---:|:---:|------| +| `window_hours` | 24 | 12 | 同簇重复推送抑制窗口（小时） | +| `cleanup_days` | 7 | 12 | 推送记录清理周期（天） | + +### 2.10 push — 推送 + +| 参数 | 默认值 | 说明 | +|------|:---:|------| +| `push.feishu.enabled` | `true` | 是否启用飞书推送 | +| `push.feishu.apiKeyName` | `FEISHU_WEBHOOK` | 飞书 Webhook 的环境变量名 | + +### 2.11 sources — 订阅源 + +| 参数 | 默认值 | 说明 | +|------|:---:|------| +| `base_opml` | `resources/rss_feeds.opml` | OPML 订阅源文件 | +| `add` | `[]` | 额外追加的 RSS 源 | +| `block` | `[]` | 屏蔽的源名称（支持 `*` 通配） | +| `block_domains` | `[]` | 屏蔽的域名（支持 `*.domain.com`） | + +--- + +## 三、常见调优场景 + +### 想要更多即时推送 + +```json +// config.json +{ "filter": { "hot_threshold": 85 } } +``` +或 `.env` 中 `HOT_THRESHOLD=85` + +### 想要更多热点簇 + +```json +// config.json +{ "heat": { "threshold_multiplier": 1.0, "threshold_clamp_min": 25 } } +``` + +### 同一热点反复推送太频繁 + +```json +// config.json +{ "memory": { "window_hours": 48 } } +``` +或 `.env` 中 `MEMORY_WINDOW_HOURS=48` + +### 簇太大（合并不够精细） + +```json +// config.json +{ "cluster": { "merge_threshold": 6 } } +``` + +### 精评分影响太大/太小 + +```json +// config.json +{ "heat": { "precise_blend_ratio": 0.2 } } +``` +精评分占比 20%（更依赖启发式）。 + +### 文章太旧不想推送 + +```json +// config.json +{ "scoring": { "half_life_hours": 6 } } +``` +半衰期从 12 小时缩短到 6 小时，旧文章衰减更快。 diff --git a/my-daily/Dockerfile b/my-daily/Dockerfile new file mode 100644 index 0000000..c216a89 --- /dev/null +++ b/my-daily/Dockerfile @@ -0,0 +1,42 @@ +# ============================================================ +# 军事科技每日资讯推送系统 - Docker 镜像 +# 基于 Python 3.11-slim，内置中文字体支持 +# ============================================================ +FROM python:3.11-slim + +LABEL maintainer="military-digest" +LABEL description="军事科技每日资讯推送系统" + +# ── 安装系统依赖（使用清华镜像加速）────────────────────────── +# tini: 信号转发（SIGTERM → 子进程安全退出） +# fontconfig + Noto Sans CJK: 网摘图片渲染中文字体 +RUN sed -i 's/deb.debian.org/mirrors.tuna.tsinghua.edu.cn/g' /etc/apt/sources.list.d/debian.sources \ + && apt-get update && apt-get install -y --no-install-recommends \ + tini \ + fontconfig \ + fonts-noto-cjk \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# ── 工作目录 ────────────────────────────────────────────────── +WORKDIR /app + +# ── 复制依赖并安装（使用清华镜像加速）──────────────────────── +COPY requirements.txt . +RUN pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple \ + && pip install --no-cache-dir -r requirements.txt + +# ── 复制项目代码 ────────────────────────────────────────────── +COPY . . + +# ── 创建运行时目录 ──────────────────────────────────────────── +RUN mkdir -p /app/data/fetch /app/data/hotalert /app/data/cluster /app/data/digest /app/data/images /app/data/cache /app/data/output /app/logs /app/cache + +# ── 入口 ────────────────────────────────────────────────────── +# tini 确保 docker stop 信号正确传递给子进程 +ENTRYPOINT ["/usr/bin/tini", "--"] +CMD ["python", "start.py"] + +# ── 健康检查 ────────────────────────────────────────────────── +HEALTHCHECK --interval=60s --timeout=10s --start-period=120s --retries=3 \ + CMD python -c "import os; exit(0 if os.path.exists('/app/data') else 1)" \ No newline at end of file diff --git a/my-daily/QUICKSTART.md b/my-daily/QUICKSTART.md new file mode 100644 index 0000000..c7e656d --- /dev/null +++ b/my-daily/QUICKSTART.md @@ -0,0 +1,69 @@ +# 快速上手指南 + +## 环境准备 + +```powershell +# 1. 切换到项目目录 +cd d:\yinpeng\documents\code\military-digest-v3-full\my-daily + +# 2. 使用 py311_1 conda 环境（依赖已安装好） +# 如果依赖缺失，运行： +# & "C:\conda\envs\py311_1\python.exe" -m pip install -r requirements.txt + +# 3. 确保 .env 和 config.json 已配置 +# .env → API Key、Webhook 等密钥 +# config.json → 调度、评分、聚类等参数 +``` + +## 一键启动（推荐） + +```powershell +python start.py +``` + +同时启动数据管线（Fetch + Push 循环）和 Web 归档站点，所有输出实时记录到 `logs/` 目录。 + +按 `Ctrl+C` 安全停止所有服务。 + +## 常用命令 + +| 命令 | 说明 | +|------|------| +| `python start.py` | 一键启动：管线 + Web 归档 + 日志 | +| `python main.py check` | 校验 LLM 接口是否可用 | +| `python main.py test-fetch` | 测试抓取（仅拉取前 5 个源，验证流程） | +| `python main.py test-fetch --max 10 --lookback 300` | 测试抓取（前 10 个源，回溯 5 小时） | +| `python main.py fetch` | 单次完整抓取 → 评分 → 缓存 | +| `python main.py fetch --lookback 120` | 单次抓取（回溯 2 小时） | +| `python main.py push` | 单次推送（早报 / 默认模式） | +| `python main.py rss` | 仅打印 RSS Digest，不推送 | +| `python main.py loop` | 长跑模式（持续运行） | + +## 首次运行建议 + +```powershell +# 1. 校验环境 +python main.py check + +# 2. 测试抓取（验证整个流程） +python main.py test-fetch + +# 3. 确认无误后，一键启动 +python start.py +``` + +## 日志查看 + +所有运行日志保存在 `logs/YYYYMMDD-HHmmSS.log`，每次启动生成一个新文件。 + +```powershell +# 查看最新日志 +Get-Content logs\*.log -Tail 50 +``` + +## 配置要点 + +- **抓取间隔**：`.env` 中 `FETCH_INTERVAL_MINUTES=30`（每 30 分钟） +- **推送时间**：`.env` 中 `PUSH_CRON="0 8 * * *"`（每天 8:00） +- **API 切换**：修改 `.env` 中 `OPENAI_API_BASE` 和 `OPENAI_MODEL` +- 更多参数见 [CONFIG.md](./CONFIG.md) \ No newline at end of file diff --git a/my-daily/README.md b/my-daily/README.md new file mode 100644 index 0000000..a7484df --- /dev/null +++ b/my-daily/README.md @@ -0,0 +1,517 @@ +# 军事科技每日资讯推送系统 + +基于大模型的军事/科技新闻采集、评分、整合与推送系统。自动抓取 RSS 订阅源，经 LLM 评分筛选与实体聚类后，生成日报并推送到飞书群。 + +## 系统架构 + +``` +┌───────────────────────────────────────────────────────────────┐ +│ Fetch 循环（每 30 分钟） │ +│ RSS 抓取 → 中英翻译 → LLM 评分 → 硬约束过滤 → 动态阈值 │ +│ → 实体提取 + 角度分类 → 实体归一化 → 实体聚类 │ +│ → 广告过滤 → 缓存 → 簇热度计算 → 即时推送（≥90 分） │ +└──────────────────┬────────────────────────────────────────────┘ + ↓ +┌───────────────────────────────────────────────────────────────┐ +│ Push 循环（每日 08:00） │ +│ 推送 1: 🔥 热点速览（簇视图） │ +│ 今日速览 + 热点洞察（含[持续跟踪]）+ 跨日追踪 + 趋势分析 │ +│ 推送 2: 📰 军事科技每日精选（单篇视图） │ +│ 网摘 Top3 + 分类概览（5 类 × 5 条） │ +│ 推送 3: 🖼️ 网摘图片（PNG 合并长图，图床上传后推送） │ +└───────────────────────────────────────────────────────────────┘ +``` + +| 核心功能 | 说明 | +|---------|------| +| 订阅源管理 | OPML 文件管理 137 个 RSS 源（英文军事 + 中文微信公众号） | +| 双语翻译 | 中英自动分离（汉字占比 > 20%），中文零 API 调用，外文并发翻译 | +| 三层评分 | 硬约束（非目标 ≤79 / KOL ≤89）+ 时间衰减 + 动态阈值 | +| 实体提取 | 5 类实体（人物/机构/装备/事件/地域）+ 6 类角度分类，双层归一化（规则 200+ 别名 + LLM 消歧） | +| 实体聚类 | 加权实体重叠 + 并查集，细粒度聚类保留多角度报道，自动命名簇 | +| 簇热度 | 四维评分（最高分 + 传播 + 角度 + 优先）+ LLM 精评融合，热度簇提升为热点 | +| 簇记忆 | SQLite 持久化已推送簇 ID，24h 内同簇不重复推送 | +| 广告过滤 | 四层防御纵深（tag 黑名单 → 动态阈值 → 空角度 → 空实体） | +| 增量缓存 | SQLite 三级缓存（文章/网摘/分类摘要），二次运行几乎零 token 消耗 | +| 飞书推送 | 即时热点推送（≥90 分）+ 每日早报三波（热点速览 + 每日精选 + 网摘图片） | +| 双视角洞察 | LLM 一次调用输出精炼摘要 + 结构化洞察（事件脉络/多角度分析/影响与展望） | +| 分类概览 | 5 类军事新闻分类（装备技术/地区安全/军事改革/产业观察/其他热点），每类 top 5 | +| 网摘图片 | PIL 渲染网摘为 PNG 合并长图，自适应高度 + 动态文本换行，图床 API 上传后推送飞书 | +| 反馈闭环 | 用户评分修正记录 + 规则自动生成，修正后的评分自动应用到后续批次 | +| 运行监控 | 12 阶段耗时 + API 调用统计 + 源抓取成功率 + 缓存命中率 | + +## 快速开始 + +### 环境要求 + +- Python 3.10+ +- 能访问 OpenAI 兼容 API（DeepSeek / 火山引擎 / OpenAI 等） + +### 安装 + +```bash +# 1. 进入项目目录 +cd my-daily + +# 2. 安装依赖 +pip install -r requirements.txt + +# 3. 配置环境变量 +cp .env.example .env +# 编辑 .env，填入 API Key 和 Webhook 地址 + +# 4. 配置系统参数（可选，已有默认值） +cp config.json.example config.json +# config.json 已存在时跳过，按需修改调度/评分/源过滤参数 +``` + +### 环境变量（.env） + +```env +# OpenAI 兼容接口（必填） +OPENAI_API_KEY=your_api_key +OPENAI_API_BASE=https://api.deepseek.com +OPENAI_MODEL=deepseek-v4-flash + +# LLM 消歧 + 网摘专用模型（可选，不填则复用 OPENAI_MODEL） +WEBZINE_MODEL=deepseek-v4-pro + +# 飞书推送（可选，不填则跳过推送） +FEISHU_WEBHOOK=your_webhook_url + +# 图床上传配置（可选，网摘图片通过图床 API 上传） +IMAGE_HOSTING_PASSWORD=your_image_hosting_password_here + +# ── 定时计划配置（可选，覆盖 config.json）── +# FETCH_INTERVAL_MINUTES=30 # RSS 拉取间隔（分钟） +# FETCH_LOOKBACK_MINUTES=60 # 回溯时间窗口（分钟） +# PUSH_CRON="0 8 * * *" # 推送定时计划（cron 表达式，多个用逗号分隔） +# TIMEZONE_HOURS=8 # 时区 + +# ── 评价管线参数覆盖（可选，最高优先级）── +# HEAT_THRESHOLD=30 # 热度阈值固定值 +# HOT_THRESHOLD=90 # 即时推送分数线 +# CLUSTER_MERGE_THRESHOLD=4 # 聚类合并阈值 +# PRECISE_BLEND_RATIO=0.4 # LLM 精评融合权重 +# MEMORY_WINDOW_HOURS=24 # 簇记忆去重时间窗口 +# CROSS_DAY_MIN_OVERLAP=0.05 # 跨日关联实体重叠阈值 +``` + +> **优先级规则**：`.env` 环境变量 > `config.json` > 源码默认值。详见 [CONFIG.md](CONFIG.md)。 + +### 配置项速查 + +所有参数及调优指南见 **[CONFIG.md](CONFIG.md)**。 + +| 配置节 | 控制环节 | 关键参数 | +|--------|----------|----------| +| **`.env`** | 接口/推送/定时/图床/评价覆盖 | `OPENAI_API_KEY`, `FEISHU_WEBHOOK`, `IMAGE_HOSTING_PASSWORD`, `FETCH_INTERVAL_MINUTES`, `PUSH_CRON` | +| `schedule` | 定时计划 | `fetch_interval_minutes`(30), `push_cron`(0 8 * * *) | +| `filter` | 过滤推送 | `hot_threshold`(90), `cluster_promotion_offset`(15) | +| `scoring` | 单篇评分 | `non_target_max_score`(79), `half_life_hours`(12), `min_age_hours`(6) | +| `cluster` | 聚类成簇 | `merge_threshold`(4), `entity_weights`, `generic_entities` | +| `heat` | 簇热度 | `score_weight`(0.4), `propagation_cap`(30), `precise_blend_ratio`(0.4) | +| `memory` | 去重记忆 | `window_hours`(24), `cleanup_days`(7) | + +## 命令用法 + +### 一键启动（推荐） + +```bash +python start.py +``` + +同时启动数据管线后端（Fetch + Push 循环）和 Web 归档站点（http://0.0.0.0:8080）。所有终端输出实时写入 `logs/YYYYMMDD-HHmmSS.log`，按 Ctrl+C 安全停止所有服务。 + +### main.py 命令 + +```bash +python main.py [options] +``` + +| 命令 | 说明 | 示例 | +|------|------|------| +| `check` | 校验 LLM 接口可达性 | `python main.py check` | +| `fetch` | 单次抓取 → 翻译 → 评分 → 缓存 | `python main.py fetch --lookback 120` | +| `test-fetch` | 测试抓取（限定源数量） | `python main.py test-fetch --max 5 --lookback 300` | +| `push` | 单次推送（含早报/默认两种模式） | `python main.py push` | +| `rss` | 仅生成 RSS Digest（不推送，打印到终端） | `python main.py rss` | +| `loop` | 长跑模式（Fetch + Push 双循环并行） | `python main.py loop` | +| `feedback` | 提交评分修正反馈 | `python main.py feedback --link URL --score 85 --reason "理由"` | + +### test-fetch / fetch 参数 + +| 参数 | 默认值 | 说明 | +|------|:---:|------| +| `--max N` | 5（仅 test-fetch） | 仅拉取前 N 个源 | +| `--lookback N` | 来自 config | 时间回溯窗口（分钟） | + +### feedback 参数 + +| 参数 | 说明 | +|------|------| +| `--link URL` | 需要修正评分的文章链接 | +| `--score N` | 修正后的分数（0-100） | +| `--reason "..."` | 修正理由 | +| `--list` | 查看所有已记录的修正 | +| `--rules` | 查看已生成的修正规则 | +| `--stats` | 查看修正统计数据 | + +## 部署 + +### Linux（systemd） + +```bash +sudo bash scripts/setup_systemd.sh # 一键安装 +bash scripts/status.sh # 查看运行状态 +sudo journalctl -u my-daily-fetch -f # 实时日志 +sudo bash scripts/uninstall_systemd.sh # 卸载 +``` + +### Windows（计划任务） + +```batch +scripts\setup_service.bat install # 安装为计划任务 +scripts\setup_service.bat start # 手动运行一次 +scripts\setup_service.bat status # 查看任务状态 +scripts\setup_service.bat loop # 前台调试 +scripts\setup_service.bat uninstall # 卸载计划任务 +``` + +### Docker（推荐，适用于绿联云 NAS / 任何 Linux 主机） + +```bash +# 1. 准备目录和配置 +mkdir -p docker/data docker/cache +cp .env docker/.env + +# 2. 构建并启动 +docker compose up -d --build + +# ── 运维 ────────────────────────────────────── +docker compose logs -f # 查看实时日志 +docker compose restart # 重启 +docker compose down # 停止并删除容器 +docker compose up -d --build # 更新代码后重建并启动 +``` + +#### 目录结构（Docker 部署） + +``` +my-daily/ +├── docker/ # Docker 持久化目录 +│ ├── .env # 配置文件（API Key, Webhook 等） +│ ├── data/ # 运行时数据（fetch/push/image 文件） +│ └── cache/ # SQLite 缓存（避免重复消费 API） +├── Dockerfile # 镜像构建文件 +├── docker-compose.yml # Docker Compose 配置 +└── .dockerignore # 构建忽略文件 +``` + +#### 在绿联云 NAS 上的操作 + +```bash +# 方案 A：SSH 直接操作（NAS 开启 SSH 后） +# 将 my-daily 目录上传到 NAS，进入目录执行： +docker compose up -d --build + +# 方案 B：绿联云 Docker 管理器（图形界面） +# 1. 在本机构建镜像并导出： +docker save -o military-digest.tar military-digest:latest + +# 2. 将 tar 文件上传到 NAS +# 3. 在绿联云 Docker > 镜像管理 > 导入镜像 +# 4. 创建容器时挂载卷： +# - /path/to/docker/.env → /app/.env（只读） +# - /path/to/docker/data → /app/data +# - /path/to/docker/cache → /app/cache +# 5. 设置重启策略：always +# 6. 启动容器 +``` + +## 运维工具 + +| 脚本 | 平台 | 功能 | +|------|:---:|------| +| [scripts/check_sources.py](scripts/check_sources.py) | 通用 | 源连通性检测：HTTP 可达性 + RSS 解析 + 条目时效分布 | +| [scripts/test_entity_extract.py](scripts/test_entity_extract.py) | 通用 | 实体提取 + 角度分类独立测试 | +| [scripts/test_normalizer.py](scripts/test_normalizer.py) | 通用 | 实体归一化效果验证 | +| [scripts/test_llm_disambiguate.py](scripts/test_llm_disambiguate.py) | 通用 | LLM 消歧层独立测试 | +| [scripts/status.sh](scripts/status.sh) | Linux | 项目状态查看 | +| [scripts/install.sh](scripts/install.sh) | Linux | 依赖安装 + systemd 服务文件生成 | +| [scripts/uninstall.sh](scripts/uninstall.sh) | Linux | systemd 服务卸载 | + +### 源连通性检测 + +```bash +python scripts/check_sources.py # 全部源 +python scripts/check_sources.py --max 50 # 前 50 个源 +python scripts/check_sources.py --failed-only # 仅显示失败源 +python scripts/check_sources.py --show-urls # 显示 URL +python scripts/check_sources.py --concurrency 20 # 自定义并发数 +``` + +## 数据与输出 + +| 目录/文件 | 说明 | 保留策略 | +|-----------|------|:---:| +| `data/fetch-YYYY-MM-DD.json` | 当日 Fetch 结果（含评分、摘要） | 7 天 | +| `data/hotalert-YYYYMMDD-HHmmSS.md` | 军事科技快讯（即时推送） | 7 天 | +| `data/cluster-YYYYMMDD-HHmmSS.md` | 热点速览（簇视图推送） | 7 天 | +| `data/digest-YYYYMMDD-HHmmSS.md` | 军事科技每日精选（单篇视图推送） | 7 天 | +| `data/images/YYYYMMDD/` | 网摘 PNG 图片 | 7 天 | +| `data/article_cache.db` | SQLite 三级缓存（文章/网摘/分类摘要） | 7 天自动清理 | +| `logs/YYYYMMDD-HHmmSS.log` | 运行日志（终端输出完整记录） | 手动清理 | + +## 评价管线详解 + +从 RSS 抓取到即时推送，文章经历 **12 个步骤**，分属两个层面：单篇评分层和簇级评价层。 + +### 第一层：单篇评分 + +#### 步骤 1 — LLM 批量评分 + +调用 `score_batch()` 对每篇文章打分（0-100），同时产出 `tags` 和 `summary`。 + +| 配置项 | 位置 | 默认值 | 说明 | +|--------|------|:---:|------| +| `max_prompt_chars` | `config.json → llm` | 10000 | 单批次 Prompt 上限，超限自动拆分 | +| `max_concurrent_batches` | `config.json → llm` | 3 | LLM 并发批次数 | +| Prompt | `prompts/score_batch.md` | — | 含核心约束：非军事≤79、KOL≤89、90+需官方首发 | + +#### 步骤 2 — 硬约束过滤 + +`apply_hard_constraints()` 对 LLM 评分施加上限： + +| 约束 | 阈值 | 作用 | +|------|:---:|------| +| 非目标域名（YouTube/Twitter/B站/微博等） | **≤79** | 社交媒体来源不可信 | +| KOL 域名（同上） | **≤89** | 个人账号非官方首发 | + +> 配置：`config.json → scoring.non_target_max_score` / `kol_max_score` + +#### 步骤 3 — 时间衰减 + +`apply_time_decay()` 指数衰减旧文章分数： + +``` +有效年龄 = max(0, 当前时间 - 发布时间 - min_age_hours) +衰减因子 = 0.5 ^ (有效年龄 / half_life_hours) +最终分数 = 原分数 × 衰减因子 +``` + +| 参数 | 默认值 | 说明 | +|------|:---:|------| +| `half_life_hours` | 12 | 半衰期 | +| `min_age_hours` | 6 | 6 小时内不衰减 | + +#### 步骤 4 — 动态阈值 + +`calculate_dynamic_threshold()` 筛选合格文章： + +``` +阈值 = 均值 + 1.5 × 标准差 +clamp[40, 90] +``` + +只有 `score ≥ 阈值` 的文章进入下一步实体提取。 + +--- + +### 第二层：簇级评价 + +#### 步骤 5 — 实体提取 + 角度分类 + +`batch_extract_all()` 对合格文章提取实体和角度，两个 LLM 调用并发执行。 + +| 类别 | Prompt | 输出 | +|------|--------|------| +| 实体（5 类） | `prompts/extract_entities.md` | `entities: {person, org, equipment, event, location}` | +| 角度（6 类） | `prompts/classify_angle.md` | `reporting_angle: 政策发布/技术突破/战略分析/舆论反应/冲突事件/人物动态` | + +#### 步骤 6 — 实体归一化 + +`normalize_entries_async()` 双层归一化： + +| 层 | 方式 | 成本 | 示例 | +|----|------|:---:|------| +| 规则映射 | `entity_aliases.json`（200+ 别名） | 零 | `PL-21 → 霹雳-21`、`DARPA → 美国国防高级研究计划局` | +| LLM 消歧 | `WEBZINE_MODEL` 批量判断未知别名 | API | `Strait of Hormuz → 霍尔木兹海峡` | + +#### 步骤 7 — 聚类成簇 + +`cluster_articles()` 加权实体重叠 + 并查集： + +| 实体类别 | 权重 | 合并条件 | +|----------|:---:|------| +| equipment / event | **3**（核心） | 共享即直接合并（泛化实体黑名单内的除外） | +| org / person | **2** | 需累计权重 ≥ 4 | +| location | **1** | 需累计权重 ≥ 4 | + +> 泛化黑名单（不触发直接合并）：`无人机`、`导弹`、`雷达`、`坦克`、`战机`、`军舰` 等 23 个类别词 + +#### 步骤 8 — 启发式热度计算 + +`calculate_cluster_heat()` 四维评分（0-100）： + +``` +热度 = min(100, + 最高分 × 0.4 × 时间衰减 + + min(报道数 × log(来源多样性) × 3, 30) + + 角度数/6 × 20 + + min(90+文章数 × 5, 10) +) +``` + +#### 步骤 9 — 热度阈值 + +`calculate_hotspot_threshold()` 动态阈值： + +``` +热度阈值 = 均值 + multiplier × σ +clamp[clamp_min, clamp_max] +``` + +| 配置 | 位置 | 默认值 | 效果 | +|------|------|:---:|------| +| `threshold_clamp_min` | `config.json → heat` | 35 | 阈值下限 | +| `threshold_clamp_max` | `config.json → heat` | 80 | 阈值上限 | +| `threshold_multiplier` | `config.json → heat` | 1.5 | σ 乘数 | +| `HEAT_THRESHOLD` | `.env` | — | **固定覆盖**（最高优先级） | + +#### 步骤 10 — LLM 精评融合 + +`precise_score_clusters()` 对 Top 30 簇调用 LLM 重新精评（军事价值40% + 时效25% + 信息密度20% + 传播15%），与启发式 6:4 融合： + +``` +融合热度 = 启发式 × 0.6 + 精评分 × 0.4 +``` + +> Prompt：`prompts/precise_heat.md`，失败时回退纯启发式热度。 + +#### 步骤 11 — 热点识别 + +热点文章 = **三类并集**： + +| 来源 | 条件 | +|------|------| +| 直接热点 | 单篇 score ≥ `hot_threshold`（90） | +| 簇提升 | 所在簇热度 ≥ 阈值，且单篇 ≥ `hot_threshold - 15`（75） | + +#### 步骤 12 — 簇记忆去重 + +`cluster_memory.is_recently_pushed()` 检查簇是否在 24h 内已推送，已推送则跳过。 + +> 存储：SQLite 表 `cluster_memory`，7 天自动清理。 + +--- + +### 配置速查 + +| 参数 | 位置 | 默认值 | 作用环节 | +|------|------|:---:|------| +| `hot_threshold` | `config.json → filter` | 90 | 步骤 11：直接热点线 | +| `non_target_max_score` | `config.json → scoring` | 79 | 步骤 2：非目标封顶 | +| `kol_max_score` | `config.json → scoring` | 89 | 步骤 2：KOL 封顶 | +| `half_life_hours` | `config.json → scoring` | 12 | 步骤 3：衰减半衰期 | +| `dynamic_threshold_multiplier` | `config.json → scoring` | 1.5 | 步骤 4：σ 乘数 | +| `threshold_clamp_min` | `config.json → heat` | 35 | 步骤 9：热度下限 | +| `threshold_multiplier` | `config.json → heat` | 1.5 | 步骤 9：热度 σ 乘数 | +| `HEAT_THRESHOLD` | `.env` | — | 步骤 9：固定覆盖 | + +## 项目文件说明 + +```text +my-daily/ +├── main.py # 入口 + 双循环架构 +├── start.py # 一键启动（含日志记录） +├── config.json # 用户配置（不入库） +├── .env # API 密钥（不入库） +├── config.json.example # 配置模板 +├── .env.example # 环境变量模板 +├── requirements.txt # Python 依赖 +├── CONFIG.md # 配置参数完整说明 +├── Dockerfile # Docker 镜像构建 +├── docker-compose.yml # Docker Compose 配置 +├── .dockerignore # Docker 构建忽略 +├── logs/ # 运行日志（YYYYMMDD-HHmmSS.log） +├── prompts/ # LLM 提示词（10 个） +│ ├── score_batch.md # 批量评分 +│ ├── digest.md # RSS Digest 生成 +│ ├── immediate_push.md # 即时推送快讯 +│ ├── webzine.md # 参考消息风格网摘 +│ ├── category_overview.md # 分类概览 +│ ├── extract_entities.md # 实体提取（5 类） +│ ├── classify_angle.md # 角度分类（6 类） +│ ├── precise_heat.md # 热点精评（四维评分） +│ ├── cluster_insight.md # 双视角洞察 +│ └── cluster_summary.md # 轻量簇摘要 +├── resources/ +│ └── rss_feeds.opml # RSS 订阅源（137 个源） +├── templates/ # 推送模板 +│ ├── cluster_view.md # 簇视图模板 +│ ├── article_view.md # 单篇视图模板 +│ └── immediate_push.md # 即时推送模板 +├── scripts/ # 运维 + 测试脚本 +│ ├── check_sources.py # 源连通性检测 +│ ├── setup_systemd.sh # Linux systemd 安装 +│ ├── setup_service.bat # Windows 计划任务管理 +│ ├── install.sh / uninstall.sh +│ └── status.sh +├── systemd/ # systemd 模板 +├── src/ # 源代码 +│ ├── config.py # 配置加载 + OPML 解析 + 环境变量覆盖 +│ ├── fetcher.py # RSS 异步抓取 +│ ├── translator.py # 中英分离翻译管线 +│ ├── llm.py # LLM 调用（评分/摘要/洞察/精评） +│ ├── scoring.py # 评分后处理（约束/衰减/阈值） +│ ├── cache.py # SQLite 三级缓存 +│ ├── storage.py # 文件存储 + Sentinel 分段 +│ ├── monitor.py # 运行监控统计 +│ ├── logger.py # 日志系统 +│ ├── processor.py # HTML → Markdown +│ ├── utils.py # 通用工具函数 +│ ├── markdown_utils.py # Markdown 解析工具 +│ ├── renderer.py # 模板渲染 +│ ├── generators/ # 图片生成模块 +│ │ ├── image_renderer.py # 网摘 PNG 图片渲染 +│ │ └── get_chinese_font.py # 中文字体管理 +│ ├── processors/ # 处理管线 +│ │ ├── entity_extractor.py # 实体提取 + 角度分类 +│ │ ├── entity_normalizer.py # 实体归一化（双层） +│ │ ├── entity_aliases.json # 别名映射表 +│ │ ├── cluster_engine.py # 实体聚类引擎 +│ │ ├── cluster_memory.py # 簇记忆系统 +│ │ ├── heat_calculator.py # 簇热度计算 +│ │ ├── rank_engine.py # 排序引擎 +│ │ ├── trend_analyzer.py # 趋势分析 +│ │ ├── feedback.py # 反馈闭环 +│ │ └── cross_day.py # 跨日关联 +│ ├── integrators/ +│ │ └── summary_integrator.py # 摘要整合编排 +│ ├── utils/ +│ │ └── prompt_checker.py # Prompt 防退化检查器 +│ ├── push/ # 推送模块 +│ │ ├── feishu.py # 飞书 Webhook 推送 +│ │ └── image_hosting.py # 图床上传模块 +│ └── sections/ # 内容板块 +│ ├── rss/section.py # RSS Digest 板块 +│ └── webzine/section.py # 网摘生成板块 +├── CHANGELOG.md # 更新日志 +├── PROGRESS.md # 开发进展追踪 +└── README.md # 本文件 +``` + +## 常见问题 + +**Q: 为什么英文源抓不到文章？** +A: 英文军事源（如 The War Zone、Defense One）更新频率低，可能 2-3 天无新文章。中文微信公众号源更新频繁，扩大 `--lookback` 可覆盖更多时间窗口。 + +**Q: 如何添加/屏蔽订阅源？** +A: OPML 文件管理为主。临时追加用 `config.json` 的 `sources.add`，屏蔽用 `sources.block` 或 `sources.block_domains`。支持 `*.domain.com` 通配。 + +**Q: 如何修改定时计划？** +A: 在 `.env` 中设置 `FETCH_INTERVAL_MINUTES`（拉取间隔）和 `PUSH_CRON`（推送定时，cron 表达式），重启服务即可生效。 + +**Q: 内存/CPU 占用如何？** +A: 异步抓取 + 并发控制（默认 10 并发），翻译/评分均分批处理。137 源全量抓取峰值内存约 200MB，日常运行稳定在 100MB 以内。 \ No newline at end of file diff --git a/my-daily/config.json.example b/my-daily/config.json.example new file mode 100644 index 0000000..abb1969 --- /dev/null +++ b/my-daily/config.json.example @@ -0,0 +1,99 @@ +{ + "filter": { + "min_score": 60, + "hot_threshold": 90, + "context_days": 1, + "keep_days": 7, + "no_content_marker": "[NO_NEW_CONTENT]", + "cluster_promotion_offset": 15 + }, + "schedule": { + "fetch_interval_minutes": 30, + "fetch_lookback_minutes": 60, + "push_cron": ["0 8 * * *"], + "timezone_hours": 8 + }, + "fetch": { + "max_workers": 10, + "timeout": 15 + }, + "translate": { + "max_concurrent": 6 + }, + "llm": { + "provider": "openai", + "model": "deepseek-v4-flash", + "baseUrl": "https://api.deepseek.com", + "apiKeyName": "OPENAI_API_KEY", + "max_prompt_chars": 10000, + "max_concurrent_batches": 3, + "prompts": { + "score_batch": "prompts/score_batch.md", + "digest": "prompts/digest.md", + "immediate_push": "prompts/immediate_push.md", + "webzine": "prompts/webzine.md", + "category_overview": "prompts/category_overview.md" + } + }, + "scoring": { + "non_target_max_score": 79, + "kol_max_score": 89, + "half_life_hours": 12, + "min_age_hours": 6, + "dynamic_threshold_multiplier": 1.5, + "fallback_threshold": 60 + }, + "cluster": { + "merge_threshold": 4, + "core_entity_weight": 3, + "entity_weights": { + "equipment": 3, + "event": 3, + "org": 2, + "person": 2, + "location": 1 + }, + "generic_entities": [ + "无人机", "导弹", "雷达", "坦克", "战机", "军舰", "潜艇", + "卫星", "火箭", "火炮", "直升机", "装甲车", "护卫舰", + "驱逐舰", "轰炸机", "巡航导弹", "弹道导弹", "防空导弹", + "战斗机", "运输机", "预警机", "加油机", "反舰导弹" + ] + }, + "heat": { + "score_weight": 0.4, + "propagation_weight": 0.3, + "angle_weight": 0.2, + "priority_weight": 0.1, + "propagation_cap": 30, + "angle_cap": 20, + "priority_cap": 10, + "priority_per_90plus": 5, + "threshold_clamp_min": 35, + "threshold_clamp_max": 80, + "threshold_multiplier": 1.5, + "precise_blend_ratio": 0.4, + "precise_top_n": 30 + }, + "memory": { + "window_hours": 24, + "cleanup_days": 7, + "cache_cleanup_days": 60 + }, + "cross_day": { + "min_overlap": 0.05, + "lookback_days": 3 + }, + "push": { + "feishu": { + "enabled": true, + "apiKeyName": "FEISHU_WEBHOOK" + } + }, + "sources": { + "base_opml": "resources/rss_feeds.opml", + "add": [], + "block": [], + "block_domains": [] + } +} diff --git a/my-daily/docker-compose.nas.yml b/my-daily/docker-compose.nas.yml new file mode 100644 index 0000000..260fa2b --- /dev/null +++ b/my-daily/docker-compose.nas.yml @@ -0,0 +1,30 @@ +services: + military-digest: + image: my-daily-military-digest:latest + container_name: military-digest + restart: unless-stopped + + # ── 文件映射 ────────────────────────────────────────────── + volumes: + # 配置文件（在 NAS 文件管理器中直接修改，修改后重启容器生效） + - /volume2/webdav/code/my-daily/docker/.env:/app/.env:ro + - /volume2/webdav/code/my-daily/docker/config.json:/app/config.json:ro + # 提示词文件 + - /volume2/webdav/code/my-daily/docker/prompts:/app/prompts:ro + # 运行时数据 + - /volume2/webdav/code/my-daily/docker/data:/app/data + # 运行日志 + - /volume2/webdav/code/my-daily/docker/logs:/app/logs + + environment: + - TZ=Asia/Shanghai + - PYTHONUNBUFFERED=1 + + ports: + - "8080:8080" + + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" \ No newline at end of file diff --git a/my-daily/docker-compose.yml b/my-daily/docker-compose.yml new file mode 100644 index 0000000..8b49dc9 --- /dev/null +++ b/my-daily/docker-compose.yml @@ -0,0 +1,39 @@ +services: + military-digest: + build: . + container_name: military-digest + restart: unless-stopped + + # ── 文件映射 ────────────────────────────────────────────── + volumes: + # 配置文件（在 NAS 上直接修改，容器内只读） + - ./docker/.env:/app/.env:ro + - ./docker/config.json:/app/config.json:ro + # 提示词文件（在 NAS 上直接修改，容器内只读） + - ./docker/prompts:/app/prompts:ro + # 运行时数据（持久化保存，含 fetch/hotalert/cluster/digest/images/cache 子目录） + - ./docker/data:/app/data + # 运行日志（持久化，方便排查问题） + - ./docker/logs:/app/logs + + # ── 环境变量 ────────────────────────────────────────────── + environment: + - TZ=Asia/Shanghai + - PYTHONUNBUFFERED=1 + + # ── 端口映射（Web 归档站点）─────────────────────────────── + ports: + - "8080:8080" + + # ── 资源限制（绿联云 NAS 建议值） ───────────────────────── + # deploy: + # resources: + # limits: + # memory: 512M + + # ── 日志轮转 ────────────────────────────────────────────── + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" \ No newline at end of file diff --git a/my-daily/docs/20260529_新闻智能分析系统开发计划_v3.md b/my-daily/docs/20260529_新闻智能分析系统开发计划_v3.md new file mode 100644 index 0000000..b35ba88 --- /dev/null +++ b/my-daily/docs/20260529_新闻智能分析系统开发计划_v3.md @@ -0,0 +1,1249 @@ +# 新闻智能分析系统开发计划 v3.0 + +> 基于大模型的时政/军事/科技新闻聚类、评价与整合系统 +> +> **迭代说明**：参考 AI Daily 项目实践与 military-digest-v3 工程落地经验，融合细粒度聚类、SQLite 增量缓存、翻译管线、网摘生成与工程化部署经验 + +--- + +## 一、项目概述 + +### 1.1 项目目标 +构建一个自动化新闻分析系统，实现： +- **细粒度聚类**：保留多角度报道，避免简单去重丢失热点 +- **动态热点识别**：基于当日新闻分布统计特征自适应判定热点 +- **三层整合**：摘要级速览（全量）+ 洞察级深度分析（Top 10）+ 参考消息风格网摘（Top 3） +- **混合处理**：实时热点感知 + 每日批量汇总生成早报 +- **跨日关联**：追踪同一事件的多日发展脉络 +- **增量缓存**：SQLite 三级缓存（文章/网摘/分类摘要），支持断点续跑，再次运行几乎零 token 消耗 +- **外文翻译**：中英分离 + 并发翻译，中文文章零 API 调用 + +### 1.2 核心约束 +| 维度 | 约束 | +|------|------| +| 数据规模 | 200-500篇/日 | +| 数据源 | RSS Feed（公众号订阅源，含外文源） | +| 输出格式 | Markdown日报 / 网摘PNG长图 / 网摘TXT文本 / 飞书推送 | +| 成本预算 | 混合模型调度，日成本约1.3-1.6元 | +| 处理时效 | 批处理15-20分钟完成（冷启动基准：5源156篇→153s） | +| 缓存命中 | 二次运行 ~5s，API 调用 0 次 | +| 部署方式 | 支持systemd服务化部署 | + +### 1.3 双循环架构（参考AI Daily优化） + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Fetch 循环（实时） │ +│ 每30分钟运行一次 │ +│ RSS抓取 → 中英分离 → 外文并发翻译 → LLM评分 → 重要性判断 │ +│ ↓ ↓ ↓ │ +│ 存入SQLite缓存 存入JSON文件(fetch-yyyy-mm-dd.json) 飞书推送│ +└─────────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────────┐ +│ Push 循环（定时） │ +│ 每日定时运行（如早8点） │ +│ 读取碎片化信息 → 细粒度聚类 → 多角度关联 → 综合评分 │ +│ → 摘要级整合(全量) → 洞察级整合(Top10) → 参考消息网摘(Top3) │ +│ → 生成日报+网摘图片 → 飞书推送 │ +└─────────────────────────────────────────────────────────────────┘ +``` + +--- + +## 二、与现有项目的对比分析 + +### 2.1 与 AI Daily 的相似性 + +| 维度 | AI Daily | 本系统 | 相似度 | +|------|----------|--------|--------| +| **数据源** | RSS订阅源 | RSS订阅源（公众号+外文） | ⭐⭐⭐⭐⭐ | +| **核心流程** | RSS→LLM评分→推送/汇总 | RSS→翻译→聚类→评分→整合→推送/汇总 | ⭐⭐⭐⭐ | +| **双循环架构** | Fetch循环+Push循环 | 实时检测+批处理 | ⭐⭐⭐⭐⭐ | +| **即时推送** | 飞书/Discord webhook | 飞书 webhook | ⭐⭐⭐⭐ | +| **定时汇总** | 每日定时推送 | 每日早报生成 | ⭐⭐⭐⭐⭐ | +| **成本预算** | 每天6毛钱 | 每天1.3-1.6元 | ⭐⭐⭐⭐ | + +### 2.2 与 military-digest-v3 的对比 + +| 维度 | military-digest-v3（现有系统） | 本系统（新架构） | 差异 | +|------|------|------|:---:| +| **架构** | 单管线（每日一次全量运行） | 双循环（Fetch 30min + Push 每日） | 🔄 架构升级 | +| **去重** | 同源同标题简单去重 | 细粒度语义聚类 + 多角度保留 | 🔄 策略升级 | +| **评分** | 四维度固定加权 | 双层筛选：硬约束 + 动态阈值融合 | 🔄 质量升级 | +| **翻译** | ✅ 中英分离 + 并发翻译 | ✅ 继承 + 统一为 Fetch 循环环节 | ✅ 复用 | +| **缓存** | ✅ SQLite 三级缓存（文章/网摘/分类摘要） | ✅ 继承 + 扩展为跨日关联数据源 | ✅ 复用 | +| **监控** | ✅ 11阶段耗时 + API统计 + 源成功率 | ✅ 继承 + 增加聚类/热点识别指标 | ✅ 复用 | +| **网摘** | ✅ 参考消息风格 + 字数校验重试 | ✅ 继承 + 作为第三层整合输出 | ✅ 复用 | +| **推送** | 企业微信 + 飞书 | **聚焦飞书 webhook** | 🔄 收束 | +| **跨日关联** | ❌ 无 | ✅ Sentinel标记 + 事件时间线追踪 | 🆕 新增 | +| **防退化** | ❌ 无 | ✅ 禁止套话 + 素材支撑要求 | 🆕 新增 | + +### 2.3 本系统的差异化优势 + +| 特性 | AI Daily | military-digest-v3 | 本系统 | 价值 | +|------|----------|------|------|------| +| **热点识别** | 单篇文章评分 | 四维度固定加权 | **双层筛选：硬约束+动态阈值融合** | 质量控制+热点发现协同 | +| **去重策略** | 简单去重 | 同源标题去重 | 细粒度聚类+多角度保留 | 不丢失多角度报道 | +| **内容整合** | 单篇摘要 | 摘要+网摘 | 三层整合（摘要+洞察+网摘） | 覆盖速览/深度/传播三种需求 | +| **翻译能力** | 无 | ✅ 中英分离并发 | ✅ 继承 | 外文源零额外成本 | +| **增量缓存** | JSON文件 | ✅ SQLite三级缓存 | ✅ 继承+扩展 | 断点续跑，二次运行~5s | +| **网摘生成** | 无 | ✅ 参考消息风格+字数校验 | ✅ 继承 | 适合直接传播的成品内容 | +| **领域聚焦** | AI领域 | 军事/科技 | 时政/军事/科技 | 专业领域实体识别 | +| **角度分类** | 无 | 无 | 6种报道角度分类 | 支持多角度整合 | +| **跨日关联** | 无 | 无 | 事件时间线追踪+Sentinel标记 | 追踪事件发展脉络 | + +### 2.4 从 AI Daily 借鉴的核心设计 + +1. **双循环架构**：Fetch循环（实时）+ Push循环（定时） +2. **systemd服务化**：使用systemd timer替代Python内部定时，提升稳定性 +3. **文件存储规范**：fetch/notify/push三种文件类型，明确保留策略 +4. **评分硬约束**：非目标领域上限、KOL转述上限，有效控制信息质量 +5. **Prompt防退化**：禁止套话、要求从素材出发，避免LLM输出风格趋同 +6. **双视角洞察**：metadata（事实压缩）+ 正文（趋势判断）解耦 +7. **Sentinel分段标记**：单文件多板块精确管理，支持跨日关联 + +### 2.5 从 military-digest-v3 继承的工程资产 + +1. **SQLite 三级缓存**：文章缓存 + 网摘缓存 + 分类摘要缓存，含自动过期清理和数据库迁移机制 +2. **翻译管线**：中英分离（汉字占比 > 20% 判定）+ 外文并发翻译（ThreadPoolExecutor），中文文章零 API 调用 +3. **监控统计体系**：11 阶段耗时 + API 调用按用途/模型分组 + 源抓取成功/失败率 + 缓存命中率 +4. **网摘生成**：参考消息风格 + 公众号源智能识别前缀 + 字数校验重试（280-320字，最多3次） +5. **异步流式管线**：生产者-消费者模型，RSS 抓取与 AI 处理流水线并行 +6. **图片生成**：Pillow 命令模式渲染，支持单篇 + TOP3 合并长图 + +--- + +## 三、开发阶段规划 + +### 阶段一：基础架构搭建（Week 1-2） +**目标**：建立数据流和存储基础，确保稳定采集 + +| 序号 | 任务 | 描述 | 输出 | 依赖 | +|------|------|------|------|------| +| 1.1 | RSS采集模块 | 解析公众号RSS Feed，提取标题、正文、来源、发布时间 | 原始文章数据库 | - | +| 1.2 | 数据标准化 | 统一文章格式，清洗HTML标签，提取纯文本 | 标准化文章表 | 1.1 | +| 1.3 | 文件存储层 | JSON文件存储(fetch-yyyy-mm-dd.json) | 存储规范 | 1.2 | +| 1.4 | **SQLite缓存系统** | 三级缓存设计（文章/网摘/分类摘要），含过期策略与迁移机制 | 缓存模块 | 1.2 | +| 1.5 | **翻译管线** | 中英分离 + 外文并发翻译 | 翻译模块 | 1.2 | +| 1.6 | **监控统计** | 阶段耗时 + API消耗 + 成功率 + 缓存命中率 | 监控模块 | 1.1 | +| 1.7 | 日志系统 | 有效日志记录，便于问题定位 | 日志模块 | 1.1 | + +#### 1.4 SQLite 缓存系统设计（继承 military-digest-v3） + +**数据库位置**：`data/article_cache.db` + +**表1：article_cache（文章处理结果缓存）** + +| 字段 | 类型 | 说明 | +|------|------|------| +| `id` | TEXT (PK) | 文章唯一标识（RSS entry id/link） | +| `source` | TEXT | 来源名称 | +| `title` | TEXT | 原始标题 | +| `link` | TEXT | 原文链接 | +| `translated_title` | TEXT | 翻译后的中文标题 | +| `content` | TEXT | 原始正文 | +| `translated_content` | TEXT | 翻译后的中文正文（前300字） | +| `ai_score` | REAL | AI 加权总分 | +| `ai_summary` | TEXT | AI 生成的中文摘要 | +| `ai_category` | TEXT | 分类（装备动态/地区冲突/战略政策） | +| `ai_scores_json` | TEXT | 四维度评分 JSON | +| `webzine_text` | TEXT | 网摘文本（迁移新增字段） | +| `published_time` | DATETIME | 文章发布时间 | +| `processed_time` | DATETIME | 处理时间（索引，用于过期判断） | +| `status` | INTEGER | 1=成功 2=失败 | +| `error_msg` | TEXT | 失败时的错误信息 | + +**索引**：`idx_processed_time`、`idx_source` + +**表2：category_summary_cache（分类摘要缓存）** + +| 字段 | 类型 | 说明 | +|------|------|------| +| `date` | TEXT (联合PK) | 日期字符串（YYYY-MM-DD） | +| `category` | TEXT (联合PK) | 分类名称 | +| `summary_text` | TEXT | 分类介绍文本 | +| `generated_time` | DATETIME | 生成时间 | + +**缓存策略**： + +| 策略项 | 设计 | +|------|------| +| 过期时间 | 24 小时（与时间窗口一致，可配置） | +| 自动清理 | 每次运行时清理 7 天前数据（`clear_expired(keep_days=7)`） | +| 写入方式 | `REPLACE INTO`（幂等，支持覆盖更新） | +| 网摘分离更新 | `save_article_webzine()` 单独更新 `webzine_text` 字段，不影响其他缓存 | +| 连接降级 | 数据库连接失败时静默降级，不影响主流程 | +| 全局单例 | `get_cache()` 全局单例模式，避免重复创建连接 | + +**数据库迁移机制**： + +```python +def _migrate_add_webzine_text(self): + """自动检测并添加 webzine_text 列""" + cursor.execute("PRAGMA table_info(article_cache)") + columns = [col[1] for col in cursor.fetchall()] + if 'webzine_text' not in columns: + cursor.execute("ALTER TABLE article_cache ADD COLUMN webzine_text TEXT") +``` + +**缓存命中流程**： + +``` +文章被抓取 → 以 article.id 查询缓存（expire_hours=24） + ├─ 命中 + status=1 → 直接使用缓存数据（from_cache=True），跳过翻译和AI分析 + └─ 未命中/过期 → 完整翻译+分析管线，结果 REPLACE INTO 写入缓存 +``` + +**实测效果**（military-digest-v3 验证数据）： + +| 指标 | 第一次（冷启动） | 第二次（含缓存） | +|------|:---:|:---:| +| 文章缓存命中 | 0 / 15 | **15 / 15** | +| API 调用次数 | 37 次 | **0 次** | +| 总耗时 | 153s | **~5s** | +| 退出码 | 0 | 0 | + +#### 1.5 翻译管线设计（继承 military-digest-v3） + +**核心策略**：中英分离，中文文章零 API 调用，外文文章并发翻译。 + +**中文检测**：汉字占比 > 20% 判定为中文（`is_chinese()` 函数）。 + +**翻译流程**： + +``` +所有文章 + ├─ 中文文章 → translated_title = title（零API调用） + │ translated_content = content[:300]（截取即可） + └─ 外文文章 → 提交到 ThreadPoolExecutor（max_workers=AI_CONCURRENCY） + ├─ translate_title()：标题翻译（temperature=0.2, max_tokens=200） + └─ translate_content()：正文前500词翻译 → 截取300字（max_tokens=800） +``` + +**并发控制**： + +| 参数 | 值 | 说明 | +|------|-----|------| +| `AI_CONCURRENCY` | 8 | 默认并发数，可通过配置调整 | +| `temperature` | 0.2 | 翻译低温度，确保准确性 | +| 失败降级 | 原文填充 | 翻译失败时 `translated_title = title`，不中断流程 | + +**关键实现**： + +```python +def batch_translate_titles(articles, api_key, base_url, model, max_concurrent=None): + """中英分离：中文直接填充，外文并发翻译标题""" + chinese_articles = [] + foreign_articles = [] + + for a in articles: + if is_chinese(a['title']): + a['translated_title'] = a['title'] + chinese_articles.append(a) + else: + foreign_articles.append(a) + + # 外文并发翻译 + with ThreadPoolExecutor(max_workers=max_concurrent) as executor: + futures = {executor.submit(translate_title, a, ...): a for a in foreign_articles} + for future in as_completed(futures): + chinese_articles.append(future.result()) + + return chinese_articles +``` + +**设计优势**： +- 中文文章零 API 调用，节省 40%+ 翻译 token +- 标题和正文分离翻译，先翻译标题再决定是否需要翻译正文（可配合关键词筛选） +- 外文并发翻译，翻译耗时 = 单篇耗时（而非 N × 单篇耗时） + +#### 1.6 监控统计设计（继承 military-digest-v3） + +**阶段耗时统计（11 阶段）**： + +| 阶段 key | 中文标签 | 统计内容 | +|------|------|------| +| `rss_fetch` | RSS抓取 | 所有源并行抓取耗时 | +| `time_filter` | 时间过滤 | 按时间窗口过滤耗时 | +| `deduplicate` | 去重 | 去重处理耗时 | +| `translate_titles` | 标题翻译 | 外文标题并发翻译耗时 | +| `keyword_filter` | 关键词筛选 | 关键词命中筛选耗时 | +| `translate_contents` | 正文翻译 | 外文正文并发翻译耗时 | +| `ai_analyze` | AI评分分类 | 批量AI评分分类耗时 | +| `webzine_generate` | 网摘生成 | TOP3网摘并发生成耗时 | +| `category_overview` | 分类介绍 | 4类分类介绍并发生成耗时 | +| `report_generate` | 报告生成 | Markdown报告拼接写入耗时 | +| `image_generate` | 图片生成 | 网摘长图渲染耗时 | + +**API 调用统计**：按用途（标题翻译/正文翻译/AI评分分类/网摘生成/分类介绍）和模型分组，记录调用次数、token 估算、重试次数、失败次数。 + +**源抓取统计**：成功/失败源计数，失败源名称列表。 + +**文章统计**：总数、缓存命中数、本次处理数、缓存命中率。 + +**报告示例**（运行结束自动打印）： + +``` +================================================================ + 📊 运行监控报告 +================================================================ + +── ⏱ 阶段耗时 ── + RSS抓取 7.5s + 时间过滤 0.0s + 去重 0.0s + 标题翻译 17.9s + 关键词筛选 0.0s + 正文翻译 9.9s + AI评分分类 30.7s + 网摘生成 53.3s + 分类介绍 33.5s + 报告生成 0.0s + 图片生成 0.3s + ──────────────────── + 合计 153.1s / 总运行 153s + +── 📡 文章统计 ── + 总计: 15 缓存命中: 0 本次处理: 15 + 缓存命中率: 0% + +── 🤖 API调用统计 ── + [按用途] + 标题翻译: 7次 / tokens≈1400 / 重试0 + 正文翻译: 6次 / tokens≈4800 / 重试0 + AI评分分类: 15次 / tokens≈12000 / 重试0 + 网摘生成: 5次 / tokens≈4000 / 重试2 + 分类介绍: 4次 / tokens≈600 / 重试0 + [按模型] + deepseek-v3: 37次 / tokens≈22800 + +── 📰 源抓取统计 ── + 源总数: 5 成功: 5 失败: 0 + +================================================================ +``` + +**数据字段规范**（扩展版）： + +```json +{ + "title": "内容标题", + "link": "原始链接", + "published": "发布时间", + "source": "来源（公众号名称）", + "content": "Markdown格式的正文内容", + "translated_title": "翻译后的中文标题", + "translated_content": "翻译后的中文正文", + "tags": "LLM识别的标签", + "score": "LLM评分（0-100）", + "summary": "LLM生成的中文摘要", + "fetched_at": "抓取时间", + "entities": ["提取的实体列表"], + "reporting_angle": "报道角度", + "webzine_text": "参考消息风格网摘文本（Top N 专属）", + "from_cache": "是否来自缓存" +} +``` + +**验收标准**： +- [ ] 稳定采集50+ RSS源 +- [ ] 数据入库成功率 > 95% +- [ ] SQLite 缓存三级覆盖（文章/网摘/分类摘要） +- [ ] 缓存命中时 API 调用为 0 +- [ ] 中英分离正确，中文文章零翻译 API 调用 +- [ ] 监控报告包含阶段耗时、API统计、源成功率、缓存命中率 +- [ ] 日志系统可定位问题 + +--- + +### 阶段二：Fetch循环与双层评分系统（Week 3-4） +**目标**：实现实时采集、双层评分（硬约束+动态阈值融合）、即时推送 + +| 序号 | 任务 | 描述 | 输出 | 依赖 | +|------|------|------|------|------| +| 2.1 | Fetch循环引擎 | 每30分钟循环一次，抓取RSS→中英分离→并发翻译→评分 | fetch引擎 | 1.7 | +| 2.2 | **第一层：单篇评分Prompt** | 单篇文章重要性评分（0-100），**含硬约束** | `prompts/article_score.txt` | 2.1 | +| 2.3 | **硬约束过滤器** | 非时政军事上限、KOL转述上限、时效衰减 | 硬约束模块 | 2.2 | +| 2.4 | 即时推送判断 | 单篇≥90分触发即时推送候选 | 推送判断模块 | 2.3 | +| 2.5 | 飞书Webhook推送 | 飞书群机器人推送（Fetch 循环即时推送 + Push 循环日报推送） | 飞书推送适配器 | 2.4 | +| 2.6 | systemd服务化 | 使用systemd timer管理服务 | systemd配置 | 2.1 | + +#### 双层评分架构设计 + +``` +第一层：单篇文章评分（硬约束） 第二层：实体簇热度（动态阈值） +├── 输入：单篇RSS文章（已翻译） ├── 输入：同一实体的多篇报道（已硬约束评分） +├── 处理：LLM评分 + 硬约束修正 ├── 处理：统计特征 + 动态阈值计算 +└── 输出：0-100分（已约束） └── 输出：热点判定 + ├── ≤79：非目标领域，过滤 ├── 低于阈值：普通关注 + ├── 80-89：一般关注，进入聚类 └── 高于阈值：热点，优先整合 + └── 90+：高优先级，即时推送候选 +``` + +#### 第一层：评分硬约束设计 + +**Prompt核心约束**： +```markdown +## 评分规则（硬性约束） + +1. **非时政/军事/科技主题上限 79 分** + - 如果内容不属于目标领域（如纯娱乐八卦、生活琐事），最高只能给 79 分 + +2. **KOL 转述上限 89 分** + - 如果只是 KOL/大V 对已有信息的转述/评论，而非原创信息，最高只能给 89 分 + +3. **时效性衰减** + - 发布超过24小时：分数×0.9 + - 发布超过48小时：分数×0.8 + +4. **90+ 分必须同时满足**： + - 重大事件/突破性进展/政策发布 + - 一手信息源（官方发布、权威媒体首发） + - 对目标领域有实质性影响 + +5. **标签禁止空泛** + - 禁止：["军事", "新闻", "热点"] + - 要求：["歼-35A", "舰载战斗机", "隐身性能", "海军航空兵"] +``` + +**代码实现**： +```python +def apply_score_constraints(entry: dict, raw_score: int) -> tuple[int, list[str]]: + """ + 应用评分硬约束 + 返回：修正后的分数，应用的约束标签列表 + """ + constraints_applied = [] + + # 约束1：非目标领域上限79 + if not is_target_domain(entry['content']): + raw_score = min(raw_score, 79) + constraints_applied.append("非目标领域") + + # 约束2：KOL转述上限89 + if is_kol_repost(entry): + raw_score = min(raw_score, 89) + constraints_applied.append("KOL转述") + + # 约束3：时效性衰减 + hours_old = get_hours_since_published(entry) + if hours_old > 24: + decay_factor = 0.9 ** (hours_old // 24) + raw_score = int(raw_score * decay_factor) + constraints_applied.append(f"时效衰减({hours_old}h)") + + return raw_score, constraints_applied +``` + +**第一层输出分级**： +| 分数段 | 处理策略 | 说明 | +|--------|---------|------| +| ≤79 | 过滤，不入库 | 非目标领域内容 | +| 80-89 | 入库，进入聚类 | 一般关注，参与动态阈值计算 | +| 90+ | 入库，即时推送候选 | 高优先级，同时触发即时推送判断 | + +**验收标准**： +- [ ] Fetch循环每30分钟稳定运行 +- [ ] 翻译管线集成正确（中英分离+并发翻译） +- [ ] 硬约束生效（非目标领域≤79，KOL转述≤89，时效衰减） +- [ ] 90+文章触发即时推送候选 +- [ ] SQLite缓存正常工作（二次运行零API调用） +- [ ] 监控统计准确记录各阶段数据 +- [ ] systemd服务可一键启动/停止 + +--- + +### 阶段三：实体提取与细粒度聚类（Week 5-6） +**目标**：实现细粒度聚类，区分同一实体的不同报道角度 + +| 序号 | 任务 | 描述 | 输出 | 依赖 | +|------|------|------|------|------| +| 3.1 | 实体提取Prompt | 设计并优化实体提取提示词 | `prompts/extract_entities.txt` | 2.6 | +| 3.2 | 角度分类Prompt | 设计报道角度分类提示词 | `prompts/classify_angle.txt` | 2.6 | +| 3.3 | 批量实体提取 | 实现50篇/批并发调用，小模型处理 | 实体提取服务 | 3.1 | +| 3.4 | 角度分类服务 | 批量角度分类，与实体提取并行 | 角度分类服务 | 3.2 | +| 3.5 | 实体归一化 | 大模型消歧，合并别名 | 实体归一化服务 | 3.3 | +| 3.6 | 聚类算法 | 基于实体+角度的细粒度聚类 | 聚类引擎 | 3.4, 3.5 | +| 3.7 | 记忆系统 | 避免同一信息反复推送 | 去重模块 | 3.6 | + +**验收标准**： +- [ ] 实体提取准确率 > 85% +- [ ] 角度分类准确率 > 80% +- [ ] 聚类后实体簇数量合理（200篇→30-50个簇） +- [ ] 同一信息不重复推送 + +--- + +### 阶段四：热度评价与动态阈值（Week 7-8） +**目标**：实现第二层筛选——基于硬约束后分数的动态热点识别 + +| 序号 | 任务 | 描述 | 输出 | 依赖 | +|------|------|------|------|------| +| 4.1 | **实体簇热度计算** | 融合硬约束分数的多维度热度计算 | 热度指标服务 | 3.6 | +| 4.2 | **第二层：动态阈值算法** | 基于硬约束后分数分布的自适应阈值 | 阈值计算模块 | 4.1 | +| 4.3 | 热点识别引擎 | 硬约束过滤 + 动态阈值筛选的双层判定 | 热点识别服务 | 4.2 | +| 4.4 | 热点精评Prompt | 大模型深度评分（Top 30热点簇） | `prompts/precise_heat.txt` | 4.3 | + +#### 第二层：动态阈值设计（融合硬约束分数） + +**实体簇热度计算**： +```python +class Cluster: + def __init__(self, entity_name: str): + self.entity_name = entity_name + self.articles = [] # 已硬约束评分的文章列表 + self.hot_score = 0 # 综合热度分 + + def calculate_hot_score(self) -> int: + """ + 计算实体簇热度，融合硬约束后的单篇分数 + """ + if not self.articles: + return 0 + + # 基础分数：最高分文章的分数（已硬约束） + max_score = max(a['score'] for a in self.articles) + + # 传播热度：报道数量 × log(来源多样性) + report_count = len(self.articles) + source_diversity = len(set(a['source'] for a in self.articles)) + propagation_heat = report_count * math.log(source_diversity + 1) + + # 角度覆盖度：不同报道角度数 / 6 + angles = set(a.get('reporting_angle', 'unknown') for a in self.articles) + angle_coverage = len(angles) / 6 + + # 高优先级文章加成（90+文章额外加权） + high_priority_count = sum(1 for a in self.articles if a['score'] >= 90) + priority_bonus = high_priority_count * 5 # 每篇90+加5分 + + # 综合计算 + self.hot_score = min(100, int( + max_score * 0.4 + # 单篇最高分权重40% + min(propagation_heat * 3, 30) + # 传播热度权重30% + angle_coverage * 20 + # 角度覆盖权重20% + priority_bonus # 高优先级加成10% + )) + + return self.hot_score +``` + +**动态阈值算法（融合版）**： +```python +def calculate_hotspot_threshold(clusters: list[Cluster], date: str) -> float: + """ + 基于硬约束后分数分布计算动态阈值 + 同时考虑统计特征和高优先级文章数量 + """ + # 获取当日所有簇的热度分数（已融合硬约束） + scores = [c.hot_score for c in clusters] + + if len(scores) < 5: + return 75 # 数据不足时使用保底阈值 + + # 统计特征 + mean_score = mean(scores) + std_score = std(scores) + median_score = median(scores) + + # 方法1：统计阈值（均值+1.5倍标准差） + threshold_stat = mean_score + 1.5 * std_score + + # 方法2：保底阈值（至少3条或前10%） + min_count = max(3, len(scores) * 0.1) + sorted_scores = sorted(scores, reverse=True) + threshold_adaptive = sorted_scores[min_count - 1] + + # 方法3：硬约束保底（考虑90+高优先级文章数量） + high_priority_count = sum( + 1 for c in clusters + if any(a['score'] >= 90 for a in c.articles) + ) + if high_priority_count >= 3: + threshold_backup = median_score + else: + threshold_backup = mean_score + 0.5 * std_score + + # 取三者中较低值，确保热点不被遗漏 + final_threshold = min(threshold_stat, threshold_adaptive, threshold_backup) + + logger.info(f"动态阈值计算: 统计={threshold_stat:.1f}, " + f"保底={threshold_adaptive:.1f}, 硬约束保底={threshold_backup:.1f}, " + f"最终={final_threshold:.1f}") + + return final_threshold +``` + +**双层热点判定流程**： +```python +def identify_hotspots(clusters: list[Cluster], date: str) -> tuple[list[Cluster], list[Cluster]]: + """ + 双层热点识别：硬约束过滤 + 动态阈值筛选 + """ + # 第一层：硬约束过滤（单篇层面已处理，此处检查） + valid_clusters = [] + for c in clusters: + if all(a['score'] <= 79 for a in c.articles): + continue + valid_clusters.append(c) + + # 计算各簇热度（融合硬约束分数） + for c in valid_clusters: + c.calculate_hot_score() + + # 第二层：计算动态阈值 + threshold = calculate_hotspot_threshold(valid_clusters, date) + + # 阈值筛选 + 额外条件 + hotspots = [] + normal = [] + for c in valid_clusters: + if c.hot_score >= threshold: + has_high_priority = any(a['score'] >= 90 for a in c.articles) + significantly_above = c.hot_score >= threshold + 10 + + if has_high_priority or significantly_above: + hotspots.append(c) + else: + normal.append(c) + else: + normal.append(c) + + hotspots.sort(key=lambda x: (x.hot_score, x.max_article_score), reverse=True) + + return hotspots, normal +``` + +**融合设计优势**： +| 层级 | 作用 | 输入 | 输出 | +|------|------|------|------| +| 第一层（硬约束） | 质量控制 | 单篇文章 | 过滤非目标领域，标记高优先级 | +| 第二层（动态阈值） | 热点发现 | 实体簇（多篇聚合） | 识别突发热点，自适应当日分布 | + +**验收标准**： +- [ ] 实体簇热度计算正确（融合硬约束分数） +- [ ] 动态阈值自适应当日分布 +- [ ] 双层筛选协同工作（硬约束过滤→动态阈值筛选） +- [ ] 热点识别召回率 > 90%，误报率 < 20% + +--- + +### 阶段五：Push循环与三层整合（Week 9-10） +**目标**：实现定时汇总、三层整合（摘要+洞察+网摘）、日报生成 + +| 序号 | 任务 | 描述 | 输出 | 依赖 | +|------|------|------|------|------| +| 5.1 | Push循环引擎 | 每日定时运行，读取碎片化信息→整合→推送 | push引擎 | 4.4 | +| 5.2 | 摘要级整合Prompt | 多源报道合并为50-80字精炼摘要 | `prompts/summary_integrate.txt` | 5.1 | +| 5.3 | 洞察级整合Prompt | Top 10生成结构化深度分析（**双视角设计**） | `prompts/insight_integrate.txt` | 5.1 | +| 5.4 | **Prompt防退化模块** | 禁止套话、要求从素材出发 | 防退化检查器 | 5.2, 5.3 | +| 5.5 | 摘要整合服务 | 小模型批量处理全量簇 | 摘要整合服务 | 5.2, 5.4 | +| 5.6 | 洞察整合服务 | 大模型逐个处理Top 10（**双视角输出**） | 洞察整合服务 | 5.3, 5.4 | +| 5.7 | **网摘生成服务** | Top 3 参考消息风格网摘（继承 military-digest-v3） | 网摘生成服务 | 5.1 | +| 5.8 | 时间线提取 | 从多源报道中提取事件时间线 | 时间线提取模块 | 5.5 | +| 5.9 | 多角度整合 | 同一事件不同角度报道的整合 | 多角度整合模块 | 5.6 | + +#### 双视角洞察设计（借鉴AI Daily） + +**设计理念**：metadata（新闻编辑视角，事实压缩）+ 正文（情报分析师视角，趋势判断）解耦 + +**Prompt设计**： +```markdown +## Part 1: Metadata（新闻编辑视角） + +请生成以下结构化信息： +- title: 10字以内的标题（事实陈述，无修饰） +- lead: 20字以内的导语（核心事实） +- highlights: 3-5个关键要点（bullet points，纯事实） + +约束：只陈述事实，不做趋势判断，不用形容词。 + +## Part 2: 洞察正文（情报分析师视角） + +基于多源报道，识别： +1. 事件脉络：关键时间节点和进展 +2. 多角度分析：技术维度、战略维度、舆论维度 +3. 影响与展望：短期影响、中长期趋势、值得关注信号 + +约束： +- 禁止空泛评论（如"意义重大""影响深远"） +- 每个观点必须有素材支撑 +- 使用具体名称（装备型号/政策名称/组织名称），不用"某些""部分"等模糊指代 +``` + +**输出示例**： +```markdown +--- +title: "歼-35A列装部队" +lead: "空军官方确认歼-35A已列装首批作战部队" +highlights: + - 首次公开确认进入实战化部署阶段 + - 隐身性能对标F-35C，可能在舰载领域形成优势 + - 多国媒体关注其对西太平洋军事平衡的影响 +--- + +## 事件脉络 +- 2024-11：珠海航展首次公开亮相 +- 2025-03：完成舰载适配测试 +- 2025-05：空军确认列装首批作战部队【新进展】 + +## 多角度分析 +- **技术维度**：隐身涂层、航电系统、舰载适配均有突破 +- **战略维度**：提升海军航空兵远海作战能力 +- **舆论维度**：外媒关注中美舰载机代差缩小 + +## 影响与展望 +短期：提升海军航母编队作战能力 +中期：可能在西太平洋形成局部优势 +值得关注：后续舰载版测试进展、出口动向 +``` + +#### 网摘生成设计（继承 military-digest-v3） + +**定位**：在三层整合中，网摘是面向直接传播的成品内容。不同于摘要（速览用）和洞察（分析用），网摘采用《参考消息》官方新闻报道格式，适合直接推送到飞书群或作为独立内容分发。 + +**核心特性**： + +| 特性 | 说明 | +|------|------| +| 风格 | 《参考消息》官方新闻报道格式：标题+原标题+发布日期+正文+价值点 | +| 来源识别 | 自动区分微信公众号源（"据公众号 XXX 报道"）和普通源（"据 XXX 报道"） | +| 字数校验 | 正文严格 280-320 字（含标点），价值点 35-45 字 | +| 重试机制 | 不达标自动重试，最多 3 次，每次带具体反馈（"正文只有 XX 字，太少！"） | +| 生成范围 | Top 3 文章，与摘要（全量）和洞察（Top 10）互补 | +| 输出格式 | 文本文件（.txt）+ Pillow 渲染长图（.png） | + +**网摘 Prompt 核心约束**： + +```markdown +标题要求 +简洁、客观、中性、信息密度高 +结构：主体 + 事件 + 核心态势 +不抒情、不夸张、不用网络用语 + +正文格式要求 +开头第一句必须加：据[来源]报道 +正文风格：客观、平实、严谨、书面化，类似外电编译稿 +必须充分展开：补充背景、说明意义、分析影响、展望前景 +全文一段到底，不分段 + +【字数强制要求 - 必须严格执行】 +正文字数严格控制在 280～320 字（含标点），一个字都不能少，一个字都不能多！ +如果内容不够，请合理补充相关背景、行业态势、同类项目对比等专业内容 + +价值点要求 +正文结束后空一行，再写价值点 +价值点严格 35～45 字（含标点） +句式结构：事件 - 影响 / 后果 - 值得关注 +``` + +**字数校验重试机制**： + +```python +def generate_webzine_for_article(article, api_key, base_url, model, max_retries=3): + """为单篇文章生成参考消息风格网摘，支持字数校验重试""" + for attempt in range(max_retries): + result = call_ai([...], purpose="网摘生成") + + # 提取正文部分 + body_text = extract_body(result) + body_len = len(body_text) + + if 280 <= body_len <= 320: + return result # 达标，直接返回 + + # 不达标，在 prompt 末尾追加反馈 + if body_len < 280: + prompt += f"\n\n【上次反馈：正文只有{body_len}字，太少！请大幅增加内容！】" + else: + prompt += f"\n\n【上次反馈：正文有{body_len}字，太多！请精简内容！】" + + return last_result # 重试耗尽，返回最后结果 +``` + +**输出示例**： + +``` +标题：歼-35A隐身舰载战斗机正式列装海军航空兵部队 +原标题：China's J-35A stealth fighter enters service with naval aviation +发布日期：2025年05月24日 +正文：据解放军报报道，中国海军航空兵部队已于近日正式列装歼-35A隐身舰载战斗机，标志着中国成为继美国之后第二个具备隐身舰载机作战能力的国家。歼-35A采用双发中型设计，配备国产涡扇-19发动机，最大起飞重量约30吨，雷达反射截面积据分析优于F-35C。该机配备有源相控阵雷达、分布式光电系统以及先进的电子战套件，可携带霹雳-15中远程空空导弹和鹰击-12超音速反舰导弹等多型武器。军事专家指出，歼-35A的列装将大幅提升中国海军航母编队的制空作战和远程打击能力，特别是在西太平洋方向形成对F-35C的局部数量优势。美国海军战争学院报告认为，中国正加速缩小与美国在舰载航空领域的技术差距，预计到2030年前后将形成至少3个歼-35A舰载机联队的规模。日本防卫省已表示将密切关注相关动向。 +价值点：歼-35A列装标志着中国成为全球第二个拥有隐身舰载机的国家，将显著改变西太平洋海上力量对比，后续量产规模和舰载适配进展值得持续关注。 +``` + +**三层整合对比**： + +| 层次 | 产品 | 覆盖范围 | 定位 | 输出格式 | +|------|------|:---:|------|------| +| 第一层 | 摘要级速览 | 全量（所有簇） | 快速浏览，50-80字精炼 | Markdown表格 | +| 第二层 | 洞察级分析 | Top 10 热点 | 深度分析，双视角（metadata+正文） | Markdown段落 | +| 第三层 | 参考消息网摘 | Top 3 热点 | 成品内容，直接传播 | TXT文本 + PNG长图 | + +#### Prompt防退化设计（借鉴AI Daily） + +**禁止套话列表**： +```python +FORBIDDEN_PHRASES = [ + "意义重大", "影响深远", "引发关注", "备受瞩目", + "深水区", "拐点", "白热化", "新纪元", "里程碑", + "不容忽视", "值得注意", "值得关注", # 除非后接具体内容 + "某些", "部分", "一些", "相关", # 模糊指代 +] + +def check_degeneration(content: str) -> list[str]: + """检查内容是否包含禁止套话""" + violations = [] + for phrase in FORBIDDEN_PHRASES: + if phrase in content: + violations.append(phrase) + return violations +``` + +**Prompt中的防退化指令**： +```markdown +## 输出约束（防退化） + +1. 禁止使用以下套话： + - "意义重大""影响深远""引发关注""备受瞩目" + - "深水区""拐点""白热化""新纪元""里程碑" + +2. 每个观点必须有素材支撑： + - 错误：该装备性能先进 + - 正确：该装备隐身系数0.01，优于F-35的0.02 + +3. 使用具体名称，禁止模糊指代： + - 错误：某些国家表示关注 + - 正确：日本防卫省发布关注声明 + +4. 从素材出发，每次措辞应不同，避免模板化输出 +``` + +**验收标准**： +- [ ] 摘要信息完整度 > 90% +- [ ] 洞察分析覆盖双视角（metadata + 正文） +- [ ] 网摘正文字数达标率 > 90%（280-320字） +- [ ] 网摘价值点字数达标率 > 90%（35-45字） +- [ ] 无禁止套话出现 +- [ ] 整合内容无明显事实错误 + +--- + +### 阶段六：日报生成与跨日关联（Week 11-12） +**目标**：生成结构化日报（含Sentinel标记），实现跨日关联 + +| 序号 | 任务 | 描述 | 输出 | 依赖 | +|------|------|------|------|------| +| 6.1 | 日报模板设计 | Markdown模板，包含热点、速览、数据概览、网摘 | `templates/daily_report.md` | 5.9 | +| 6.2 | **Sentinel分段标记** | 单文件多板块精确管理 | `src/utils/sentinel.py` | 6.1 | +| 6.3 | **跨日关联模块** | 从历史日报提取上下文，追踪事件脉络 | `src/processors/cross_day.py` | 6.2 | +| 6.4 | 排序算法 | 综合热度、时效、重要性的排序逻辑 | 排序模块 | 4.4 | +| 6.5 | 日报渲染服务 | 按模板组装数据，生成Markdown（含Sentinel） | 日报渲染服务 | 6.1, 6.2, 6.4 | +| 6.6 | 网摘图片渲染 | Pillow渲染TOP3合并长图，含Banner标题 | 图片渲染服务 | 5.7 | +| 6.7 | Web归档站点 | 部署在线阅览站点 | Web站点 | 6.5 | +| 6.8 | 数据保留策略 | fetch保留2天，notify保留2天，push保留5天 | 清理策略 | 6.7 | + +#### 简化版Sentinel分段标记设计 + +**标记格式**： +```markdown +--- +title: "每日情报早报" +date: "2026-05-24" +stats: + total_entities: 87 + hotspot_count: 10 +--- + + +## 📋 全量新闻速览 + +| # | 实体 | 摘要 | 来源数 | 热度 | +|---|------|------|--------|------| +| 1 | 歼-35A | 空军确认列装\|隐身性能对标F-35C | 8 | ★★★★★ | + + + +## 🔥 热点深度洞察 + +### 1. 歼-35A列装进展 [持续跟踪] +> 速览：空军官方确认歼-35A已列装首批作战部队 [8源报道] + +**事件脉络** +- 5月20日：首次公开亮相 +- 5月24日：【新进展】确认列装部队 +... + + + +## 📰 参考消息网摘 + +（网摘文本内容，含标题/原标题/发布日期/正文/价值点） + +``` + +**核心代码**： +```python +import re +from datetime import datetime, timedelta + +def extract_section(content: str, section_name: str) -> str: + """从日报内容中提取指定section""" + pattern = rf'(.*?)' + match = re.search(pattern, content, re.DOTALL | re.IGNORECASE) + return match.group(1).strip() if match else "" + +def get_historical_context(entity_name: str, days: int = 7) -> dict: + """获取某实体近N天的历史报道上下文""" + context = { + "first_seen": None, + "previous_summaries": [], + "previous_insights": [], + "mention_count": 0 + } + + for i in range(days): + date = (datetime.now() - timedelta(days=i)).strftime("%Y-%m-%d") + file_path = f"data/reports/daily-{date}.md" + + try: + with open(file_path, 'r', encoding='utf-8') as f: + content = f.read() + + if entity_name in content: + context["mention_count"] += 1 + if context["first_seen"] is None: + context["first_seen"] = date + + summary_section = extract_section(content, "summary") + for line in summary_section.split('\n'): + if entity_name in line and line.strip().startswith('|'): + context["previous_summaries"].append({ + "date": date, + "content": line.strip() + }) + + insights_section = extract_section(content, "insights") + if entity_name in insights_section: + insight_blocks = re.split(r'### \d+\.', insights_section) + for block in insight_blocks: + if entity_name in block: + context["previous_insights"].append({ + "date": date, + "content": block.strip()[:500] + }) + break + except FileNotFoundError: + continue + + return context + +def generate_cross_day_marker(entity_name: str) -> str: + """生成跨日标记""" + history = get_historical_context(entity_name, days=7) + + if history["mention_count"] == 0: + return "首次报道" + elif history["mention_count"] == 1: + return "持续跟踪" + else: + return f"持续跟踪({history['mention_count']}次)" +``` + +**验收标准**： +- [ ] 日报生成时间 < 20分钟 +- [ ] Sentinel标记正确，可精确提取summary/insights/webzine +- [ ] 跨日关联可追踪实体历史报道 +- [ ] 网摘长图渲染正常（合并多篇 + Banner标题） +- [ ] Web站点可正常访问 + +--- + +### 阶段七：优化与迭代（Week 13-14） +**目标**：持续优化，功能扩展 + +| 序号 | 任务 | 描述 | 优先级 | +|------|------|------|--------| +| 7.1 | Prompt优化 | 基于实际效果迭代优化提示词 | 高 | +| 7.2 | 反馈闭环 | 收集用户反馈，标注数据，优化模型 | 高 | +| 7.3 | 趋势分析 | 7日热点趋势图，周期性报告 | 中 | +| 7.4 | 个性化推荐 | 基于用户阅读历史的个性化排序 | 低 | + +--- + +## 四、技术栈建议 + +### 4.1 核心组件 +| 组件 | 推荐方案 | 说明 | +|------|----------|------| +| 数据采集 | Python + feedparser | RSS解析 | +| 数据存储 | SQLite + JSON文件 | 数据库+文件双存储 | +| LLM调用 | OpenAI SDK / 国产模型SDK | 统一接口封装 | +| 翻译 | 中英分离 + ThreadPoolExecutor并发 | 中文文章零API调用 | +| 缓存 | SQLite 三级缓存 + 自动迁移 | 断点续跑，二次运行 ~5s | +| 图片渲染 | Pillow + 中文字体fallback | 网摘长图生成 | +| 任务调度 | systemd timer | 系统级服务管理 | +| Web展示 | Hugo / Flask | 静态/动态站点 | +| 推送 | 飞书 Webhook | 群机器人消息推送 | +| 日志 | Python logging + systemd journal | 集中日志管理 | +| 监控 | 11阶段耗时 + API统计 + 源成功率 | 运行结束自动打印报表 | + +### 4.2 模型选择 +| 任务类型 | 推荐模型 | 预估成本 | +|----------|----------|----------| +| 标题翻译 | DeepSeek-V3 / Qwen-Plus | ~0.001元/次 | +| 正文翻译 | DeepSeek-V3 / Qwen-Plus | ~0.002元/次 | +| 实体提取 | DeepSeek-V3 / Qwen-Plus | ~0.001元/次 | +| 角度分类 | DeepSeek-V3 / Qwen-Plus | ~0.001元/次 | +| 单篇评分 | DeepSeek-V3 / Qwen-Plus | ~0.001元/次 | +| 摘要整合 | DeepSeek-V3 / Qwen-Plus | ~0.002元/次 | +| 网摘生成 | DeepSeek-V3 / Qwen-Plus | ~0.005元/次（含重试） | +| 实体消歧 | GPT-4o / Claude-3.5-Sonnet | ~0.01元/次 | +| 热点精评 | GPT-4o / Claude-3.5-Sonnet | ~0.02元/次 | +| 洞察整合 | GPT-4o / Claude-3.5-Sonnet | ~0.1元/次 | + +**日成本估算**： +- 200-500篇/日 × 0.001元（小模型）= 0.2-0.5元 +- 外文翻译（按30%外文率）：60-150篇 × 0.003元 = 0.18-0.45元 +- 100个簇 × 0.001元（小模型）= 0.1元 +- 3篇网摘 × 0.005元 = 0.015元 +- 10个洞察 × 0.1元（大模型）= 1元 +- **总计：约1.5-2.1元/日** + +--- + +## 五、项目结构 + +``` +news-intelligence-system/ +├── config/ +│ ├── rss_sources.json # RSS源配置 +│ ├── models.yaml # 模型配置 +│ └── systemd/ # systemd服务配置 +├── src/ +│ ├── collectors/ # 数据采集 +│ │ └── rss_collector.py +│ ├── processors/ # 处理管道 +│ │ ├── translator.py # 翻译管线（中英分离+并发翻译） +│ │ ├── entity_extractor.py +│ │ ├── angle_classifier.py +│ │ ├── cluster_engine.py +│ │ ├── heat_calculator.py +│ │ └── cross_day.py # 跨日关联 +│ ├── integrators/ # 内容整合 +│ │ ├── summary_integrator.py +│ │ ├── insight_integrator.py +│ │ └── webzine_generator.py # 网摘生成（参考消息风格+字数校验） +│ ├── generators/ # 报告生成 +│ │ ├── daily_report_generator.py +│ │ ├── image_renderer.py # 网摘图片渲染（Pillow） +│ │ └── web_renderer.py +│ ├── push/ # 推送平台 +│ │ ├── base.py +│ │ └── feishu.py # 飞书 webhook（即时+日报双模式） +│ ├── storage/ # 存储层 +│ │ ├── cache.py # SQLite三级缓存（文章/网摘/分类摘要） +│ │ └── file_storage.py # JSON文件读写 +│ ├── monitor/ # 监控统计 +│ │ └── monitor.py # 阶段耗时+API消耗+成功率+缓存命中率 +│ └── utils/ # 工具函数 +│ ├── sentinel.py # Sentinel分段标记 +│ ├── prompt_checker.py # Prompt防退化检查 +│ └── language.py # 中文检测（is_chinese） +├── prompts/ # 提示词文件 +│ ├── article_score.txt # 单篇评分（含硬约束） +│ ├── extract_entities.txt +│ ├── classify_angle.txt +│ ├── quick_heat.txt +│ ├── precise_heat.txt +│ ├── summary_integrate.txt +│ ├── insight_integrate.txt # 双视角洞察 +│ └── webzine_generate.txt # 参考消息网摘（含字数约束） +├── templates/ +│ └── daily_report.md # 含Sentinel标记（summary/insights/webzine） +├── data/ +│ ├── cache.db # SQLite缓存数据库 +│ ├── fetch/ # fetch-yyyy-mm-dd.json +│ ├── notify/ # notify-yyyy-mm-dd.json +│ └── reports/ # daily-yyyy-mm-dd.md +├── output/ # 输出文件 +│ ├── report_YYYYMMDD.md # Markdown日报 +│ ├── webzine_YYYYMMDD.txt # 网摘文本 +│ └── webzine_YYYYMMDD.png # 网摘合并长图 +├── web/ # Web归档站点 +├── tests/ +├── scripts/ +│ ├── fetch_loop.py +│ ├── push_loop.py +│ └── setup_systemd.sh +├── requirements.txt +└── README.md +``` + +--- + +## 六、关键风险与应对 + +| 风险 | 影响 | 应对措施 | +|------|------|----------| +| RSS源失效 | 数据缺失 | 多源备份，监控告警 | +| LLM API限流 | 处理延迟 | 批量处理，重试机制，降级策略 | +| 翻译质量不稳定 | 后续分析偏差 | 低温度(0.2)翻译，失败时原文填充 | +| 缓存数据库损坏 | 缓存失效 | 连接降级不影响主流程，自动重建 | +| 网摘字数不达标 | 输出质量下降 | 3次重试校验，每次带具体反馈 | +| 实体识别错误 | 聚类偏差 | 人工标注反馈，Prompt迭代 | +| 热点漏识别 | 信息缺失 | 动态阈值保底机制，人工复核 | +| 成本超支 | 预算超支 | Token监控，模型降级，配额控制 | +| 进程崩溃 | 服务中断 | systemd自动重启 | +| LLM输出退化 | 质量下降 | **Prompt防退化检查** | + +--- + +## 七、里程碑与交付物 + +| 里程碑 | 时间 | 交付物 | +|--------|------|--------| +| M1 | Week 2 | 稳定运行的数据采集系统（含翻译管线、SQLite缓存、监控统计） | +| M2 | Week 4 | Fetch循环上线，**评分硬约束生效**，飞书即时推送可用 | +| M3 | Week 6 | 细粒度聚类引擎，实体提取准确率>85% | +| M4 | Week 8 | 动态热点识别系统，召回率>90% | +| M5 | Week 10 | Push循环上线，**三层整合（摘要+洞察+网摘）+Prompt防退化** | +| M6 | Week 12 | **Sentinel跨日关联上线**，网摘长图渲染，Web归档站点 | +| M7 | Week 14 | 持续优化迭代 | + +--- + +## 八、借鉴与继承总结 + +### 8.1 架构层面（借鉴 AI Daily） +1. **双循环架构**：Fetch循环（实时）+ Push循环（定时） +2. **systemd服务化**：替代Python内部定时，提升稳定性 +3. **文件存储规范**：fetch/notify/reports三种文件类型，明确保留策略 + +### 8.2 LLM应用层面（借鉴 AI Daily） +1. **评分硬约束**：非目标领域≤79、KOL转述≤89，有效控制信息质量 +2. **Prompt防退化**：禁止套话、要求从素材出发，避免LLM输出风格趋同 +3. **双视角洞察**：metadata（事实压缩）+ 正文（趋势判断）解耦 + +### 8.3 数据管理层面（借鉴 AI Daily） +1. **Sentinel分段标记**：单文件多板块精确管理，支持跨日关联 +2. **跨日关联**：从历史日报提取上下文，追踪事件发展脉络 +3. **数据保留策略**：自动清理过期文件 + +### 8.4 工程资产层面（继承 military-digest-v3） +1. **SQLite三级缓存**：文章+网摘+分类摘要，含自动迁移和过期清理，二次运行 ~5s +2. **翻译管线**：中英分离（汉字占比>20%判定）+ 外文并发翻译，中文零API调用 +3. **监控统计**：11阶段耗时 + API按用途/模型分组 + 源抓取成功率 + 缓存命中率 +4. **网摘生成**：参考消息风格 + 公众号源智能识别 + 280-320字校验重试（3次） +5. **图片渲染**：Pillow命令模式 + 中文字体多级fallback +6. **飞书推送**：webhook文本推送 + 5级异常分层捕获 + 失败优雅降级 + +### 8.5 工程层面 +1. **日志集中管理**：systemd journal便于问题定位 +2. **推送聚焦飞书**：webhook消息推送，覆盖即时快讯和日报两种场景 +3. **成本可控**：每天约1.5-2.1元（含翻译），缓存命中时几乎零成本 + +--- + +## 附录A：military-digest-v3 实际运行验证数据 + +> 以下数据来自 military-digest-v3 系统 2026-05-14 的实际运行验证，为新系统的设计和成本估算提供基准参考。 + +### A.1 全链路冷启动验证 + +清除所有缓存和输出文件后运行，完整链路通过： + +| 阶段 | 耗时 | 状态 | +|------|:---:|:---:| +| RSS 抓取（5源） | 7.5s | ✅ | +| 时间过滤 (156→22篇) | 0.0s | ✅ | +| 去重 | 0.0s | ✅ | +| 标题翻译（7篇外文） | 17.9s | ✅ | +| 关键词筛选 (22→15篇) | 0.0s | ✅ | +| 正文翻译（6篇外文） | 9.9s | ✅ | +| AI评分分类（15篇） | 30.7s | ✅ | +| 网摘生成（3篇，含2次重试） | 53.3s | ✅ | +| 分类介绍生成（4类） | 33.5s | ✅ | +| 图片生成 | 0.3s | ✅ | +| 报告生成 | 0.0s | ✅ | +| **总耗时** | **153s** | ✅ | + +**API 消耗统计：** + +| 用途 | 次数 | tokens 估算 | +|------|:---:|:---:| +| 标题翻译 | 7 | 1,400 | +| 正文翻译 | 6 | 4,800 | +| AI评分分类 | 15 | 12,000 | +| 网摘生成 | 5 (含2次重试) | 4,000 | +| 分类介绍 | 4 | 600 | +| **合计** | **37** | **~22,800** | + +**输出文件验证：** + +``` +military_report_20260514.md 10.6 KB ✅ Markdown 格式完整 +military_webzine_20260514.txt 4.9 KB ✅ TOP3 网摘文本完整 +military_webzine_20260514.png 868.5 KB ✅ 合并长图渲染正常 +article_cache.db 60.0 KB ✅ 数据完整无异常 +``` + +### A.2 SQLite 缓存数据完整性验证 + +冷启动全链路运行后，直接查询 `article_cache.db` 验证： + +**基本信息：** + +| 表 | 行数 | 说明 | +|------|:---:|------| +| `article_cache` | 15 | 与关键词筛选后文章数完全一致 | +| `category_summary_cache` | 4 | 今日必看/装备动态/地区冲突/战略政策全覆盖 | + +**逐项检查：** + +| 检查项 | 结果 | 判定 | +|------|:---:|:---:| +| id 重复 | 0 条 | ✅ | +| 失败条目 (status≠1) | 0 条 | ✅ | +| translated_title 缺失 | 0/15 | ✅ | +| translated_content 缺失 | 0/15 | ✅ | +| ai_summary 缺失 | 0/15 | ✅ | +| ai_category 缺失 | 0/15 | ✅ | +| published_time 缺失 | 0/15 | ✅ | +| webzine_text 覆盖率 | 3/15 | ✅ (仅TOP3需要) | +| ai_score 范围 | 3.3 ~ 6.5 (avg 5.3) | ✅ 正态分布 | +| 分类分布 | 装备7 / 战略5 / 冲突3 | ✅ 合理 | + +### A.3 缓存命中验证（连续运行2次） + +| 指标 | 第一次（冷启动） | 第二次（含缓存） | +|------|:---:|:---:| +| 文章缓存命中 | 0 / 15 | **15 / 15** ✅ | +| API 调用次数 | 37 次 | **0 次** ✅ | +| 总耗时 | 153s | **~5s** ✅ | +| 退出码 | 0 | 0 | + +> 第二次运行时文章缓存、网摘缓存、分类介绍缓存全部命中，跳过所有 AI 调用，仅做 RSS 抓取 + 文件排版，几乎零 token 消耗。 + +### A.4 飞书推送验证 + +| 场景 | 配置 | 结果 | 主流程 | +|------|------|------|:---:| +| 推送关闭 | `enable_feishu_push: false` | "未启用任何推送渠道，跳过" | exit 0 ✅ | +| **飞书（真实webhook）** | `enable_feishu_push: true` | **飞书推送成功** 🎉 | exit 0 ✅ | + +**错误处理矩阵（5级分层）：** + +| 异常类型 | 处理方式 | 主流程影响 | +|------|------|:---:| +| `ConnectionError` | warning 日志 + 返回 False | 无 | +| `Timeout` (连接5s/读取15s) | warning 日志 + 返回 False | 无 | +| `HTTPError` (4xx/5xx) | warning 日志 + 返回 False | 无 | +| `JSONDecodeError` | warning 日志 + 返回 False | 无 | +| 其他 `Exception` | warning 日志 + 返回 False | 无 | + +--- + +*文档版本：v3.0* +*最后更新：2026-05-29* +*状态：融合 AI Daily 四大核心设计 + military-digest-v3 六大工程资产* diff --git a/my-daily/docs/PROGRESS.md b/my-daily/docs/PROGRESS.md new file mode 100644 index 0000000..a4d7920 --- /dev/null +++ b/my-daily/docs/PROGRESS.md @@ -0,0 +1,227 @@ +# 开发进展追踪 + +--- + +> 对照 [20260529_新闻智能分析系统开发计划_v3.md](./20260529_新闻智能分析系统开发计划_v3.md) +> +> 最后更新：2026-06-30 + +## 总览 + +| 阶段 | 名称 | 进度 | 状态 | +|------|------|:---:|:---:| +| 一 | 基础架构搭建 | 7/7 | ✅ 完成 | +| 二 | Fetch循环与双层评分系统 | 6/6 | ✅ 完成 | +| 三 | 实体提取与细粒度聚类 | 7/7 | ✅ 完成 | +| 四 | 热度评价与动态阈值 | 4/4 | ✅ 完成 | +| 五 | Push循环与三层整合 | 9/9 | ✅ 完成 | +| 六 | 日报生成与跨日关联 | 7/8 | ⚠️ 部分完成 | +| 七 | 优化与迭代 | 3/4 | ⚠️ 部分完成 | + +--- + +## 阶段一：基础架构搭建 + +| 序号 | 任务 | 状态 | 实现位置 | 备注 | +|:---:|------|:---:|------|------| +| 1.1 | RSS 采集模块 | ✅ | [src/fetcher.py](src/fetcher.py) | aiohttp + feedparser 异步抓取，137 源 OPML | +| 1.2 | 数据标准化 | ✅ | [src/processor.py](src/processor.py) | HTML → Markdown（markdownify） | +| 1.3 | 文件存储层 | ✅ | [src/storage.py](src/storage.py) | JSON 文件读写，fetch/hotalert/cluster/digest 四类，时间戳格式 `YYYYMMDD-HHmmSS` | +| 1.4 | SQLite 缓存系统 | ✅ | [src/cache.py](src/cache.py) | 三级缓存（article / category_summary / webzine_text），含迁移机制 | +| 1.5 | 翻译管线 | ✅ | [src/translator.py](src/translator.py) | 中英分离（hanzi>20%）+ ThreadPoolExecutor 并发 | +| 1.6 | 监控统计 | ✅ | [src/monitor.py](src/monitor.py) | 12 阶段耗时 + API 统计 + 源成功率 + 缓存命中率 | +| 1.7 | 日志系统 | ✅ | [src/logger.py](src/logger.py) | Python logging，分级输出 | + +**验收状态**： +- [x] 稳定采集 50+ RSS 源（实际 137 源） +- [x] SQLite 缓存三级覆盖（文章/网摘/分类摘要） +- [x] 中英分离正确，中文文章零翻译 API 调用 +- [x] 监控报告（阶段耗时、API统计、源成功率、缓存命中率） +- [x] 日志系统可定位问题 + +--- + +## 阶段二：Fetch 循环与双层评分系统 + +| 序号 | 任务 | 状态 | 实现位置 | 备注 | +|:---:|------|:---:|------|------| +| 2.1 | Fetch 循环引擎 | ✅ | [main.py](main.py) `fetch_loop()` / `run_fetch_job()` | 可配置间隔（默认 30 分钟），支持 `--max` / `--lookback` 测试参数 | +| 2.2 | 单篇评分 Prompt | ✅ | [prompts/score_batch.md](prompts/score_batch.md) | 批量评分，0-100 分含核心约束与分档规则 | +| 2.3 | 硬约束过滤器 | ✅ | [src/scoring.py](src/scoring.py) `apply_hard_constraints()` | 非目标域名封顶 79 / KOL 来源封顶 89 / 时间衰减（含 min_age_hours） | +| 2.4 | 即时推送判断 | ✅ | [main.py](main.py) `run_fetch_job()` / [prompts/immediate_push.md](prompts/immediate_push.md) | 高评分条目触发即时推送候选（hot_threshold=90） | +| 2.5 | 飞书 Webhook 推送 | ✅ | [src/push/feishu.py](src/push/feishu.py) | 卡片消息 V2，8000 字符自动分片 | +| 2.6 | 部署脚本 | ✅ | [systemd/](systemd/) + [scripts/setup_systemd.sh](scripts/setup_systemd.sh) + [scripts/setup_service.bat](scripts/setup_service.bat) + [Dockerfile](Dockerfile) | Linux systemd + Windows 计划任务 + Docker 容器化 | + +**验收状态**： +- [x] Fetch 循环稳定运行 +- [x] 翻译管线集成正确（中英分离+并发翻译） +- [x] 硬约束生效（非目标领域封顶 79，KOL 转述封顶 89，时间衰减含最小年龄阈值） +- [x] 90+ 文章触发即时推送候选 +- [x] SQLite 缓存正常工作 +- [x] 监控统计准确记录各阶段数据 +- [x] 部署脚本可一键启动/停止 + +--- + +## 阶段三：实体提取与细粒度聚类 + +| 序号 | 任务 | 状态 | 备注 | +|:---:|------|:---:|------| +| 3.1 | 实体提取 Prompt | ✅ | [prompts/extract_entities.md](prompts/extract_entities.md) — 5 类实体，双列示例（军事/科技 + 政策/福利） | +| 3.2 | 角度分类 Prompt | ✅ | [prompts/classify_angle.md](prompts/classify_angle.md) — 6 类角度 + 空值（广告），决策树 + 反例表 | +| 3.3 | 批量实体提取 | ✅ | [src/processors/entity_extractor.py](src/processors/entity_extractor.py) — 实体+角度 asyncio.gather 并发，已集成 fetch 流程 | +| 3.4 | 角度分类服务 | ✅ | 与 3.3 合并实现。广告→空角度，全链路四层广告过滤 | +| 3.5 | 实体归一化 | ✅ | [src/processors/entity_normalizer.py](src/processors/entity_normalizer.py) + [src/processors/entity_aliases.json](src/processors/entity_aliases.json) — 双层归一化（规则映射 + LLM 消歧），Pickle 缓存加速，版本追踪 | +| 3.6 | 聚类算法 | ✅ | [src/processors/cluster_engine.py](src/processors/cluster_engine.py) — 加权实体重叠 + 并查集，装备/事件实体直接合并，泛化黑名单防误合并 | +| 3.7 | 记忆系统 | ✅ | [src/processors/cluster_memory.py](src/processors/cluster_memory.py) — SQLite 持久化已推送簇 ID，24h 内同簇不重复推送 | + +**验收状态**： +- [x] 实体提取对军事科技文章准确（歼-35、辽宁舰、驻欧美军），广告文章正确返回空 +- [x] 角度分类准确率：战略分析纯度从 20% 提升至 100%，广告识别为空的准确率 100% +- [x] 实体归一化双层架构（规则 200+ 别名 + LLM 消歧）运行正常 +- [x] 聚类后实体簇数量合理，泛化实体黑名单防误合并 + +--- + +## 阶段四：热度评价与动态阈值 + +| 序号 | 任务 | 状态 | 实现位置 | 备注 | +|:---:|------|:---:|------|------| +| 4.1 | 实体簇热度计算 | ✅ | [src/processors/heat_calculator.py](src/processors/heat_calculator.py) — 四维评分（最高分40% + 传播30% + 角度20% + 优先10%），含时间衰减和动态阈值 | +| 4.2 | 动态阈值算法 | ✅ | [src/processors/heat_calculator.py](src/processors/heat_calculator.py) `calculate_hotspot_threshold()` | 均值+N×σ + clamp，支持 env/config 覆盖 | +| 4.3 | 热点识别引擎 | ✅ | [src/processors/heat_calculator.py](src/processors/heat_calculator.py) + [main.py](main.py) | 硬约束过滤 + 簇热度阈值 + 簇记忆去重，三条链路完整 | +| 4.4 | 热点精评 Prompt | ✅ | [prompts/precise_heat.md](prompts/precise_heat.md) + [src/llm.py](src/llm.py) `precise_score_clusters()` | Top 30 簇 LLM 精评，四维评分，与启发式 6:4 融合 | + +**验收状态**： +- [x] 实体簇热度计算正确（融合硬约束分数） +- [x] 动态阈值自适应当日分布（单篇层 + 实体簇层均支持） +- [x] 双层筛选协同工作（硬约束过滤→动态阈值筛选） +- [x] 热点精评融合正确（启发式 × 0.6 + 精评 × 0.4） + +--- + +## 阶段五：Push 循环与三层整合 + +| 序号 | 任务 | 状态 | 实现位置 | 备注 | +|:---:|------|:---:|------|------| +| 5.1 | Push 循环引擎 | ✅ | [main.py](main.py) `push_loop()` / `run_push_job()` | 按 cron 表达式定时，区分早报/默认推送 | +| 5.2 | 摘要级整合 Prompt | ✅ | [prompts/cluster_insight.md](prompts/cluster_insight.md) — 合并 5.3 双视角设计 | 多源报道 → 50-80 字精炼摘要 | +| 5.3 | 洞察级整合 Prompt | ✅ | [prompts/cluster_insight.md](prompts/cluster_insight.md) — 与 5.2 合并为单次调用 | 双视角（metadata + 正文） | +| 5.4 | Prompt 防退化模块 | ✅ | [src/utils/prompt_checker.py](src/utils/prompt_checker.py) — 45 条禁止套话正则 + 特异性检查 + 违规重试 | 洞察生成后自动验证 | +| 5.5 | 摘要整合服务 | ✅ | [src/integrators/summary_integrator.py](src/integrators/summary_integrator.py) — 编排层：复用洞察摘要 + 轻量 LLM 补缺 | 全量热点簇摘要生成 | +| 5.6 | 洞察整合服务 | ✅ | [main.py](main.py) `_generate_insights_section()` + `_generate_insight_with_retry()` | Top 5 簇深度洞察 + 防退化验证 + 违规重试 | +| 5.7 | 网摘生成服务 | ✅ | [src/sections/webzine/section.py](src/sections/webzine/section.py) | 参考消息风格，280-320 字校验 + 3 次重试 | +| 5.8 | 时间线提取 | ✅ | [prompts/cluster_insight.md](prompts/cluster_insight.md) — 事件脉络段，含时间节点 + [持续跟踪] 跨日标记 | 已通过双视角洞察实现 | +| 5.9 | 多角度整合 | ✅ | [prompts/cluster_insight.md](prompts/cluster_insight.md) — 多角度分析段（技术/战略/舆论维度） | 已通过双视角洞察实现 | + +**验收状态**： +- [x] 网摘正文字数达标率（280-320字） +- [x] 网摘价值点字数达标率（35-45字） +- [x] 摘要整合编排服务正常运行（复用洞察 + flash 补缺） +- [x] 洞察分析覆盖双视角（metadata + 正文），含防退化验证 +- [x] 无禁止套话出现 + +--- + +## 阶段六：日报生成与跨日关联 + +| 序号 | 任务 | 状态 | 实现位置 | 备注 | +|:---:|------|:---:|------|------| +| 6.1 | 日报模板设计 | ✅ | [templates/cluster_view.md](templates/cluster_view.md) + [templates/article_view.md](templates/article_view.md) | 变量替换 + Sentinel 分段，模板驱动组装 | +| 6.2 | Sentinel 分段标记 | ✅ | [src/storage.py](src/storage.py) `assemble_with_sentinels()` | 支持多板块 | +| 6.3 | 跨日关联模块 | ✅ | [src/processors/cross_day.py](src/processors/cross_day.py) — 实体重叠匹配 + 热度/报道量趋势 | 已集成到簇视图，新增 📈 跨日追踪板块 | +| 6.4 | 排序算法 | ✅ | [src/processors/rank_engine.py](src/processors/rank_engine.py) | 三维加权排序（score 0.6 + freshness 0.2 + density 0.2），已集成到推送流程 | +| 6.5 | 日报渲染服务 | ✅ | [src/renderer.py](src/renderer.py) `render_template()` | 模板加载 + 变量替换，已集成到早报两波推送 | +| 6.6 | 网摘图片渲染 | ✅ | [src/generators/image_renderer.py](src/generators/image_renderer.py) + [src/push/image_hosting.py](src/push/image_hosting.py) | 基于网摘 JSON 生成 PNG 合并长图，图床 API 上传，飞书群推送 | +| 6.7 | Web 归档站点 | ❌ | — | — | +| 6.8 | 数据保留策略 | ✅ | [src/storage.py](src/storage.py) | 统一保留 7 天（fetch/hotalert/cluster/digest/images），自动清理 | + +**验收状态**： +- [x] Sentinel 标记正确，可提取 summary / webzine / category_overview +- [x] 跨日关联可追踪实体历史报道 + 趋势分析 +- [x] 网摘图片渲染正常（PIL 渲染 + 图床上传 + 飞书推送） +- [ ] Web 站点可正常访问（未实现，计划 6.7） + +--- + +## 阶段七：优化与迭代 + +| 序号 | 任务 | 状态 | 备注 | +|:---:|------|:---:|------| +| 7.1 | Prompt 优化 | ✅ | [docs/prompt_review_7.1.md](docs/prompt_review_7.1.md) | 7 个 Prompt 全面优化：军事时政档位、信息战维度、中文环境适配、术语准确性要求 | +| 7.2 | 反馈闭环 | ✅ | [src/processors/feedback.py](src/processors/feedback.py) | 用户评分修正记录、规则自动生成、评分后处理应用，支持 `feedback` 命令行 | +| 7.3 | 趋势分析 | ✅ | [src/processors/trend_analyzer.py](src/processors/trend_analyzer.py) | 跨日热度趋势追踪，集成到簇视图推送 | +| 7.4 | 个性化推荐 | ❌ | — | 基于用户阅读历史的排序（未实现） | + +--- + +## 额外实现（超出 V3 计划） + +| 功能 | 实现位置 | 备注 | +|------|------|------| +| Docker 容器化部署 | [Dockerfile](Dockerfile) + [docker-compose.yml](docker-compose.yml) + [.dockerignore](.dockerignore) | 基于 Python 3.11-slim，内置 Noto Sans CJK 中文字体，支持绿联云 NAS | +| 定时计划环境变量配置 | [.env.example](.env.example) + [src/config.py](src/config.py) `_apply_env_overrides()` | FETCH_INTERVAL_MINUTES / FETCH_LOOKBACK_MINUTES / PUSH_CRON / TIMEZONE_HOURS | +| 图床集成 | [src/push/image_hosting.py](src/push/image_hosting.py) | 公网图床 API 认证 → 上传 → URL 获取，支持密码认证 | +| 飞书图片推送 | [src/push/feishu.py](src/push/feishu.py) `send_image()` | 图床上传后构造 Markdown 消息推送飞书群 | + +--- + +## 已实现模块（28 个源文件） + +| 文件 | 功能 | +|------|------| +| [src/config.py](src/config.py) | 配置加载 + OPML 解析 + 源合并 + 环境变量覆盖 | +| [src/fetcher.py](src/fetcher.py) | RSS 异步抓取 | +| [src/processor.py](src/processor.py) | HTML → Markdown 转换 | +| [src/translator.py](src/translator.py) | 中英分离翻译管线 | +| [src/processors/entity_extractor.py](src/processors/entity_extractor.py) | 实体提取 + 角度分类批量服务 | +| [src/processors/entity_normalizer.py](src/processors/entity_normalizer.py) | 实体归一化（双层：规则映射 + LLM 消歧） | +| [src/processors/cluster_engine.py](src/processors/cluster_engine.py) | 细粒度聚类引擎（加权实体重叠 + 并查集） | +| [src/processors/cluster_memory.py](src/processors/cluster_memory.py) | 簇记忆系统（已推送簇持久化，24h 去重） | +| [src/processors/heat_calculator.py](src/processors/heat_calculator.py) | 簇热度计算（四维评分 + 动态阈值） | +| [src/processors/rank_engine.py](src/processors/rank_engine.py) | 排序引擎（三维加权排序） | +| [src/processors/trend_analyzer.py](src/processors/trend_analyzer.py) | 趋势分析（跨日热度追踪） | +| [src/processors/feedback.py](src/processors/feedback.py) | 反馈闭环（评分修正记录与规则生成） | +| [src/processors/cross_day.py](src/processors/cross_day.py) | 跨日关联（实体重叠 + 趋势） | +| [src/generators/image_renderer.py](src/generators/image_renderer.py) | 网摘图片生成（单篇/合并长图） | +| [src/generators/get_chinese_font.py](src/generators/get_chinese_font.py) | 中文字体加载（内置字体 + 系统fallback） | +| [src/llm.py](src/llm.py) | LLM 调用（评分、Digest、即时推送、洞察、精评） | +| [src/scoring.py](src/scoring.py) | 评分后处理（硬约束 + 时间衰减 + 动态阈值） | +| [src/cache.py](src/cache.py) | SQLite 三级缓存 | +| [src/storage.py](src/storage.py) | 文件存储 + Sentinel 分段标记 | +| [src/monitor.py](src/monitor.py) | 运行监控统计 | +| [src/logger.py](src/logger.py) | 日志系统 | +| [src/utils.py](src/utils.py) | 通用工具函数 | +| [src/markdown_utils.py](src/markdown_utils.py) | Markdown 格式工具 | +| [src/renderer.py](src/renderer.py) | 模板渲染 | +| [src/push/feishu.py](src/push/feishu.py) | 飞书 Webhook 推送（含图片消息） | +| [src/push/image_hosting.py](src/push/image_hosting.py) | 图床上传模块 | +| [src/sections/rss/section.py](src/sections/rss/section.py) | RSS Digest 板块 | +| [src/sections/webzine/section.py](src/sections/webzine/section.py) | 网摘生成板块 | +| [src/utils/prompt_checker.py](src/utils/prompt_checker.py) | Prompt 防退化检查器 | +| [src/integrators/summary_integrator.py](src/integrators/summary_integrator.py) | 摘要整合编排服务 | +| [main.py](main.py) | 入口 + 双循环架构 | + +## 提示词文件（10 个） + +| 文件 | 用途 | +|------|------| +| [prompts/score_batch.md](prompts/score_batch.md) | 批量评分（0-100 分） | +| [prompts/digest.md](prompts/digest.md) | RSS Digest 生成 | +| [prompts/immediate_push.md](prompts/immediate_push.md) | 即时推送快讯 | +| [prompts/webzine.md](prompts/webzine.md) | 参考消息风格网摘 | +| [prompts/category_overview.md](prompts/category_overview.md) | 分类概览 | +| [prompts/extract_entities.md](prompts/extract_entities.md) | 实体提取（5 类：人物/机构/装备/事件/地域） | +| [prompts/classify_angle.md](prompts/classify_angle.md) | 角度分类（6 类+空值/广告） | +| [prompts/precise_heat.md](prompts/precise_heat.md) | 热点精评（四维评分） | +| [prompts/cluster_insight.md](prompts/cluster_insight.md) | 双视角洞察（精炼摘要 + 结构化洞察） | +| [prompts/cluster_summary.md](prompts/cluster_summary.md) | 轻量簇摘要（flash 模型，50-80 字纯文本） | + +## 下一步建议 + +1. **Web 归档站点（6.7）**：部署在线阅览站点，提供历史日报浏览 +2. **个性化推荐（7.4）**：基于用户阅读历史和评分反馈的个性化排序 + +--- + +*每次完成阶段性开发后，请同步更新本文件中的对应条目状态。* \ No newline at end of file diff --git a/my-daily/docs/prompt_diff_7.1.md b/my-daily/docs/prompt_diff_7.1.md new file mode 100644 index 0000000..1e8bb88 --- /dev/null +++ b/my-daily/docs/prompt_diff_7.1.md @@ -0,0 +1,158 @@ +# Prompt 修改前后对比报告 + +## 1. score_batch.md + +### 90+ 条件 +| 修改前 | 修改后 | +|--------|--------| +| 来源为当事人本人的官方账号或官方博客（非KOL、非媒体） | 来源为当事方官方账号或权威防务专业媒体（Defense News / The War Zone / Janes / 新华社等） | + +### KOL 封顶规则 +| 修改前 | 修改后 | +|--------|--------| +| 来源是KOL/媒体/分析师转述（即使转述的是新装备发布）→ **上限89分** | 低质KOL（纯转述无增量、个人自媒体、无专业背景的营销号）→ **上限89分**。有深度分析/独家信息/专业背景的KOL不在此限 | + +### tags 字符限制 +| 修改前 | 修改后 | +|--------|--------| +| 2-12 个字符 | 2-16 个字符 | + +### 分档新增 +| 修改前 | 修改后 | +|--------|--------| +| 90-100: 军事/科技 + 官方首发 + 里程碑 | 90-100: 军事/科技 + 官方或权威媒体首发 + 里程碑。**新增：军事时政重磅事件也在此档** | + +--- + +## 2. precise_heat.md + +### 军事价值维度 +| 修改前 | 修改后 | +|--------|--------| +| 装备首次曝光=35-40, 技术验证=25-35, 演习=15-25, 非军事=0-15 | **新增军控谈判/重大制裁=25-35**，**新增外交斡旋=15-25**。**新增注意**：芯片管制/AI军事化/太空军备等交叉领域属于军事，不归入非军事 | + +### 传播广度 +| 修改前 | 修改后 | +|--------|--------| +| 5+源 × 3+角度 = 12-15 | **新增**：新华社/央视/国防部发布的一手通稿视为等效多源，不因"单源"而扣分 | + +### 信息密度新增 +| 修改前 | 修改后 | +|--------|--------| +| — | **新增加分项**：含卫星图像/电子信号数据/OSINT分析/部署热力图等情报级信息 +3 | + +--- + +## 3. cluster_insight.md + +### 分析维度 +| 修改前 | 修改后 | +|--------|--------| +| 技术维度 / 战略维度 / **舆论维度** | 技术维度（增加"战术/战役层面应用价值"） / 战略维度 / **信息战/认知战维度**（各方信息释放、叙事冲突、舆论引导策略） | + +### 摘要字数 +| 修改前 | 修改后 | +|--------|--------| +| 50-80 字 | 50-100 字。**新增**：摘要应体现该簇最核心的军事/科技/时政价值点 | + +### 约束新增 +| 修改前 | 修改后 | +|--------|--------| +| — | 装备型号必须完整（如"F-35A"不能简写为"F35"），兵力/规模要有具体数字 | + +--- + +## 4. webzine.md + +### 来源格式 +| 修改前 | 修改后 | +|--------|--------| +| 据公众号 XXX 报道 | 据**微信**公众号 XXX | + +### 字数区间 +| 修改前 | 修改后 | +|--------|--------| +| 280～320 字 | 250～350 字 | + +### 新增要求 +| 修改前 | 修改后 | +|--------|--------| +| — | 装备型号必须完整准确（如"F-35A"不简写为"F35"），兵力规模使用具体数字 | + +--- + +## 5. immediate_push.md + +### 语气风格 +| 修改前 | 修改后 | +|--------|--------| +| 客观、犀利、克制 | 客观、**准确**、克制 | + +### Why it matters +| 修改前 | 修改后 | +|--------|--------| +| 1-2句洞察分析 | 应涉及对**地区安全架构/军力平衡/技术格局的影响** | + +### 新增规则 +| 修改前 | 修改后 | +|--------|--------| +| — | **规则5**：型号完整性（"阿利·伯克级"不简写"伯克级"），兵力/射程/吨位使用具体数字 | + +--- + +## 6. cluster_summary.md + +### 字数 +| 修改前 | 修改后 | +|--------|--------| +| 50-80 字 | 50-100 字 | + +### 新增规则 +| 修改前 | 修改后 | +|--------|--------| +| 5条规则 | **新增规则6**：装备型号必须完整，地点精确到城市/基地级别 | + +--- + +## 7. classify_angle.md + +### 角度数量 +| 修改前 | 修改后 | +|--------|--------| +| **7 个选项**（6类 + 空值） | **8 个选项**（7类 + 空值），新增 `情报信号` | + +### 新增角度 +| 角度 | 说明 | +|------|------| +| `情报信号` | 基于OSINT的发现：卫星图像识别/电子信号截获/部队部署热力图/新基地建设/异常调动 | + +### 决策顺序变更 +| 步骤 | 修改前 | 修改后 | +|:---:|------|------| +| 0 | 广告→空 | 不变 | +| 1 | 表态→舆论反应 | 不变 | +| 2 | 冲突/演习→冲突事件 | 不变 | +| **3** | 政策→政策发布 | **信息源为卫星图/AIS/电子信号→情报信号** | +| **4** | 技术首破→技术突破 | **装备首破/列装/测试→技术突破** | +| **5** | 人物→人物动态 | **政策/法规/预算出台→政策发布** | +| 6 | 战略分析兜底 | 人物→人物动态 | +| **7** | — | **有主观论断→战略分析** | +| **8** | — | **以上不符→归入最接近角度** | + +### 歧义修复 +| 修改前 | 修改后 | +|--------|--------| +| 无判定标准 | 政策发布 vs 技术突破：信息源头是官方公告→政策发布；是测试/列装/验证→技术突破 | + +--- + +## 跨 Prompt 影响总览 + +| 维度 | 修改前 | 修改后 | +|------|--------|--------| +| 信息源认知 | 官方首发 > 一切 | 权威防务专业媒体 = 官方首发的等效价值 | +| 军事时政 | 无专门覆盖 | score/precise_heat 均新增军事时政档位 | +| 信息战维度 | 无 | cluster_insight 新增，替代传统舆论维度 | +| 情报信号 | 无独立角度 | classify_angle 新增 `情报信号` 角度 | +| 中文环境适配 | KOL一律封顶 | 区分低质KOL vs 专业KOL；通稿等效多源 | +| 术语准确性 | 无要求 | 4个Prompt统一要求型号完整/数字具体 | diff --git a/my-daily/docs/prompt_review_7.1.md b/my-daily/docs/prompt_review_7.1.md new file mode 100644 index 0000000..a3ef597 --- /dev/null +++ b/my-daily/docs/prompt_review_7.1.md @@ -0,0 +1,433 @@ +# Prompt 审查修改汇总（7.1） + +> 修改原则： +> 1. 消除"官方首发 > 专业媒体"的认知偏差——防务专业媒体首发与官方首发同等价值 +> 2. 军事科技、军事时政、信息战/认知战维度在所有Prompt中补齐 +> 3. 中文语境适配（微信公众号转述不降分、KOL封顶只对低质KOL） +> 4. 统一术语准确性要求（型号完整、兵力具体、地名精确） + +--- + +## 1. prompts/score_batch.md + +**修改点**: +- 90+ 条件去掉"当事方官方首发"，改为"官方或权威专业防务媒体首发" +- KOL 封顶改为"低质KOL / 纯转述无增量信息"，有深度分析的KOL不限 +- tags 字符限制从 2-12 放宽到 2-16 +- 新增"军事时政"为独立价值维度 + +```markdown +你是一个专业且严苛的军事科技新闻主编。请对抓取到的碎片化信息进行过滤、评分和信息提取。 + +## 任务与评分标准 +请根据以下标准为每条信息打分（0-100）。 + +**核心约束（先判这三条，再进入分档）**： +1. 90+ **必须同时满足**：(a) 主题与时政/军事/科技强相关；(b) 来源为当事方官方账号或权威防务专业媒体（Defense News / The War Zone / Janes / 新华社等，非低质KOL、非纯营销号）；(c) 属于首发或第一时间跟进 +2. 非时政/军事/科技主题（纯娱乐八卦、生活琐事、纯体育等）无论多重大、即使来自官方，**上限79分** +3. 时政/军事/科技新闻若来源是低质KOL（纯转述无增量信息、个人自媒体、无专业背景的营销号），**上限89分**。有深度分析、独家信息或专业背景（退役军官/OSINT分析师/行业研究员）的KOL不在此限。 + +**分档**： +- 【90-100分】军事/科技领域 + 官方或权威防务专业媒体首发 + 里程碑级装备/政策/事件。军事时政重磅事件（军控条约签署、重大制裁、香格里拉级别对话会）也在此档。 +- 【80-89分】重要军事科技进展、深度技术分析、军事时政重要进展；或重磅新闻但通过普通媒体转述。 +- 【70-79分】实用装备分析、技术教程、行业报告；**非目标领域**的重磅新闻；低质KOL转述的军事内容。 +- 【60-69分】二手信息、一般性新闻、小道消息。 +- 【<60分】低价值内容：纯情绪宣泄、无营养评价、广告。 + +## 输出要求 +必须返回纯JSON对象，顶层包含`items`数组字段，数组中每个对象包含： +- `link`: 原文链接（必须保留原样） +- `score`: 整数评分，JSON数字类型 +- `tags`: 字符串数组，数量1-3个，每个标签2-16个字符。必须是新闻中具体的关键词（装备型号/政策名称/技术特性/关键人物/地缘事件），禁止空泛分类标签。 + - ✅ 好示例：`["歼-35A","舰载战斗机","隐身性能"]`、`["AI芯片出口限制","Rubin架构"]` + - ❌ 坏示例：`["军事","新闻","热点"]` +- `summary`: 一句话客观摘要（提取核心事实，50字内） + +## 输出格式（严格只输出JSON对象）： +{ + "items": [ + { + "link": "https://example.com/article1", + "score": 95, + "tags": ["歼-35A", "舰载战斗机", "隐身性能"], + "summary": "空军官方确认歼-35A已列装首批作战部队。" + } + ] +} + +## 重要提示 +1. items数组长度必须与输入相同 +2. link字段必须与输入一一对应 +3. 只返回JSON对象，不要添加任何额外说明文字 + +## 输入数据 +以下是一个JSON数组，包含抓取到的内容： +每个对象包含：link, title, source, published, content + +```json +{entries_json} +``` +``` + +--- + +## 2. prompts/precise_heat.md + +**修改点**: +- 军事价值增加"军事时政"档位（制裁/军控/外交战略部署） +- 传播广度适配中文环境（新华社通稿视为等效多源） +- 新增"情报价值"维度提示（卫星图/OSINT/电子信号/热力图） + +```markdown +你是一个资深的军事新闻主编。请对以下热点话题簇进行精评分（0-100）。 + +## 评分维度 + +请从以下四个维度综合打分： + +1. **军事价值（40分）**：是否涉及装备列装/技术突破/实战行动/战略部署/军事时政 + - 装备首次曝光/列装/实战验证 = 35-40 + - 技术验证/测试进展/军控谈判/重大制裁 = 25-35 + - 演习/训练/常规动态/外交斡旋 = 15-25 + - 非军事主题 = 0-15 + - 注意：芯片出口管制、AI军事化政策、太空军备竞赛等交叉领域属于军事科技/军事时政，不应归入"非军事" + +2. **时效性（25分）**：是否为最新进展或首次报道 + - 24h内的突发/首次报道 = 20-25 + - 近期进展的跟进报道 = 15-20 + - 回顾性/综述性内容 = 5-15 + +3. **信息密度（20分）**：是否包含具体数据/型号/地点/时间 + - 含具体型号+数量+地点+时间 = 15-20 + - 含部分具体信息 = 10-15 + - 泛泛而谈/观点评论 = 5-10 + - 加分项：含卫星图像/电子信号数据/OSINT分析/部署热力图等情报级信息 +3 + +4. **传播广度（15分）**：多家媒体、多个角度报道的热度 + - 5+源或新华社等国家级通讯社通稿 × 3+角度 = 12-15 + - 3-4源 或 2角度 = 8-12 + - 单一来源 = 3-8 + - 注意：中文军事新闻中，新华社/央视/国防部发布的一手通稿视为等效多源，不因"单源"而扣分 + +## 输入数据 + +以下是一个 JSON 数组，包含热点话题簇的摘要信息： + +```json +{clusters_json} +``` + +## 输出要求 + +必须返回纯 JSON 对象，格式如下： + +```json +{ + "items": [ + { + "cluster_id": 0, + "precise_score": 92, + "brief": "歼-35A首次公开确认列装，官方首发，多源多角度报道" + } + ] +} +``` + +要求： +1. items 数组长度必须与输入相同 +2. cluster_id 必须与输入一一对应 +3. precise_score 为 0-100 整数 +4. brief 为 25 字以内的评分依据简述 +5. 只返回 JSON 对象，不要添加额外说明 +``` + +--- + +## 3. prompts/cluster_insight.md + +**修改点**: +- "舆论维度"改为"信息战/认知战维度" +- summary 段与 body 建立呼应关系（摘要应体现洞察的核心发现） +- 字数区间扩大到 50-100 字 + +```markdown +你是一个资深军事新闻主编兼情报分析师。请对以下热点话题簇进行整合分析，同时输出精炼摘要和结构化洞察。 + +## 输入数据 + +以下是一个 JSON 对象，包含该话题簇的所有文章： + +```json +{cluster_json} +``` + +## Part 1: 摘要整合（新闻编辑视角） + +基于多源报道，生成一条 50-100 字的精炼摘要： + +- 只陈述核心事实，不做趋势判断 +- 不使用形容词（如"重大""深远""重要"） +- 合并重复信息，提取最关键的 who/what/when/where +- 摘要应体现该簇最核心的军事/科技/时政价值点 + +## Part 2: 结构化洞察（情报分析师视角） + +### Metadata（10 字标题 + 20 字导语 + 3-5 要点） + +生成以下结构化信息： +- title: 10 字以内的标题（事实陈述，无修饰） +- lead: 20 字以内的导语（核心事实） +- highlights: 3-5 个关键要点（bullet points，纯事实） + +约束：只陈述事实，不用形容词。 + +### 洞察正文 + +基于多源报道，识别： +1. **事件脉络**：关键时间节点和进展。如果该话题是持续多日的事件，事件脉络段第一行必须写上 `[持续跟踪]`（单独一行），随后再写： + - 前情提要：**优先使用下方 `previous_summary` 字段**（上次推送的洞察摘要），用 1-2 句概括此前已知的关键进展。仅当 `previous_summary` 为空或不存在时，才从素材文章的早期报道中提取前情。禁止写"无显著前情" + - 最新突破：今天的最新进展（具体事实） +2. 多角度分析： + - **技术维度**：装备性能、技术水平、技术创新点及其在战术/战役层面的应用价值 + - **战略维度**：地缘政治影响、军力平衡变化、联盟关系调整 + - **信息战/认知战维度**：各方如何通过信息释放塑造战场叙事、舆论引导策略、叙事冲突（替代传统"舆论维度"） +3. 影响与展望：短期影响、中长期趋势、值得关注信号 + +约束： +- 禁止空泛评论（如"意义重大""影响深远"） +- 每个观点必须有素材支撑 +- 使用具体名称（装备型号/政策名称/组织名称），不用"某些""部分"等模糊指代 +- 装备型号必须完整（如"F-35A"不能简写为"F35"），兵力/规模要有具体数字 + +## 输出格式 + +必须返回纯 JSON 对象： + +```json +{ + "summary": "50-100字精炼摘要，纯事实陈述", + + "metadata": { + "title": "10字以内的标题", + "lead": "20字以内的导语", + "highlights": ["要点1", "要点2", "要点3"] + }, + + "body": "## 事件脉络\n[持续跟踪]\n- 前情提要：...\n- 最新突破：...\n\n## 多角度分析\n- **技术维度**：...\n- **战略维度**：...\n- **信息战/认知战维度**：...\n\n## 影响与展望\n短期：...\n中期：...\n值得关注：..." + +注意：如果该话题是单日事件（所有报道在同一天），事件脉络段**不加** `[持续跟踪]` 标记。 +} +``` + +要求： +1. summary 必须 50-100 字（中文字符计数） +2. metadata.highlights 至少 2 条，至多 5 条 +3. body 必须包含三个段落（事件脉络/多角度分析/影响与展望） +4. 只返回 JSON 对象，不要添加额外说明 +``` + +--- + +## 4. prompts/webzine.md + +**修改点**: +- "据公众号XXX报道" → "据微信公众号XXX" +- 字数从 280-320 放宽到 250-350 +- 新增专业术语准确性要求 + +```markdown +# 军事科技网摘生成 + +你是一名资深军事编辑，请你严格按照《参考消息》官方新闻报道格式、标题风格，对我提供的新闻内容进行改写，严格遵守以下所有规则： + +## 文章信息 +- 标题：{title} +- 来源：{source} +- 发布时间：{published} +- AI摘要：{summary} +- 原文内容：{content} + +## 标题要求 +- 简洁、客观、中性、信息密度高 +- 结构：主体 + 事件 + 核心态势 +- 不抒情、不夸张、不用网络用语 + +## 价值点要求 +- 价值点严格 35～45 字（含标点） +- 句式结构：事件 - 影响/后果 - 值得关注 +- 凝练、客观、不发散 + +## 正文要求 +- 开头第一句必须加：据 XXX 报道 +- 如果新闻来源是微信公众号，开头第一句必须加：据微信公众号 XXX +- 正文风格：客观、平实、严谨、书面化，类似外电编译稿 +- 只保留核心信息：时间、地点、人物、事件、内容、前景/影响 +- 装备型号必须完整准确（如"F-35A"不简写为"F35"），兵力规模使用具体数字 +- 全文一段到底，不分段 +- 字数强制要求：正文字数严格控制在 250～350 字（含标点），不许超、不许少 + +## 输出格式 + +严格输出 JSON，包含六个字段： + +```json +{ + "title": "网摘标题", + "value": "价值点", + "body": "正文", + "source": "新华社", + "original_title": "原文原标题", + "published": "2026-06-02T10:00:00" +} +``` + +- source: 来源名称 +- original_title: 未经改写的原标题 +- published: 发布时间 + +只输出 JSON，不要任何前缀或后缀文字。 +``` + +--- + +## 5. prompts/immediate_push.md + +**修改点**: +- 去掉"犀利"，改为"客观、准确、克制" +- 新增装备型号完整性和Why it matters地缘后果要求 + +```markdown +你是一位顶级的军事科技观察家与专业的军事新闻记者。你需要将最新发生的高分重磅消息整理成即时推送快讯。 + +## 任务要求： +1. **严格查重与阻断机制**：仔细对比原始数据与【过去已推送历史】。如果事件已经推送过且无重大新进展，请直接抛弃。如果所有输入的事件都被判定为重复，你必须且只能输出[NO_NEW_CONTENT]。 +2. **专业聚合**：如果输入数据中有多条讨论同一个事件，请将它们融合为一条快讯。 +3. **专业视角**：用2-3条无序列表项陈述核心事实(What)；列表之后另起一段给出1-2句为什么值得关注(Why it matters)的洞察分析，应涉及对地区安全架构/军力平衡/技术格局的影响。 +4. **语气风格**：客观、准确、克制。多用短句，重点内容加粗。避免空洞的行业宏大叙事和未来预测。 +5. **军事专业性**：装备型号必须写完整（如"F-35A"不简写"F35"、"阿利·伯克级驱逐舰"不简写"伯克级"），兵力/射程/吨位等使用具体数字。 + +## 输出格式： +{output_format} + +## 过去N天已推送事件清单（仅供查重，严禁模仿）： +{recent_push_context} + +## 本次需要推送的原始数据： +```json +{entries} +``` +``` + +--- + +## 6. prompts/cluster_summary.md + +**修改点**: +- 字数放宽到 50-100 字，与 cluster_insight 对齐 +- 增加具体化要求（型号/地点/数字） + +```markdown +你是一个军事新闻编辑。请将以下多源报道合并为一条 50-100 字的中文精炼摘要。 + +## 规则 + +1. 只陈述核心事实（who/what/when/where），不做趋势判断 +2. 不使用形容词（如"重大""深远""重要""显著"） +3. 不使用模糊指代（如"某些""相关""有关"），必须用具体名称 +4. 合并重复信息，提取最关键的进展 +5. 如果同一事件有多个角度，用 1-2 句话概括主要维度 +6. 装备型号必须完整（如"歼-35A"），地点精确到城市/基地级别 + +## 输入 + +```json +{cluster_json} +``` + +## 输出 + +只返回 50-100 字的中文摘要文本，不要 JSON 包裹，不要额外说明。 +``` + +--- + +## 7. prompts/classify_angle.md + +**修改点**: +- 新增"情报信号"角度（卫星图/OSINT/电子信号/部署热力图） +- 修复政策发布 vs 技术突破的歧义：以"信息源头是官方公告还是技术验证"为判定标准 +- 战略分析兜底规则收紧：必须有"分析/认为/判断"等关键词否则不归入 + +```markdown +你是一名军事科技新闻编辑。请判断以下文章的报道角度。 + +## 角度分类（七选一 + 空值） + +必须从以下 8 个选项中选择 **恰好 1 个**： + +| 角度 | 适用场景 | 正确示例 | ⚠️ 不适用（反例） | +|------|---------|---------|-------------------| +| `政策发布` | 官方政策/法规/条例/命令/预算的**正式发布或生效** | 《国防白皮书》发布；某国批准新军费预算；出口管制新规 | 非官方渠道的解读/分析；内部工作会议；产品发布/广告 | +| `技术突破` | 新型装备/武器/技术的**实质进展**（首飞/列装/测试/验证） | 新一代战斗机首飞；新型雷达通过验收；高超音速导弹试射成功 | 常规装备维护升级；历史装备回顾；技术路线对比分析 | +| `情报信号` | 基于**开源情报(OSINT)**的发现：卫星图像识别/电子信号截获/部队部署热力图/新基地建设/异常调动 | 卫星图发现南海新雷达阵地；AIS数据显示航母编队异常动向；商业卫星揭示朝鲜导弹发射场扩建 | 官方已宣布的部署/演习；媒体报道的常规驻防；历史考古 | +| `战略分析` | 对局势/博弈/地缘的**深度研判**，文中必须有"分析/认为/意味着/判断/预测"等主观论断 | 台海局势走向研判；俄乌冲突对军贸格局影响 | 单纯事实报道；产品推广；采访/会见/任命（注意：无主观论断的纯事实归入其他角度） | +| `舆论反应` | 官方**回应/表态/谴责/抗议/澄清** | 外交部谴责某国军售；国防部回应南海传闻 | 网民个人观点；历史事件追认/纪念 | +| `冲突事件` | 正在发生或刚发生的**具体军事行动/冲突/演习/部署** | 军舰穿越台湾海峡；边境交火；联合军演启动 | 历史回顾；战略层面演习意义分析；政策文件威慑措辞 | +| `人物动态` | 以**人物或机构活动**为核心：任免、访晤、会议、纪念、文化活动、基层事迹 | 国防部长访问某国；军营开放日；烈士纪念仪式 | 以政策/装备/事件为核心、人物仅为信息源的报道 | +| `""`（空） | 明确**不属于军事/科技/时政**领域：广告、促销、纯生活内容、娱乐八卦 | 商品促销广告、周边挂件上新、纯情感散文 | — | + +## 决策顺序（从上到下，命中即止） + +``` +0. 是否明确不属于军事/科技/时政（广告/促销/纯娱乐）？→ 返回空字符串 "" +1. 文中是否有"谴责/驳斥/抗议/回应/声明"等官方表态？→ 舆论反应 +2. 是否在报道某场具体的战斗/冲突/演习/部署过程？→ 冲突事件 +3. 信息源是否为卫星图像/AIS/电子信号等OSINT数据？→ 情报信号 +4. 是否在报道某项技术/装备的首次突破/列装/测试/验证？→ 技术突破 +5. 是否在宣布某个政策/法规/条例/命令/预算的正式出台？→ 政策发布 +6. 文章核心是人物（任免/访问/活动）还是机构内部事务？→ 人物动态 +7. 文中是否有明确的"分析/认为/判断/预测/意味着"等主观论断？→ 战略分析 +8. 以上均不符合 → 根据内容归入最接近的角度（避免都进战略分析） +``` + +## 强制规则 + +1. 只看标题和正文前 500 字做判断 +2. **情报信号**必须基于开源情报(OSINT)数据源（卫星图/AIS/电子信号/热力图等），普通新闻配图不算 +3. **政策发布 vs 技术突破**判别标准：信息源头是官方公告 → 政策发布；是测试/列装/验证的实际进展 → 技术突破 +4. **战略分析**必须在文中找到至少 1 处明确的"分析/判断/预测/认为/意味着"等主观论断，否则不选 +5. 一条输出对应一条输入，链接丢失的输出项将被丢弃 + +## 输出格式 + +严格 JSON： + +```json +{{ + "items": [ + {{"link": "原文链接", "angle": "技术突破"}}, + {{"link": "原文链接2", "angle": "情报信号"}} + ] +}} +``` + +## 输入数据 + +```json +{entries_json} +``` +``` + +--- + +## 汇总：修改统计 + +| Prompt | 主要改动 | 影响 | +|------|------|:---:| +| score_batch.md | 90+放宽权威来源 + KOL分级 + tags放宽 | 🔴 核心 | +| precise_heat.md | 军事时政档位 + 中文传播适配 + 情报加分 | 🔴 核心 | +| cluster_insight.md | 信息战维度 + 摘要/洞察呼应 + 字数放宽 | 🔴 核心 | +| webzine.md | 公众号格式 + 字数放宽 + 术语准确性 | 🟡 中等 | +| immediate_push.md | 风格修正 + 型号完整性 + 地缘后果引导 | 🟡 中等 | +| cluster_summary.md | 字数对齐 + 具体化要求 | 🟢 轻量 | +| classify_angle.md | 情报信号角度 + 歧义修复 + 决策顺序 | 🟡 中等 | diff --git a/my-daily/docs/web_archive_design.md b/my-daily/docs/web_archive_design.md new file mode 100644 index 0000000..7d48c92 --- /dev/null +++ b/my-daily/docs/web_archive_design.md @@ -0,0 +1,522 @@ +# Web 归档站点设计方案 + +> 版本：2.0 | 日期：2026-06-30 | 对应 V3 计划 6.7 + +## 一、项目定位 + +整个项目分为两个独立子系统： + +| 子系统 | 位置 | 角色 | +|--------|------|------| +| **数据管线后端** | `my-daily/` 根目录（`main.py` + `src/`） | 数据源：RSS 抓取 → LLM 评分 → 推送飞书 → 生成 `data/*.md` | +| **Web 归档站点** | `my-daily/web_archive/` | 前端展示：读取 `data/*.md` → 渲染 HTML 页面 → 浏览器访问 | + +**关系**：数据管线是生产者，Web 归档站点是消费者。两者通过 `data/` 目录共享数据，互不依赖。 + +**三种启动方式**： + +| 方式 | 命令 | 说明 | +|------|------|------| +| 全栈启动 | `python start.py` | 同时启动数据管线 + Web 归档站点 | +| 仅数据管线 | `python main.py loop` | 只运行 RSS 抓取、评分、推送 | +| 仅 Web 归档 | `python -m web_archive` | 独立运行，只读已有 `data/`，数据管线可关闭 | + +## 二、目录结构 + +``` +my-daily/ +├── main.py # [现有] 数据管线后端入口（loop/fetch/push） +├── start.py # [新增] 一键启动脚本（双进程） +│ +├── web_archive/ # [新增] Web 归档站点（前端 + Flask 后端，可独立启动） +│ ├── __init__.py +│ ├── __main__.py # 入口：python -m web_archive 即可独立启动 +│ ├── server.py # Flask 应用（路由 + API） +│ ├── data_parser.py # 数据解析（读取 data/*.md → 结构化 JSON） +│ ├── templates/ # Jinja2 模板 +│ │ └── archive.html # 归档站点主页面 +│ └── static/ # 静态资源 +│ ├── css/ +│ │ └── archive.css # 响应式样式 +│ └── js/ +│ └── archive.js # 前端交互（Tab 切换 / 日期切换） +│ +├── data/ # [现有] 数据目录（两个子系统共享） +│ ├── cluster-*.md # 热点速览 Markdown +│ ├── digest-*.md # 每日精选 Markdown +│ └── images/YYYYMMDD/ # 网摘图片 +│ +├── src/ # [现有] 数据管线核心模块 +│ ├── storage.py # 文件读写 +│ ├── markdown_utils.py # frontmatter 解析 +│ └── ... +│ +└── requirements.txt # [修改] 新增 flask, markdown +``` + +## 三、架构设计 + +### 3.1 整体架构 + +``` +┌──────────────────────────────────────────────────┐ +│ start.py（一键全栈启动） │ +│ multiprocessing 启动两个独立子进程 │ +│ │ +│ ┌──────────────────┐ ┌────────────────────┐ │ +│ │ 数据管线后端 │ │ Web 归档站点 │ │ +│ │ main.py loop │ │ web_archive (独立) │ │ +│ │ Fetch + Push │ │ Flask :8080 │ │ +│ │ 写入 data/*.md │ │ 只读 data/*.md │ │ +│ └────────┬─────────┘ └─────────┬──────────┘ │ +│ │ │ │ +│ └────── data/ ──────────┘ │ +│ (共享数据目录) │ +│ │ +│ ★ Web 归档站点可独立启动：python -m web_archive │ +│ 数据管线关闭时，站点仍可正常浏览已有数据 │ +└──────────────────────────────────────────────────┘ +``` + +### 3.2 数据流 + +``` +[数据管线] [Web 归档站点] +main.py loop web_archive/server.py + │ │ + ├─ Fetch RSS │ + ├─ LLM 评分 │ + ├─ 推送飞书 │ + └─ 写入 data/ ────── 共享 ──────→ 读取 data/ + ├─ cluster-*.md ├─ 解析 frontmatter + ├─ digest-*.md ├─ 提取 Sentinel 分段 + └─ images//*.png ├─ Markdown → HTML + └─ 渲染 archive.html + │ + 浏览器访问 :8080 +``` + +### 3.3 技术选型 + +| 层 | 技术 | 说明 | +|----|------|------| +| Web 框架 | Flask 3.x | 轻量，与 Python 生态一致 | +| 模板引擎 | Jinja2（Flask 内置） | 服务端渲染，SEO 友好 | +| Markdown 渲染 | `markdown` + `pymdown-extensions` | 支持表格、代码块、扩展语法 | +| 前端 | 原生 HTML/CSS/JS | 零 npm 依赖，极致轻量 | +| 图片服务 | Flask `send_from_directory` | 从 `data/images/` 读取 | +| 进程管理 | `multiprocessing.Process` | 一键启动双进程 | +| 部署 | 嵌入现有 Docker 容器 | 新增端口 8080 | + +## 四、页面设计 + +### 4.1 整体布局 + +``` +┌──────────────────────────────────────────────────────────┐ +│ 🪖 军事科技每日摘要 │ +│ [◀ 前一天] 2026年6月30日 [后一天 ▶] │ +│ ┌──────────┬──────────┬──────────┐ │ +│ │ 🔥 热点速览 │ 📰 每日精选 │ 🖼️ 网摘图片 │ ← Tab 导航 │ +│ └──────────┴──────────┴──────────┘ │ +├──────────────────────────────────────────────────────────┤ +│ │ +│ [热点速览 Tab] │ +│ ┌──────────────────────────────────────────────────┐ │ +│ │ 📋 今日速览 │ │ +│ │ （summary 内容，多源报道摘要列表） │ │ +│ ├──────────────────────────────────────────────────┤ │ +│ │ 🔍 热点洞察 │ │ +│ │ ┌─────────────────────────────────────────┐ │ │ +│ │ │ 簇标题（含 [持续跟踪] 标记） │ │ │ +│ │ │ 摘要 + 要点列表 │ │ │ +│ │ │ 📅 事件脉络（前情提要 → 最新突破） │ │ │ +│ │ │ 🔬 多角度分析（技术/战略/舆论） │ │ │ +│ │ │ 📈 影响与展望 │ │ │ +│ │ └─────────────────────────────────────────┘ │ │ +│ ├──────────────────────────────────────────────────┤ │ +│ │ 📈 跨日追踪 │ │ +│ │ 📊 趋势分析 │ │ +│ └──────────────────────────────────────────────────┘ │ +│ │ +│ [每日精选 Tab] │ +│ ┌──────────────────────────────────────────────────┐ │ +│ │ 📋 今日网摘 Top 3 │ │ +│ │ 每篇：标题 + 来源 + 正文 + 价值点 + 时间 │ │ +│ ├──────────────────────────────────────────────────┤ │ +│ │ 📊 分类概览（5 类 × 5 条） │ │ +│ │ 装备技术 / 地区安全 / 军事改革 / 产业观察 / 其他 │ │ +│ │ 每篇：评分 + 可点击原文链接 │ │ +│ └──────────────────────────────────────────────────┘ │ +│ │ +│ [网摘图片 Tab] │ +│ ┌──────────────────────────────────────────────────┐ │ +│ │ 合并长图（点击放大、手势缩放） │ │ +│ │ [图片预览] │ │ +│ └──────────────────────────────────────────────────┘ │ +│ │ +└──────────────────────────────────────────────────────────┘ +``` + +### 4.2 响应式策略 + +| 屏幕宽度 | 布局 | Tab 位置 | +|---------|------|---------| +| > 768px（PC） | 最大宽度 960px 居中，左右留白 | 顶部水平 Tab | +| ≤ 768px（手机） | 全宽，卡片式布局 | 底部固定 Tab 栏 | + +**PC 端特点**： +- 洞察卡片双列网格布局 `grid-template-columns: repeat(auto-fit, minmax(400px, 1fr))` +- 图片预览限制最大宽度 800px +- 日期导航栏在顶部水平排列 + +**手机端特点**： +- 单列全宽布局 +- 图片 `max-width: 100%` 自适应 +- 触摸友好的按钮尺寸（≥ 44px） +- 底部固定 Tab 导航 +- 日期选择器占据全宽，左右箭头 + 日期文字 + +## 五、API 设计 + +### 5.1 页面路由 + +| 方法 | 路径 | 说明 | +|------|------|------| +| GET | `/` | 渲染归档主页，默认显示当天数据 | +| GET | `/?date=2026-06-30` | 渲染指定日期的归档页 | + +### 5.2 数据 API + +| 方法 | 路径 | 返回 | 说明 | +|------|------|------|------| +| GET | `/api/dates` | `{"dates": [...], "today": "..."}` | 可用日期列表 | +| GET | `/api/content?date=2026-06-30` | 结构化 JSON | 指定日期的全部内容 | +| GET | `/images//` | 图片二进制 | 网摘图片文件 | + +### 5.3 API 响应格式 + +```json +// GET /api/content?date=2026-06-30 +{ + "date": "2026-06-30", + "has_data": true, + "cluster": { + "title": "热点速览 | 2026-06-30", + "summary": "...", + "insights": [ + { + "title": "美陆军AI无人艇测试", + "tracking": true, + "summary": "...", + "highlights": ["...", "..."], + "timeline": "...", + "angles": "...", + "outlook": "..." + } + ], + "cross_day": "...", + "trend": "..." + }, + "digest": { + "title": "军事科技每日精选 | 2026-06-30", + "webzine": [ + { + "title": "美陆军利用AI与无人艇提升太平洋后勤能力", + "source": "Defense One", + "body": "...", + "value": "...", + "time": "2026-06-30T19:22:25+08:00" + } + ], + "categories": [ + { + "name": "装备技术", + "articles": [ + {"score": 85, "title": "...", "link": "https://..."} + ] + } + ] + }, + "images": [ + { + "name": "webzine_combined_20260630_215533.png", + "url": "/images/20260630/webzine_combined_20260630_215533.png" + } + ] +} +``` + +## 六、模块详细设计 + +### 6.1 server.py（Flask 应用） + +```python +"""Web 归档站点 - Flask 应用""" +import re +from datetime import date +from pathlib import Path + +from flask import Flask, render_template, send_from_directory, request, jsonify + +from web_archive.data_parser import ( + scan_available_dates, parse_cluster_content, parse_digest_content, + scan_images, md_to_html +) + +# Flask 应用，模板和静态资源在 web_archive/ 内 +app = Flask(__name__, + template_folder="templates", + static_folder="static") + +# 图片目录（相对于项目根目录，即 my-daily/） +PROJECT_ROOT = Path(__file__).resolve().parent.parent +IMAGE_DIR = PROJECT_ROOT / "data" / "images" +DATA_DIR = PROJECT_ROOT / "data" + + +@app.route("/") +def index(): + """渲染归档主页，默认当天""" + target_date = request.args.get("date", date.today().isoformat()) + return render_template("archive.html", date=target_date) + + +@app.route("/api/dates") +def api_dates(): + """返回可用日期列表""" + dates = scan_available_dates(DATA_DIR) + return jsonify({ + "dates": dates, + "today": date.today().isoformat() + }) + + +@app.route("/api/content") +def api_content(): + """返回指定日期的内容 JSON""" + target_date = request.args.get("date", date.today().isoformat()) + cluster = parse_cluster_content(DATA_DIR, target_date) + digest = parse_digest_content(DATA_DIR, target_date) + images = scan_images(IMAGE_DIR, target_date) + + return jsonify({ + "date": target_date, + "has_data": cluster is not None or digest is not None, + "cluster": cluster, + "digest": digest, + "images": images + }) + + +@app.route("/images/") +def serve_image(subpath): + return send_from_directory(IMAGE_DIR, subpath) +``` + +### 6.1.1 __main__.py（独立启动入口） + +```python +"""Web 归档站点 - 独立启动入口 +用法: python -m web_archive [--port PORT] [--host HOST] +""" +import argparse +from web_archive.server import app + + +def main(): + parser = argparse.ArgumentParser(description="Web 归档站点") + parser.add_argument("--port", type=int, default=8080, help="监听端口（默认 8080）") + parser.add_argument("--host", type=str, default="0.0.0.0", help="监听地址（默认 0.0.0.0）") + args = parser.parse_args() + + print("🪖 Web 归档站点 - 独立模式") + print(f" 地址: http://{args.host}:{args.port}") + print(f" 数据目录: data/") + print(" 按 Ctrl+C 停止") + app.run(host=args.host, port=args.port, debug=False) + + +if __name__ == "__main__": + main() +``` + +### 6.2 data_parser.py（数据解析模块） + +复用 `src/markdown_utils.py` 的 `parse_frontmatter()` 函数，实现以下功能： + +| 函数 | 输入 | 输出 | 说明 | +|------|------|------|------| +| `scan_available_dates(data_dir)` | `data/` 目录 | `List[str]` | 扫描 `cluster-*.md` 文件名提取日期，降序排列 | +| `parse_cluster_content(data_dir, date)` | 日期 | `Dict` 或 None | 解析 cluster-*.md → frontmatter + Sentinel 分段 → HTML | +| `parse_digest_content(data_dir, date)` | 日期 | `Dict` 或 None | 解析 digest-*.md → frontmatter + Sentinel 分段 → 结构化 JSON | +| `extract_section(text, name)` | 文本 + 段名 | `str` | 提取 `...END -->` 之间的内容 | +| `md_to_html(text)` | Markdown | `str` | 渲染 Markdown 为 HTML | +| `scan_images(image_dir, date)` | 日期 | `List[Dict]` | 扫描 `images/YYYYMMDD/` 下的图片文件 | + +### 6.3 前端模板（archive.html） + +使用 Jinja2 模板，单文件包含所有 Tab 内容区： + +``` +archive.html +├── 响应式 meta + CSS 引入 +├──
标题 + 日期导航栏 +├──