#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ 军事科技每日摘报 - V3 正式版(模块化重构) 完整流程:RSS抓取 → 时间过滤 → 去重 → 批量翻译外文 → 关键词筛选 → AI评分分类摘要 → 生成分类总体介绍 → 网摘生成 → 报告输出 """ import asyncio from datetime import datetime, timedelta from pathlib import Path from modules.config import TIME_WINDOW_HOURS, load_ai_config, load_push_config, RSS_FEEDS from modules.logger import get_logger from modules.cache import get_cache from modules.keyword_filter import keyword_filter from modules.ai_analyzer import ( score_and_classify_article, calculate_weighted_score, generate_category_overview, generate_webzine_for_article, batch_score_and_classify_articles, ) from modules.image_generator import create_combined_webzine_image from modules.monitor import get_monitor from modules.pusher import send_daily_push from modules.async_pipeline import ( process_articles_streaming, generate_webzines_parallel, generate_overviews_parallel, cleanup_executor, ) logger = get_logger() OUTPUT_DIR = Path(__file__).resolve().parent.parent / "output" def merge_from_cache(cached, processed): """合并缓存和新处理文章,按评分降序排序;无文章时返回 None""" merged = cached + processed if not merged: logger.info("没有可用的文章,流程结束") return None merged.sort(key=lambda x: x['final_score'], reverse=True) logger.info("总文章数: %d 篇,最高分: %.1f/10", len(merged), merged[0]['final_score']) return merged def process_new_articles(articles, api_key, base_url, model, monitor, cache): """新文章处理管线:排序 → 标题翻译 → 关键词筛选 → 正文翻译 → AI分析 → 写缓存""" articles = sorted(articles, key=lambda x: x['published'], reverse=True) logger.info("开始批量翻译所有文章标题...") with monitor.stage("translate_titles"): articles = batch_translate_titles(articles, api_key, base_url, model) logger.info("标题翻译完成,共 %d 篇文章", len(articles)) with monitor.stage("keyword_filter"): if len(articles) > 15: logger.info("文章数超过15篇,执行关键词筛选(基于标题)...") articles = keyword_filter(articles) hit_dist = [a['keyword_hits'] for a in articles[:10]] logger.info("关键词筛选后: %d 篇,命中数分布: %s", len(articles), hit_dist) else: logger.info("文章数 %d <= 15,全部保留", len(articles)) if not articles: logger.info("筛选后无符合条件文章") return [] logger.info("开始批量翻译筛选通过的文章正文...") with monitor.stage("translate_contents"): articles = batch_translate_contents(articles, api_key, base_url, model) logger.info("正文翻译完成,共 %d 篇文章", len(articles)) logger.info("开始 AI 智能分析(使用 %s)...", model) with monitor.stage("ai_analyze"): articles = batch_score_and_classify_articles(articles, api_key, base_url, model) logger.info("保存 %d 篇新处理的文章到缓存...", len(articles)) for a in articles: cache.save_article(a) return articles def deduplicate_articles(articles): """同源标题去重:同一源+相同标题仅保留一篇,过滤短标题""" seen = set() result = [] for a in articles: title = a['title'].strip() if len(title) <= 5: continue key = (a['source'].strip(), title) if key not in seen: seen.add(key) result.append(a) return result def write_markdown_report(all_articles, today_top3, equipment, conflict, strategy, today_overview, equipment_overview, conflict_overview, strategy_overview, now, report_file): """拼接并写入 Markdown 报告文件""" md_lines = [] md_lines.append(f"# 🔥 {now.year}年{now.month}月{now.day}日 军事科技每日摘报\n") md_lines.append(f"📊 今日共 {len(all_articles)} 篇精选文章 | ⏰ {now.strftime('%Y-%m-%d %H:%M')}\n") md_lines.append(f"## 🔥 今日必看\n") if today_overview: md_lines.append(f"> {today_overview}\n") for i, a in enumerate(today_top3, 1): md_lines.append(f"### {i}. {a['translated_title']}\n") md_lines.append(f"- 📡 来源: {a['source']}") md_lines.append(f"- ⏰ 时间: {a['published'].strftime('%m-%d %H:%M')}") md_lines.append(f"- ⭐ 评分: {a['final_score']}/10") md_lines.append(f"- 📝 {a['summary']}") md_lines.append(f"- 🔗 [阅读原文]({a['link']})\n") def _append_category(title, icon, overview, articles): if not articles: return md_lines.append(f"## {icon} {title}\n") if overview: md_lines.append(f"> {overview}\n") for i, a in enumerate(articles, 1): md_lines.append(f"### {i}. {a['translated_title']}\n") md_lines.append(f"- 📡 {a['source']} | ⏰ {a['published'].strftime('%m-%d %H:%M')}") md_lines.append(f"- 📝 {a['summary']}") md_lines.append(f"- 🔗 [阅读原文]({a['link']})\n") _append_category("装备动态", "⚙️", equipment_overview, equipment) _append_category("地区冲突", "💥", conflict_overview, conflict) _append_category("战略政策", "🎯", strategy_overview, strategy) with open(report_file, 'w', encoding='utf-8') as f: f.write('\n'.join(md_lines)) logger.info("报告文件已保存: %s", report_file) def build_webzine_content(top3, now, api_key, base_url, webzine_model, cache): """生成TOP3网摘文本,优先取缓存,写回新生成的网摘""" webzine_content = [] webzine_content.append(f"═{'═'*60}═") webzine_content.append(f" 🔥 军事科技每日摘报 - 今日必看 TOP3 网摘") webzine_content.append(f" 📅 {now.strftime('%Y年%m月%d日')}") webzine_content.append(f"═{'═'*60}═") webzine_content.append("") webzine_texts = [] for i, article in enumerate(top3, 1): cached_webzine = article.get('webzine_text', '') if cached_webzine: logger.info("网摘 %d/3(缓存命中): %s...", i, article['translated_title'][:35]) webzine = cached_webzine else: logger.info("生成网摘 %d/3: %s...", i, article['translated_title'][:35]) webzine = generate_webzine_for_article(article, api_key, base_url, webzine_model) cache.save_article_webzine(article['id'], webzine) webzine_texts.append(webzine) webzine_content.append(f"{'='*62}") webzine_content.append(f" 📌 第 {i} 篇") webzine_content.append(f"{'='*62}") webzine_content.append("") webzine_content.append(webzine) webzine_content.append("") webzine_content.append(f"🔗 原文链接:{article['link']}") webzine_content.append("") webzine_content.append("") return webzine_content, webzine_texts def get_or_generate_overview(cache, date_str, cache_key, label, articles, api_key, base_url, model): """获取分类介绍:有缓存则直接返回,否则AI生成后写缓存""" cached = cache.get_category_summary(date_str, cache_key) if cached is not None: logger.info("分类介绍 [%s](缓存命中)", label) return cached overview = generate_category_overview(label, articles, api_key, base_url, model) cache.save_category_summary(date_str, cache_key, overview) return overview def categorize_articles(articles): """按分类整理文章,返回 (装备动态, 地区冲突, 战略政策) 三元组""" equipment, conflict, strategy = [], [], [] for a in articles: cat = a.get('category', '装备动态') if '冲突' in cat or '战争' in cat or '地区' in cat: conflict.append(a) elif '战略' in cat or '政策' in cat or '外交' in cat: strategy.append(a) else: equipment.append(a) return equipment, conflict, strategy def build_webzine_text_content(now, today_top3, webzine_texts): """根据已并行生成的网摘文本构建格式化输出文件内容""" lines = [] lines.append(f"═{'═'*60}═") lines.append(f" 🔥 军事科技每日摘报 - 今日必看 TOP3 网摘") lines.append(f" 📅 {now.strftime('%Y年%m月%d日')}") lines.append(f"═{'═'*60}═") lines.append("") for i, (article, wz_text) in enumerate(zip(today_top3, webzine_texts), 1): lines.append(f"{'='*62}") lines.append(f" 📌 第 {i} 篇") lines.append(f"{'='*62}") lines.append("") lines.append(wz_text) lines.append("") lines.append(f"🔗 原文链接:{article['link']}") lines.append("") lines.append("") return lines async def async_main(): """异步主流程:流水线抓取处理 + 并行网摘 + 并行分类介绍""" now = datetime.now() cutoff = now - timedelta(hours=TIME_WINDOW_HOURS) date_str = now.strftime('%Y%m%d') cache = get_cache() cache.clear_expired(keep_days=7) monitor = get_monitor() logger.info("开始异步流式抓取 %d 个军事科技 RSS 订阅源...", len(RSS_FEEDS)) logger.info("时间窗口: %s ~ %s", cutoff.strftime('%m-%d %H:%M'), now.strftime('%m-%d %H:%M')) api_key, base_url, model, webzine_model = load_ai_config() if not api_key: logger.error("未找到OPENAI_API_KEY,退出") cache.close() return # ========================================================================= # 异步流水线:RSS抓取 → 时间过滤 → 缓存检查 → 标题翻译 → 筛选 → 正文翻译 → AI分析 # 关键收益: # 1. feed_1 的文章在 feed_2 仍在抓取时已开始处理(流水线并行) # 2. AI_CONCURRENCY 个worker同时处理多篇文章(阶段内并行) # ========================================================================= with monitor.stage("streaming_pipeline"): cached_articles, processed_articles = await process_articles_streaming( RSS_FEEDS, cutoff, api_key, base_url, model, cache ) logger.info("流水线完成:缓存命中 %d 篇,新处理 %d 篇", len(cached_articles), len(processed_articles)) # ========================================================================= # 同源标题去重(流水线内无法跨feed去重,此处统一处理) # ========================================================================= with monitor.stage("deduplicate"): all_raw = deduplicate_articles(cached_articles + processed_articles) all_articles = merge_from_cache([], all_raw) if not all_articles: logger.info("去重后无有效文章") cache.close() cleanup_executor() return logger.info("去重后总文章数: %d 篇", len(all_articles)) # ========================================================================= # 提取TOP3 + 分类整理 # ========================================================================= today_top3 = all_articles[:3] OUTPUT_DIR.mkdir(parents=True, exist_ok=True) webzine_file = OUTPUT_DIR / f"military_webzine_{date_str}.txt" webzine_image = OUTPUT_DIR / f"military_webzine_{date_str}.png" equipment_articles, conflict_articles, strategy_articles = categorize_articles(all_articles[3:]) # ========================================================================= # 并行生成 TOP3 网摘(3篇文章同时调用AI,原串行 53s → 并行 ~18s) # ========================================================================= all_webzine_cached = all(bool(a.get('webzine_text', '')) for a in today_top3) if all_webzine_cached and webzine_file.exists() and webzine_image.exists(): logger.info("今日必看 TOP3 网摘文件及长图已存在(全部缓存命中),跳过生成") webzine_texts = [a['webzine_text'] for a in today_top3] else: with monitor.stage("webzine_generate"): logger.info("并行生成今日必看 TOP3 网摘...") webzine_texts = await generate_webzines_parallel( today_top3, api_key, base_url, webzine_model, cache ) webzine_content = build_webzine_text_content(now, today_top3, webzine_texts) with open(webzine_file, 'w', encoding='utf-8') as f: f.write('\n'.join(webzine_content)) logger.info("网摘文件已保存: %s", webzine_file) with monitor.stage("image_generate"): create_combined_webzine_image(webzine_texts, str(webzine_image), now) # ========================================================================= # 并行生成各分类总体介绍(4类同时调用AI,原串行 33s → 并行 ~9s) # ========================================================================= overview_tasks = [ ("today_must_read", "今日必看", today_top3), ("equipment", "装备动态", equipment_articles), ("conflict", "地区冲突", conflict_articles), ("strategy", "战略政策", strategy_articles), ] with monitor.stage("category_overview"): logger.info("并行生成各分类总体情况介绍...") overviews = await generate_overviews_parallel( date_str, overview_tasks, cache, api_key, base_url, model ) today_overview, equipment_overview, conflict_overview, strategy_overview = overviews # ========================================================================= # 生成 Markdown 报告 # ========================================================================= report_file = OUTPUT_DIR / f"military_report_{date_str}.md" all_articles_cached = all(a.get('from_cache') for a in all_articles) with monitor.stage("report_generate"): if all_articles_cached and report_file.exists(): logger.info("报告文件已存在且全部文章来自缓存,跳过生成") else: write_markdown_report(all_articles, today_top3, equipment_articles, conflict_articles, strategy_articles, today_overview, equipment_overview, conflict_overview, strategy_overview, now, report_file) monitor.record_article_stats(len(all_articles), len(cached_articles), len(processed_articles)) monitor.report() # ========================================================================= # 多渠道推送 # ========================================================================= push_config = load_push_config() send_daily_push(now, date_str, len(all_articles), today_top3, OUTPUT_DIR, push_config) cleanup_executor() cache.close() def main(): asyncio.run(async_main()) if __name__ == '__main__': main()