354 lines
15 KiB
Python
354 lines
15 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""
|
||
军事科技每日摘报 - V3 正式版（模块化重构）
|
||
完整流程：RSS抓取 → 时间过滤 → 去重 → 批量翻译外文 → 关键词筛选 →
|
||
AI评分分类摘要 → 生成分类总体介绍 → 网摘生成 → 报告输出
|
||
"""
|
||
|
||
import asyncio
|
||
from datetime import datetime, timedelta
|
||
from pathlib import Path
|
||
|
||
from modules.config import TIME_WINDOW_HOURS, load_ai_config, load_push_config, RSS_FEEDS
|
||
from modules.logger import get_logger
|
||
from modules.cache import get_cache
|
||
from modules.keyword_filter import keyword_filter
|
||
from modules.ai_analyzer import (
|
||
score_and_classify_article,
|
||
calculate_weighted_score,
|
||
generate_category_overview,
|
||
generate_webzine_for_article,
|
||
batch_score_and_classify_articles,
|
||
)
|
||
from modules.image_generator import create_combined_webzine_image
|
||
from modules.monitor import get_monitor
|
||
from modules.pusher import send_daily_push
|
||
from modules.async_pipeline import (
|
||
process_articles_streaming,
|
||
generate_webzines_parallel,
|
||
generate_overviews_parallel,
|
||
cleanup_executor,
|
||
)
|
||
|
||
logger = get_logger()
|
||
|
||
OUTPUT_DIR = Path(__file__).resolve().parent.parent / "output"
|
||
|
||
|
||
def merge_from_cache(cached, processed):
|
||
"""合并缓存和新处理文章，按评分降序排序；无文章时返回 None"""
|
||
merged = cached + processed
|
||
if not merged:
|
||
logger.info("没有可用的文章，流程结束")
|
||
return None
|
||
merged.sort(key=lambda x: x['final_score'], reverse=True)
|
||
logger.info("总文章数: %d 篇，最高分: %.1f/10", len(merged), merged[0]['final_score'])
|
||
return merged
|
||
|
||
|
||
def process_new_articles(articles, api_key, base_url, model, monitor, cache):
|
||
"""新文章处理管线：排序 → 标题翻译 → 关键词筛选 → 正文翻译 → AI分析 → 写缓存"""
|
||
articles = sorted(articles, key=lambda x: x['published'], reverse=True)
|
||
|
||
logger.info("开始批量翻译所有文章标题...")
|
||
with monitor.stage("translate_titles"):
|
||
articles = batch_translate_titles(articles, api_key, base_url, model)
|
||
logger.info("标题翻译完成，共 %d 篇文章", len(articles))
|
||
|
||
with monitor.stage("keyword_filter"):
|
||
if len(articles) > 15:
|
||
logger.info("文章数超过15篇，执行关键词筛选（基于标题）...")
|
||
articles = keyword_filter(articles)
|
||
hit_dist = [a['keyword_hits'] for a in articles[:10]]
|
||
logger.info("关键词筛选后: %d 篇，命中数分布: %s", len(articles), hit_dist)
|
||
else:
|
||
logger.info("文章数 %d <= 15，全部保留", len(articles))
|
||
|
||
if not articles:
|
||
logger.info("筛选后无符合条件文章")
|
||
return []
|
||
|
||
logger.info("开始批量翻译筛选通过的文章正文...")
|
||
with monitor.stage("translate_contents"):
|
||
articles = batch_translate_contents(articles, api_key, base_url, model)
|
||
logger.info("正文翻译完成，共 %d 篇文章", len(articles))
|
||
|
||
logger.info("开始 AI 智能分析（使用 %s）...", model)
|
||
with monitor.stage("ai_analyze"):
|
||
articles = batch_score_and_classify_articles(articles, api_key, base_url, model)
|
||
|
||
logger.info("保存 %d 篇新处理的文章到缓存...", len(articles))
|
||
for a in articles:
|
||
cache.save_article(a)
|
||
|
||
return articles
|
||
|
||
|
||
def deduplicate_articles(articles):
|
||
"""同源标题去重：同一源+相同标题仅保留一篇，过滤短标题"""
|
||
seen = set()
|
||
result = []
|
||
for a in articles:
|
||
title = a['title'].strip()
|
||
if len(title) <= 5:
|
||
continue
|
||
key = (a['source'].strip(), title)
|
||
if key not in seen:
|
||
seen.add(key)
|
||
result.append(a)
|
||
return result
|
||
|
||
|
||
def write_markdown_report(all_articles, today_top3, equipment, conflict, strategy,
|
||
today_overview, equipment_overview, conflict_overview,
|
||
strategy_overview, now, report_file):
|
||
"""拼接并写入 Markdown 报告文件"""
|
||
md_lines = []
|
||
md_lines.append(f"# 🔥 {now.year}年{now.month}月{now.day}日 军事科技每日摘报\n")
|
||
md_lines.append(f"📊 今日共 {len(all_articles)} 篇精选文章 | ⏰ {now.strftime('%Y-%m-%d %H:%M')}\n")
|
||
|
||
md_lines.append(f"## 🔥 今日必看\n")
|
||
if today_overview:
|
||
md_lines.append(f"> {today_overview}\n")
|
||
for i, a in enumerate(today_top3, 1):
|
||
md_lines.append(f"### {i}. {a['translated_title']}\n")
|
||
md_lines.append(f"- 📡 来源: {a['source']}")
|
||
md_lines.append(f"- ⏰ 时间: {a['published'].strftime('%m-%d %H:%M')}")
|
||
md_lines.append(f"- ⭐ 评分: {a['final_score']}/10")
|
||
md_lines.append(f"- 📝 {a['summary']}")
|
||
md_lines.append(f"- 🔗 [阅读原文]({a['link']})\n")
|
||
|
||
def _append_category(title, icon, overview, articles):
|
||
if not articles:
|
||
return
|
||
md_lines.append(f"## {icon} {title}\n")
|
||
if overview:
|
||
md_lines.append(f"> {overview}\n")
|
||
for i, a in enumerate(articles, 1):
|
||
md_lines.append(f"### {i}. {a['translated_title']}\n")
|
||
md_lines.append(f"- 📡 {a['source']} | ⏰ {a['published'].strftime('%m-%d %H:%M')}")
|
||
md_lines.append(f"- 📝 {a['summary']}")
|
||
md_lines.append(f"- 🔗 [阅读原文]({a['link']})\n")
|
||
|
||
_append_category("装备动态", "⚙️", equipment_overview, equipment)
|
||
_append_category("地区冲突", "💥", conflict_overview, conflict)
|
||
_append_category("战略政策", "🎯", strategy_overview, strategy)
|
||
|
||
with open(report_file, 'w', encoding='utf-8') as f:
|
||
f.write('\n'.join(md_lines))
|
||
logger.info("报告文件已保存: %s", report_file)
|
||
|
||
|
||
def build_webzine_content(top3, now, api_key, base_url, webzine_model, cache):
|
||
"""生成TOP3网摘文本，优先取缓存，写回新生成的网摘"""
|
||
webzine_content = []
|
||
webzine_content.append(f"═{'═'*60}═")
|
||
webzine_content.append(f" 🔥 军事科技每日摘报 - 今日必看 TOP3 网摘")
|
||
webzine_content.append(f" 📅 {now.strftime('%Y年%m月%d日')}")
|
||
webzine_content.append(f"═{'═'*60}═")
|
||
webzine_content.append("")
|
||
|
||
webzine_texts = []
|
||
for i, article in enumerate(top3, 1):
|
||
cached_webzine = article.get('webzine_text', '')
|
||
if cached_webzine:
|
||
logger.info("网摘 %d/3（缓存命中）: %s...", i, article['translated_title'][:35])
|
||
webzine = cached_webzine
|
||
else:
|
||
logger.info("生成网摘 %d/3: %s...", i, article['translated_title'][:35])
|
||
webzine = generate_webzine_for_article(article, api_key, base_url, webzine_model)
|
||
cache.save_article_webzine(article['id'], webzine)
|
||
webzine_texts.append(webzine)
|
||
webzine_content.append(f"{'='*62}")
|
||
webzine_content.append(f" 📌 第 {i} 篇")
|
||
webzine_content.append(f"{'='*62}")
|
||
webzine_content.append("")
|
||
webzine_content.append(webzine)
|
||
webzine_content.append("")
|
||
webzine_content.append(f"🔗 原文链接：{article['link']}")
|
||
webzine_content.append("")
|
||
webzine_content.append("")
|
||
|
||
return webzine_content, webzine_texts
|
||
|
||
|
||
def get_or_generate_overview(cache, date_str, cache_key, label, articles, api_key, base_url, model):
|
||
"""获取分类介绍：有缓存则直接返回，否则AI生成后写缓存"""
|
||
cached = cache.get_category_summary(date_str, cache_key)
|
||
if cached is not None:
|
||
logger.info("分类介绍 [%s]（缓存命中）", label)
|
||
return cached
|
||
overview = generate_category_overview(label, articles, api_key, base_url, model)
|
||
cache.save_category_summary(date_str, cache_key, overview)
|
||
return overview
|
||
|
||
|
||
def categorize_articles(articles):
|
||
"""按分类整理文章，返回 (装备动态, 地区冲突, 战略政策) 三元组"""
|
||
equipment, conflict, strategy = [], [], []
|
||
for a in articles:
|
||
cat = a.get('category', '装备动态')
|
||
if '冲突' in cat or '战争' in cat or '地区' in cat:
|
||
conflict.append(a)
|
||
elif '战略' in cat or '政策' in cat or '外交' in cat:
|
||
strategy.append(a)
|
||
else:
|
||
equipment.append(a)
|
||
return equipment, conflict, strategy
|
||
|
||
|
||
def build_webzine_text_content(now, today_top3, webzine_texts):
|
||
"""根据已并行生成的网摘文本构建格式化输出文件内容"""
|
||
lines = []
|
||
lines.append(f"═{'═'*60}═")
|
||
lines.append(f" 🔥 军事科技每日摘报 - 今日必看 TOP3 网摘")
|
||
lines.append(f" 📅 {now.strftime('%Y年%m月%d日')}")
|
||
lines.append(f"═{'═'*60}═")
|
||
lines.append("")
|
||
for i, (article, wz_text) in enumerate(zip(today_top3, webzine_texts), 1):
|
||
lines.append(f"{'='*62}")
|
||
lines.append(f" 📌 第 {i} 篇")
|
||
lines.append(f"{'='*62}")
|
||
lines.append("")
|
||
lines.append(wz_text)
|
||
lines.append("")
|
||
lines.append(f"🔗 原文链接：{article['link']}")
|
||
lines.append("")
|
||
lines.append("")
|
||
return lines
|
||
|
||
|
||
async def async_main():
|
||
"""异步主流程：流水线抓取处理 + 并行网摘 + 并行分类介绍"""
|
||
now = datetime.now()
|
||
cutoff = now - timedelta(hours=TIME_WINDOW_HOURS)
|
||
date_str = now.strftime('%Y%m%d')
|
||
cache = get_cache()
|
||
cache.clear_expired(keep_days=7)
|
||
|
||
monitor = get_monitor()
|
||
|
||
logger.info("开始异步流式抓取 %d 个军事科技 RSS 订阅源...", len(RSS_FEEDS))
|
||
logger.info("时间窗口: %s ~ %s", cutoff.strftime('%m-%d %H:%M'), now.strftime('%m-%d %H:%M'))
|
||
|
||
api_key, base_url, model, webzine_model = load_ai_config()
|
||
if not api_key:
|
||
logger.error("未找到OPENAI_API_KEY，退出")
|
||
cache.close()
|
||
return
|
||
|
||
# =========================================================================
|
||
# 异步流水线：RSS抓取 → 时间过滤 → 缓存检查 → 标题翻译 → 筛选 → 正文翻译 → AI分析
|
||
# 关键收益：
|
||
# 1. feed_1 的文章在 feed_2 仍在抓取时已开始处理（流水线并行）
|
||
# 2. AI_CONCURRENCY 个worker同时处理多篇文章（阶段内并行）
|
||
# =========================================================================
|
||
with monitor.stage("streaming_pipeline"):
|
||
cached_articles, processed_articles = await process_articles_streaming(
|
||
RSS_FEEDS, cutoff, api_key, base_url, model, cache
|
||
)
|
||
|
||
logger.info("流水线完成：缓存命中 %d 篇，新处理 %d 篇",
|
||
len(cached_articles), len(processed_articles))
|
||
|
||
# =========================================================================
|
||
# 同源标题去重（流水线内无法跨feed去重，此处统一处理）
|
||
# =========================================================================
|
||
with monitor.stage("deduplicate"):
|
||
all_raw = deduplicate_articles(cached_articles + processed_articles)
|
||
|
||
all_articles = merge_from_cache([], all_raw)
|
||
if not all_articles:
|
||
logger.info("去重后无有效文章")
|
||
cache.close()
|
||
cleanup_executor()
|
||
return
|
||
|
||
logger.info("去重后总文章数: %d 篇", len(all_articles))
|
||
|
||
# =========================================================================
|
||
# 提取TOP3 + 分类整理
|
||
# =========================================================================
|
||
today_top3 = all_articles[:3]
|
||
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
|
||
webzine_file = OUTPUT_DIR / f"military_webzine_{date_str}.txt"
|
||
webzine_image = OUTPUT_DIR / f"military_webzine_{date_str}.png"
|
||
|
||
equipment_articles, conflict_articles, strategy_articles = categorize_articles(all_articles[3:])
|
||
|
||
# =========================================================================
|
||
# 并行生成 TOP3 网摘（3篇文章同时调用AI，原串行 53s → 并行 ~18s）
|
||
# =========================================================================
|
||
all_webzine_cached = all(bool(a.get('webzine_text', '')) for a in today_top3)
|
||
|
||
if all_webzine_cached and webzine_file.exists() and webzine_image.exists():
|
||
logger.info("今日必看 TOP3 网摘文件及长图已存在（全部缓存命中），跳过生成")
|
||
webzine_texts = [a['webzine_text'] for a in today_top3]
|
||
else:
|
||
with monitor.stage("webzine_generate"):
|
||
logger.info("并行生成今日必看 TOP3 网摘...")
|
||
webzine_texts = await generate_webzines_parallel(
|
||
today_top3, api_key, base_url, webzine_model, cache
|
||
)
|
||
|
||
webzine_content = build_webzine_text_content(now, today_top3, webzine_texts)
|
||
with open(webzine_file, 'w', encoding='utf-8') as f:
|
||
f.write('\n'.join(webzine_content))
|
||
logger.info("网摘文件已保存: %s", webzine_file)
|
||
|
||
with monitor.stage("image_generate"):
|
||
create_combined_webzine_image(webzine_texts, str(webzine_image), now)
|
||
|
||
# =========================================================================
|
||
# 并行生成各分类总体介绍（4类同时调用AI，原串行 33s → 并行 ~9s）
|
||
# =========================================================================
|
||
overview_tasks = [
|
||
("today_must_read", "今日必看", today_top3),
|
||
("equipment", "装备动态", equipment_articles),
|
||
("conflict", "地区冲突", conflict_articles),
|
||
("strategy", "战略政策", strategy_articles),
|
||
]
|
||
|
||
with monitor.stage("category_overview"):
|
||
logger.info("并行生成各分类总体情况介绍...")
|
||
overviews = await generate_overviews_parallel(
|
||
date_str, overview_tasks, cache, api_key, base_url, model
|
||
)
|
||
today_overview, equipment_overview, conflict_overview, strategy_overview = overviews
|
||
|
||
# =========================================================================
|
||
# 生成 Markdown 报告
|
||
# =========================================================================
|
||
report_file = OUTPUT_DIR / f"military_report_{date_str}.md"
|
||
all_articles_cached = all(a.get('from_cache') for a in all_articles)
|
||
|
||
with monitor.stage("report_generate"):
|
||
if all_articles_cached and report_file.exists():
|
||
logger.info("报告文件已存在且全部文章来自缓存，跳过生成")
|
||
else:
|
||
write_markdown_report(all_articles, today_top3,
|
||
equipment_articles, conflict_articles, strategy_articles,
|
||
today_overview, equipment_overview, conflict_overview,
|
||
strategy_overview, now, report_file)
|
||
|
||
monitor.record_article_stats(len(all_articles), len(cached_articles), len(processed_articles))
|
||
monitor.report()
|
||
|
||
# =========================================================================
|
||
# 多渠道推送
|
||
# =========================================================================
|
||
push_config = load_push_config()
|
||
send_daily_push(now, date_str, len(all_articles), today_top3, OUTPUT_DIR, push_config)
|
||
|
||
cleanup_executor()
|
||
cache.close()
|
||
|
||
|
||
def main():
|
||
asyncio.run(async_main())
|
||
|
||
|
||
if __name__ == '__main__':
|
||
main()
|