feat: 军事科技每日资讯推送系统 - Docker部署 + 日志系统 + 数据目录重组
This commit is contained in:
poiuy
@@ -0,0 +1,353 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
军事科技每日摘报 - V3 正式版（模块化重构）
|
||||
完整流程：RSS抓取 → 时间过滤 → 去重 → 批量翻译外文 → 关键词筛选 →
|
||||
AI评分分类摘要 → 生成分类总体介绍 → 网摘生成 → 报告输出
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
from modules.config import TIME_WINDOW_HOURS, load_ai_config, load_push_config, RSS_FEEDS
|
||||
from modules.logger import get_logger
|
||||
from modules.cache import get_cache
|
||||
from modules.keyword_filter import keyword_filter
|
||||
from modules.ai_analyzer import (
|
||||
score_and_classify_article,
|
||||
calculate_weighted_score,
|
||||
generate_category_overview,
|
||||
generate_webzine_for_article,
|
||||
batch_score_and_classify_articles,
|
||||
)
|
||||
from modules.image_generator import create_combined_webzine_image
|
||||
from modules.monitor import get_monitor
|
||||
from modules.pusher import send_daily_push
|
||||
from modules.async_pipeline import (
|
||||
process_articles_streaming,
|
||||
generate_webzines_parallel,
|
||||
generate_overviews_parallel,
|
||||
cleanup_executor,
|
||||
)
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
OUTPUT_DIR = Path(__file__).resolve().parent.parent / "output"
|
||||
|
||||
|
||||
def merge_from_cache(cached, processed):
|
||||
"""合并缓存和新处理文章，按评分降序排序；无文章时返回 None"""
|
||||
merged = cached + processed
|
||||
if not merged:
|
||||
logger.info("没有可用的文章，流程结束")
|
||||
return None
|
||||
merged.sort(key=lambda x: x['final_score'], reverse=True)
|
||||
logger.info("总文章数: %d 篇，最高分: %.1f/10", len(merged), merged[0]['final_score'])
|
||||
return merged
|
||||
|
||||
|
||||
def process_new_articles(articles, api_key, base_url, model, monitor, cache):
|
||||
"""新文章处理管线：排序 → 标题翻译 → 关键词筛选 → 正文翻译 → AI分析 → 写缓存"""
|
||||
articles = sorted(articles, key=lambda x: x['published'], reverse=True)
|
||||
|
||||
logger.info("开始批量翻译所有文章标题...")
|
||||
with monitor.stage("translate_titles"):
|
||||
articles = batch_translate_titles(articles, api_key, base_url, model)
|
||||
logger.info("标题翻译完成，共 %d 篇文章", len(articles))
|
||||
|
||||
with monitor.stage("keyword_filter"):
|
||||
if len(articles) > 15:
|
||||
logger.info("文章数超过15篇，执行关键词筛选（基于标题）...")
|
||||
articles = keyword_filter(articles)
|
||||
hit_dist = [a['keyword_hits'] for a in articles[:10]]
|
||||
logger.info("关键词筛选后: %d 篇，命中数分布: %s", len(articles), hit_dist)
|
||||
else:
|
||||
logger.info("文章数 %d <= 15，全部保留", len(articles))
|
||||
|
||||
if not articles:
|
||||
logger.info("筛选后无符合条件文章")
|
||||
return []
|
||||
|
||||
logger.info("开始批量翻译筛选通过的文章正文...")
|
||||
with monitor.stage("translate_contents"):
|
||||
articles = batch_translate_contents(articles, api_key, base_url, model)
|
||||
logger.info("正文翻译完成，共 %d 篇文章", len(articles))
|
||||
|
||||
logger.info("开始 AI 智能分析（使用 %s）...", model)
|
||||
with monitor.stage("ai_analyze"):
|
||||
articles = batch_score_and_classify_articles(articles, api_key, base_url, model)
|
||||
|
||||
logger.info("保存 %d 篇新处理的文章到缓存...", len(articles))
|
||||
for a in articles:
|
||||
cache.save_article(a)
|
||||
|
||||
return articles
|
||||
|
||||
|
||||
def deduplicate_articles(articles):
|
||||
"""同源标题去重：同一源+相同标题仅保留一篇，过滤短标题"""
|
||||
seen = set()
|
||||
result = []
|
||||
for a in articles:
|
||||
title = a['title'].strip()
|
||||
if len(title) <= 5:
|
||||
continue
|
||||
key = (a['source'].strip(), title)
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
result.append(a)
|
||||
return result
|
||||
|
||||
|
||||
def write_markdown_report(all_articles, today_top3, equipment, conflict, strategy,
|
||||
today_overview, equipment_overview, conflict_overview,
|
||||
strategy_overview, now, report_file):
|
||||
"""拼接并写入 Markdown 报告文件"""
|
||||
md_lines = []
|
||||
md_lines.append(f"# 🔥 {now.year}年{now.month}月{now.day}日 军事科技每日摘报\n")
|
||||
md_lines.append(f"📊 今日共 {len(all_articles)} 篇精选文章 | ⏰ {now.strftime('%Y-%m-%d %H:%M')}\n")
|
||||
|
||||
md_lines.append(f"## 🔥 今日必看\n")
|
||||
if today_overview:
|
||||
md_lines.append(f"> {today_overview}\n")
|
||||
for i, a in enumerate(today_top3, 1):
|
||||
md_lines.append(f"### {i}. {a['translated_title']}\n")
|
||||
md_lines.append(f"- 📡 来源: {a['source']}")
|
||||
md_lines.append(f"- ⏰ 时间: {a['published'].strftime('%m-%d %H:%M')}")
|
||||
md_lines.append(f"- ⭐ 评分: {a['final_score']}/10")
|
||||
md_lines.append(f"- 📝 {a['summary']}")
|
||||
md_lines.append(f"- 🔗 [阅读原文]({a['link']})\n")
|
||||
|
||||
def _append_category(title, icon, overview, articles):
|
||||
if not articles:
|
||||
return
|
||||
md_lines.append(f"## {icon} {title}\n")
|
||||
if overview:
|
||||
md_lines.append(f"> {overview}\n")
|
||||
for i, a in enumerate(articles, 1):
|
||||
md_lines.append(f"### {i}. {a['translated_title']}\n")
|
||||
md_lines.append(f"- 📡 {a['source']} | ⏰ {a['published'].strftime('%m-%d %H:%M')}")
|
||||
md_lines.append(f"- 📝 {a['summary']}")
|
||||
md_lines.append(f"- 🔗 [阅读原文]({a['link']})\n")
|
||||
|
||||
_append_category("装备动态", "⚙️", equipment_overview, equipment)
|
||||
_append_category("地区冲突", "💥", conflict_overview, conflict)
|
||||
_append_category("战略政策", "🎯", strategy_overview, strategy)
|
||||
|
||||
with open(report_file, 'w', encoding='utf-8') as f:
|
||||
f.write('\n'.join(md_lines))
|
||||
logger.info("报告文件已保存: %s", report_file)
|
||||
|
||||
|
||||
def build_webzine_content(top3, now, api_key, base_url, webzine_model, cache):
|
||||
"""生成TOP3网摘文本，优先取缓存，写回新生成的网摘"""
|
||||
webzine_content = []
|
||||
webzine_content.append(f"═{'═'*60}═")
|
||||
webzine_content.append(f" 🔥 军事科技每日摘报 - 今日必看 TOP3 网摘")
|
||||
webzine_content.append(f" 📅 {now.strftime('%Y年%m月%d日')}")
|
||||
webzine_content.append(f"═{'═'*60}═")
|
||||
webzine_content.append("")
|
||||
|
||||
webzine_texts = []
|
||||
for i, article in enumerate(top3, 1):
|
||||
cached_webzine = article.get('webzine_text', '')
|
||||
if cached_webzine:
|
||||
logger.info("网摘 %d/3（缓存命中）: %s...", i, article['translated_title'][:35])
|
||||
webzine = cached_webzine
|
||||
else:
|
||||
logger.info("生成网摘 %d/3: %s...", i, article['translated_title'][:35])
|
||||
webzine = generate_webzine_for_article(article, api_key, base_url, webzine_model)
|
||||
cache.save_article_webzine(article['id'], webzine)
|
||||
webzine_texts.append(webzine)
|
||||
webzine_content.append(f"{'='*62}")
|
||||
webzine_content.append(f" 📌 第 {i} 篇")
|
||||
webzine_content.append(f"{'='*62}")
|
||||
webzine_content.append("")
|
||||
webzine_content.append(webzine)
|
||||
webzine_content.append("")
|
||||
webzine_content.append(f"🔗 原文链接：{article['link']}")
|
||||
webzine_content.append("")
|
||||
webzine_content.append("")
|
||||
|
||||
return webzine_content, webzine_texts
|
||||
|
||||
|
||||
def get_or_generate_overview(cache, date_str, cache_key, label, articles, api_key, base_url, model):
|
||||
"""获取分类介绍：有缓存则直接返回，否则AI生成后写缓存"""
|
||||
cached = cache.get_category_summary(date_str, cache_key)
|
||||
if cached is not None:
|
||||
logger.info("分类介绍 [%s]（缓存命中）", label)
|
||||
return cached
|
||||
overview = generate_category_overview(label, articles, api_key, base_url, model)
|
||||
cache.save_category_summary(date_str, cache_key, overview)
|
||||
return overview
|
||||
|
||||
|
||||
def categorize_articles(articles):
|
||||
"""按分类整理文章，返回 (装备动态, 地区冲突, 战略政策) 三元组"""
|
||||
equipment, conflict, strategy = [], [], []
|
||||
for a in articles:
|
||||
cat = a.get('category', '装备动态')
|
||||
if '冲突' in cat or '战争' in cat or '地区' in cat:
|
||||
conflict.append(a)
|
||||
elif '战略' in cat or '政策' in cat or '外交' in cat:
|
||||
strategy.append(a)
|
||||
else:
|
||||
equipment.append(a)
|
||||
return equipment, conflict, strategy
|
||||
|
||||
|
||||
def build_webzine_text_content(now, today_top3, webzine_texts):
|
||||
"""根据已并行生成的网摘文本构建格式化输出文件内容"""
|
||||
lines = []
|
||||
lines.append(f"═{'═'*60}═")
|
||||
lines.append(f" 🔥 军事科技每日摘报 - 今日必看 TOP3 网摘")
|
||||
lines.append(f" 📅 {now.strftime('%Y年%m月%d日')}")
|
||||
lines.append(f"═{'═'*60}═")
|
||||
lines.append("")
|
||||
for i, (article, wz_text) in enumerate(zip(today_top3, webzine_texts), 1):
|
||||
lines.append(f"{'='*62}")
|
||||
lines.append(f" 📌 第 {i} 篇")
|
||||
lines.append(f"{'='*62}")
|
||||
lines.append("")
|
||||
lines.append(wz_text)
|
||||
lines.append("")
|
||||
lines.append(f"🔗 原文链接：{article['link']}")
|
||||
lines.append("")
|
||||
lines.append("")
|
||||
return lines
|
||||
|
||||
|
||||
async def async_main():
|
||||
"""异步主流程：流水线抓取处理 + 并行网摘 + 并行分类介绍"""
|
||||
now = datetime.now()
|
||||
cutoff = now - timedelta(hours=TIME_WINDOW_HOURS)
|
||||
date_str = now.strftime('%Y%m%d')
|
||||
cache = get_cache()
|
||||
cache.clear_expired(keep_days=7)
|
||||
|
||||
monitor = get_monitor()
|
||||
|
||||
logger.info("开始异步流式抓取 %d 个军事科技 RSS 订阅源...", len(RSS_FEEDS))
|
||||
logger.info("时间窗口: %s ~ %s", cutoff.strftime('%m-%d %H:%M'), now.strftime('%m-%d %H:%M'))
|
||||
|
||||
api_key, base_url, model, webzine_model = load_ai_config()
|
||||
if not api_key:
|
||||
logger.error("未找到OPENAI_API_KEY，退出")
|
||||
cache.close()
|
||||
return
|
||||
|
||||
# =========================================================================
|
||||
# 异步流水线：RSS抓取 → 时间过滤 → 缓存检查 → 标题翻译 → 筛选 → 正文翻译 → AI分析
|
||||
# 关键收益：
|
||||
# 1. feed_1 的文章在 feed_2 仍在抓取时已开始处理（流水线并行）
|
||||
# 2. AI_CONCURRENCY 个worker同时处理多篇文章（阶段内并行）
|
||||
# =========================================================================
|
||||
with monitor.stage("streaming_pipeline"):
|
||||
cached_articles, processed_articles = await process_articles_streaming(
|
||||
RSS_FEEDS, cutoff, api_key, base_url, model, cache
|
||||
)
|
||||
|
||||
logger.info("流水线完成：缓存命中 %d 篇，新处理 %d 篇",
|
||||
len(cached_articles), len(processed_articles))
|
||||
|
||||
# =========================================================================
|
||||
# 同源标题去重（流水线内无法跨feed去重，此处统一处理）
|
||||
# =========================================================================
|
||||
with monitor.stage("deduplicate"):
|
||||
all_raw = deduplicate_articles(cached_articles + processed_articles)
|
||||
|
||||
all_articles = merge_from_cache([], all_raw)
|
||||
if not all_articles:
|
||||
logger.info("去重后无有效文章")
|
||||
cache.close()
|
||||
cleanup_executor()
|
||||
return
|
||||
|
||||
logger.info("去重后总文章数: %d 篇", len(all_articles))
|
||||
|
||||
# =========================================================================
|
||||
# 提取TOP3 + 分类整理
|
||||
# =========================================================================
|
||||
today_top3 = all_articles[:3]
|
||||
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
webzine_file = OUTPUT_DIR / f"military_webzine_{date_str}.txt"
|
||||
webzine_image = OUTPUT_DIR / f"military_webzine_{date_str}.png"
|
||||
|
||||
equipment_articles, conflict_articles, strategy_articles = categorize_articles(all_articles[3:])
|
||||
|
||||
# =========================================================================
|
||||
# 并行生成 TOP3 网摘（3篇文章同时调用AI，原串行 53s → 并行 ~18s）
|
||||
# =========================================================================
|
||||
all_webzine_cached = all(bool(a.get('webzine_text', '')) for a in today_top3)
|
||||
|
||||
if all_webzine_cached and webzine_file.exists() and webzine_image.exists():
|
||||
logger.info("今日必看 TOP3 网摘文件及长图已存在（全部缓存命中），跳过生成")
|
||||
webzine_texts = [a['webzine_text'] for a in today_top3]
|
||||
else:
|
||||
with monitor.stage("webzine_generate"):
|
||||
logger.info("并行生成今日必看 TOP3 网摘...")
|
||||
webzine_texts = await generate_webzines_parallel(
|
||||
today_top3, api_key, base_url, webzine_model, cache
|
||||
)
|
||||
|
||||
webzine_content = build_webzine_text_content(now, today_top3, webzine_texts)
|
||||
with open(webzine_file, 'w', encoding='utf-8') as f:
|
||||
f.write('\n'.join(webzine_content))
|
||||
logger.info("网摘文件已保存: %s", webzine_file)
|
||||
|
||||
with monitor.stage("image_generate"):
|
||||
create_combined_webzine_image(webzine_texts, str(webzine_image), now)
|
||||
|
||||
# =========================================================================
|
||||
# 并行生成各分类总体介绍（4类同时调用AI，原串行 33s → 并行 ~9s）
|
||||
# =========================================================================
|
||||
overview_tasks = [
|
||||
("today_must_read", "今日必看", today_top3),
|
||||
("equipment", "装备动态", equipment_articles),
|
||||
("conflict", "地区冲突", conflict_articles),
|
||||
("strategy", "战略政策", strategy_articles),
|
||||
]
|
||||
|
||||
with monitor.stage("category_overview"):
|
||||
logger.info("并行生成各分类总体情况介绍...")
|
||||
overviews = await generate_overviews_parallel(
|
||||
date_str, overview_tasks, cache, api_key, base_url, model
|
||||
)
|
||||
today_overview, equipment_overview, conflict_overview, strategy_overview = overviews
|
||||
|
||||
# =========================================================================
|
||||
# 生成 Markdown 报告
|
||||
# =========================================================================
|
||||
report_file = OUTPUT_DIR / f"military_report_{date_str}.md"
|
||||
all_articles_cached = all(a.get('from_cache') for a in all_articles)
|
||||
|
||||
with monitor.stage("report_generate"):
|
||||
if all_articles_cached and report_file.exists():
|
||||
logger.info("报告文件已存在且全部文章来自缓存，跳过生成")
|
||||
else:
|
||||
write_markdown_report(all_articles, today_top3,
|
||||
equipment_articles, conflict_articles, strategy_articles,
|
||||
today_overview, equipment_overview, conflict_overview,
|
||||
strategy_overview, now, report_file)
|
||||
|
||||
monitor.record_article_stats(len(all_articles), len(cached_articles), len(processed_articles))
|
||||
monitor.report()
|
||||
|
||||
# =========================================================================
|
||||
# 多渠道推送
|
||||
# =========================================================================
|
||||
push_config = load_push_config()
|
||||
send_daily_push(now, date_str, len(all_articles), today_top3, OUTPUT_DIR, push_config)
|
||||
|
||||
cleanup_executor()
|
||||
cache.close()
|
||||
|
||||
|
||||
def main():
|
||||
asyncio.run(async_main())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,22 @@
|
||||
# 军事科技每日摘报 - 功能模块包
|
||||
# V3 正式版，模块化重组
|
||||
|
||||
from .config import (
|
||||
RSS_FEEDS, FETCH_TIMEOUT, MAX_THREADS, TIME_WINDOW_HOURS,
|
||||
KEYWORDS, SCORE_DIMENSIONS, CATEGORIES, load_ai_config, AI_CONCURRENCY,
|
||||
)
|
||||
from .utils import (
|
||||
is_chinese, clean_html, extract_content, parse_pub_time,
|
||||
call_ai, wrap_text,
|
||||
)
|
||||
from .logger import get_logger
|
||||
from .cache import get_cache
|
||||
from .rss_fetcher import fetch_all_feeds
|
||||
from .translator import batch_translate_articles, batch_translate_titles, batch_translate_contents
|
||||
from .keyword_filter import keyword_filter
|
||||
from .ai_analyzer import (
|
||||
score_and_classify_article, calculate_weighted_score,
|
||||
generate_category_overview, generate_webzine_for_article,
|
||||
batch_score_and_classify_articles,
|
||||
)
|
||||
from .image_generator import create_webzine_image, create_combined_webzine_image
|
||||
@@ -0,0 +1,241 @@
|
||||
"""
|
||||
AI分析模块：四维度评分、智能分类、中文摘要、分类洞察、网摘生成
|
||||
"""
|
||||
import re
|
||||
import json
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from .config import SCORE_DIMENSIONS, WECHAT_SOURCES, AI_CONCURRENCY
|
||||
from .utils import call_ai
|
||||
from .logger import get_logger
|
||||
|
||||
|
||||
def score_and_classify_article(article, api_key, base_url, model, index=0, total=0):
|
||||
"""
|
||||
AI三合一分析：四维度评分 + 中文摘要 + 智能分类
|
||||
一次API调用完成三项任务，返回包含 scores/summary/category 的字典
|
||||
"""
|
||||
logger = get_logger()
|
||||
title = article['translated_title']
|
||||
content = article['translated_content'][:1000] if article['translated_content'] else article['translated_title']
|
||||
source = article['source']
|
||||
|
||||
dimensions_desc = '\n'.join(
|
||||
f'- {d["name"]} (权重 {int(d["weight"]*100)}%): {d["description"]}'
|
||||
for d in SCORE_DIMENSIONS
|
||||
)
|
||||
|
||||
prompt = f"""请对以下军事科技文章进行分析。
|
||||
|
||||
【文章标题】：{title}
|
||||
|
||||
【文章内容摘要】：{content}
|
||||
|
||||
【文章来源】：{source}
|
||||
|
||||
【评分维度】：
|
||||
{dimensions_desc}
|
||||
|
||||
【输出要求】：
|
||||
请严格按照以下 JSON 格式返回（不要其他文字说明）：
|
||||
{{
|
||||
"scores": {{
|
||||
"维度名称1": 分数（1-10的整数）,
|
||||
"维度名称2": 分数（1-10的整数）
|
||||
}},
|
||||
"summary": "2-3句话的高质量中文摘要，突出技术参数、战略影响、国际竞争格局",
|
||||
"category": "文章分类，必须是以下之一：装备动态/地区冲突/战略政策"
|
||||
}}
|
||||
|
||||
只输出 JSON，不要其他内容。"""
|
||||
|
||||
try:
|
||||
result = call_ai([
|
||||
{'role': 'system', 'content': '你是专业的军事科技分析专家，擅长武器装备分析、战略评估。'},
|
||||
{'role': 'user', 'content': prompt}
|
||||
], api_key, base_url, model, temperature=0.2, max_tokens=800, purpose="AI评分分类")
|
||||
|
||||
json_match = re.search(r'\{[\s\S]*\}', result)
|
||||
if not json_match:
|
||||
raise ValueError('未找到JSON输出')
|
||||
|
||||
parsed = json.loads(json_match[0])
|
||||
if index > 0 and total > 0:
|
||||
logger.info("AI分析完成 %d/%d: %s...", index, total, title[:30])
|
||||
return (article, parsed)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning("AI处理失败 '%s...': %s", title[:30], e)
|
||||
default_scores = {d['name']: 5 for d in SCORE_DIMENSIONS}
|
||||
default_result = {
|
||||
'scores': default_scores,
|
||||
'summary': title,
|
||||
'category': '装备动态'
|
||||
}
|
||||
return (article, default_result)
|
||||
|
||||
|
||||
def batch_score_and_classify_articles(articles, api_key, base_url, model, max_concurrent=None):
|
||||
"""批量对文章进行AI评分与分类，并发执行
|
||||
返回处理后的文章列表，文章已包含ai_result、final_score、summary、category等字段
|
||||
"""
|
||||
logger = get_logger()
|
||||
if not articles:
|
||||
return []
|
||||
|
||||
if max_concurrent is None:
|
||||
max_concurrent = AI_CONCURRENCY
|
||||
|
||||
logger.info("开始批量AI分析，共%d篇文章，并发数：%d", len(articles), max_concurrent)
|
||||
processed_articles = []
|
||||
|
||||
with ThreadPoolExecutor(max_workers=max_concurrent) as executor:
|
||||
future_to_article = {
|
||||
executor.submit(score_and_classify_article, article, api_key, base_url, model, i+1, len(articles)): article
|
||||
for i, article in enumerate(articles)
|
||||
}
|
||||
|
||||
for future in as_completed(future_to_article):
|
||||
article, ai_result = future.result()
|
||||
article['ai_result'] = ai_result
|
||||
article['final_score'] = calculate_weighted_score(ai_result)
|
||||
article['summary'] = ai_result.get('summary', article['translated_title'])
|
||||
article['category'] = ai_result.get('category', '装备动态')
|
||||
article['scores'] = ai_result.get('scores', {})
|
||||
processed_articles.append(article)
|
||||
|
||||
logger.info("批量AI分析完成，共处理%d篇文章", len(processed_articles))
|
||||
return processed_articles
|
||||
|
||||
|
||||
def calculate_weighted_score(ai_result):
|
||||
"""根据四维度分数和权重计算加权总分"""
|
||||
scores = ai_result.get('scores', {})
|
||||
total = 0.0
|
||||
for dim in SCORE_DIMENSIONS:
|
||||
score = scores.get(dim['name'], 5)
|
||||
total += score * dim['weight']
|
||||
return round(total, 1)
|
||||
|
||||
|
||||
def generate_category_overview(category_name, articles, api_key, base_url, model):
|
||||
"""为某分类生成50字左右的总体情况介绍"""
|
||||
logger = get_logger()
|
||||
if not articles:
|
||||
return ""
|
||||
|
||||
article_titles = "\u3001".join(a['translated_title'][:15] for a in articles[:5])
|
||||
|
||||
prompt = f"""请为以下军事科技新闻分类生成50字左右的总体情况介绍。
|
||||
|
||||
【分类名称】：{category_name}
|
||||
|
||||
【本分类文章】：{article_titles}
|
||||
|
||||
【要求】：
|
||||
- 50字左右，简洁专业
|
||||
- 概括本分类今日核心看点或趋势
|
||||
- 军事科技专业风格，不要口语化
|
||||
|
||||
直接输出文字，不要其他格式。"""
|
||||
|
||||
try:
|
||||
result = call_ai([
|
||||
{'role': 'system', 'content': '你是专业的军事科技编辑，擅长撰写简洁专业的新闻综述。'},
|
||||
{'role': 'user', 'content': prompt}
|
||||
], api_key, base_url, model, temperature=0.3, max_tokens=150, purpose="分类介绍")
|
||||
return result.strip()
|
||||
except Exception as e:
|
||||
logger.warning("生成分类介绍失败: %s", e)
|
||||
return ""
|
||||
|
||||
|
||||
def generate_webzine_for_article(article, api_key, base_url, model, max_retries=3):
|
||||
"""
|
||||
为单篇文章生成《参考消息》风格的网摘
|
||||
支持字数校验重试（正文280-320字、价值点35-45字），最多重试3次
|
||||
"""
|
||||
logger = get_logger()
|
||||
source_name = article['source']
|
||||
publish_time = article['published'].strftime('%Y年%m月%d日')
|
||||
|
||||
is_wechat = source_name in WECHAT_SOURCES
|
||||
source_prefix = f"据公众号 {source_name} 报道" if is_wechat else f"据 {source_name} 报道"
|
||||
|
||||
prompt = f"""文章标题：{article['translated_title']}
|
||||
文章来源：{source_name}
|
||||
文章发布时间：{publish_time}
|
||||
文章内容：{article.get('summary', article.get('content', '')[:800])}
|
||||
文章链接：{article['link']}
|
||||
|
||||
【重要警告：字数必须严格达标，否则视为不合格！】
|
||||
|
||||
请你严格按照 《参考消息》官方新闻报道格式、标题风格 ，对我提供的新闻内容进行改写，严格遵守以下所有规则：
|
||||
|
||||
标题要求
|
||||
简洁、客观、中性、信息密度高
|
||||
结构：主体 + 事件 + 核心态势
|
||||
不抒情、不夸张、不用网络用语
|
||||
|
||||
正文格式要求
|
||||
开头第一句必须加：{source_prefix}
|
||||
正文风格：客观、平实、严谨、书面化，类似外电编译稿
|
||||
必须充分展开：补充背景、说明意义、分析影响、展望前景，确保内容充实
|
||||
只保留核心信息：时间、地点、人物、事件、内容、前景
|
||||
全文一段到底，不分段
|
||||
|
||||
【字数强制要求 - 必须严格执行】
|
||||
正文字数严格控制在 280～320 字（含标点），一个字都不能少，一个字都不能多！
|
||||
300字左右的正文示例长度：大概5-6个完整的长句，涵盖事件背景、核心内容、技术细节、战略意义、未来影响五个层面。
|
||||
如果内容不够，请合理补充相关背景、行业态势、同类项目对比等专业内容，确保达到字数要求。
|
||||
|
||||
价值点要求
|
||||
正文结束后空一行，再写价值点
|
||||
价值点严格 35～45 字（含标点）
|
||||
句式结构：事件 - 影响 / 后果 - 值得关注
|
||||
凝练、客观、不发散
|
||||
|
||||
输出格式
|
||||
标题：新闻标题
|
||||
原标题：原文章标题
|
||||
发布日期：严格使用上面提供的「{publish_time}」，不要修改
|
||||
正文：改写后的正文内容
|
||||
价值点：价值点内容
|
||||
|
||||
注意：每一行都必须以「类别：内容」的格式输出，类别严格使用上面的5个名称，不要修改。"""
|
||||
|
||||
last_result = None
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
result = call_ai([
|
||||
{'role': 'system', 'content': '你是《参考消息》资深编辑，擅长撰写客观、严谨、专业的新闻报道，对字数控制极其精准。'},
|
||||
{'role': 'user', 'content': prompt}
|
||||
], api_key, base_url, model, temperature=0.4, max_tokens=800, purpose="网摘生成")
|
||||
|
||||
result = result.strip()
|
||||
last_result = result
|
||||
|
||||
lines = result.split('\n')
|
||||
body_text = ''
|
||||
for line in lines:
|
||||
if line.startswith('正文：'):
|
||||
body_text = line[3:].strip()
|
||||
break
|
||||
|
||||
body_len = len(body_text)
|
||||
if 280 <= body_len <= 320:
|
||||
if attempt > 0:
|
||||
logger.info("第%d次尝试成功，字数：%d", attempt + 1, body_len)
|
||||
return result
|
||||
else:
|
||||
logger.warning("第%d次尝试字数不达标：%d字（要求280-320），重试中...", attempt + 1, body_len)
|
||||
if body_len < 280:
|
||||
prompt += f"\n\n【上次反馈：正文只有{body_len}字，太少！请大幅增加内容，补充更多背景、细节和分析，确保达到280-320字！】"
|
||||
else:
|
||||
prompt += f"\n\n【上次反馈：正文有{body_len}字，太多！请精简内容，删除冗余描述，控制在280-320字以内！】"
|
||||
|
||||
except Exception as e:
|
||||
logger.warning("第%d次尝试失败: %s", attempt + 1, e)
|
||||
|
||||
logger.warning("重试%d次仍未达标，返回最后结果", max_retries)
|
||||
return last_result if last_result else ""
|
||||
@@ -0,0 +1,228 @@
|
||||
"""
|
||||
异步流式处理管线模块
|
||||
实现三级并行：RSS抓取与文章处理流水线 / 网摘并行生成 / 分类介绍并行生成
|
||||
"""
|
||||
import asyncio
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from .config import AI_CONCURRENCY, MAX_THREADS
|
||||
from .logger import get_logger
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
_shared_executor = None
|
||||
|
||||
|
||||
def _get_executor():
|
||||
global _shared_executor
|
||||
if _shared_executor is None:
|
||||
_shared_executor = ThreadPoolExecutor(max_workers=max(MAX_THREADS, AI_CONCURRENCY) * 2)
|
||||
return _shared_executor
|
||||
|
||||
|
||||
def cleanup_executor():
|
||||
global _shared_executor
|
||||
if _shared_executor:
|
||||
_shared_executor.shutdown(wait=True)
|
||||
_shared_executor = None
|
||||
|
||||
|
||||
async def fetch_all_feeds_async(feeds):
|
||||
"""异步并行抓取所有RSS源，每个feed独立提交到线程池"""
|
||||
from .rss_fetcher import _fetch_single_feed
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
executor = _get_executor()
|
||||
|
||||
async def fetch_one(feed):
|
||||
return await loop.run_in_executor(executor, _fetch_single_feed, feed)
|
||||
|
||||
tasks = [asyncio.create_task(fetch_one(f)) for f in feeds]
|
||||
results = await asyncio.gather(*tasks)
|
||||
|
||||
all_articles = []
|
||||
for articles in results:
|
||||
all_articles.extend(articles)
|
||||
return all_articles
|
||||
|
||||
|
||||
async def process_articles_streaming(feeds, cutoff, api_key, base_url, model, cache):
|
||||
"""
|
||||
流水线处理：抓取与AI处理并行执行
|
||||
使用生产者-消费者模型：
|
||||
- 上游：每个feed抓取完成后文章立即入队
|
||||
- 下游：AI_CONCURRENCY个worker从队列取文章，执行翻译+分析
|
||||
- 两级队列：标题翻译 → 收集过滤 → 正文翻译+AI分析
|
||||
"""
|
||||
from .rss_fetcher import _fetch_single_feed
|
||||
from .translator import translate_title, translate_content
|
||||
from .ai_analyzer import score_and_classify_article, calculate_weighted_score
|
||||
from .keyword_filter import keyword_filter
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
executor = _get_executor()
|
||||
|
||||
fetch_queue = asyncio.Queue(maxsize=200)
|
||||
|
||||
# =========================================================================
|
||||
# Phase 1: 流水线抓取 + 标题翻译
|
||||
# =========================================================================
|
||||
|
||||
async def produce(feed):
|
||||
articles = await loop.run_in_executor(executor, _fetch_single_feed, feed)
|
||||
for a in articles:
|
||||
if a['published'] >= cutoff:
|
||||
await fetch_queue.put(a)
|
||||
|
||||
async def worker_title(worker_id, local_cached, local_new):
|
||||
"""第一级worker：标题翻译 + 缓存检查"""
|
||||
while True:
|
||||
article = await fetch_queue.get()
|
||||
if article is None:
|
||||
fetch_queue.task_done()
|
||||
break
|
||||
|
||||
cached = cache.get_article(article['id'], expire_hours=24)
|
||||
if cached:
|
||||
local_cached.append(cached)
|
||||
fetch_queue.task_done()
|
||||
continue
|
||||
|
||||
try:
|
||||
article = await loop.run_in_executor(
|
||||
executor, translate_title, article, api_key, base_url, model
|
||||
)
|
||||
local_new.append(article)
|
||||
except Exception as e:
|
||||
logger.warning("Worker%d 标题翻译失败: %s", worker_id, e)
|
||||
fetch_queue.task_done()
|
||||
|
||||
producer_tasks = [asyncio.create_task(produce(f)) for f in feeds]
|
||||
|
||||
cached_all = []
|
||||
title_translated = []
|
||||
worker_tasks_title = [
|
||||
asyncio.create_task(worker_title(i, cached_all, title_translated))
|
||||
for i in range(AI_CONCURRENCY)
|
||||
]
|
||||
|
||||
await asyncio.gather(*producer_tasks)
|
||||
for _ in range(AI_CONCURRENCY):
|
||||
await fetch_queue.put(None)
|
||||
await asyncio.gather(*worker_tasks_title)
|
||||
|
||||
logger.info("流水线第一阶段完成：缓存命中 %d 篇，标题翻译 %d 篇",
|
||||
len(cached_all), len(title_translated))
|
||||
|
||||
if not title_translated:
|
||||
return cached_all, []
|
||||
|
||||
# =========================================================================
|
||||
# Phase 2: 关键词筛选
|
||||
# =========================================================================
|
||||
process_queue = asyncio.Queue(maxsize=200)
|
||||
|
||||
if len(title_translated) > 15:
|
||||
logger.info("文章数超过15篇，执行关键词筛选...")
|
||||
filtered = keyword_filter(title_translated)
|
||||
logger.info("关键词筛选: %d → %d 篇", len(title_translated), len(filtered))
|
||||
else:
|
||||
filtered = title_translated
|
||||
|
||||
if not filtered:
|
||||
return cached_all, []
|
||||
|
||||
# =========================================================================
|
||||
# Phase 3: 流水线正文翻译 + AI分析
|
||||
# =========================================================================
|
||||
for a in filtered:
|
||||
await process_queue.put(a)
|
||||
for _ in range(AI_CONCURRENCY):
|
||||
await process_queue.put(None)
|
||||
|
||||
async def worker_full(worker_id, local_processed):
|
||||
"""第二级worker：正文翻译 + AI评分分类 + 写缓存"""
|
||||
while True:
|
||||
article = await process_queue.get()
|
||||
if article is None:
|
||||
process_queue.task_done()
|
||||
break
|
||||
|
||||
try:
|
||||
article = await loop.run_in_executor(
|
||||
executor, translate_content, article, api_key, base_url, model
|
||||
)
|
||||
article, ai_result = await loop.run_in_executor(
|
||||
executor, score_and_classify_article, article, api_key, base_url, model
|
||||
)
|
||||
|
||||
article['ai_result'] = ai_result
|
||||
article['final_score'] = calculate_weighted_score(ai_result)
|
||||
article['summary'] = ai_result.get('summary', article['translated_title'])
|
||||
article['category'] = ai_result.get('category', '装备动态')
|
||||
article['scores'] = ai_result.get('scores', {})
|
||||
|
||||
cache.save_article(article)
|
||||
local_processed.append(article)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning("Worker%d 文章处理失败: %s", worker_id, e)
|
||||
process_queue.task_done()
|
||||
|
||||
processed_all = []
|
||||
worker_tasks_full = [
|
||||
asyncio.create_task(worker_full(i, processed_all))
|
||||
for i in range(AI_CONCURRENCY)
|
||||
]
|
||||
await asyncio.gather(*worker_tasks_full)
|
||||
|
||||
logger.info("流水线第二阶段完成：新处理 %d 篇文章", len(processed_all))
|
||||
return cached_all, processed_all
|
||||
|
||||
|
||||
async def generate_webzines_parallel(top3, api_key, base_url, webzine_model, cache, max_concurrent=1):
|
||||
"""受控并行生成TOP3网摘（信号量限流，避免API 429）"""
|
||||
from .ai_analyzer import generate_webzine_for_article
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
executor = _get_executor()
|
||||
semaphore = asyncio.Semaphore(max_concurrent)
|
||||
|
||||
async def gen_one(index, article):
|
||||
cached_wz = article.get('webzine_text', '')
|
||||
if cached_wz:
|
||||
logger.info("网摘 %d/3（缓存命中）: %s...", index, article['translated_title'][:35])
|
||||
return cached_wz
|
||||
async with semaphore:
|
||||
logger.info("生成网摘 %d/3: %s...", index, article['translated_title'][:35])
|
||||
webzine = await loop.run_in_executor(
|
||||
executor, generate_webzine_for_article, article, api_key, base_url, webzine_model
|
||||
)
|
||||
cache.save_article_webzine(article['id'], webzine)
|
||||
return webzine
|
||||
|
||||
tasks = [asyncio.create_task(gen_one(i + 1, a)) for i, a in enumerate(top3)]
|
||||
return await asyncio.gather(*tasks)
|
||||
|
||||
|
||||
async def generate_overviews_parallel(date_str, overview_tasks, cache, api_key, base_url, model, max_concurrent=2):
|
||||
"""受控并行生成各分类总体介绍（信号量限流，避免API 429）"""
|
||||
from .ai_analyzer import generate_category_overview
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
executor = _get_executor()
|
||||
semaphore = asyncio.Semaphore(max_concurrent)
|
||||
|
||||
async def gen_one(cache_key, label, articles):
|
||||
cached = cache.get_category_summary(date_str, cache_key)
|
||||
if cached is not None:
|
||||
logger.info("分类介绍 [%s]（缓存命中）", label)
|
||||
return cached
|
||||
async with semaphore:
|
||||
overview = await loop.run_in_executor(
|
||||
executor, generate_category_overview, label, articles, api_key, base_url, model
|
||||
)
|
||||
cache.save_category_summary(date_str, cache_key, overview)
|
||||
return overview
|
||||
|
||||
tasks = [asyncio.create_task(gen_one(k, l, a)) for k, l, a in overview_tasks]
|
||||
return await asyncio.gather(*tasks)
|
||||
@@ -0,0 +1,310 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
SQLite增量缓存模块
|
||||
实现文章处理结果的本地存储和复用，避免重复翻译和AI分析，支持断点续跑
|
||||
"""
|
||||
import sqlite3
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from .logger import get_logger
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
# 数据库文件路径，放在项目根目录下的data目录
|
||||
DB_PATH = Path(__file__).resolve().parent.parent.parent / "data" / "article_cache.db"
|
||||
|
||||
class ArticleCache:
|
||||
def __init__(self):
|
||||
# 确保数据目录存在
|
||||
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.conn = None
|
||||
self._connect()
|
||||
self._init_table()
|
||||
|
||||
def _connect(self):
|
||||
"""连接数据库，失败降级为不使用缓存"""
|
||||
try:
|
||||
self.conn = sqlite3.connect(str(DB_PATH), check_same_thread=False)
|
||||
logger.debug("缓存数据库连接成功")
|
||||
except Exception as e:
|
||||
logger.warning(f"缓存数据库连接失败，将不使用缓存功能: {e}")
|
||||
self.conn = None
|
||||
|
||||
def _init_table(self):
|
||||
"""初始化缓存表"""
|
||||
if not self.conn:
|
||||
return
|
||||
try:
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS article_cache (
|
||||
id TEXT PRIMARY KEY,
|
||||
source TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
link TEXT NOT NULL,
|
||||
translated_title TEXT,
|
||||
content TEXT,
|
||||
translated_content TEXT,
|
||||
ai_score REAL,
|
||||
ai_summary TEXT,
|
||||
ai_category TEXT,
|
||||
ai_scores_json TEXT,
|
||||
published_time DATETIME,
|
||||
processed_time DATETIME NOT NULL,
|
||||
status INTEGER NOT NULL DEFAULT 1,
|
||||
error_msg TEXT
|
||||
)
|
||||
""")
|
||||
# 创建索引提升查询速度
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_processed_time ON article_cache(processed_time)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_source ON article_cache(source)")
|
||||
self.conn.commit()
|
||||
logger.debug("缓存表初始化完成")
|
||||
except Exception as e:
|
||||
logger.warning(f"缓存表初始化失败，将不使用缓存功能: {e}")
|
||||
self.conn = None
|
||||
|
||||
self._migrate_add_webzine_text()
|
||||
self._init_summary_table()
|
||||
|
||||
def _init_summary_table(self):
|
||||
if not self.conn:
|
||||
return
|
||||
try:
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS category_summary_cache (
|
||||
date TEXT NOT NULL,
|
||||
category TEXT NOT NULL,
|
||||
summary_text TEXT,
|
||||
generated_time DATETIME NOT NULL,
|
||||
PRIMARY KEY (date, category)
|
||||
)
|
||||
""")
|
||||
self.conn.commit()
|
||||
logger.debug("分类摘要缓存表初始化完成")
|
||||
except Exception as e:
|
||||
logger.warning("分类摘要缓存表初始化失败: %s", e)
|
||||
|
||||
def _migrate_add_webzine_text(self):
|
||||
if not self.conn:
|
||||
return
|
||||
try:
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute("PRAGMA table_info(article_cache)")
|
||||
columns = [col[1] for col in cursor.fetchall()]
|
||||
if 'webzine_text' not in columns:
|
||||
cursor.execute("ALTER TABLE article_cache ADD COLUMN webzine_text TEXT")
|
||||
self.conn.commit()
|
||||
logger.debug("缓存表已添加 webzine_text 列")
|
||||
except Exception as e:
|
||||
logger.debug("添加 webzine_text 列跳过: %s", e)
|
||||
|
||||
def get_article(self, article_id, expire_hours=24):
|
||||
"""
|
||||
查询文章缓存
|
||||
:param article_id: 文章ID
|
||||
:param expire_hours: 缓存过期时间，默认24小时，和时间窗口一致
|
||||
:return: 缓存的文章数据，不存在或过期返回None
|
||||
"""
|
||||
if not self.conn:
|
||||
return None
|
||||
try:
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute("""
|
||||
SELECT id, source, title, link, translated_title, content, translated_content,
|
||||
ai_score, ai_summary, ai_category, ai_scores_json, published_time,
|
||||
processed_time, status, error_msg, webzine_text
|
||||
FROM article_cache
|
||||
WHERE id = ?
|
||||
AND processed_time >= ?
|
||||
AND status = 1
|
||||
""", (
|
||||
article_id,
|
||||
(datetime.now() - timedelta(hours=expire_hours)).strftime("%Y-%m-%d %H:%M:%S")
|
||||
))
|
||||
row = cursor.fetchone()
|
||||
if not row:
|
||||
return None
|
||||
# 转换为和现有article结构一致的字典
|
||||
article = {
|
||||
"id": row[0],
|
||||
"source": row[1],
|
||||
"title": row[2],
|
||||
"link": row[3],
|
||||
"translated_title": row[4],
|
||||
"content": row[5],
|
||||
"translated_content": row[6],
|
||||
"final_score": row[7],
|
||||
"summary": row[8],
|
||||
"category": row[9],
|
||||
"scores": json.loads(row[10]) if row[10] else {},
|
||||
"published": datetime.strptime(row[11], "%Y-%m-%d %H:%M:%S") if row[11] else None,
|
||||
"processed_time": datetime.strptime(row[12], "%Y-%m-%d %H:%M:%S"),
|
||||
"webzine_text": row[15] or "",
|
||||
"from_cache": True
|
||||
}
|
||||
return article
|
||||
except Exception as e:
|
||||
logger.warning(f"查询缓存失败: {e}")
|
||||
return None
|
||||
|
||||
def save_article(self, article, status=1, error_msg=""):
|
||||
"""
|
||||
保存文章处理结果到缓存
|
||||
:param article: 文章字典
|
||||
:param status: 处理状态，1-成功 2-失败
|
||||
:param error_msg: 失败时的错误信息
|
||||
"""
|
||||
if not self.conn:
|
||||
return
|
||||
try:
|
||||
cursor = self.conn.cursor()
|
||||
# 处理JSON字段
|
||||
ai_scores_json = json.dumps(article.get("scores", {}), ensure_ascii=False)
|
||||
# 发布时间转字符串
|
||||
published_str = article["published"].strftime("%Y-%m-%d %H:%M:%S") if article.get("published") else None
|
||||
processed_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
cursor.execute("""
|
||||
REPLACE INTO article_cache (
|
||||
id, source, title, link, translated_title, content, translated_content,
|
||||
ai_score, ai_summary, ai_category, ai_scores_json, published_time,
|
||||
processed_time, status, error_msg, webzine_text
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (
|
||||
article["id"],
|
||||
article["source"],
|
||||
article["title"],
|
||||
article["link"],
|
||||
article.get("translated_title", ""),
|
||||
article.get("content", ""),
|
||||
article.get("translated_content", ""),
|
||||
article.get("final_score", 0),
|
||||
article.get("summary", ""),
|
||||
article.get("category", ""),
|
||||
ai_scores_json,
|
||||
published_str,
|
||||
processed_str,
|
||||
status,
|
||||
error_msg,
|
||||
article.get("webzine_text", "")
|
||||
))
|
||||
self.conn.commit()
|
||||
logger.debug(f"文章 {article['id'][:20]}... 缓存保存成功")
|
||||
except Exception as e:
|
||||
logger.warning(f"保存缓存失败: {e}")
|
||||
# 写入失败不影响主流程，忽略错误
|
||||
|
||||
def save_article_webzine(self, article_id, webzine_text):
|
||||
|
||||
if not self.conn or not webzine_text:
|
||||
return
|
||||
try:
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute("""
|
||||
UPDATE article_cache SET webzine_text = ?
|
||||
WHERE id = ? AND status = 1
|
||||
""", (webzine_text, article_id))
|
||||
self.conn.commit()
|
||||
logger.debug(f"文章 {article_id[:20]}... 网摘缓存更新成功")
|
||||
except Exception as e:
|
||||
logger.warning(f"保存网摘缓存失败: {e}")
|
||||
|
||||
def get_category_summary(self, date_str, category, expire_hours=24):
|
||||
if not self.conn:
|
||||
return None
|
||||
try:
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute("""
|
||||
SELECT summary_text FROM category_summary_cache
|
||||
WHERE date = ? AND category = ?
|
||||
AND generated_time >= ?
|
||||
""", (date_str, category,
|
||||
(datetime.now() - timedelta(hours=expire_hours)).strftime("%Y-%m-%d %H:%M:%S")))
|
||||
row = cursor.fetchone()
|
||||
return row[0] if row else None
|
||||
except Exception as e:
|
||||
logger.warning("查询分类摘要缓存失败: %s", e)
|
||||
return None
|
||||
|
||||
def save_category_summary(self, date_str, category, summary_text):
|
||||
if not self.conn or not summary_text:
|
||||
return
|
||||
try:
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute("""
|
||||
REPLACE INTO category_summary_cache (date, category, summary_text, generated_time)
|
||||
VALUES (?, ?, ?, ?)
|
||||
""", (date_str, category, summary_text,
|
||||
datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
|
||||
self.conn.commit()
|
||||
logger.debug("分类摘要 [%s/%s] 缓存保存成功", date_str, category)
|
||||
except Exception as e:
|
||||
logger.warning("保存分类摘要缓存失败: %s", e)
|
||||
|
||||
def is_processed(self, article_id, expire_hours=24):
|
||||
"""判断文章是否已处理且未过期"""
|
||||
if not self.conn:
|
||||
return False
|
||||
try:
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute("""
|
||||
SELECT 1 FROM article_cache
|
||||
WHERE id = ?
|
||||
AND processed_time >= ?
|
||||
AND status = 1
|
||||
LIMIT 1
|
||||
""", (
|
||||
article_id,
|
||||
(datetime.now() - timedelta(hours=expire_hours)).strftime("%Y-%m-%d %H:%M:%S")
|
||||
))
|
||||
return cursor.fetchone() is not None
|
||||
except Exception as e:
|
||||
logger.warning(f"查询处理状态失败: {e}")
|
||||
return False
|
||||
|
||||
def clear_expired(self, keep_days=7):
|
||||
"""清理过期缓存，默认保留7天数据"""
|
||||
if not self.conn:
|
||||
return
|
||||
try:
|
||||
cursor = self.conn.cursor()
|
||||
# 清理文章缓存
|
||||
cursor.execute("""
|
||||
DELETE FROM article_cache
|
||||
WHERE processed_time < ?
|
||||
""", ((datetime.now() - timedelta(days=keep_days)).strftime("%Y-%m-%d %H:%M:%S"),))
|
||||
deleted = cursor.rowcount
|
||||
# 清理分类摘要缓存
|
||||
cursor.execute("""
|
||||
DELETE FROM category_summary_cache
|
||||
WHERE generated_time < ?
|
||||
""", ((datetime.now() - timedelta(days=keep_days)).strftime("%Y-%m-%d %H:%M:%S"),))
|
||||
deleted += cursor.rowcount
|
||||
self.conn.commit()
|
||||
if deleted > 0:
|
||||
logger.info(f"清理了 {deleted} 条过期缓存数据")
|
||||
except Exception as e:
|
||||
logger.warning(f"清理过期缓存失败: {e}")
|
||||
|
||||
def close(self):
|
||||
"""关闭数据库连接"""
|
||||
if self.conn:
|
||||
try:
|
||||
self.conn.close()
|
||||
logger.debug("缓存数据库连接已关闭")
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
# 全局缓存实例
|
||||
_cache = None
|
||||
|
||||
def get_cache():
|
||||
"""获取全局缓存单例"""
|
||||
global _cache
|
||||
if _cache is None:
|
||||
_cache = ArticleCache()
|
||||
return _cache
|
||||
@@ -0,0 +1,198 @@
|
||||
"""
|
||||
配置模块：系统常量、RSS源、关键词库、评分维度、分类定义、AI配置加载
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
# =============================================================================
|
||||
# 军事科技 RSS 订阅源
|
||||
# 优先读取 config/rss_feeds.txt 文件，文件不存在则使用默认12个源
|
||||
# =============================================================================
|
||||
def load_rss_feeds() -> list:
|
||||
"""从配置文件读取RSS源，格式每行：名称|URL，#开头为注释"""
|
||||
default_feeds = [
|
||||
{"name": "Defense One", "url": "https://www.defenseone.com/rss/all/"},
|
||||
{"name": "NEWUAS", "url": f"{RSS_PROXY_BASE}/feed/MP_WXS_3246130840.rss"},
|
||||
{"name": "Seapower", "url": "https://seapowermagazine.org/feed/"},
|
||||
{"name": "The War Zone", "url": "https://www.twz.com/feed"},
|
||||
{"name": "国防科技要闻", "url": f"{RSS_PROXY_BASE}/feed/MP_WXS_3921202880.rss"},
|
||||
{"name": "战略前沿技术", "url": f"{RSS_PROXY_BASE}/feed/MP_WXS_3271903347.rss"},
|
||||
{"name": "无人机邦", "url": f"{RSS_PROXY_BASE}/feed/MP_WXS_3957361285.rss"},
|
||||
{"name": "浮空飞行器", "url": f"{RSS_PROXY_BASE}/feed/MP_WXS_3291643707.rss"},
|
||||
{"name": "海鹰资讯", "url": f"{RSS_PROXY_BASE}/feed/MP_WXS_3091381580.rss"},
|
||||
{"name": "渊亭防务", "url": f"{RSS_PROXY_BASE}/feed/MP_WXS_3865628826.rss"},
|
||||
{"name": "电波之矛", "url": f"{RSS_PROXY_BASE}/feed/MP_WXS_3004998095.rss"},
|
||||
{"name": "龙牙的一座山", "url": f"{RSS_PROXY_BASE}/feed/MP_WXS_3517933009.rss"},
|
||||
]
|
||||
|
||||
# RSS配置文件路径：项目根目录/config/rss_feeds.txt
|
||||
config_file = Path(__file__).resolve().parent.parent.parent / "config" / "rss_feeds.txt"
|
||||
|
||||
if not config_file.exists():
|
||||
return default_feeds
|
||||
|
||||
feeds = []
|
||||
with open(config_file, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
# 跳过空行和注释行
|
||||
if not line or line.startswith('#'):
|
||||
continue
|
||||
# 按|分割，确保格式正确
|
||||
parts = line.split('|', 1)
|
||||
if len(parts) == 2:
|
||||
name = parts[0].strip()
|
||||
url = parts[1].strip()
|
||||
if name and url:
|
||||
feeds.append({"name": name, "url": url})
|
||||
|
||||
# 如果读取到的源为空，返回默认源
|
||||
return feeds if feeds else default_feeds
|
||||
|
||||
# =============================================================================
|
||||
# RSS 代理基础 URL（read_env 尚未定义，直接读文件/file和ennv）
|
||||
# =============================================================================
|
||||
def _load_rss_proxy_base():
|
||||
default = "https://werss.yynnice.top"
|
||||
env_val = os.environ.get("RSS_PROXY_BASE")
|
||||
if env_val:
|
||||
return env_val
|
||||
env_file = Path(__file__).resolve().parent.parent.parent / "config" / ".env"
|
||||
if env_file.exists():
|
||||
with open(env_file, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line and not line.startswith('#') and line.startswith('RSS_PROXY_BASE='):
|
||||
_, _, val = line.partition('=')
|
||||
return val.strip().strip('"').strip("'")
|
||||
return default
|
||||
|
||||
RSS_PROXY_BASE = _load_rss_proxy_base()
|
||||
|
||||
RSS_FEEDS = load_rss_feeds()
|
||||
|
||||
WECHAT_SOURCES = {'NEWUAS', '国防科技要闻', '战略前沿技术', '无人机邦', '浮空飞行器', '海鹰资讯', '渊亭防务', '电波之矛', '龙牙的一座山'}
|
||||
|
||||
# =============================================================================
|
||||
# 抓取配置
|
||||
# =============================================================================
|
||||
FETCH_TIMEOUT = 30
|
||||
MAX_THREADS = 15 # 抓取并发数，100+源建议15-20
|
||||
AI_CONCURRENCY = 8 # AI调用并发数，根据API限流调整
|
||||
TIME_WINDOW_HOURS = 24
|
||||
|
||||
# =============================================================================
|
||||
# 关键词库（44个）
|
||||
# =============================================================================
|
||||
KEYWORDS = [
|
||||
"首飞", "曝光", "美军", "解放军", "新型", "研发", "部署", "军演", "导弹", "战机",
|
||||
"舰艇", "雷达", "芯片", "突破", "测试", "服役", "列装", "武器", "装备", "军事",
|
||||
"武器系统", "冲突", "封锁", "威慑", "无人机", "AI", "人工智能", "战争", "以色列",
|
||||
"伊朗", "俄罗斯", "乌克兰", "航母", "潜艇", "坦克", "防空", "反导", "激光", "电磁",
|
||||
"卫星", "航天", "蜂群", "拦截", "打击", "攻击", "作战", "战略", "战术", "军工"
|
||||
]
|
||||
|
||||
# =============================================================================
|
||||
# AI评分维度（4维度加权）
|
||||
# =============================================================================
|
||||
SCORE_DIMENSIONS = [
|
||||
{"name": "技术突破性", "weight": 0.35, "description": "是否有新技术、新武器、新架构曝光，1-10分"},
|
||||
{"name": "战略价值", "weight": 0.30, "description": "对地缘政治、军事格局的影响程度，1-10分"},
|
||||
{"name": "信息可信度", "weight": 0.20, "description": "来源权威性、数据可验证性，1-10分"},
|
||||
{"name": "时效性", "weight": 0.15, "description": "事件的新鲜程度，1-10分"},
|
||||
]
|
||||
|
||||
# =============================================================================
|
||||
# 分类定义
|
||||
# =============================================================================
|
||||
CATEGORIES = {
|
||||
"today_must_read": {"name": "🔥 今日必看", "desc": "最重磅、最紧急的头条要闻"},
|
||||
"equipment": {"name": "⚙️ 装备动态", "desc": "新型武器、技术突破、军备交付"},
|
||||
"conflict": {"name": "💥 地区冲突", "desc": "热点战事、边境对峙、反恐行动"},
|
||||
"strategy": {"name": "🎯 战略政策", "desc": "国防战略、军演部署、外交安全"},
|
||||
}
|
||||
|
||||
|
||||
def _read_env_file(file_path):
|
||||
"""从.env文件读取配置，返回 {key: value} 字典"""
|
||||
result = {}
|
||||
if file_path.exists():
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line and not line.startswith('#') and '=' in line:
|
||||
key, _, value = line.partition('=')
|
||||
value = value.strip().strip('"').strip("'")
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def load_ai_config():
|
||||
"""加载AI配置（环境变量 > 项目config/.env > 硬编码默认值）"""
|
||||
api_key = None
|
||||
base_url = "https://api.openai.com/v1"
|
||||
model = "gpt-4o-mini"
|
||||
webzine_model = "deepseek-v3.2"
|
||||
|
||||
local_env = Path(__file__).resolve().parent.parent.parent / "config" / ".env"
|
||||
values = _read_env_file(local_env)
|
||||
if values.get('OPENAI_API_KEY'):
|
||||
api_key = values['OPENAI_API_KEY']
|
||||
if values.get('OPENAI_API_BASE'):
|
||||
base_url = values['OPENAI_API_BASE']
|
||||
if values.get('OPENAI_MODEL'):
|
||||
model = values['OPENAI_MODEL']
|
||||
if values.get('WEBZINE_MODEL'):
|
||||
webzine_model = values['WEBZINE_MODEL']
|
||||
|
||||
if os.environ.get('OPENAI_API_KEY'):
|
||||
api_key = os.environ['OPENAI_API_KEY']
|
||||
if os.environ.get('OPENAI_API_BASE'):
|
||||
base_url = os.environ['OPENAI_API_BASE']
|
||||
if os.environ.get('OPENAI_MODEL'):
|
||||
model = os.environ['OPENAI_MODEL']
|
||||
if os.environ.get('WEBZINE_MODEL'):
|
||||
webzine_model = os.environ['WEBZINE_MODEL']
|
||||
|
||||
return api_key, base_url, model, webzine_model
|
||||
|
||||
|
||||
def load_push_config():
|
||||
"""加载推送配置（环境变量 > config.json > 默认值）"""
|
||||
config = {
|
||||
"enable_wechat_push": False,
|
||||
"enable_feishu_push": False,
|
||||
"wechat_key": "",
|
||||
"wechat_target": "",
|
||||
"feishu_webhook": "",
|
||||
}
|
||||
|
||||
config_file = Path(__file__).resolve().parent.parent.parent / "config" / "config.json"
|
||||
values = _read_env_file(Path(__file__).resolve().parent.parent.parent / "config" / ".env")
|
||||
|
||||
if config_file.exists():
|
||||
try:
|
||||
with open(config_file, 'r', encoding='utf-8') as f:
|
||||
json_config = json.load(f)
|
||||
config["enable_wechat_push"] = json_config.get("enable_wechat_push", False)
|
||||
config["enable_feishu_push"] = json_config.get("enable_feishu_push", False)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if values.get("WECHAT_ACCOUNT"):
|
||||
config["wechat_key"] = values["WECHAT_ACCOUNT"]
|
||||
if values.get("WECHAT_TARGET"):
|
||||
config["wechat_target"] = values["WECHAT_TARGET"]
|
||||
if values.get("FEISHU_WEBHOOK"):
|
||||
config["feishu_webhook"] = values["FEISHU_WEBHOOK"]
|
||||
|
||||
if os.environ.get("WECHAT_ACCOUNT"):
|
||||
config["wechat_key"] = os.environ["WECHAT_ACCOUNT"]
|
||||
if os.environ.get("WECHAT_TARGET"):
|
||||
config["wechat_target"] = os.environ["WECHAT_TARGET"]
|
||||
if os.environ.get("FEISHU_WEBHOOK"):
|
||||
config["feishu_webhook"] = os.environ["FEISHU_WEBHOOK"]
|
||||
|
||||
return config
|
||||
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""自带NotoSansCJK中文字体，完全不依赖系统"""
|
||||
import os
|
||||
import sys
|
||||
from PIL import ImageFont
|
||||
|
||||
# 字体文件路径（项目内置开源无版权NotoSansCJK精简版）
|
||||
FONT_DIR = os.path.join(os.path.dirname(__file__), "assets", "fonts")
|
||||
BOLD_FONT = os.path.join(FONT_DIR, "NotoSansCJK-Bold.ttc")
|
||||
REGULAR_FONT = os.path.join(FONT_DIR, "NotoSansCJK-Regular.ttc")
|
||||
|
||||
def get_chinese_font(size: int, bold: bool = False) -> ImageFont.FreeTypeFont:
|
||||
"""优先使用项目内置字体，完全兼容所有系统"""
|
||||
try:
|
||||
if bold and os.path.exists(BOLD_FONT):
|
||||
return ImageFont.truetype(BOLD_FONT, size)
|
||||
elif not bold and os.path.exists(REGULAR_FONT):
|
||||
return ImageFont.truetype(REGULAR_FONT, size)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 内置字体找不到，再尝试系统字体
|
||||
font_candidates = [
|
||||
# Windows
|
||||
"simhei.ttf" if bold else "simsun.ttc",
|
||||
"msyh.ttc" if bold else "msyh.ttf",
|
||||
# Linux
|
||||
"/usr/share/fonts/opentype/noto/NotoSansCJK-Bold.ttc" if bold else "/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc",
|
||||
"/usr/share/fonts/truetype/arphic/uming.ttc",
|
||||
# macOS
|
||||
"/System/Library/Fonts/PingFang.ttc" if bold else "/System/Library/Fonts/STHeiti Light.ttc",
|
||||
]
|
||||
|
||||
for font_name in font_candidates:
|
||||
try:
|
||||
return ImageFont.truetype(font_name, size)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 最后 fallback
|
||||
return ImageFont.load_default(size)
|
||||
|
||||
# 自动创建assets/fonts目录
|
||||
os.makedirs(FONT_DIR, exist_ok=True)
|
||||
@@ -0,0 +1,208 @@
|
||||
"""
|
||||
图片生成模块：创建军事主题网摘图片（单篇/合并长图）
|
||||
"""
|
||||
|
||||
import re
|
||||
from datetime import datetime
|
||||
|
||||
try:
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
except ImportError:
|
||||
Image, ImageDraw, ImageFont = None, None, None
|
||||
|
||||
from .utils import wrap_text
|
||||
from .logger import get_logger
|
||||
from .get_chinese_font import get_chinese_font
|
||||
|
||||
|
||||
def _parse_webzine_line(line):
|
||||
"""解析网摘单行文本，返回 (前缀类型, 前缀原文, 内容文本)"""
|
||||
if not line.strip():
|
||||
return ('empty', '', '')
|
||||
|
||||
for prefix in ['标题：', '原标题：', '发布日期：', '正文：', '价值点：']:
|
||||
if line.startswith(prefix):
|
||||
return (prefix.rstrip('：'), prefix, line[len(prefix):])
|
||||
|
||||
return ('normal', '', line)
|
||||
|
||||
|
||||
def _load_webzine_fonts():
|
||||
"""加载网摘图片所需的所有字体"""
|
||||
return {
|
||||
'banner': get_chinese_font(size=40, bold=True),
|
||||
'article_title': get_chinese_font(size=36, bold=True),
|
||||
'bold': get_chinese_font(size=32, bold=True),
|
||||
'content': get_chinese_font(size=32, bold=False)
|
||||
}
|
||||
|
||||
|
||||
def _build_render_commands(webzine_content, fonts, content_width, temp_draw, skip_empty=False):
|
||||
"""将网摘文本转换为渲染命令列表"""
|
||||
commands = []
|
||||
|
||||
for line in webzine_content.split('\n'):
|
||||
line_type, prefix, content = _parse_webzine_line(line)
|
||||
|
||||
if line_type == 'empty':
|
||||
if not skip_empty:
|
||||
commands.append(('empty',))
|
||||
continue
|
||||
|
||||
if line_type == '标题':
|
||||
full_title = prefix + content
|
||||
wrapped = wrap_text(full_title, fonts['article_title'], content_width, temp_draw)
|
||||
for wl in wrapped:
|
||||
commands.append(('article_title', wl))
|
||||
elif line_type in ('原标题', '发布日期', '正文', '价值点'):
|
||||
test_line = prefix + content
|
||||
test_bbox = temp_draw.textbbox((0, 0), test_line, font=fonts['content'])
|
||||
test_width = test_bbox[2] - test_bbox[0]
|
||||
if test_width <= content_width:
|
||||
commands.append(('prefix_sameline', prefix, content))
|
||||
else:
|
||||
commands.append(('prefix_only', prefix))
|
||||
wrapped = wrap_text(content, fonts['content'], content_width, temp_draw)
|
||||
for wl in wrapped:
|
||||
commands.append(('content_indented', wl))
|
||||
else:
|
||||
wrapped = wrap_text(content, fonts['content'], content_width, temp_draw)
|
||||
for wl in wrapped:
|
||||
commands.append(('normal', wl))
|
||||
|
||||
return commands
|
||||
|
||||
|
||||
def _build_combined_render_commands(webzine_texts, fonts, content_width, temp_draw):
|
||||
"""将多篇网摘文本合并转换为渲染命令列表（篇间以分隔线分隔）"""
|
||||
commands = []
|
||||
|
||||
for idx, wz in enumerate(webzine_texts, 1):
|
||||
if idx > 1:
|
||||
commands.append(('separator',))
|
||||
commands.extend(_build_render_commands(wz, fonts, content_width, temp_draw, skip_empty=True))
|
||||
|
||||
return commands
|
||||
|
||||
|
||||
def _execute_render_commands(draw, commands, fonts, padding, line_height, content_width):
|
||||
"""执行渲染命令列表，在指定画布上绘制内容，返回最终y坐标"""
|
||||
y = 135
|
||||
|
||||
for cmd in commands:
|
||||
cmd_type = cmd[0]
|
||||
|
||||
if cmd_type == 'separator':
|
||||
y += 30
|
||||
draw.line([(padding, y), (content_width + padding, y)], fill=(200, 200, 200), width=1)
|
||||
y += 30
|
||||
elif cmd_type == 'empty':
|
||||
y += line_height
|
||||
elif cmd_type == 'normal':
|
||||
draw.text((padding, y), cmd[1], fill=(0, 0, 0), font=fonts['content'])
|
||||
y += line_height
|
||||
elif cmd_type == 'article_title':
|
||||
draw.text((padding, y), cmd[1], fill=(0, 0, 0), font=fonts['article_title'])
|
||||
y += line_height
|
||||
elif cmd_type == 'prefix_sameline':
|
||||
label_bbox = draw.textbbox((0, 0), cmd[1], font=fonts['bold'])
|
||||
label_width = label_bbox[2] - label_bbox[0]
|
||||
draw.text((padding, y), cmd[1], fill=(0, 0, 0), font=fonts['bold'])
|
||||
draw.text((padding + label_width, y), cmd[2], fill=(0, 0, 0), font=fonts['content'])
|
||||
y += line_height
|
||||
elif cmd_type == 'prefix_only':
|
||||
draw.text((padding, y), cmd[1], fill=(0, 0, 0), font=fonts['bold'])
|
||||
y += line_height
|
||||
elif cmd_type == 'content_indented':
|
||||
draw.text((padding, y), cmd[1], fill=(0, 0, 0), font=fonts['content'])
|
||||
y += line_height
|
||||
|
||||
return y
|
||||
|
||||
|
||||
def create_webzine_image(webzine_content, output_path, title_text):
|
||||
"""将网摘文本内容渲染为PNG图片（800px宽，自适应高度）"""
|
||||
logger = get_logger()
|
||||
if Image is None:
|
||||
logger.warning("PIL 未安装，跳过图片生成")
|
||||
return None
|
||||
|
||||
try:
|
||||
width = 800
|
||||
line_height = 40
|
||||
padding = 50
|
||||
content_width = width - padding * 2
|
||||
|
||||
date_match = re.search(r'\d{4}年\d{2}月\d{2}日', webzine_content)
|
||||
if date_match:
|
||||
date_str = date_match.group(0).replace('年', '.').replace('月', '.').replace('日', '')
|
||||
else:
|
||||
date_str = datetime.now().strftime('%Y.%m.%d')
|
||||
banner_text = f'网摘 {date_str}'
|
||||
|
||||
fonts = _load_webzine_fonts()
|
||||
|
||||
temp_img = Image.new('RGB', (100, 100), color=(255, 255, 255))
|
||||
temp_draw = ImageDraw.Draw(temp_img)
|
||||
commands = _build_render_commands(webzine_content, fonts, content_width, temp_draw)
|
||||
|
||||
actual_lines_count = len([c for c in commands if c[0] != 'separator'])
|
||||
img_height = 140 + actual_lines_count * line_height + padding * 2
|
||||
img_height = int(img_height * 1.15)
|
||||
|
||||
img = Image.new('RGB', (width, img_height), color=(255, 255, 255))
|
||||
draw = ImageDraw.Draw(img)
|
||||
|
||||
draw.rectangle([(0, 0), (width, 90)], fill=(26, 72, 144))
|
||||
draw.text((padding, 25), banner_text, fill=(255, 255, 255), font=fonts['banner'])
|
||||
draw.line([(padding, 100), (width - padding, 100)], fill=(180, 180, 180), width=2)
|
||||
|
||||
_execute_render_commands(draw, commands, fonts, padding, line_height, content_width)
|
||||
|
||||
img.save(output_path, 'PNG', quality=95)
|
||||
logger.info("网摘图片已保存: %s (%d条命令, %dpx)", output_path, len(commands), img_height)
|
||||
return output_path
|
||||
|
||||
except Exception as e:
|
||||
logger.warning("图片生成失败: %s", e)
|
||||
return None
|
||||
|
||||
|
||||
def create_combined_webzine_image(webzine_texts, output_path, now):
|
||||
"""将已生成的网摘文本合并渲染为一张3800px长图"""
|
||||
logger = get_logger()
|
||||
if Image is None:
|
||||
logger.warning("PIL 未安装，跳过图片生成")
|
||||
return None
|
||||
|
||||
try:
|
||||
width = 800
|
||||
padding = 50
|
||||
content_width = width - padding * 2
|
||||
line_height = 45
|
||||
|
||||
fonts = _load_webzine_fonts()
|
||||
|
||||
temp_img = Image.new('RGB', (100, 100), color=(255, 255, 255))
|
||||
temp_draw = ImageDraw.Draw(temp_img)
|
||||
commands = _build_combined_render_commands(webzine_texts, fonts, content_width, temp_draw)
|
||||
|
||||
fixed_height = 3800
|
||||
|
||||
img = Image.new('RGB', (width, fixed_height), color=(255, 255, 255))
|
||||
draw = ImageDraw.Draw(img)
|
||||
|
||||
draw.rectangle([(0, 0), (width, 90)], fill=(26, 72, 144))
|
||||
date_str = now.strftime('%Y.%m.%d')
|
||||
draw.text((padding, 25), f'今日必看TOP3网摘 {date_str}', fill=(255, 255, 255), font=fonts['banner'])
|
||||
draw.line([(padding, 100), (width - padding, 100)], fill=(180, 180, 180), width=2)
|
||||
|
||||
_execute_render_commands(draw, commands, fonts, padding, line_height, content_width)
|
||||
|
||||
img.save(output_path, 'PNG', quality=95)
|
||||
logger.info("合并长图已保存: %s (%d条指令, %dpx)", output_path, len(commands), fixed_height)
|
||||
return output_path
|
||||
|
||||
except Exception as e:
|
||||
logger.warning("合并长图生成失败: %s", e)
|
||||
return None
|
||||
@@ -0,0 +1,24 @@
|
||||
"""
|
||||
关键词筛选模块：基于44个军事科技关键词对文章进行命中筛选与排序
|
||||
"""
|
||||
|
||||
from .config import KEYWORDS
|
||||
|
||||
|
||||
def keyword_filter(articles):
|
||||
"""
|
||||
关键词筛选：
|
||||
1. 对每篇文章计算标题+正文前300字中命中的关键词数量
|
||||
2. 按命中数降序排序
|
||||
3. 只保留命中数 > 0 的文章
|
||||
4. 最多返回前15篇
|
||||
"""
|
||||
for a in articles:
|
||||
content = a.get('translated_content') or a.get('content') or ''
|
||||
text = (a['translated_title'] + ' ' + content[:300]).lower()
|
||||
hit_count = sum(1 for k in KEYWORDS if k.lower() in text)
|
||||
a['keyword_hits'] = hit_count
|
||||
|
||||
articles.sort(key=lambda x: x['keyword_hits'], reverse=True)
|
||||
filtered = [a for a in articles if a['keyword_hits'] > 0]
|
||||
return filtered[:15]
|
||||
@@ -0,0 +1,29 @@
|
||||
"""
|
||||
标准日志模块：统一管理 info / warning / error 输出
|
||||
"""
|
||||
|
||||
import logging
|
||||
import sys
|
||||
|
||||
_logger = None
|
||||
|
||||
|
||||
def get_logger(name="military_digest"):
|
||||
"""获取或创建全局 logger 实例"""
|
||||
global _logger
|
||||
if _logger is not None:
|
||||
return _logger
|
||||
|
||||
_logger = logging.getLogger(name)
|
||||
_logger.setLevel(logging.INFO)
|
||||
|
||||
fmt = logging.Formatter(
|
||||
"%(asctime)s [%(levelname)s] %(message)s",
|
||||
datefmt="%H:%M:%S",
|
||||
)
|
||||
|
||||
handler = logging.StreamHandler(sys.stdout)
|
||||
handler.setFormatter(fmt)
|
||||
_logger.addHandler(handler)
|
||||
|
||||
return _logger
|
||||
@@ -0,0 +1,162 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
运行时监控模块
|
||||
统计各环节耗时、API调用量/token消耗、成功率，运行结束输出统计报表
|
||||
"""
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from contextlib import contextmanager
|
||||
from .logger import get_logger
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
STAGE_ORDER = [
|
||||
"rss_fetch", "time_filter", "deduplicate", "translate_titles",
|
||||
"keyword_filter", "translate_contents", "ai_analyze",
|
||||
"webzine_generate", "category_overview", "report_generate", "image_generate"
|
||||
]
|
||||
|
||||
STAGE_LABELS = {
|
||||
"rss_fetch": "RSS抓取",
|
||||
"time_filter": "时间过滤",
|
||||
"deduplicate": "去重",
|
||||
"translate_titles": "标题翻译",
|
||||
"keyword_filter": "关键词筛选",
|
||||
"translate_contents": "正文翻译",
|
||||
"ai_analyze": "AI评分分类",
|
||||
"webzine_generate": "网摘生成",
|
||||
"category_overview": "分类介绍",
|
||||
"report_generate": "报告生成",
|
||||
"image_generate": "图片生成",
|
||||
}
|
||||
|
||||
|
||||
class RunMonitor:
|
||||
def __init__(self):
|
||||
self.start_time = time.time()
|
||||
self.stage_times = {}
|
||||
self.stage_active = {}
|
||||
self.api_calls = []
|
||||
self.source_success = defaultdict(int)
|
||||
self.source_failure = defaultdict(int)
|
||||
self.articles_total = 0
|
||||
self.articles_cached = 0
|
||||
self.articles_processed = 0
|
||||
|
||||
@contextmanager
|
||||
def stage(self, name):
|
||||
"""上下文管理器，自动记录阶段耗时"""
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
elapsed = time.perf_counter() - start
|
||||
self.stage_times[name] = elapsed
|
||||
|
||||
def record_api_call(self, model, purpose, max_tokens, success, retries=0):
|
||||
"""记录一次API调用"""
|
||||
self.api_calls.append({
|
||||
"model": model,
|
||||
"purpose": purpose,
|
||||
"max_tokens": max_tokens,
|
||||
"success": success,
|
||||
"retries": retries,
|
||||
})
|
||||
|
||||
def record_article_stats(self, total, cached, processed):
|
||||
self.articles_total = total
|
||||
self.articles_cached = cached
|
||||
self.articles_processed = processed
|
||||
|
||||
def record_source_result(self, source, success):
|
||||
if success:
|
||||
self.source_success[source] += 1
|
||||
else:
|
||||
self.source_failure[source] += 1
|
||||
|
||||
def report(self):
|
||||
total_elapsed = time.time() - self.start_time
|
||||
|
||||
lines = []
|
||||
lines.append("")
|
||||
lines.append("=" * 64)
|
||||
lines.append(" 📊 运行监控报告")
|
||||
lines.append("=" * 64)
|
||||
|
||||
lines.append("")
|
||||
lines.append("── ⏱ 阶段耗时 ──")
|
||||
stage_total = 0.0
|
||||
for key in STAGE_ORDER:
|
||||
if key in self.stage_times:
|
||||
t = self.stage_times[key]
|
||||
stage_total += t
|
||||
label = STAGE_LABELS.get(key, key)
|
||||
lines.append(f" {label: <10s} {t:6.1f}s")
|
||||
lines.append(f" {'─' * 20}")
|
||||
lines.append(f" {'合计': <10s} {stage_total:6.1f}s / 总运行 {total_elapsed:.0f}s")
|
||||
|
||||
lines.append("")
|
||||
lines.append("── 📡 文章统计 ──")
|
||||
lines.append(f" 总计: {self.articles_total} 缓存命中: {self.articles_cached} 本次处理: {self.articles_processed}")
|
||||
if self.articles_total > 0:
|
||||
cache_rate = self.articles_cached / self.articles_total * 100
|
||||
lines.append(f" 缓存命中率: {cache_rate:.0f}%")
|
||||
|
||||
lines.append("")
|
||||
lines.append("── 🤖 API调用统计 ──")
|
||||
if not self.api_calls:
|
||||
lines.append(" 无API调用（全部命中缓存）")
|
||||
else:
|
||||
by_purpose = defaultdict(lambda: {"count": 0, "tokens": 0, "retries": 0, "failed": 0})
|
||||
by_model = defaultdict(lambda: {"count": 0, "tokens": 0})
|
||||
for call in self.api_calls:
|
||||
p = call["purpose"]
|
||||
by_purpose[p]["count"] += 1
|
||||
by_purpose[p]["tokens"] += call["max_tokens"]
|
||||
by_purpose[p]["retries"] += call["retries"]
|
||||
if not call["success"]:
|
||||
by_purpose[p]["failed"] += 1
|
||||
|
||||
m = call["model"]
|
||||
by_model[m]["count"] += 1
|
||||
by_model[m]["tokens"] += call["max_tokens"]
|
||||
|
||||
lines.append(" [按用途]")
|
||||
for purpose, stats in by_purpose.items():
|
||||
fail_info = f" 失败{stats['failed']}" if stats['failed'] > 0 else ""
|
||||
lines.append(f" {purpose}: {stats['count']}次 / tokens≈{stats['tokens']} / 重试{stats['retries']}{fail_info}")
|
||||
|
||||
lines.append(" [按模型]")
|
||||
for model, stats in by_model.items():
|
||||
lines.append(f" {model}: {stats['count']}次 / tokens≈{stats['tokens']}")
|
||||
|
||||
lines.append("")
|
||||
lines.append("── 📰 源抓取统计 ──")
|
||||
total_sources = len(self.source_success) + len(self.source_failure)
|
||||
failed_names = [src for src in self.source_failure if self.source_failure[src] > 0]
|
||||
success_count = total_sources - len(failed_names)
|
||||
lines.append(f" 源总数: {total_sources} 成功: {success_count} 失败: {len(failed_names)}")
|
||||
if failed_names:
|
||||
lines.append(f" 失败源: {', '.join(failed_names)}")
|
||||
|
||||
lines.append("")
|
||||
lines.append("=" * 64)
|
||||
|
||||
report_text = "\n".join(lines)
|
||||
logger.info(report_text)
|
||||
return report_text
|
||||
|
||||
|
||||
_monitor = None
|
||||
|
||||
|
||||
def get_monitor():
|
||||
global _monitor
|
||||
if _monitor is None:
|
||||
_monitor = RunMonitor()
|
||||
return _monitor
|
||||
|
||||
|
||||
def reset_monitor():
|
||||
global _monitor
|
||||
_monitor = RunMonitor()
|
||||
@@ -0,0 +1,154 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
推送模块：企业微信 Webhook + 飞书 Webhook
|
||||
支持推送文本消息、图片消息、文件消息
|
||||
"""
|
||||
import json
|
||||
import base64
|
||||
import requests
|
||||
from pathlib import Path
|
||||
from .logger import get_logger
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
WECHAT_WEBHOOK_BASE = "https://qyapi.weixin.qq.com/cgi-bin/webhook"
|
||||
|
||||
|
||||
def _send_wechat_message(payload, webhook_key, target):
|
||||
url = f"{WECHAT_WEBHOOK_BASE}/send?key={webhook_key}"
|
||||
if target:
|
||||
payload["chatid"] = target
|
||||
try:
|
||||
resp = requests.post(url, json=payload, timeout=(5, 15))
|
||||
resp.raise_for_status()
|
||||
result = resp.json()
|
||||
if result.get("errcode") == 0:
|
||||
logger.info("企业微信推送成功")
|
||||
return True
|
||||
else:
|
||||
logger.warning("企业微信推送失败: errcode=%s, errmsg=%s",
|
||||
result.get("errcode"), result.get("errmsg"))
|
||||
return False
|
||||
except requests.exceptions.ConnectionError:
|
||||
logger.warning("企业微信推送失败：无法连接到服务器")
|
||||
return False
|
||||
except requests.exceptions.Timeout:
|
||||
logger.warning("企业微信推送失败：请求超时")
|
||||
return False
|
||||
except requests.exceptions.HTTPError as e:
|
||||
logger.warning("企业微信推送失败：HTTP错误 %s", e)
|
||||
return False
|
||||
except (ValueError, requests.exceptions.JSONDecodeError):
|
||||
logger.warning("企业微信推送失败：服务器返回非JSON响应")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.warning("企业微信推送异常: %s", e)
|
||||
return False
|
||||
|
||||
|
||||
def push_wechat_text(content, webhook_key, target):
|
||||
payload = {
|
||||
"msgtype": "text",
|
||||
"text": {"content": content}
|
||||
}
|
||||
return _send_wechat_message(payload, webhook_key, target)
|
||||
|
||||
|
||||
def push_wechat_image(image_path, webhook_key, target):
|
||||
if not Path(image_path).exists():
|
||||
logger.warning("图片文件不存在，跳过推送: %s", image_path)
|
||||
return False
|
||||
with open(image_path, "rb") as f:
|
||||
img_b64 = base64.b64encode(f.read()).decode("utf-8")
|
||||
img_md5 = __import__("hashlib").md5(
|
||||
base64.b64decode(img_b64)
|
||||
).hexdigest()
|
||||
payload = {
|
||||
"msgtype": "image",
|
||||
"image": {"base64": img_b64, "md5": img_md5}
|
||||
}
|
||||
return _send_wechat_message(payload, webhook_key, target)
|
||||
|
||||
|
||||
def push_wechat_markdown(content, webhook_key, target):
|
||||
payload = {
|
||||
"msgtype": "markdown",
|
||||
"markdown": {"content": content}
|
||||
}
|
||||
return _send_wechat_message(payload, webhook_key, target)
|
||||
|
||||
|
||||
def push_feishu_text(content, webhook_url):
|
||||
if not webhook_url:
|
||||
logger.info("飞书 webhook 未配置，跳过推送")
|
||||
return False
|
||||
try:
|
||||
payload = {"msg_type": "text", "content": {"text": content}}
|
||||
resp = requests.post(webhook_url, json=payload, timeout=(5, 15))
|
||||
resp.raise_for_status()
|
||||
result = resp.json()
|
||||
if result.get("code") == 0:
|
||||
logger.info("飞书推送成功")
|
||||
return True
|
||||
else:
|
||||
logger.warning("飞书推送失败: code=%s, msg=%s",
|
||||
result.get("code"), result.get("msg"))
|
||||
return False
|
||||
except requests.exceptions.ConnectionError:
|
||||
logger.warning("飞书推送失败：无法连接到飞书服务器")
|
||||
return False
|
||||
except requests.exceptions.Timeout:
|
||||
logger.warning("飞书推送失败：请求超时")
|
||||
return False
|
||||
except requests.exceptions.HTTPError as e:
|
||||
logger.warning("飞书推送失败：HTTP错误 %s", e)
|
||||
return False
|
||||
except (ValueError, requests.exceptions.JSONDecodeError):
|
||||
logger.warning("飞书推送失败：服务器返回非JSON响应")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.warning("飞书推送异常: %s", e)
|
||||
return False
|
||||
|
||||
|
||||
def send_daily_push(now, date_str, total_count, top3_articles, output_dir, push_config):
|
||||
webhook_key = push_config.get("wechat_key", "")
|
||||
target = push_config.get("wechat_target", "")
|
||||
enable_wechat = push_config.get("enable_wechat_push", False)
|
||||
enable_feishu = push_config.get("enable_feishu_push", False)
|
||||
feishu_url = push_config.get("feishu_webhook", "")
|
||||
|
||||
if not enable_wechat and not enable_feishu:
|
||||
logger.info("未启用任何推送渠道，跳过")
|
||||
return
|
||||
|
||||
if not webhook_key and not feishu_url:
|
||||
logger.info("推送已启用但均未配置 webhook")
|
||||
return
|
||||
|
||||
date_label = now.strftime("%m月%d日")
|
||||
|
||||
# 文本摘要
|
||||
summary_lines = [
|
||||
f"🪖 军事科技每日摘报 — {date_label}",
|
||||
f"📊 今日共 {total_count} 篇精选文章",
|
||||
"",
|
||||
"🔥 今日必看 TOP3：",
|
||||
]
|
||||
for i, a in enumerate(top3_articles, 1):
|
||||
title = a.get("translated_title") or a.get("title", "")[:40]
|
||||
score = a.get("final_score", 0)
|
||||
summary_lines.append(f" {i}. [{score:.1f}] {title}")
|
||||
summary_lines.append("")
|
||||
summary_lines.append(f"📄 完整报告已生成，包含装备动态/地区冲突/战略政策三分类深度洞察。")
|
||||
|
||||
text_content = "\n".join(summary_lines)
|
||||
|
||||
if enable_wechat and webhook_key:
|
||||
push_wechat_text(text_content, webhook_key, target)
|
||||
|
||||
webzine_image = output_dir / f"military_webzine_{date_str}.png"
|
||||
push_wechat_image(str(webzine_image), webhook_key, target)
|
||||
|
||||
if enable_feishu and feishu_url:
|
||||
push_feishu_text(text_content, feishu_url)
|
||||
@@ -0,0 +1,99 @@
|
||||
"""
|
||||
RSS抓取模块：多线程并发抓取RSS源，解析并提取文章信息
|
||||
支持失败自动重试、并发控制、超时保护
|
||||
"""
|
||||
import time
|
||||
import requests
|
||||
import feedparser
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from .config import RSS_FEEDS, FETCH_TIMEOUT, MAX_THREADS
|
||||
from .utils import parse_pub_time, extract_content
|
||||
from .logger import get_logger
|
||||
|
||||
|
||||
def _fetch_single_feed(feed_info):
|
||||
"""抓取单个RSS源，返回该源的所有文章列表，内置2次重试机制"""
|
||||
logger = get_logger()
|
||||
name = feed_info['name']
|
||||
url = feed_info['url']
|
||||
max_retries = 2
|
||||
retry_delay = 1
|
||||
|
||||
for retry in range(max_retries + 1):
|
||||
try:
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
|
||||
}
|
||||
response = requests.get(url, headers=headers, timeout=FETCH_TIMEOUT)
|
||||
response.raise_for_status()
|
||||
feed = feedparser.parse(response.content)
|
||||
|
||||
if feed.bozo != 0:
|
||||
logger.warning("%s: 解析警告", name)
|
||||
|
||||
articles = []
|
||||
for entry in feed.entries:
|
||||
pub_time = parse_pub_time(entry)
|
||||
content = extract_content(entry)
|
||||
|
||||
article = {
|
||||
'id': entry.get('id', entry.get('link', '')),
|
||||
'title': entry.get('title', '无标题').strip(),
|
||||
'link': entry.get('link', ''),
|
||||
'content': content,
|
||||
'published': pub_time,
|
||||
'source': name,
|
||||
}
|
||||
articles.append(article)
|
||||
|
||||
logger.info("%s: 抓取成功，获取 %d 篇文章", name, len(articles))
|
||||
_record_source_success(name)
|
||||
return articles
|
||||
|
||||
except Exception as e:
|
||||
if retry < max_retries:
|
||||
logger.warning("%s: 抓取失败，第%d次重试: %s", name, retry + 1, e)
|
||||
time.sleep(retry_delay)
|
||||
else:
|
||||
logger.error("%s: 抓取失败，已重试%d次，放弃: %s", name, max_retries, e)
|
||||
_record_source_failure(name)
|
||||
return []
|
||||
|
||||
|
||||
def fetch_all_feeds():
|
||||
"""并行抓取所有RSS源，返回文章列表
|
||||
采用标准线程池实现，自动管理并发，避免线程泄漏
|
||||
"""
|
||||
logger = get_logger()
|
||||
logger.info("开始抓取 %d 个RSS源，并发数：%d", len(RSS_FEEDS), MAX_THREADS)
|
||||
|
||||
all_articles = []
|
||||
with ThreadPoolExecutor(max_workers=MAX_THREADS) as executor:
|
||||
future_to_feed = {executor.submit(_fetch_single_feed, feed): feed['name'] for feed in RSS_FEEDS}
|
||||
|
||||
for future in as_completed(future_to_feed):
|
||||
feed_name = future_to_feed[future]
|
||||
try:
|
||||
articles = future.result()
|
||||
all_articles.extend(articles)
|
||||
except Exception as e:
|
||||
logger.error("%s: 抓取任务异常: %s", feed_name, e)
|
||||
|
||||
logger.info("所有源抓取完成，共获取 %d 篇文章", len(all_articles))
|
||||
return all_articles
|
||||
|
||||
|
||||
def _record_source_success(name):
|
||||
try:
|
||||
from .monitor import get_monitor
|
||||
get_monitor().record_source_result(name, True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _record_source_failure(name):
|
||||
try:
|
||||
from .monitor import get_monitor
|
||||
get_monitor().record_source_result(name, False)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1,194 @@
|
||||
"""
|
||||
翻译处理模块：自动识别外文文章，通过AI并发生成中文翻译
|
||||
"""
|
||||
import re
|
||||
import json
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from .utils import is_chinese, call_ai
|
||||
from .config import AI_CONCURRENCY
|
||||
from .logger import get_logger
|
||||
|
||||
|
||||
def translate_title(article, api_key, base_url, model, index=0, total=0):
|
||||
"""仅翻译外文文章的标题，返回更新后的文章"""
|
||||
logger = get_logger()
|
||||
try:
|
||||
title = article['title']
|
||||
prompt = f"""请将以下军事相关的英文标题翻译成专业中文，仅返回翻译结果，不要其他内容：
|
||||
{title}"""
|
||||
|
||||
result = call_ai([
|
||||
{'role': 'system', 'content': '你是专业的军事科技翻译专家，翻译准确、专业、简洁，只返回翻译结果。'},
|
||||
{'role': 'user', 'content': prompt}
|
||||
], api_key, base_url, model, temperature=0.2, max_tokens=200, purpose="标题翻译")
|
||||
|
||||
article['translated_title'] = result.strip()
|
||||
if index > 0 and total > 0:
|
||||
logger.info("标题翻译完成 %d/%d: %s", index, total, article['translated_title'][:30])
|
||||
return article
|
||||
|
||||
except Exception as e:
|
||||
article['translated_title'] = article['title']
|
||||
if index > 0 and total > 0:
|
||||
logger.warning("标题翻译失败 %d/%d: %s", index, total, e)
|
||||
return article
|
||||
|
||||
|
||||
def translate_content(article, api_key, base_url, model, index=0, total=0):
|
||||
"""翻译外文文章的正文前500词，返回更新后的文章"""
|
||||
logger = get_logger()
|
||||
try:
|
||||
content = article['content'][:500] if article['content'] else ''
|
||||
prompt = f"""请将以下军事相关的英文正文翻译成专业中文，仅返回翻译结果：
|
||||
{content}"""
|
||||
|
||||
result = call_ai([
|
||||
{'role': 'system', 'content': '你是专业的军事科技翻译专家，翻译准确、专业、简洁，只返回翻译结果。'},
|
||||
{'role': 'user', 'content': prompt}
|
||||
], api_key, base_url, model, temperature=0.2, max_tokens=800, purpose="正文翻译")
|
||||
|
||||
article['translated_content'] = result.strip()[:300]
|
||||
if index > 0 and total > 0:
|
||||
logger.info("正文翻译完成 %d/%d: %s", index, total, article['translated_title'][:30])
|
||||
return article
|
||||
|
||||
except Exception as e:
|
||||
article['translated_content'] = article['content'][:300] if article['content'] else ''
|
||||
if index > 0 and total > 0:
|
||||
logger.warning("正文翻译失败 %d/%d: %s", index, total, e)
|
||||
return article
|
||||
|
||||
|
||||
def _translate_single_article(article, api_key, base_url, model, index=0, total=0):
|
||||
"""翻译单篇外文文章的标题和正文前500词，返回处理后的文章（兼容旧接口）"""
|
||||
article = translate_title(article, api_key, base_url, model, index, total)
|
||||
article = translate_content(article, api_key, base_url, model, index, total)
|
||||
return article
|
||||
|
||||
|
||||
def batch_translate_titles(articles, api_key, base_url, model, max_concurrent=None):
|
||||
"""批量翻译所有外文文章的标题，中文文章直接填充translated_title
|
||||
所有文章处理后都包含translated_title字段，无需后续判断
|
||||
"""
|
||||
logger = get_logger()
|
||||
processed = []
|
||||
chinese_articles = []
|
||||
foreign_articles = []
|
||||
|
||||
for a in articles:
|
||||
if is_chinese(a['title']):
|
||||
a['translated_title'] = a['title']
|
||||
chinese_articles.append(a)
|
||||
else:
|
||||
foreign_articles.append(a)
|
||||
|
||||
processed.extend(chinese_articles)
|
||||
logger.info("中文文章：%d 篇，待翻译标题的外文：%d 篇", len(chinese_articles), len(foreign_articles))
|
||||
|
||||
if not foreign_articles:
|
||||
return processed
|
||||
|
||||
if max_concurrent is None:
|
||||
max_concurrent = AI_CONCURRENCY
|
||||
|
||||
logger.info("开始并发翻译标题，并发数：%d", max_concurrent)
|
||||
translated_results = []
|
||||
|
||||
with ThreadPoolExecutor(max_workers=max_concurrent) as executor:
|
||||
future_to_index = {
|
||||
executor.submit(translate_title, article, api_key, base_url, model, i+1, len(foreign_articles)): i
|
||||
for i, article in enumerate(foreign_articles)
|
||||
}
|
||||
|
||||
for future in as_completed(future_to_index):
|
||||
translated_article = future.result()
|
||||
translated_results.append(translated_article)
|
||||
|
||||
processed.extend(translated_results)
|
||||
logger.info("标题翻译完成，共处理 %d 篇文章", len(processed))
|
||||
return processed
|
||||
|
||||
|
||||
def batch_translate_contents(articles, api_key, base_url, model, max_concurrent=None):
|
||||
"""批量翻译指定文章的正文，仅翻译需要的文章
|
||||
调用前需要确保文章已经翻译了标题，并且经过筛选确实需要翻译正文
|
||||
"""
|
||||
logger = get_logger()
|
||||
if not articles:
|
||||
return []
|
||||
|
||||
# 分离不需要翻译正文的中文和需要翻译的外文
|
||||
chinese_articles = []
|
||||
foreign_articles = []
|
||||
for a in articles:
|
||||
if is_chinese(a['title']):
|
||||
a['translated_content'] = a['content'][:300] if a['content'] else ''
|
||||
chinese_articles.append(a)
|
||||
else:
|
||||
foreign_articles.append(a)
|
||||
|
||||
logger.info("无需翻译正文的中文文章：%d 篇，待翻译正文的外文：%d 篇", len(chinese_articles), len(foreign_articles))
|
||||
|
||||
if not foreign_articles:
|
||||
return chinese_articles
|
||||
|
||||
if max_concurrent is None:
|
||||
max_concurrent = AI_CONCURRENCY
|
||||
|
||||
logger.info("开始并发翻译正文，并发数：%d", max_concurrent)
|
||||
translated_results = []
|
||||
|
||||
with ThreadPoolExecutor(max_workers=max_concurrent) as executor:
|
||||
future_to_index = {
|
||||
executor.submit(translate_content, article, api_key, base_url, model, i+1, len(foreign_articles)): i
|
||||
for i, article in enumerate(foreign_articles)
|
||||
}
|
||||
|
||||
for future in as_completed(future_to_index):
|
||||
translated_article = future.result()
|
||||
translated_results.append(translated_article)
|
||||
|
||||
all_articles = chinese_articles + translated_results
|
||||
logger.info("正文翻译完成，共处理 %d 篇文章", len(all_articles))
|
||||
return all_articles
|
||||
|
||||
|
||||
def batch_translate_articles(articles, api_key, base_url, model, max_concurrent=None):
|
||||
"""批量翻译外文文章：中英分离，外文并发翻译（兼容旧接口）
|
||||
默认并发数使用配置中的AI_CONCURRENCY
|
||||
"""
|
||||
logger = get_logger()
|
||||
chinese_articles = []
|
||||
foreign_articles = []
|
||||
|
||||
for a in articles:
|
||||
if is_chinese(a['title']) and is_chinese(a['content'][:100]):
|
||||
a['translated_title'] = a['title']
|
||||
a['translated_content'] = a['content'][:300] if a['content'] else ''
|
||||
chinese_articles.append(a)
|
||||
else:
|
||||
foreign_articles.append(a)
|
||||
|
||||
logger.info("中文文章：%d 篇，待翻译外文：%d 篇", len(chinese_articles), len(foreign_articles))
|
||||
|
||||
if not foreign_articles:
|
||||
return articles
|
||||
|
||||
if max_concurrent is None:
|
||||
max_concurrent = AI_CONCURRENCY
|
||||
|
||||
logger.info("开始并发翻译，并发数：%d", max_concurrent)
|
||||
translated_results = []
|
||||
|
||||
with ThreadPoolExecutor(max_workers=max_concurrent) as executor:
|
||||
future_to_index = {
|
||||
executor.submit(_translate_single_article, article, api_key, base_url, model, i+1, len(foreign_articles)): i
|
||||
for i, article in enumerate(foreign_articles)
|
||||
}
|
||||
|
||||
for future in as_completed(future_to_index):
|
||||
translated_article = future.result()
|
||||
translated_results.append(translated_article)
|
||||
|
||||
logger.info("所有翻译任务完成，共翻译 %d 篇外文", len(translated_results))
|
||||
return chinese_articles + translated_results
|
||||
@@ -0,0 +1,133 @@
|
||||
"""
|
||||
工具函数模块：语言检测、HTML清洗、内容提取、时间解析、AI API调用、文本换行
|
||||
"""
|
||||
|
||||
import re
|
||||
import requests
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from email.utils import parsedate_to_datetime
|
||||
|
||||
|
||||
def is_chinese(text):
|
||||
"""判断文本是否为中文（汉字占比 > 20%）"""
|
||||
if not text:
|
||||
return True
|
||||
chinese_chars = sum(1 for c in text if '\u4e00' <= c <= '\u9fff')
|
||||
return chinese_chars > len(text) * 0.2
|
||||
|
||||
|
||||
def clean_html(html_text):
|
||||
"""清理HTML标签，合并空白字符"""
|
||||
if not html_text:
|
||||
return ""
|
||||
text = re.sub(r'<[^>]+>', ' ', html_text)
|
||||
text = re.sub(r'\s+', ' ', text)
|
||||
return text.strip()
|
||||
|
||||
|
||||
def extract_content(entry):
|
||||
"""从RSS条目提取完整正文内容"""
|
||||
content = entry.get('summary', '') or entry.get('description', '')
|
||||
if not content:
|
||||
content_list = entry.get('content', [{}])
|
||||
if content_list and isinstance(content_list, list):
|
||||
content = content_list[0].get('value', '')
|
||||
return clean_html(content)
|
||||
|
||||
|
||||
def parse_pub_time(entry):
|
||||
"""解析发布时间，统一转换为北京时间（UTC+8）"""
|
||||
beijing_tz = timezone(timedelta(hours=8))
|
||||
|
||||
time_fields = ['published_parsed', 'updated_parsed', 'created_parsed']
|
||||
for field in time_fields:
|
||||
if entry.get(field):
|
||||
try:
|
||||
utc_time = datetime(*entry[field][:6], tzinfo=timezone.utc)
|
||||
return utc_time.astimezone(beijing_tz).replace(tzinfo=None)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
for field in ['published', 'updated', 'created']:
|
||||
if entry.get(field):
|
||||
try:
|
||||
dt = parsedate_to_datetime(entry[field])
|
||||
if dt.tzinfo is not None:
|
||||
return dt.astimezone(beijing_tz).replace(tzinfo=None)
|
||||
return dt
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
return datetime.now()
|
||||
|
||||
|
||||
def call_ai(messages, api_key, base_url, model, temperature=0.3, max_tokens=800, purpose=None):
|
||||
"""调用 OpenAI 兼容的 Chat Completion API"""
|
||||
if not api_key:
|
||||
raise ValueError('OPENAI_API_KEY 未配置')
|
||||
|
||||
headers = {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': f'Bearer {api_key}'
|
||||
}
|
||||
|
||||
data = {
|
||||
'model': model,
|
||||
'messages': messages,
|
||||
'temperature': temperature,
|
||||
'max_tokens': max_tokens,
|
||||
}
|
||||
|
||||
try:
|
||||
response = requests.post(
|
||||
f'{base_url}/chat/completions',
|
||||
headers=headers,
|
||||
json=data,
|
||||
timeout=120
|
||||
)
|
||||
response.raise_for_status()
|
||||
result = response.json()['choices'][0]['message']['content']
|
||||
_record_api_success(model, purpose, max_tokens)
|
||||
return result
|
||||
except Exception:
|
||||
_record_api_failure(model, purpose, max_tokens)
|
||||
raise
|
||||
|
||||
|
||||
def wrap_text(text, font, max_width, draw):
|
||||
"""将长文本按像素宽度自动换行，返回行列表"""
|
||||
lines = []
|
||||
current_line = ""
|
||||
|
||||
for char in text:
|
||||
test_line = current_line + char
|
||||
bbox = draw.textbbox((0, 0), test_line, font=font)
|
||||
line_width = bbox[2] - bbox[0]
|
||||
|
||||
if line_width <= max_width:
|
||||
current_line = test_line
|
||||
else:
|
||||
if current_line:
|
||||
lines.append(current_line)
|
||||
current_line = char
|
||||
|
||||
if current_line:
|
||||
lines.append(current_line)
|
||||
|
||||
return lines
|
||||
|
||||
|
||||
def _record_api_success(model, purpose, max_tokens):
|
||||
try:
|
||||
from .monitor import get_monitor
|
||||
get_monitor().record_api_call(model or "unknown", purpose or "unknown", max_tokens, True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _record_api_failure(model, purpose, max_tokens):
|
||||
try:
|
||||
from .monitor import get_monitor
|
||||
get_monitor().record_api_call(model or "unknown", purpose or "unknown", max_tokens, False)
|
||||
except Exception:
|
||||
pass
|
||||
Reference in New Issue
Block a user