feat: 军事科技每日资讯推送系统 - Docker部署 + 日志系统 + 数据目录重组
This commit is contained in:
poiuy
@@ -0,0 +1,194 @@
|
||||
"""
|
||||
翻译处理模块：自动识别外文文章，通过AI并发生成中文翻译
|
||||
"""
|
||||
import re
|
||||
import json
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from .utils import is_chinese, call_ai
|
||||
from .config import AI_CONCURRENCY
|
||||
from .logger import get_logger
|
||||
|
||||
|
||||
def translate_title(article, api_key, base_url, model, index=0, total=0):
|
||||
"""仅翻译外文文章的标题，返回更新后的文章"""
|
||||
logger = get_logger()
|
||||
try:
|
||||
title = article['title']
|
||||
prompt = f"""请将以下军事相关的英文标题翻译成专业中文，仅返回翻译结果，不要其他内容：
|
||||
{title}"""
|
||||
|
||||
result = call_ai([
|
||||
{'role': 'system', 'content': '你是专业的军事科技翻译专家，翻译准确、专业、简洁，只返回翻译结果。'},
|
||||
{'role': 'user', 'content': prompt}
|
||||
], api_key, base_url, model, temperature=0.2, max_tokens=200, purpose="标题翻译")
|
||||
|
||||
article['translated_title'] = result.strip()
|
||||
if index > 0 and total > 0:
|
||||
logger.info("标题翻译完成 %d/%d: %s", index, total, article['translated_title'][:30])
|
||||
return article
|
||||
|
||||
except Exception as e:
|
||||
article['translated_title'] = article['title']
|
||||
if index > 0 and total > 0:
|
||||
logger.warning("标题翻译失败 %d/%d: %s", index, total, e)
|
||||
return article
|
||||
|
||||
|
||||
def translate_content(article, api_key, base_url, model, index=0, total=0):
|
||||
"""翻译外文文章的正文前500词，返回更新后的文章"""
|
||||
logger = get_logger()
|
||||
try:
|
||||
content = article['content'][:500] if article['content'] else ''
|
||||
prompt = f"""请将以下军事相关的英文正文翻译成专业中文，仅返回翻译结果：
|
||||
{content}"""
|
||||
|
||||
result = call_ai([
|
||||
{'role': 'system', 'content': '你是专业的军事科技翻译专家，翻译准确、专业、简洁，只返回翻译结果。'},
|
||||
{'role': 'user', 'content': prompt}
|
||||
], api_key, base_url, model, temperature=0.2, max_tokens=800, purpose="正文翻译")
|
||||
|
||||
article['translated_content'] = result.strip()[:300]
|
||||
if index > 0 and total > 0:
|
||||
logger.info("正文翻译完成 %d/%d: %s", index, total, article['translated_title'][:30])
|
||||
return article
|
||||
|
||||
except Exception as e:
|
||||
article['translated_content'] = article['content'][:300] if article['content'] else ''
|
||||
if index > 0 and total > 0:
|
||||
logger.warning("正文翻译失败 %d/%d: %s", index, total, e)
|
||||
return article
|
||||
|
||||
|
||||
def _translate_single_article(article, api_key, base_url, model, index=0, total=0):
|
||||
"""翻译单篇外文文章的标题和正文前500词，返回处理后的文章（兼容旧接口）"""
|
||||
article = translate_title(article, api_key, base_url, model, index, total)
|
||||
article = translate_content(article, api_key, base_url, model, index, total)
|
||||
return article
|
||||
|
||||
|
||||
def batch_translate_titles(articles, api_key, base_url, model, max_concurrent=None):
|
||||
"""批量翻译所有外文文章的标题，中文文章直接填充translated_title
|
||||
所有文章处理后都包含translated_title字段，无需后续判断
|
||||
"""
|
||||
logger = get_logger()
|
||||
processed = []
|
||||
chinese_articles = []
|
||||
foreign_articles = []
|
||||
|
||||
for a in articles:
|
||||
if is_chinese(a['title']):
|
||||
a['translated_title'] = a['title']
|
||||
chinese_articles.append(a)
|
||||
else:
|
||||
foreign_articles.append(a)
|
||||
|
||||
processed.extend(chinese_articles)
|
||||
logger.info("中文文章：%d 篇，待翻译标题的外文：%d 篇", len(chinese_articles), len(foreign_articles))
|
||||
|
||||
if not foreign_articles:
|
||||
return processed
|
||||
|
||||
if max_concurrent is None:
|
||||
max_concurrent = AI_CONCURRENCY
|
||||
|
||||
logger.info("开始并发翻译标题，并发数：%d", max_concurrent)
|
||||
translated_results = []
|
||||
|
||||
with ThreadPoolExecutor(max_workers=max_concurrent) as executor:
|
||||
future_to_index = {
|
||||
executor.submit(translate_title, article, api_key, base_url, model, i+1, len(foreign_articles)): i
|
||||
for i, article in enumerate(foreign_articles)
|
||||
}
|
||||
|
||||
for future in as_completed(future_to_index):
|
||||
translated_article = future.result()
|
||||
translated_results.append(translated_article)
|
||||
|
||||
processed.extend(translated_results)
|
||||
logger.info("标题翻译完成，共处理 %d 篇文章", len(processed))
|
||||
return processed
|
||||
|
||||
|
||||
def batch_translate_contents(articles, api_key, base_url, model, max_concurrent=None):
|
||||
"""批量翻译指定文章的正文，仅翻译需要的文章
|
||||
调用前需要确保文章已经翻译了标题，并且经过筛选确实需要翻译正文
|
||||
"""
|
||||
logger = get_logger()
|
||||
if not articles:
|
||||
return []
|
||||
|
||||
# 分离不需要翻译正文的中文和需要翻译的外文
|
||||
chinese_articles = []
|
||||
foreign_articles = []
|
||||
for a in articles:
|
||||
if is_chinese(a['title']):
|
||||
a['translated_content'] = a['content'][:300] if a['content'] else ''
|
||||
chinese_articles.append(a)
|
||||
else:
|
||||
foreign_articles.append(a)
|
||||
|
||||
logger.info("无需翻译正文的中文文章：%d 篇，待翻译正文的外文：%d 篇", len(chinese_articles), len(foreign_articles))
|
||||
|
||||
if not foreign_articles:
|
||||
return chinese_articles
|
||||
|
||||
if max_concurrent is None:
|
||||
max_concurrent = AI_CONCURRENCY
|
||||
|
||||
logger.info("开始并发翻译正文，并发数：%d", max_concurrent)
|
||||
translated_results = []
|
||||
|
||||
with ThreadPoolExecutor(max_workers=max_concurrent) as executor:
|
||||
future_to_index = {
|
||||
executor.submit(translate_content, article, api_key, base_url, model, i+1, len(foreign_articles)): i
|
||||
for i, article in enumerate(foreign_articles)
|
||||
}
|
||||
|
||||
for future in as_completed(future_to_index):
|
||||
translated_article = future.result()
|
||||
translated_results.append(translated_article)
|
||||
|
||||
all_articles = chinese_articles + translated_results
|
||||
logger.info("正文翻译完成，共处理 %d 篇文章", len(all_articles))
|
||||
return all_articles
|
||||
|
||||
|
||||
def batch_translate_articles(articles, api_key, base_url, model, max_concurrent=None):
|
||||
"""批量翻译外文文章：中英分离，外文并发翻译（兼容旧接口）
|
||||
默认并发数使用配置中的AI_CONCURRENCY
|
||||
"""
|
||||
logger = get_logger()
|
||||
chinese_articles = []
|
||||
foreign_articles = []
|
||||
|
||||
for a in articles:
|
||||
if is_chinese(a['title']) and is_chinese(a['content'][:100]):
|
||||
a['translated_title'] = a['title']
|
||||
a['translated_content'] = a['content'][:300] if a['content'] else ''
|
||||
chinese_articles.append(a)
|
||||
else:
|
||||
foreign_articles.append(a)
|
||||
|
||||
logger.info("中文文章：%d 篇，待翻译外文：%d 篇", len(chinese_articles), len(foreign_articles))
|
||||
|
||||
if not foreign_articles:
|
||||
return articles
|
||||
|
||||
if max_concurrent is None:
|
||||
max_concurrent = AI_CONCURRENCY
|
||||
|
||||
logger.info("开始并发翻译，并发数：%d", max_concurrent)
|
||||
translated_results = []
|
||||
|
||||
with ThreadPoolExecutor(max_workers=max_concurrent) as executor:
|
||||
future_to_index = {
|
||||
executor.submit(_translate_single_article, article, api_key, base_url, model, i+1, len(foreign_articles)): i
|
||||
for i, article in enumerate(foreign_articles)
|
||||
}
|
||||
|
||||
for future in as_completed(future_to_index):
|
||||
translated_article = future.result()
|
||||
translated_results.append(translated_article)
|
||||
|
||||
logger.info("所有翻译任务完成，共翻译 %d 篇外文", len(translated_results))
|
||||
return chinese_articles + translated_results
|
||||
Reference in New Issue
Block a user