feat: 军事科技每日资讯推送系统 - Docker部署 + 日志系统 + 数据目录重组
This commit is contained in:
poiuy
@@ -0,0 +1,310 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
SQLite增量缓存模块
|
||||
实现文章处理结果的本地存储和复用，避免重复翻译和AI分析，支持断点续跑
|
||||
"""
|
||||
import sqlite3
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from .logger import get_logger
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
# 数据库文件路径，放在项目根目录下的data目录
|
||||
DB_PATH = Path(__file__).resolve().parent.parent.parent / "data" / "article_cache.db"
|
||||
|
||||
class ArticleCache:
|
||||
def __init__(self):
|
||||
# 确保数据目录存在
|
||||
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.conn = None
|
||||
self._connect()
|
||||
self._init_table()
|
||||
|
||||
def _connect(self):
|
||||
"""连接数据库，失败降级为不使用缓存"""
|
||||
try:
|
||||
self.conn = sqlite3.connect(str(DB_PATH), check_same_thread=False)
|
||||
logger.debug("缓存数据库连接成功")
|
||||
except Exception as e:
|
||||
logger.warning(f"缓存数据库连接失败，将不使用缓存功能: {e}")
|
||||
self.conn = None
|
||||
|
||||
def _init_table(self):
|
||||
"""初始化缓存表"""
|
||||
if not self.conn:
|
||||
return
|
||||
try:
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS article_cache (
|
||||
id TEXT PRIMARY KEY,
|
||||
source TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
link TEXT NOT NULL,
|
||||
translated_title TEXT,
|
||||
content TEXT,
|
||||
translated_content TEXT,
|
||||
ai_score REAL,
|
||||
ai_summary TEXT,
|
||||
ai_category TEXT,
|
||||
ai_scores_json TEXT,
|
||||
published_time DATETIME,
|
||||
processed_time DATETIME NOT NULL,
|
||||
status INTEGER NOT NULL DEFAULT 1,
|
||||
error_msg TEXT
|
||||
)
|
||||
""")
|
||||
# 创建索引提升查询速度
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_processed_time ON article_cache(processed_time)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_source ON article_cache(source)")
|
||||
self.conn.commit()
|
||||
logger.debug("缓存表初始化完成")
|
||||
except Exception as e:
|
||||
logger.warning(f"缓存表初始化失败，将不使用缓存功能: {e}")
|
||||
self.conn = None
|
||||
|
||||
self._migrate_add_webzine_text()
|
||||
self._init_summary_table()
|
||||
|
||||
def _init_summary_table(self):
|
||||
if not self.conn:
|
||||
return
|
||||
try:
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS category_summary_cache (
|
||||
date TEXT NOT NULL,
|
||||
category TEXT NOT NULL,
|
||||
summary_text TEXT,
|
||||
generated_time DATETIME NOT NULL,
|
||||
PRIMARY KEY (date, category)
|
||||
)
|
||||
""")
|
||||
self.conn.commit()
|
||||
logger.debug("分类摘要缓存表初始化完成")
|
||||
except Exception as e:
|
||||
logger.warning("分类摘要缓存表初始化失败: %s", e)
|
||||
|
||||
def _migrate_add_webzine_text(self):
|
||||
if not self.conn:
|
||||
return
|
||||
try:
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute("PRAGMA table_info(article_cache)")
|
||||
columns = [col[1] for col in cursor.fetchall()]
|
||||
if 'webzine_text' not in columns:
|
||||
cursor.execute("ALTER TABLE article_cache ADD COLUMN webzine_text TEXT")
|
||||
self.conn.commit()
|
||||
logger.debug("缓存表已添加 webzine_text 列")
|
||||
except Exception as e:
|
||||
logger.debug("添加 webzine_text 列跳过: %s", e)
|
||||
|
||||
def get_article(self, article_id, expire_hours=24):
|
||||
"""
|
||||
查询文章缓存
|
||||
:param article_id: 文章ID
|
||||
:param expire_hours: 缓存过期时间，默认24小时，和时间窗口一致
|
||||
:return: 缓存的文章数据，不存在或过期返回None
|
||||
"""
|
||||
if not self.conn:
|
||||
return None
|
||||
try:
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute("""
|
||||
SELECT id, source, title, link, translated_title, content, translated_content,
|
||||
ai_score, ai_summary, ai_category, ai_scores_json, published_time,
|
||||
processed_time, status, error_msg, webzine_text
|
||||
FROM article_cache
|
||||
WHERE id = ?
|
||||
AND processed_time >= ?
|
||||
AND status = 1
|
||||
""", (
|
||||
article_id,
|
||||
(datetime.now() - timedelta(hours=expire_hours)).strftime("%Y-%m-%d %H:%M:%S")
|
||||
))
|
||||
row = cursor.fetchone()
|
||||
if not row:
|
||||
return None
|
||||
# 转换为和现有article结构一致的字典
|
||||
article = {
|
||||
"id": row[0],
|
||||
"source": row[1],
|
||||
"title": row[2],
|
||||
"link": row[3],
|
||||
"translated_title": row[4],
|
||||
"content": row[5],
|
||||
"translated_content": row[6],
|
||||
"final_score": row[7],
|
||||
"summary": row[8],
|
||||
"category": row[9],
|
||||
"scores": json.loads(row[10]) if row[10] else {},
|
||||
"published": datetime.strptime(row[11], "%Y-%m-%d %H:%M:%S") if row[11] else None,
|
||||
"processed_time": datetime.strptime(row[12], "%Y-%m-%d %H:%M:%S"),
|
||||
"webzine_text": row[15] or "",
|
||||
"from_cache": True
|
||||
}
|
||||
return article
|
||||
except Exception as e:
|
||||
logger.warning(f"查询缓存失败: {e}")
|
||||
return None
|
||||
|
||||
def save_article(self, article, status=1, error_msg=""):
|
||||
"""
|
||||
保存文章处理结果到缓存
|
||||
:param article: 文章字典
|
||||
:param status: 处理状态，1-成功 2-失败
|
||||
:param error_msg: 失败时的错误信息
|
||||
"""
|
||||
if not self.conn:
|
||||
return
|
||||
try:
|
||||
cursor = self.conn.cursor()
|
||||
# 处理JSON字段
|
||||
ai_scores_json = json.dumps(article.get("scores", {}), ensure_ascii=False)
|
||||
# 发布时间转字符串
|
||||
published_str = article["published"].strftime("%Y-%m-%d %H:%M:%S") if article.get("published") else None
|
||||
processed_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
cursor.execute("""
|
||||
REPLACE INTO article_cache (
|
||||
id, source, title, link, translated_title, content, translated_content,
|
||||
ai_score, ai_summary, ai_category, ai_scores_json, published_time,
|
||||
processed_time, status, error_msg, webzine_text
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (
|
||||
article["id"],
|
||||
article["source"],
|
||||
article["title"],
|
||||
article["link"],
|
||||
article.get("translated_title", ""),
|
||||
article.get("content", ""),
|
||||
article.get("translated_content", ""),
|
||||
article.get("final_score", 0),
|
||||
article.get("summary", ""),
|
||||
article.get("category", ""),
|
||||
ai_scores_json,
|
||||
published_str,
|
||||
processed_str,
|
||||
status,
|
||||
error_msg,
|
||||
article.get("webzine_text", "")
|
||||
))
|
||||
self.conn.commit()
|
||||
logger.debug(f"文章 {article['id'][:20]}... 缓存保存成功")
|
||||
except Exception as e:
|
||||
logger.warning(f"保存缓存失败: {e}")
|
||||
# 写入失败不影响主流程，忽略错误
|
||||
|
||||
def save_article_webzine(self, article_id, webzine_text):
|
||||
|
||||
if not self.conn or not webzine_text:
|
||||
return
|
||||
try:
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute("""
|
||||
UPDATE article_cache SET webzine_text = ?
|
||||
WHERE id = ? AND status = 1
|
||||
""", (webzine_text, article_id))
|
||||
self.conn.commit()
|
||||
logger.debug(f"文章 {article_id[:20]}... 网摘缓存更新成功")
|
||||
except Exception as e:
|
||||
logger.warning(f"保存网摘缓存失败: {e}")
|
||||
|
||||
def get_category_summary(self, date_str, category, expire_hours=24):
|
||||
if not self.conn:
|
||||
return None
|
||||
try:
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute("""
|
||||
SELECT summary_text FROM category_summary_cache
|
||||
WHERE date = ? AND category = ?
|
||||
AND generated_time >= ?
|
||||
""", (date_str, category,
|
||||
(datetime.now() - timedelta(hours=expire_hours)).strftime("%Y-%m-%d %H:%M:%S")))
|
||||
row = cursor.fetchone()
|
||||
return row[0] if row else None
|
||||
except Exception as e:
|
||||
logger.warning("查询分类摘要缓存失败: %s", e)
|
||||
return None
|
||||
|
||||
def save_category_summary(self, date_str, category, summary_text):
|
||||
if not self.conn or not summary_text:
|
||||
return
|
||||
try:
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute("""
|
||||
REPLACE INTO category_summary_cache (date, category, summary_text, generated_time)
|
||||
VALUES (?, ?, ?, ?)
|
||||
""", (date_str, category, summary_text,
|
||||
datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
|
||||
self.conn.commit()
|
||||
logger.debug("分类摘要 [%s/%s] 缓存保存成功", date_str, category)
|
||||
except Exception as e:
|
||||
logger.warning("保存分类摘要缓存失败: %s", e)
|
||||
|
||||
def is_processed(self, article_id, expire_hours=24):
|
||||
"""判断文章是否已处理且未过期"""
|
||||
if not self.conn:
|
||||
return False
|
||||
try:
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute("""
|
||||
SELECT 1 FROM article_cache
|
||||
WHERE id = ?
|
||||
AND processed_time >= ?
|
||||
AND status = 1
|
||||
LIMIT 1
|
||||
""", (
|
||||
article_id,
|
||||
(datetime.now() - timedelta(hours=expire_hours)).strftime("%Y-%m-%d %H:%M:%S")
|
||||
))
|
||||
return cursor.fetchone() is not None
|
||||
except Exception as e:
|
||||
logger.warning(f"查询处理状态失败: {e}")
|
||||
return False
|
||||
|
||||
def clear_expired(self, keep_days=7):
|
||||
"""清理过期缓存，默认保留7天数据"""
|
||||
if not self.conn:
|
||||
return
|
||||
try:
|
||||
cursor = self.conn.cursor()
|
||||
# 清理文章缓存
|
||||
cursor.execute("""
|
||||
DELETE FROM article_cache
|
||||
WHERE processed_time < ?
|
||||
""", ((datetime.now() - timedelta(days=keep_days)).strftime("%Y-%m-%d %H:%M:%S"),))
|
||||
deleted = cursor.rowcount
|
||||
# 清理分类摘要缓存
|
||||
cursor.execute("""
|
||||
DELETE FROM category_summary_cache
|
||||
WHERE generated_time < ?
|
||||
""", ((datetime.now() - timedelta(days=keep_days)).strftime("%Y-%m-%d %H:%M:%S"),))
|
||||
deleted += cursor.rowcount
|
||||
self.conn.commit()
|
||||
if deleted > 0:
|
||||
logger.info(f"清理了 {deleted} 条过期缓存数据")
|
||||
except Exception as e:
|
||||
logger.warning(f"清理过期缓存失败: {e}")
|
||||
|
||||
def close(self):
|
||||
"""关闭数据库连接"""
|
||||
if self.conn:
|
||||
try:
|
||||
self.conn.close()
|
||||
logger.debug("缓存数据库连接已关闭")
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
# 全局缓存实例
|
||||
_cache = None
|
||||
|
||||
def get_cache():
|
||||
"""获取全局缓存单例"""
|
||||
global _cache
|
||||
if _cache is None:
|
||||
_cache = ArticleCache()
|
||||
return _cache
|
||||
Reference in New Issue
Block a user