311 lines
11 KiB
Python
311 lines
11 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""
|
|
SQLite增量缓存模块
|
|
实现文章处理结果的本地存储和复用，避免重复翻译和AI分析，支持断点续跑
|
|
"""
|
|
import sqlite3
|
|
import json
|
|
import os
|
|
from datetime import datetime, timedelta
|
|
from pathlib import Path
|
|
from .logger import get_logger
|
|
|
|
logger = get_logger()
|
|
|
|
# 数据库文件路径，放在项目根目录下的data目录
|
|
DB_PATH = Path(__file__).resolve().parent.parent.parent / "data" / "article_cache.db"
|
|
|
|
class ArticleCache:
|
|
def __init__(self):
|
|
# 确保数据目录存在
|
|
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
self.conn = None
|
|
self._connect()
|
|
self._init_table()
|
|
|
|
def _connect(self):
|
|
"""连接数据库，失败降级为不使用缓存"""
|
|
try:
|
|
self.conn = sqlite3.connect(str(DB_PATH), check_same_thread=False)
|
|
logger.debug("缓存数据库连接成功")
|
|
except Exception as e:
|
|
logger.warning(f"缓存数据库连接失败，将不使用缓存功能: {e}")
|
|
self.conn = None
|
|
|
|
def _init_table(self):
|
|
"""初始化缓存表"""
|
|
if not self.conn:
|
|
return
|
|
try:
|
|
cursor = self.conn.cursor()
|
|
cursor.execute("""
|
|
CREATE TABLE IF NOT EXISTS article_cache (
|
|
id TEXT PRIMARY KEY,
|
|
source TEXT NOT NULL,
|
|
title TEXT NOT NULL,
|
|
link TEXT NOT NULL,
|
|
translated_title TEXT,
|
|
content TEXT,
|
|
translated_content TEXT,
|
|
ai_score REAL,
|
|
ai_summary TEXT,
|
|
ai_category TEXT,
|
|
ai_scores_json TEXT,
|
|
published_time DATETIME,
|
|
processed_time DATETIME NOT NULL,
|
|
status INTEGER NOT NULL DEFAULT 1,
|
|
error_msg TEXT
|
|
)
|
|
""")
|
|
# 创建索引提升查询速度
|
|
cursor.execute("CREATE INDEX IF NOT EXISTS idx_processed_time ON article_cache(processed_time)")
|
|
cursor.execute("CREATE INDEX IF NOT EXISTS idx_source ON article_cache(source)")
|
|
self.conn.commit()
|
|
logger.debug("缓存表初始化完成")
|
|
except Exception as e:
|
|
logger.warning(f"缓存表初始化失败，将不使用缓存功能: {e}")
|
|
self.conn = None
|
|
|
|
self._migrate_add_webzine_text()
|
|
self._init_summary_table()
|
|
|
|
def _init_summary_table(self):
|
|
if not self.conn:
|
|
return
|
|
try:
|
|
cursor = self.conn.cursor()
|
|
cursor.execute("""
|
|
CREATE TABLE IF NOT EXISTS category_summary_cache (
|
|
date TEXT NOT NULL,
|
|
category TEXT NOT NULL,
|
|
summary_text TEXT,
|
|
generated_time DATETIME NOT NULL,
|
|
PRIMARY KEY (date, category)
|
|
)
|
|
""")
|
|
self.conn.commit()
|
|
logger.debug("分类摘要缓存表初始化完成")
|
|
except Exception as e:
|
|
logger.warning("分类摘要缓存表初始化失败: %s", e)
|
|
|
|
def _migrate_add_webzine_text(self):
|
|
if not self.conn:
|
|
return
|
|
try:
|
|
cursor = self.conn.cursor()
|
|
cursor.execute("PRAGMA table_info(article_cache)")
|
|
columns = [col[1] for col in cursor.fetchall()]
|
|
if 'webzine_text' not in columns:
|
|
cursor.execute("ALTER TABLE article_cache ADD COLUMN webzine_text TEXT")
|
|
self.conn.commit()
|
|
logger.debug("缓存表已添加 webzine_text 列")
|
|
except Exception as e:
|
|
logger.debug("添加 webzine_text 列跳过: %s", e)
|
|
|
|
def get_article(self, article_id, expire_hours=24):
|
|
"""
|
|
查询文章缓存
|
|
:param article_id: 文章ID
|
|
:param expire_hours: 缓存过期时间，默认24小时，和时间窗口一致
|
|
:return: 缓存的文章数据，不存在或过期返回None
|
|
"""
|
|
if not self.conn:
|
|
return None
|
|
try:
|
|
cursor = self.conn.cursor()
|
|
cursor.execute("""
|
|
SELECT id, source, title, link, translated_title, content, translated_content,
|
|
ai_score, ai_summary, ai_category, ai_scores_json, published_time,
|
|
processed_time, status, error_msg, webzine_text
|
|
FROM article_cache
|
|
WHERE id = ?
|
|
AND processed_time >= ?
|
|
AND status = 1
|
|
""", (
|
|
article_id,
|
|
(datetime.now() - timedelta(hours=expire_hours)).strftime("%Y-%m-%d %H:%M:%S")
|
|
))
|
|
row = cursor.fetchone()
|
|
if not row:
|
|
return None
|
|
# 转换为和现有article结构一致的字典
|
|
article = {
|
|
"id": row[0],
|
|
"source": row[1],
|
|
"title": row[2],
|
|
"link": row[3],
|
|
"translated_title": row[4],
|
|
"content": row[5],
|
|
"translated_content": row[6],
|
|
"final_score": row[7],
|
|
"summary": row[8],
|
|
"category": row[9],
|
|
"scores": json.loads(row[10]) if row[10] else {},
|
|
"published": datetime.strptime(row[11], "%Y-%m-%d %H:%M:%S") if row[11] else None,
|
|
"processed_time": datetime.strptime(row[12], "%Y-%m-%d %H:%M:%S"),
|
|
"webzine_text": row[15] or "",
|
|
"from_cache": True
|
|
}
|
|
return article
|
|
except Exception as e:
|
|
logger.warning(f"查询缓存失败: {e}")
|
|
return None
|
|
|
|
def save_article(self, article, status=1, error_msg=""):
|
|
"""
|
|
保存文章处理结果到缓存
|
|
:param article: 文章字典
|
|
:param status: 处理状态，1-成功 2-失败
|
|
:param error_msg: 失败时的错误信息
|
|
"""
|
|
if not self.conn:
|
|
return
|
|
try:
|
|
cursor = self.conn.cursor()
|
|
# 处理JSON字段
|
|
ai_scores_json = json.dumps(article.get("scores", {}), ensure_ascii=False)
|
|
# 发布时间转字符串
|
|
published_str = article["published"].strftime("%Y-%m-%d %H:%M:%S") if article.get("published") else None
|
|
processed_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
|
|
|
cursor.execute("""
|
|
REPLACE INTO article_cache (
|
|
id, source, title, link, translated_title, content, translated_content,
|
|
ai_score, ai_summary, ai_category, ai_scores_json, published_time,
|
|
processed_time, status, error_msg, webzine_text
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
""", (
|
|
article["id"],
|
|
article["source"],
|
|
article["title"],
|
|
article["link"],
|
|
article.get("translated_title", ""),
|
|
article.get("content", ""),
|
|
article.get("translated_content", ""),
|
|
article.get("final_score", 0),
|
|
article.get("summary", ""),
|
|
article.get("category", ""),
|
|
ai_scores_json,
|
|
published_str,
|
|
processed_str,
|
|
status,
|
|
error_msg,
|
|
article.get("webzine_text", "")
|
|
))
|
|
self.conn.commit()
|
|
logger.debug(f"文章 {article['id'][:20]}... 缓存保存成功")
|
|
except Exception as e:
|
|
logger.warning(f"保存缓存失败: {e}")
|
|
# 写入失败不影响主流程，忽略错误
|
|
|
|
def save_article_webzine(self, article_id, webzine_text):
|
|
|
|
if not self.conn or not webzine_text:
|
|
return
|
|
try:
|
|
cursor = self.conn.cursor()
|
|
cursor.execute("""
|
|
UPDATE article_cache SET webzine_text = ?
|
|
WHERE id = ? AND status = 1
|
|
""", (webzine_text, article_id))
|
|
self.conn.commit()
|
|
logger.debug(f"文章 {article_id[:20]}... 网摘缓存更新成功")
|
|
except Exception as e:
|
|
logger.warning(f"保存网摘缓存失败: {e}")
|
|
|
|
def get_category_summary(self, date_str, category, expire_hours=24):
|
|
if not self.conn:
|
|
return None
|
|
try:
|
|
cursor = self.conn.cursor()
|
|
cursor.execute("""
|
|
SELECT summary_text FROM category_summary_cache
|
|
WHERE date = ? AND category = ?
|
|
AND generated_time >= ?
|
|
""", (date_str, category,
|
|
(datetime.now() - timedelta(hours=expire_hours)).strftime("%Y-%m-%d %H:%M:%S")))
|
|
row = cursor.fetchone()
|
|
return row[0] if row else None
|
|
except Exception as e:
|
|
logger.warning("查询分类摘要缓存失败: %s", e)
|
|
return None
|
|
|
|
def save_category_summary(self, date_str, category, summary_text):
|
|
if not self.conn or not summary_text:
|
|
return
|
|
try:
|
|
cursor = self.conn.cursor()
|
|
cursor.execute("""
|
|
REPLACE INTO category_summary_cache (date, category, summary_text, generated_time)
|
|
VALUES (?, ?, ?, ?)
|
|
""", (date_str, category, summary_text,
|
|
datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
|
|
self.conn.commit()
|
|
logger.debug("分类摘要 [%s/%s] 缓存保存成功", date_str, category)
|
|
except Exception as e:
|
|
logger.warning("保存分类摘要缓存失败: %s", e)
|
|
|
|
def is_processed(self, article_id, expire_hours=24):
|
|
"""判断文章是否已处理且未过期"""
|
|
if not self.conn:
|
|
return False
|
|
try:
|
|
cursor = self.conn.cursor()
|
|
cursor.execute("""
|
|
SELECT 1 FROM article_cache
|
|
WHERE id = ?
|
|
AND processed_time >= ?
|
|
AND status = 1
|
|
LIMIT 1
|
|
""", (
|
|
article_id,
|
|
(datetime.now() - timedelta(hours=expire_hours)).strftime("%Y-%m-%d %H:%M:%S")
|
|
))
|
|
return cursor.fetchone() is not None
|
|
except Exception as e:
|
|
logger.warning(f"查询处理状态失败: {e}")
|
|
return False
|
|
|
|
def clear_expired(self, keep_days=7):
|
|
"""清理过期缓存，默认保留7天数据"""
|
|
if not self.conn:
|
|
return
|
|
try:
|
|
cursor = self.conn.cursor()
|
|
# 清理文章缓存
|
|
cursor.execute("""
|
|
DELETE FROM article_cache
|
|
WHERE processed_time < ?
|
|
""", ((datetime.now() - timedelta(days=keep_days)).strftime("%Y-%m-%d %H:%M:%S"),))
|
|
deleted = cursor.rowcount
|
|
# 清理分类摘要缓存
|
|
cursor.execute("""
|
|
DELETE FROM category_summary_cache
|
|
WHERE generated_time < ?
|
|
""", ((datetime.now() - timedelta(days=keep_days)).strftime("%Y-%m-%d %H:%M:%S"),))
|
|
deleted += cursor.rowcount
|
|
self.conn.commit()
|
|
if deleted > 0:
|
|
logger.info(f"清理了 {deleted} 条过期缓存数据")
|
|
except Exception as e:
|
|
logger.warning(f"清理过期缓存失败: {e}")
|
|
|
|
def close(self):
|
|
"""关闭数据库连接"""
|
|
if self.conn:
|
|
try:
|
|
self.conn.close()
|
|
logger.debug("缓存数据库连接已关闭")
|
|
except Exception as e:
|
|
pass
|
|
|
|
# 全局缓存实例
|
|
_cache = None
|
|
|
|
def get_cache():
|
|
"""获取全局缓存单例"""
|
|
global _cache
|
|
if _cache is None:
|
|
_cache = ArticleCache()
|
|
return _cache
|