# -*- coding: utf-8 -*- """ SQLite增量缓存模块 实现文章处理结果的本地存储和复用,避免重复翻译和AI分析,支持断点续跑 """ import sqlite3 import json import os from datetime import datetime, timedelta from pathlib import Path from .logger import get_logger logger = get_logger() # 数据库文件路径,放在项目根目录下的data目录 DB_PATH = Path(__file__).resolve().parent.parent.parent / "data" / "article_cache.db" class ArticleCache: def __init__(self): # 确保数据目录存在 DB_PATH.parent.mkdir(parents=True, exist_ok=True) self.conn = None self._connect() self._init_table() def _connect(self): """连接数据库,失败降级为不使用缓存""" try: self.conn = sqlite3.connect(str(DB_PATH), check_same_thread=False) logger.debug("缓存数据库连接成功") except Exception as e: logger.warning(f"缓存数据库连接失败,将不使用缓存功能: {e}") self.conn = None def _init_table(self): """初始化缓存表""" if not self.conn: return try: cursor = self.conn.cursor() cursor.execute(""" CREATE TABLE IF NOT EXISTS article_cache ( id TEXT PRIMARY KEY, source TEXT NOT NULL, title TEXT NOT NULL, link TEXT NOT NULL, translated_title TEXT, content TEXT, translated_content TEXT, ai_score REAL, ai_summary TEXT, ai_category TEXT, ai_scores_json TEXT, published_time DATETIME, processed_time DATETIME NOT NULL, status INTEGER NOT NULL DEFAULT 1, error_msg TEXT ) """) # 创建索引提升查询速度 cursor.execute("CREATE INDEX IF NOT EXISTS idx_processed_time ON article_cache(processed_time)") cursor.execute("CREATE INDEX IF NOT EXISTS idx_source ON article_cache(source)") self.conn.commit() logger.debug("缓存表初始化完成") except Exception as e: logger.warning(f"缓存表初始化失败,将不使用缓存功能: {e}") self.conn = None self._migrate_add_webzine_text() self._init_summary_table() def _init_summary_table(self): if not self.conn: return try: cursor = self.conn.cursor() cursor.execute(""" CREATE TABLE IF NOT EXISTS category_summary_cache ( date TEXT NOT NULL, category TEXT NOT NULL, summary_text TEXT, generated_time DATETIME NOT NULL, PRIMARY KEY (date, category) ) """) self.conn.commit() logger.debug("分类摘要缓存表初始化完成") except Exception as e: logger.warning("分类摘要缓存表初始化失败: %s", e) def _migrate_add_webzine_text(self): if not self.conn: return try: cursor = self.conn.cursor() cursor.execute("PRAGMA table_info(article_cache)") columns = [col[1] for col in cursor.fetchall()] if 'webzine_text' not in columns: cursor.execute("ALTER TABLE article_cache ADD COLUMN webzine_text TEXT") self.conn.commit() logger.debug("缓存表已添加 webzine_text 列") except Exception as e: logger.debug("添加 webzine_text 列跳过: %s", e) def get_article(self, article_id, expire_hours=24): """ 查询文章缓存 :param article_id: 文章ID :param expire_hours: 缓存过期时间,默认24小时,和时间窗口一致 :return: 缓存的文章数据,不存在或过期返回None """ if not self.conn: return None try: cursor = self.conn.cursor() cursor.execute(""" SELECT id, source, title, link, translated_title, content, translated_content, ai_score, ai_summary, ai_category, ai_scores_json, published_time, processed_time, status, error_msg, webzine_text FROM article_cache WHERE id = ? AND processed_time >= ? AND status = 1 """, ( article_id, (datetime.now() - timedelta(hours=expire_hours)).strftime("%Y-%m-%d %H:%M:%S") )) row = cursor.fetchone() if not row: return None # 转换为和现有article结构一致的字典 article = { "id": row[0], "source": row[1], "title": row[2], "link": row[3], "translated_title": row[4], "content": row[5], "translated_content": row[6], "final_score": row[7], "summary": row[8], "category": row[9], "scores": json.loads(row[10]) if row[10] else {}, "published": datetime.strptime(row[11], "%Y-%m-%d %H:%M:%S") if row[11] else None, "processed_time": datetime.strptime(row[12], "%Y-%m-%d %H:%M:%S"), "webzine_text": row[15] or "", "from_cache": True } return article except Exception as e: logger.warning(f"查询缓存失败: {e}") return None def save_article(self, article, status=1, error_msg=""): """ 保存文章处理结果到缓存 :param article: 文章字典 :param status: 处理状态,1-成功 2-失败 :param error_msg: 失败时的错误信息 """ if not self.conn: return try: cursor = self.conn.cursor() # 处理JSON字段 ai_scores_json = json.dumps(article.get("scores", {}), ensure_ascii=False) # 发布时间转字符串 published_str = article["published"].strftime("%Y-%m-%d %H:%M:%S") if article.get("published") else None processed_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") cursor.execute(""" REPLACE INTO article_cache ( id, source, title, link, translated_title, content, translated_content, ai_score, ai_summary, ai_category, ai_scores_json, published_time, processed_time, status, error_msg, webzine_text ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( article["id"], article["source"], article["title"], article["link"], article.get("translated_title", ""), article.get("content", ""), article.get("translated_content", ""), article.get("final_score", 0), article.get("summary", ""), article.get("category", ""), ai_scores_json, published_str, processed_str, status, error_msg, article.get("webzine_text", "") )) self.conn.commit() logger.debug(f"文章 {article['id'][:20]}... 缓存保存成功") except Exception as e: logger.warning(f"保存缓存失败: {e}") # 写入失败不影响主流程,忽略错误 def save_article_webzine(self, article_id, webzine_text): if not self.conn or not webzine_text: return try: cursor = self.conn.cursor() cursor.execute(""" UPDATE article_cache SET webzine_text = ? WHERE id = ? AND status = 1 """, (webzine_text, article_id)) self.conn.commit() logger.debug(f"文章 {article_id[:20]}... 网摘缓存更新成功") except Exception as e: logger.warning(f"保存网摘缓存失败: {e}") def get_category_summary(self, date_str, category, expire_hours=24): if not self.conn: return None try: cursor = self.conn.cursor() cursor.execute(""" SELECT summary_text FROM category_summary_cache WHERE date = ? AND category = ? AND generated_time >= ? """, (date_str, category, (datetime.now() - timedelta(hours=expire_hours)).strftime("%Y-%m-%d %H:%M:%S"))) row = cursor.fetchone() return row[0] if row else None except Exception as e: logger.warning("查询分类摘要缓存失败: %s", e) return None def save_category_summary(self, date_str, category, summary_text): if not self.conn or not summary_text: return try: cursor = self.conn.cursor() cursor.execute(""" REPLACE INTO category_summary_cache (date, category, summary_text, generated_time) VALUES (?, ?, ?, ?) """, (date_str, category, summary_text, datetime.now().strftime("%Y-%m-%d %H:%M:%S"))) self.conn.commit() logger.debug("分类摘要 [%s/%s] 缓存保存成功", date_str, category) except Exception as e: logger.warning("保存分类摘要缓存失败: %s", e) def is_processed(self, article_id, expire_hours=24): """判断文章是否已处理且未过期""" if not self.conn: return False try: cursor = self.conn.cursor() cursor.execute(""" SELECT 1 FROM article_cache WHERE id = ? AND processed_time >= ? AND status = 1 LIMIT 1 """, ( article_id, (datetime.now() - timedelta(hours=expire_hours)).strftime("%Y-%m-%d %H:%M:%S") )) return cursor.fetchone() is not None except Exception as e: logger.warning(f"查询处理状态失败: {e}") return False def clear_expired(self, keep_days=7): """清理过期缓存,默认保留7天数据""" if not self.conn: return try: cursor = self.conn.cursor() # 清理文章缓存 cursor.execute(""" DELETE FROM article_cache WHERE processed_time < ? """, ((datetime.now() - timedelta(days=keep_days)).strftime("%Y-%m-%d %H:%M:%S"),)) deleted = cursor.rowcount # 清理分类摘要缓存 cursor.execute(""" DELETE FROM category_summary_cache WHERE generated_time < ? """, ((datetime.now() - timedelta(days=keep_days)).strftime("%Y-%m-%d %H:%M:%S"),)) deleted += cursor.rowcount self.conn.commit() if deleted > 0: logger.info(f"清理了 {deleted} 条过期缓存数据") except Exception as e: logger.warning(f"清理过期缓存失败: {e}") def close(self): """关闭数据库连接""" if self.conn: try: self.conn.close() logger.debug("缓存数据库连接已关闭") except Exception as e: pass # 全局缓存实例 _cache = None def get_cache(): """获取全局缓存单例""" global _cache if _cache is None: _cache = ArticleCache() return _cache