feat: 军事科技每日资讯推送系统 - Docker部署 + 日志系统 + 数据目录重组
This commit is contained in:
poiuy
@@ -0,0 +1 @@
|
||||
"""板块模块包。每个子模块导出 run_<board>_section(config, now) -> (markdown, error)"""
|
||||
@@ -0,0 +1,111 @@
|
||||
"""GitHub REST API enrich:metadata + README → enriched repo dict
|
||||
|
||||
匿名调用受 60 req/hr 限,设置 GITHUB_TOKEN 环境变量后走 5000 req/hr。
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import os
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
import aiohttp
|
||||
|
||||
API_BASE = "https://api.github.com"
|
||||
|
||||
|
||||
def _auth_headers(token_env: str = "GITHUB_TOKEN") -> Dict[str, str]:
|
||||
headers = {"Accept": "application/vnd.github+json"}
|
||||
token = os.environ.get(token_env)
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
print(f"🔑 GH: 已配置 {token_env},鉴权调用 (5000 req/hr)")
|
||||
else:
|
||||
print(f"⚠️ GH: 未配置 {token_env},匿名调用 (60 req/hr)")
|
||||
return headers
|
||||
|
||||
|
||||
async def _get_json(
|
||||
session: aiohttp.ClientSession, url: str, timeout: int = 10
|
||||
) -> Optional[Dict]:
|
||||
async with session.get(
|
||||
url, timeout=aiohttp.ClientTimeout(total=timeout)
|
||||
) as resp:
|
||||
if resp.status == 404:
|
||||
return None
|
||||
if resp.status != 200:
|
||||
raise RuntimeError(f"GitHub API {resp.status} for {url}")
|
||||
return await resp.json()
|
||||
|
||||
|
||||
async def enrich_repo(
|
||||
session: aiohttp.ClientSession,
|
||||
repo: Dict,
|
||||
token_env: str = "GITHUB_TOKEN",
|
||||
readme_max_chars: int = 3000,
|
||||
timeout: int = 10,
|
||||
) -> Optional[Dict]:
|
||||
"""单 repo 双调用 enrich。返回 None 表示该 repo 应剔除(archived 或 metadata 不可达)。
|
||||
|
||||
任一调用失败 raise → 调用方按 return_exceptions 模式聚合错误。
|
||||
"""
|
||||
full_name = repo["full_name"]
|
||||
meta_url = f"{API_BASE}/repos/{full_name}"
|
||||
readme_url = f"{API_BASE}/repos/{full_name}/readme"
|
||||
|
||||
meta, readme = await asyncio.gather(
|
||||
_get_json(session, meta_url, timeout=timeout),
|
||||
_get_json(session, readme_url, timeout=timeout),
|
||||
)
|
||||
|
||||
if meta is None:
|
||||
return None
|
||||
if meta.get("archived"):
|
||||
return None
|
||||
|
||||
license_spdx = ""
|
||||
if isinstance(meta.get("license"), dict):
|
||||
license_spdx = meta["license"].get("spdx_id") or ""
|
||||
|
||||
readme_excerpt = ""
|
||||
if readme and readme.get("content"):
|
||||
try:
|
||||
raw = base64.b64decode(readme["content"]).decode("utf-8", errors="replace")
|
||||
readme_excerpt = raw[:readme_max_chars]
|
||||
except Exception:
|
||||
readme_excerpt = ""
|
||||
|
||||
return {
|
||||
**repo,
|
||||
"topics": meta.get("topics") or [],
|
||||
"license": license_spdx,
|
||||
"pushed_at": meta.get("pushed_at") or "",
|
||||
"readme_excerpt": readme_excerpt,
|
||||
}
|
||||
|
||||
|
||||
async def enrich_repos(
|
||||
candidates: List[Dict],
|
||||
token_env: str = "GITHUB_TOKEN",
|
||||
readme_max_chars: int = 3000,
|
||||
timeout: int = 10,
|
||||
) -> Tuple[List[Dict], List[str]]:
|
||||
"""并发 enrich 多个 repo。返回 (enriched_list_with_archived_filtered, errors)"""
|
||||
errors: List[str] = []
|
||||
headers = _auth_headers(token_env)
|
||||
|
||||
async with aiohttp.ClientSession(headers=headers) as session:
|
||||
results = await asyncio.gather(
|
||||
*[
|
||||
enrich_repo(session, r, token_env, readme_max_chars, timeout)
|
||||
for r in candidates
|
||||
],
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
enriched: List[Dict] = []
|
||||
for r, candidate in zip(results, candidates):
|
||||
if isinstance(r, Exception):
|
||||
errors.append(f"enrich {candidate['full_name']} 失败: {r}")
|
||||
elif r is not None:
|
||||
enriched.append(r)
|
||||
return enriched, errors
|
||||
@@ -0,0 +1,104 @@
|
||||
"""GitHub Trending 板块入口。
|
||||
|
||||
流程:trending 抓取 → history 过滤 → 候选写回 history → deep-dive → LLM 总结
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Dict, Optional, Tuple
|
||||
|
||||
from src.config import get_timezone
|
||||
from src.sections.github.repo_enricher import enrich_repos
|
||||
from src.sections.github.trending_scraper import (
|
||||
fetch_trending_page,
|
||||
parse_trending_html,
|
||||
)
|
||||
from src.storage import load_trending_history
|
||||
|
||||
|
||||
async def run_github_section(
|
||||
config: Dict, now: Optional[datetime] = None
|
||||
) -> Tuple[str, Optional[str]]:
|
||||
cfg = config.get("sections", {}).get("github_trending", {})
|
||||
if not cfg.get("enabled", False):
|
||||
return "", None
|
||||
|
||||
# 延迟 import,Task 11 才提供 summarize_github_trending
|
||||
from src.llm import summarize_github_trending
|
||||
|
||||
today = (now or datetime.now(get_timezone())).date()
|
||||
keep_days = config["filter"]["keep_days"]
|
||||
timeout = cfg.get("request_timeout", 10)
|
||||
max_deep_dive = cfg.get("max_deep_dive", 10)
|
||||
readme_max_chars = cfg.get("readme_max_chars", 3000)
|
||||
history_path = cfg.get("history_file", "news-data/trending-history.json")
|
||||
token_env = cfg.get("tokenName", "GITHUB_TOKEN")
|
||||
|
||||
# 1. 抓取
|
||||
print("📥 GH: 抓取 trending 页...")
|
||||
try:
|
||||
html = await fetch_trending_page(timeout=timeout)
|
||||
except Exception as e:
|
||||
return "", f"GH 抓取失败: {e}"
|
||||
|
||||
all_repos = parse_trending_html(html)
|
||||
print(f"📋 GH: 解析 {len(all_repos)} 个 repo")
|
||||
if not all_repos:
|
||||
return "", None
|
||||
|
||||
# 2. history 加载 + 清理
|
||||
history = load_trending_history(history_path)
|
||||
before_cleanup = len(history.repos)
|
||||
history.cleanup(today=today, keep_days=keep_days)
|
||||
after_cleanup = len(history.repos)
|
||||
if before_cleanup != after_cleanup:
|
||||
print(
|
||||
f"🧹 GH: history 清理过期 {before_cleanup - after_cleanup} 条 (剩 {after_cleanup})"
|
||||
)
|
||||
|
||||
# 3. 候选筛选(按 spec §4.3 语义)
|
||||
candidates = []
|
||||
already_seen = 0
|
||||
for repo in all_repos:
|
||||
if repo["url"] in history:
|
||||
history.touch(repo["url"], today)
|
||||
already_seen += 1
|
||||
else:
|
||||
candidates.append(repo)
|
||||
print(f"🔍 GH: history 过滤掉 {already_seen} 条已见,新候选 {len(candidates)} 条")
|
||||
|
||||
if not candidates:
|
||||
print("ℹ️ GH: 无新候选,跳过")
|
||||
return "", None
|
||||
if len(candidates) > max_deep_dive:
|
||||
print(f"✂️ GH: 候选 {len(candidates)} 超 max_deep_dive={max_deep_dive},截断")
|
||||
candidates = candidates[:max_deep_dive]
|
||||
|
||||
# 4. 候选写回 history + 持久化
|
||||
for repo in candidates:
|
||||
history.touch(repo["url"], today)
|
||||
history.save()
|
||||
|
||||
# 5. 并发 enrich
|
||||
print(f"🌐 GH: 并发 enrich {len(candidates)} 个 repo (REST API)...")
|
||||
enriched, enrich_errors = await enrich_repos(
|
||||
candidates,
|
||||
token_env=token_env,
|
||||
readme_max_chars=readme_max_chars,
|
||||
timeout=timeout,
|
||||
)
|
||||
for e in enrich_errors:
|
||||
print(f"⚠️ GH enrich: {e}")
|
||||
print(
|
||||
f"✅ GH: enrich 成功 {len(enriched)} / 失败 {len(enrich_errors)} / "
|
||||
f"输入 {len(candidates)}"
|
||||
)
|
||||
if not enriched:
|
||||
return "", None
|
||||
|
||||
# 6. LLM 总结
|
||||
print(f"🤖 GH: summarize {len(enriched)} 个候选...")
|
||||
md, err = await summarize_github_trending(enriched, config["llm"])
|
||||
if err:
|
||||
return "", f"summarize_github_trending: {err}"
|
||||
print(f"✅ GH: 板块输出 {len(md or '')} chars")
|
||||
return md or "", None
|
||||
@@ -0,0 +1,98 @@
|
||||
"""GitHub Trending 单页 HTML 抓取与解析。
|
||||
|
||||
数据源: https://github.com/trending (无 language / since 过滤)
|
||||
"""
|
||||
|
||||
import re
|
||||
from typing import Dict, List
|
||||
|
||||
import aiohttp
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
TRENDING_URL = "https://github.com/trending"
|
||||
USER_AGENT = (
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
)
|
||||
_NUM_RE = re.compile(r"[\d,]+")
|
||||
|
||||
|
||||
def _parse_int(s: str) -> int:
|
||||
m = _NUM_RE.search(s or "")
|
||||
if not m:
|
||||
return 0
|
||||
return int(m.group(0).replace(",", ""))
|
||||
|
||||
|
||||
async def fetch_trending_page(timeout: int = 10) -> str:
|
||||
"""抓取 trending 页 HTML;非 200 抛 RuntimeError"""
|
||||
async with aiohttp.ClientSession(
|
||||
headers={"User-Agent": USER_AGENT}
|
||||
) as session:
|
||||
async with session.get(
|
||||
TRENDING_URL, timeout=aiohttp.ClientTimeout(total=timeout)
|
||||
) as resp:
|
||||
if resp.status != 200:
|
||||
text = await resp.text()
|
||||
raise RuntimeError(
|
||||
f"GitHub trending 返回 {resp.status}: {text[:200]}"
|
||||
)
|
||||
return await resp.text()
|
||||
|
||||
|
||||
def parse_trending_html(html: str) -> List[Dict]:
|
||||
"""解析 trending HTML,返回去重后的 repo 字典数组。"""
|
||||
if not html:
|
||||
return []
|
||||
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
seen_urls = set()
|
||||
repos: List[Dict] = []
|
||||
|
||||
for article in soup.select("article.Box-row"):
|
||||
h2 = article.find("h2")
|
||||
a = h2.find("a") if h2 else None
|
||||
if not a or not a.get("href"):
|
||||
continue
|
||||
|
||||
href = a["href"].strip()
|
||||
full_name = href.lstrip("/")
|
||||
url = f"https://github.com/{full_name}"
|
||||
if url in seen_urls:
|
||||
continue
|
||||
seen_urls.add(url)
|
||||
|
||||
# description
|
||||
desc_tag = article.find("p")
|
||||
description = (desc_tag.get_text(strip=True) if desc_tag else "") or ""
|
||||
|
||||
# language
|
||||
lang_tag = article.find("span", attrs={"itemprop": "programmingLanguage"})
|
||||
language = (lang_tag.get_text(strip=True) if lang_tag else "") or ""
|
||||
|
||||
# stars_total: 第一个指向 /stargazers 的链接
|
||||
stars_total = 0
|
||||
star_a = article.find("a", href=re.compile(r"/stargazers$"))
|
||||
if star_a:
|
||||
stars_total = _parse_int(star_a.get_text(strip=True))
|
||||
|
||||
# stars_today: 末尾的 "N stars today" span
|
||||
stars_today = 0
|
||||
for span in article.find_all("span"):
|
||||
t = span.get_text(strip=True)
|
||||
if "stars today" in t or "stars this week" in t or "stars this month" in t:
|
||||
stars_today = _parse_int(t)
|
||||
break
|
||||
|
||||
repos.append(
|
||||
{
|
||||
"url": url,
|
||||
"full_name": full_name,
|
||||
"description": description,
|
||||
"language": language,
|
||||
"stars_today": stars_today,
|
||||
"stars_total": stars_total,
|
||||
}
|
||||
)
|
||||
|
||||
return repos
|
||||
@@ -0,0 +1,91 @@
|
||||
"""HN 首页 HTML 抓取与解析。
|
||||
|
||||
数据源: https://news.ycombinator.com/news (30 条)
|
||||
"""
|
||||
|
||||
import re
|
||||
from typing import Dict, List
|
||||
|
||||
import aiohttp
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
FRONTPAGE_URL = "https://news.ycombinator.com/news"
|
||||
USER_AGENT = (
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
)
|
||||
_NUM_RE = re.compile(r"\d+")
|
||||
|
||||
|
||||
def _first_int(text: str) -> int:
|
||||
m = _NUM_RE.search(text or "")
|
||||
return int(m.group(0)) if m else 0
|
||||
|
||||
|
||||
async def fetch_frontpage(timeout: int = 10) -> str:
|
||||
async with aiohttp.ClientSession(headers={"User-Agent": USER_AGENT}) as session:
|
||||
async with session.get(
|
||||
FRONTPAGE_URL, timeout=aiohttp.ClientTimeout(total=timeout)
|
||||
) as resp:
|
||||
if resp.status != 200:
|
||||
raise RuntimeError(f"HN frontpage 返回 {resp.status}")
|
||||
return await resp.text()
|
||||
|
||||
|
||||
def parse_frontpage_html(html: str) -> List[Dict]:
|
||||
"""解析首页 HTML,返回 [{id, title, url, site, points, comments, comments_url}]
|
||||
|
||||
注:HN frontpage 的 athing 行的实际 class 是 'athing submission' (多类),
|
||||
用 CSS 选择器 'tr.athing' 仍然匹配。
|
||||
"""
|
||||
if not html:
|
||||
return []
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
stories: List[Dict] = []
|
||||
|
||||
for athing in soup.select("tr.athing"):
|
||||
item_id = athing.get("id")
|
||||
if not item_id:
|
||||
continue
|
||||
|
||||
title_a = athing.select_one("span.titleline > a")
|
||||
if not title_a:
|
||||
continue
|
||||
title = title_a.get_text(strip=True)
|
||||
href = title_a.get("href", "")
|
||||
if href.startswith("item?id="):
|
||||
url = f"https://news.ycombinator.com/{href}"
|
||||
site = ""
|
||||
else:
|
||||
url = href
|
||||
site_tag = athing.select_one("span.sitestr")
|
||||
site = site_tag.get_text(strip=True) if site_tag else ""
|
||||
|
||||
sub_tr = athing.find_next_sibling("tr")
|
||||
points = 0
|
||||
comments = 0
|
||||
comments_url = f"https://news.ycombinator.com/item?id={item_id}"
|
||||
if sub_tr:
|
||||
score = sub_tr.select_one("span.score")
|
||||
if score:
|
||||
points = _first_int(score.get_text(strip=True))
|
||||
comment_a = None
|
||||
for a in sub_tr.find_all("a", href=re.compile(r"^item\?id=")):
|
||||
comment_a = a
|
||||
if comment_a:
|
||||
comments = _first_int(comment_a.get_text(strip=True))
|
||||
comments_url = f"https://news.ycombinator.com/{comment_a['href']}"
|
||||
|
||||
stories.append(
|
||||
{
|
||||
"id": item_id,
|
||||
"title": title,
|
||||
"url": url,
|
||||
"site": site,
|
||||
"points": points,
|
||||
"comments": comments,
|
||||
"comments_url": comments_url,
|
||||
}
|
||||
)
|
||||
|
||||
return stories
|
||||
@@ -0,0 +1,237 @@
|
||||
"""HN 单 story enrich:Algolia 评论树 + 外链正文。
|
||||
|
||||
Algolia API: GET /api/v1/items/{id}
|
||||
- root.text 是 Show HN / Ask HN 的 post 正文
|
||||
- root.children[] 是顶层评论(按 HN ranking 排序)
|
||||
- 每条 child 自己还有 children[],承载嵌套回复
|
||||
|
||||
enrich 策略:取 L1 + 每个 L1 下前 N 条 L2 回复,合并为 tree JSON:
|
||||
[{"l1": "顶层评论", "replies": ["回复 1", "回复 2"]}, ...]
|
||||
|
||||
外链正文优先走 Jina Reader (https://r.jina.ai/<url>, 返回 markdown),
|
||||
失败回退到直接 GET + html_to_markdown。Jina API key 可选(配置后免费额度更高),
|
||||
环境变量名通过 `sections.hackernews.jinaTokenName` 配置(默认 JINA_API_KEY)。
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
import aiohttp
|
||||
|
||||
from src.processor import html_to_markdown
|
||||
|
||||
USER_AGENT = (
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
)
|
||||
|
||||
JINA_READER_BASE = "https://r.jina.ai"
|
||||
|
||||
|
||||
def _is_internal_hn_url(url: str) -> bool:
|
||||
return url.startswith("https://news.ycombinator.com/item?id=")
|
||||
|
||||
|
||||
async def _fetch_algolia_item(
|
||||
session: aiohttp.ClientSession, item_id: str, algolia_base: str, timeout: int
|
||||
) -> Dict:
|
||||
url = f"{algolia_base}/items/{item_id}"
|
||||
async with session.get(
|
||||
url, timeout=aiohttp.ClientTimeout(total=timeout)
|
||||
) as resp:
|
||||
if resp.status != 200:
|
||||
raise RuntimeError(f"Algolia /items/{item_id} 返回 {resp.status}")
|
||||
return await resp.json()
|
||||
|
||||
|
||||
async def _fetch_url_html(
|
||||
session: aiohttp.ClientSession, url: str, timeout: int
|
||||
) -> str:
|
||||
async with session.get(
|
||||
url, timeout=aiohttp.ClientTimeout(total=timeout)
|
||||
) as resp:
|
||||
if resp.status != 200:
|
||||
raise RuntimeError(f"外链 {url} 返回 {resp.status}")
|
||||
return await resp.text()
|
||||
|
||||
|
||||
async def _fetch_via_jina(
|
||||
session: aiohttp.ClientSession,
|
||||
url: str,
|
||||
timeout: int,
|
||||
jina_token_env: str = "JINA_API_KEY",
|
||||
) -> str:
|
||||
"""通过 Jina Reader 拉取外链 markdown。`jina_token_env` 指定 API key 环境变量名,配置后免费额度更高。"""
|
||||
jina_url = f"{JINA_READER_BASE}/{url}"
|
||||
headers = {"Accept": "text/markdown"}
|
||||
api_key = os.environ.get(jina_token_env) if jina_token_env else None
|
||||
if api_key:
|
||||
headers["Authorization"] = f"Bearer {api_key}"
|
||||
async with session.get(
|
||||
jina_url, headers=headers, timeout=aiohttp.ClientTimeout(total=timeout)
|
||||
) as resp:
|
||||
if resp.status != 200:
|
||||
raise RuntimeError(f"Jina Reader {url} 返回 {resp.status}")
|
||||
return await resp.text()
|
||||
|
||||
|
||||
async def _fetch_external_markdown(
|
||||
session: aiohttp.ClientSession,
|
||||
url: str,
|
||||
timeout: int,
|
||||
jina_token_env: str = "JINA_API_KEY",
|
||||
) -> str:
|
||||
"""获取外链正文 markdown:先走 Jina Reader,失败回退到直接 GET + html_to_markdown。"""
|
||||
try:
|
||||
return await _fetch_via_jina(session, url, timeout, jina_token_env=jina_token_env)
|
||||
except Exception:
|
||||
html = await _fetch_url_html(session, url, timeout)
|
||||
return html_to_markdown(html, base_url=url)
|
||||
|
||||
|
||||
def _collect_comments_tree(
|
||||
root: Dict,
|
||||
top_comments: int,
|
||||
top_l2_per_l1: int,
|
||||
comment_max_chars: int,
|
||||
comments_total_chars: int,
|
||||
) -> List[Dict]:
|
||||
"""从 Algolia 根节点提取评论树,返回 [{l1, replies}]。
|
||||
|
||||
规则:
|
||||
- L1 上限 `top_comments`,每个 L1 下取前 `top_l2_per_l1` 条 L2 作为 replies
|
||||
- 每条 text 过 html_to_markdown,单条截断到 `comment_max_chars`
|
||||
- 累计字符达 `comments_total_chars` 时立即停止(防离群 story 撑爆 prompt)
|
||||
- 跳过空 text;空 replies 仍保留 `replies: []`,schema 一致
|
||||
"""
|
||||
out: List[Dict] = []
|
||||
consumed = 0
|
||||
l1_children = (root.get("children") or [])[:top_comments]
|
||||
for l1 in l1_children:
|
||||
if consumed >= comments_total_chars:
|
||||
break
|
||||
l1_raw = (l1 or {}).get("text") or ""
|
||||
if not l1_raw:
|
||||
continue
|
||||
l1_md = html_to_markdown(l1_raw)[:comment_max_chars]
|
||||
consumed += len(l1_md)
|
||||
replies: List[str] = []
|
||||
for l2 in ((l1 or {}).get("children") or [])[:top_l2_per_l1]:
|
||||
if consumed >= comments_total_chars:
|
||||
break
|
||||
l2_raw = (l2 or {}).get("text") or ""
|
||||
if not l2_raw:
|
||||
continue
|
||||
l2_md = html_to_markdown(l2_raw)[:comment_max_chars]
|
||||
consumed += len(l2_md)
|
||||
replies.append(l2_md)
|
||||
out.append({"l1": l1_md, "replies": replies})
|
||||
return out
|
||||
|
||||
|
||||
async def enrich_story(
|
||||
session: aiohttp.ClientSession,
|
||||
story: Dict,
|
||||
top_comments: int,
|
||||
top_l2_per_l1: int,
|
||||
comment_max_chars: int,
|
||||
comments_total_chars: int,
|
||||
link_content_max_chars: int,
|
||||
algolia_base: str,
|
||||
timeout: int,
|
||||
jina_token_env: str = "JINA_API_KEY",
|
||||
) -> Dict:
|
||||
"""对单 story enrich。任一子任务失败 → 对应字段留空,不抛。"""
|
||||
item_id = story["id"]
|
||||
is_internal = _is_internal_hn_url(story["url"])
|
||||
|
||||
tasks = [
|
||||
_fetch_algolia_item(
|
||||
session, item_id, algolia_base=algolia_base, timeout=timeout
|
||||
)
|
||||
]
|
||||
if not is_internal:
|
||||
tasks.append(
|
||||
_fetch_external_markdown(
|
||||
session, story["url"], timeout=timeout, jina_token_env=jina_token_env
|
||||
)
|
||||
)
|
||||
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
algolia_result = results[0]
|
||||
external_markdown_result = results[1] if not is_internal else None
|
||||
|
||||
comments_tree: List[Dict] = []
|
||||
post_text = ""
|
||||
if not isinstance(algolia_result, Exception) and algolia_result:
|
||||
post_text = algolia_result.get("text") or ""
|
||||
comments_tree = _collect_comments_tree(
|
||||
algolia_result,
|
||||
top_comments=top_comments,
|
||||
top_l2_per_l1=top_l2_per_l1,
|
||||
comment_max_chars=comment_max_chars,
|
||||
comments_total_chars=comments_total_chars,
|
||||
)
|
||||
|
||||
link_content = ""
|
||||
if is_internal:
|
||||
if post_text:
|
||||
link_content = html_to_markdown(post_text)[:link_content_max_chars]
|
||||
else:
|
||||
if (
|
||||
not isinstance(external_markdown_result, Exception)
|
||||
and external_markdown_result
|
||||
):
|
||||
link_content = external_markdown_result[:link_content_max_chars]
|
||||
|
||||
return {
|
||||
**story,
|
||||
"link_content": link_content,
|
||||
"top_comments": comments_tree,
|
||||
}
|
||||
|
||||
|
||||
async def enrich_stories(
|
||||
stories: List[Dict],
|
||||
top_comments: int,
|
||||
top_l2_per_l1: int,
|
||||
comment_max_chars: int,
|
||||
comments_total_chars: int,
|
||||
link_content_max_chars: int,
|
||||
algolia_base: str = "https://hn.algolia.com/api/v1",
|
||||
timeout: int = 10,
|
||||
jina_token_env: str = "JINA_API_KEY",
|
||||
) -> Tuple[List[Dict], List[str]]:
|
||||
"""并发 enrich 多个 stories。"""
|
||||
errors: List[str] = []
|
||||
if os.environ.get(jina_token_env):
|
||||
print(f"🔑 HN: 已配置 {jina_token_env},Jina Reader 鉴权调用")
|
||||
else:
|
||||
print(f"⚠️ HN: 未配置 {jina_token_env},Jina Reader 匿名调用 (额度受限)")
|
||||
async with aiohttp.ClientSession(headers={"User-Agent": USER_AGENT}) as session:
|
||||
results = await asyncio.gather(
|
||||
*[
|
||||
enrich_story(
|
||||
session,
|
||||
s,
|
||||
top_comments=top_comments,
|
||||
top_l2_per_l1=top_l2_per_l1,
|
||||
comment_max_chars=comment_max_chars,
|
||||
comments_total_chars=comments_total_chars,
|
||||
link_content_max_chars=link_content_max_chars,
|
||||
algolia_base=algolia_base,
|
||||
timeout=timeout,
|
||||
jina_token_env=jina_token_env,
|
||||
)
|
||||
for s in stories
|
||||
],
|
||||
return_exceptions=True,
|
||||
)
|
||||
enriched: List[Dict] = []
|
||||
for r, src in zip(results, stories):
|
||||
if isinstance(r, Exception):
|
||||
errors.append(f"enrich story {src['id']} 失败: {r}")
|
||||
else:
|
||||
enriched.append(r)
|
||||
return enriched, errors
|
||||
@@ -0,0 +1,90 @@
|
||||
"""HN 板块入口。流程:首页 → 轻 LLM 选 K → enrich → 最终 LLM 行文"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Dict, Optional, Tuple
|
||||
|
||||
from src.sections.hackernews.frontpage_scraper import (
|
||||
fetch_frontpage,
|
||||
parse_frontpage_html,
|
||||
)
|
||||
from src.sections.hackernews.item_enricher import enrich_stories
|
||||
|
||||
|
||||
async def run_hackernews_section(
|
||||
config: Dict, now: Optional[datetime] = None
|
||||
) -> Tuple[str, Optional[str]]:
|
||||
cfg = config.get("sections", {}).get("hackernews", {})
|
||||
if not cfg.get("enabled", False):
|
||||
return "", None
|
||||
|
||||
# 延迟 import (Task 16 提供这两个函数)
|
||||
from src.llm import select_ai_related_hn, summarize_hackernews
|
||||
|
||||
timeout = cfg.get("request_timeout", 10)
|
||||
select_k = cfg.get("select_k", 1)
|
||||
top_comments = cfg.get("top_comments", 30)
|
||||
top_l2_per_l1 = cfg.get("top_l2_per_l1", 3)
|
||||
comment_max_chars = cfg.get("comment_max_chars", 2000)
|
||||
comments_total_chars = cfg.get("comments_total_chars", 60000)
|
||||
link_content_max_chars = cfg.get("link_content_max_chars", 50000)
|
||||
algolia_base = cfg.get("algolia_base", "https://hn.algolia.com/api/v1")
|
||||
jina_token_env = cfg.get("jinaTokenName", "JINA_API_KEY")
|
||||
|
||||
# 1. 抓首页
|
||||
print("📥 HN: 抓取首页...")
|
||||
try:
|
||||
html = await fetch_frontpage(timeout=timeout)
|
||||
except Exception as e:
|
||||
return "", f"HN 首页抓取失败: {e}"
|
||||
|
||||
front = parse_frontpage_html(html)
|
||||
print(f"📋 HN: 解析 {len(front)} 条 frontpage stories")
|
||||
if not front:
|
||||
return "", None
|
||||
|
||||
# 2. 轻 LLM 初筛
|
||||
print(f"🤖 HN: 轻 LLM 初筛 (k={select_k})...")
|
||||
selected_ids, select_err = await select_ai_related_hn(front, k=select_k, config=config["llm"])
|
||||
if select_err:
|
||||
return "", f"select_ai_related_hn: {select_err}"
|
||||
if not selected_ids:
|
||||
print("ℹ️ HN: 初筛未挑出 AI 相关内容,跳过")
|
||||
return "", None
|
||||
|
||||
selected = [s for s in front if s["id"] in set(selected_ids)]
|
||||
if not selected:
|
||||
return "", None
|
||||
print(
|
||||
f"🎯 HN: 初筛选出 {len(selected)} 个: "
|
||||
f"{', '.join(s['id'] for s in selected)}"
|
||||
)
|
||||
|
||||
# 3. enrich
|
||||
print(f"🌐 HN: enrich {len(selected)} 个 story (Algolia + 外链)...")
|
||||
enriched, enrich_errors = await enrich_stories(
|
||||
selected,
|
||||
top_comments=top_comments,
|
||||
top_l2_per_l1=top_l2_per_l1,
|
||||
comment_max_chars=comment_max_chars,
|
||||
comments_total_chars=comments_total_chars,
|
||||
link_content_max_chars=link_content_max_chars,
|
||||
algolia_base=algolia_base,
|
||||
timeout=timeout,
|
||||
jina_token_env=jina_token_env,
|
||||
)
|
||||
for e in enrich_errors:
|
||||
print(f"⚠️ HN enrich: {e}")
|
||||
print(
|
||||
f"✅ HN: enrich 成功 {len(enriched)} / 失败 {len(enrich_errors)} / "
|
||||
f"输入 {len(selected)}"
|
||||
)
|
||||
if not enriched:
|
||||
return "", None
|
||||
|
||||
# 4. LLM 总结
|
||||
print(f"🤖 HN: summarize {len(enriched)} 个 enriched story...")
|
||||
md, err = await summarize_hackernews(enriched, config["llm"])
|
||||
if err:
|
||||
return "", f"summarize_hackernews: {err}"
|
||||
print(f"✅ HN: 板块输出 {len(md or '')} chars")
|
||||
return md or "", None
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Insights 板块:基于 RSS/GH/HN 三段成品做跨板块小结"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Dict, Optional, Tuple
|
||||
|
||||
EMPTY_MARKER = "(本次无内容)"
|
||||
|
||||
|
||||
async def run_insights_section(
|
||||
rss_md: str,
|
||||
gh_md: str,
|
||||
hn_md: str,
|
||||
config: Dict,
|
||||
now: Optional[datetime] = None,
|
||||
) -> Tuple[str, Optional[Dict], Optional[str]]:
|
||||
cfg = config.get("sections", {}).get("insights", {})
|
||||
if not cfg.get("enabled", False):
|
||||
return "", None, None
|
||||
|
||||
from src.llm import generate_trend_insights, parse_insights_with_metadata
|
||||
|
||||
sections = {
|
||||
"rss": rss_md or EMPTY_MARKER,
|
||||
"github": gh_md or EMPTY_MARKER,
|
||||
"hackernews": hn_md or EMPTY_MARKER,
|
||||
}
|
||||
|
||||
md, err = await generate_trend_insights(sections, config["llm"])
|
||||
if err:
|
||||
return "", None, f"generate_trend_insights: {err}"
|
||||
|
||||
if now is None:
|
||||
now = datetime.now()
|
||||
date_str = now.strftime("%Y-%m-%d")
|
||||
insights_md, metadata = parse_insights_with_metadata(md or "", date_str)
|
||||
|
||||
return insights_md, metadata, None
|
||||
@@ -0,0 +1,3 @@
|
||||
from src.sections.rss.section import run_rss_section
|
||||
|
||||
__all__ = ["run_rss_section"]
|
||||
@@ -0,0 +1,62 @@
|
||||
"""RSS 板块:沿用现有 collect_entries_for_push + compose_digest 流程
|
||||
|
||||
迁移自 src/main.py::run_push_job 中 RSS digest 部分,行为保持一致。
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Dict, Optional, Tuple
|
||||
|
||||
from src.llm import compose_digest, parse_digest_with_metadata
|
||||
from src.storage import (
|
||||
extract_push_time,
|
||||
get_last_push_file,
|
||||
load_recent_push_content,
|
||||
)
|
||||
|
||||
|
||||
async def run_rss_section(
|
||||
config: Dict, now: Optional[datetime] = None
|
||||
) -> Tuple[str, Optional[Dict], Optional[str]]:
|
||||
"""生成 RSS digest markdown 段(不含 sentinel)。
|
||||
|
||||
返回:
|
||||
(markdown_body, metadata, error)
|
||||
- 无新内容时返回 ("", None, None)
|
||||
- compose_digest 失败时返回 ("", None, error_message)
|
||||
- metadata 字段:title / lead / highlights / profile=default / date
|
||||
早报场景下调用方可丢弃 metadata(由 insights 段覆盖)
|
||||
"""
|
||||
# 延迟 import 避免循环:Task 20-21 后 main.py 会反向 import run_rss_section
|
||||
from src.main import collect_entries_for_push
|
||||
|
||||
last_push_file = get_last_push_file()
|
||||
last_push_time = extract_push_time(last_push_file) if last_push_file else None
|
||||
|
||||
min_score = config["filter"]["min_score"]
|
||||
context_days = config["filter"]["context_days"]
|
||||
|
||||
to_push, context = collect_entries_for_push(
|
||||
last_push_time=last_push_time,
|
||||
context_days=context_days,
|
||||
min_score=min_score,
|
||||
)
|
||||
|
||||
if not to_push:
|
||||
print("ℹ️ RSS: 无新消息")
|
||||
return "", None, None
|
||||
|
||||
push_context_days = config["filter"].get("push_context_days", 5)
|
||||
recent = load_recent_push_content(push_context_days)
|
||||
|
||||
try:
|
||||
raw = await compose_digest(
|
||||
to_push, context, config["llm"], recent_push_context=recent
|
||||
)
|
||||
except Exception as e:
|
||||
msg = f"compose_digest 失败: {e}"
|
||||
print(f"⚠️ RSS: {msg}")
|
||||
return "", None, msg
|
||||
|
||||
date_str = (now or datetime.now()).strftime("%Y-%m-%d")
|
||||
body, metadata = parse_digest_with_metadata(raw or "", date_str)
|
||||
return body, metadata, None
|
||||
Reference in New Issue
Block a user