feat: 军事科技每日资讯推送系统 - Docker部署 + 日志系统 + 数据目录重组

2026-07-12 20:01:02 +08:00
commit 54ca4b1b6a
267 changed files with 47047 additions and 0 deletions
@@ -0,0 +1,140 @@
+"""Probe HN: 抓取首页前 10 条,统计每个 story 评论树的深度/数量/字符数。
+
+用于判断 enrich 策略:是否需要 L2 回复、压缩、过滤短评等。
+
+字符数说明:
+- "raw" = Algolia 返回的 text 字段(HTML)
+- "md"  = html_to_markdown 后的字符数(LLM 实际看到的)
+"""
+
+import asyncio
+import json
+import os
+import sys
+from typing import Dict, List, Tuple
+
+import aiohttp
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from src.processor import html_to_markdown
+from src.sections.hackernews.frontpage_scraper import (
+    fetch_frontpage,
+    parse_frontpage_html,
+)
+
+ALGOLIA = "https://hn.algolia.com/api/v1/items"
+TIMEOUT = 30
+
+
+async def fetch_item(session: aiohttp.ClientSession, item_id: str) -> Dict:
+    async with session.get(
+        f"{ALGOLIA}/{item_id}",
+        timeout=aiohttp.ClientTimeout(total=TIMEOUT),
+    ) as resp:
+        if resp.status != 200:
+            raise RuntimeError(f"{item_id} -> {resp.status}")
+        return await resp.json()
+
+
+def walk_tree(node: Dict, depth: int = 0) -> List[Tuple[int, str]]:
+    """递归遍历评论树,返回 [(depth, text_html), ...]。depth=0 是 story 自身,1 是顶层评论。"""
+    result: List[Tuple[int, str]] = []
+    text = (node or {}).get("text") or ""
+    if depth > 0 and text:
+        result.append((depth, text))
+    for child in (node or {}).get("children") or []:
+        result.extend(walk_tree(child, depth + 1))
+    return result
+
+
+def stats(samples: List[str]) -> Dict:
+    if not samples:
+        return {"count": 0, "raw_chars": 0, "md_chars": 0, "avg_md": 0, "max_md": 0}
+    md_lens = [len(html_to_markdown(s)) for s in samples]
+    raw_lens = [len(s) for s in samples]
+    return {
+        "count": len(samples),
+        "raw_chars": sum(raw_lens),
+        "md_chars": sum(md_lens),
+        "avg_md": round(sum(md_lens) / len(md_lens)),
+        "max_md": max(md_lens),
+    }
+
+
+def by_depth(nodes: List[Tuple[int, str]], d: int) -> List[str]:
+    return [t for depth, t in nodes if depth == d]
+
+
+async def probe_one(
+    session: aiohttp.ClientSession, story: Dict
+) -> Dict:
+    item_id = story["id"]
+    data = await fetch_item(session, item_id)
+    flat = walk_tree(data)
+    l1 = by_depth(flat, 1)
+    l2 = by_depth(flat, 2)
+    l3plus = [t for d, t in flat if d >= 3]
+    all_comments = [t for _, t in flat]
+    return {
+        "id": item_id,
+        "title": story["title"][:60],
+        "page_comments": story["comments"],
+        "L1": stats(l1),
+        "L2": stats(l2),
+        "L3+": stats(l3plus),
+        "ALL": stats(all_comments),
+    }
+
+
+def print_row(label: str, s: Dict):
+    print(
+        f"  {label:5s} count={s['count']:4d}  md_total={s['md_chars']:7d}  "
+        f"avg={s['avg_md']:5d}  max={s['max_md']:6d}"
+    )
+
+
+async def main():
+    print("📥 抓取 HN 首页...")
+    html = await fetch_frontpage(timeout=15)
+    stories = parse_frontpage_html(html)[:10]
+    print(f"📋 取前 10 条 (实际 {len(stories)} 条)\n")
+
+    async with aiohttp.ClientSession() as session:
+        results = await asyncio.gather(
+            *[probe_one(session, s) for s in stories],
+            return_exceptions=True,
+        )
+
+    summary = {"L1": [], "L2": [], "L3+": [], "ALL": []}
+    for r in results:
+        if isinstance(r, Exception):
+            print(f"❌ {r}\n")
+            continue
+        print(
+            f"#{r['id']}  page_comments={r['page_comments']}\n  {r['title']}"
+        )
+        print_row("L1", r["L1"])
+        print_row("L2", r["L2"])
+        print_row("L3+", r["L3+"])
+        print_row("ALL", r["ALL"])
+        print()
+        for key in summary:
+            summary[key].append(r[key])
+
+    print("=" * 70)
+    print("📊 10 条 story 汇总(平均/总和)\n")
+    for key in ["L1", "L2", "L3+", "ALL"]:
+        rows = summary[key]
+        total_count = sum(r["count"] for r in rows)
+        total_md = sum(r["md_chars"] for r in rows)
+        avg_count_per_story = round(total_count / len(rows), 1) if rows else 0
+        avg_md_per_story = round(total_md / len(rows)) if rows else 0
+        print(
+            f"  {key:5s} 总数={total_count:4d}  总md字符={total_md:8d}  "
+            f"平均每story count={avg_count_per_story:5.1f} md_chars={avg_md_per_story:6d}"
+        )
+
+
+if __name__ == "__main__":
+    asyncio.run(main())