From 04001f273a9947aa1159672d86815359b5a8e76b Mon Sep 17 00:00:00 2001 From: admin Date: Sat, 29 Aug 2026 21:18:59 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E6=94=B9blogwatcher-daily=E8=84=9A?= =?UTF-8?q?=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 + blogwatcher-daily/SKILL.md | 128 +++++---- .../blogwatcher-daily.cpython-311.pyc | Bin 23285 -> 0 bytes .../scripts/blogwatcher-daily.py | 266 ++++++++++-------- 4 files changed, 230 insertions(+), 166 deletions(-) delete mode 100644 blogwatcher-daily/scripts/__pycache__/blogwatcher-daily.cpython-311.pyc diff --git a/.gitignore b/.gitignore index 74c13cb..4f12652 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ .DS_Store **/.DS_Store + +__pycache__/ \ No newline at end of file diff --git a/blogwatcher-daily/SKILL.md b/blogwatcher-daily/SKILL.md index 28f8fb8..b04b765 100644 --- a/blogwatcher-daily/SKILL.md +++ b/blogwatcher-daily/SKILL.md @@ -1,12 +1,12 @@ --- name: blogwatcher-daily -description: RSS 订阅监控 + 每日笔记生成。使用 RSSHub + feedparser 抓取 31 个订阅，自动去重并存入 SQLite，新文章追加写入 Markdown 笔记。 -version: 1.0 +description: RSS 订阅监控 + 文章导出。使用 RSSHub + feedparser 抓取 31 个订阅，自动去重并存入 SQLite；可将文章按 txt/markdown/html 三种格式导出。 +version: 1.3 category: custom -tags: [rss, blog, monitoring, automation] +tags: [rss, blog, monitoring, automation, export] metadata: author: Hermes Agent - last_updated: 2025-04-19 + last_updated: 2026-08-29 platform: macos, ubuntu custom_skill_path: /Users/weishen/.hermes/skills/custom/ installation_note: "blogwatcher-daily 脚本在 Mac mini 本地运行（依赖 feedparser）。需要 RSSHub 服务（http://192.168.3.45:1200）访问 YouTube/Bilibili 等被墙源。" @@ -14,7 +14,7 @@ metadata: # Blogwatcher Daily -RSS 订阅监控 + 每日笔记生成自动化。 +RSS 订阅监控自动化，抓取后去重入库，可按 txt / markdown / html 三种格式导出。 ## 依赖 @@ -22,7 +22,7 @@ RSS 订阅监控 + 每日笔记生成自动化。 pip3 install feedparser ``` -> feedparser 是 Python 最成熟的 RSS 解析库，支持 RSS 1.0/2.0/Atom、任意编码、畸形 XML。 +> feedparser 是 Python 最成熟的 RSS 解析库，支持 RSS 1.0/2.0/Atom、任意编码、畸形 XML。HTML 导出使用 Python 标准库 `html.escape`，无第三方依赖。 ## 实际抓取架构（重要） @@ -48,35 +48,13 @@ curl → 目标网站 - 已有的 RSSHub URL → 直接使用 - 其他 → 直接访问原始 URL -``` -YouTube 频道 URL - ↓ 脚本自动识别并转为 RSSHub 格式 -http://192.168.3.45:1200/youtube/channel/{id} - ↓ -curl → RSSHub → YouTube - -普通 RSS（Engadget, Slashdot 等） - ↓ 脚本直接访问（绕过 RSSHub /rss/ 路由） -原始 RSS URL - ↓ -curl → 目标网站 -``` - -**关键发现**：RSSHub 的 `/rss/{url}` 路由不稳定（返回 RSSHub 欢迎页）， -因此普通 RSS 源直接访问，不走 RSSHub 代理。 - -脚本内部 `build_fetch_url()` 根据 URL 类型自动选择路由： -- YouTube → RSSHub -- 已有的 RSSHub URL → 直接使用 -- 其他 → 直接访问原始 URL - ## 目录结构 ``` ~/.hermes/skills/custom/blogwatcher-daily/ ├── SKILL.md # 本文件 ├── scripts/ -│ └── blogwatcher-daily.py # 主脚本 +│ └── blogwatcher-daily.py # 扫描入库 + 订阅管理 + 文章导出（单文件） ├── subscriptions.txt # 订阅列表（name|URL） └── blogwatcher.db # SQLite 数据库（自动创建） ``` @@ -90,7 +68,7 @@ python3 ~/.hermes/skills/custom/blogwatcher-daily/scripts/blogwatcher-daily.py ``` - 扫描所有订阅，新增文章存入数据库 -- 生成今日笔记：`~/Workspace/nexus/ishenwei/blogwatcher/YYYY-MM-DD.md` +- 控制台打印每个订阅新增文章数量及总计 ### 添加订阅 @@ -115,24 +93,13 @@ python3 ~/.hermes/skills/custom/blogwatcher-daily/scripts/blogwatcher-daily.py \ python3 ~/.hermes/skills/custom/blogwatcher-daily/scripts/blogwatcher-daily.py --list ``` -### 仅扫描（不写文件） - -```bash -python3 ~/.hermes/skills/custom/blogwatcher-daily/scripts/blogwatcher-daily.py --scan-only -``` - ### 强制回扫（`--all`） ```bash -# 强制抓取每频道10篇（忽略已读状态），写入独立文件 +# 强制抓取每频道10篇（忽略已读状态） python3 ~/.hermes/skills/custom/blogwatcher-daily/scripts/blogwatcher-daily.py --all - -# 测试模式：只打印，不写文件 -python3 ~/.hermes/skills/custom/blogwatcher-daily/scripts/blogwatcher-daily.py --scan-only --all ``` -- 输出文件：`~/Workspace/nexus/ishenwei/blogwatcher/all-YYYY-MM-DD.md`（**覆盖模式**，每次运行覆盖） -- **不混入**每日报告 `YYYY-MM-DD.md` - 每频道最多10篇 - 用途：历史内容回扫、测试订阅状态 @@ -142,6 +109,51 @@ python3 ~/.hermes/skills/custom/blogwatcher-daily/scripts/blogwatcher-daily.py - python3 ~/.hermes/skills/custom/blogwatcher-daily/scripts/blogwatcher-daily.py --mark-read ``` +### 导出文章（`--export`） + +从 SQLite 里读取文章并输出到 stdout，格式通过 `--export {txt,markdown,html}` 选择： + +```bash +# 今天所有文章 → Markdown +python3 ~/.hermes/skills/custom/blogwatcher-daily/scripts/blogwatcher-daily.py --export markdown + +# 指定日期 → HTML +python3 ~/.hermes/skills/custom/blogwatcher-daily/scripts/blogwatcher-daily.py \ + --export html --date 2026-08-28 > digest.html + +# 全库最新 20 篇 → 纯文本 +python3 ~/.hermes/skills/custom/blogwatcher-daily/scripts/blogwatcher-daily.py \ + --export txt --limit 20 +``` + +**过滤规则**（`--date` 与 `--limit` 的组合语义） + +| 参数组合 | 行为 | +|---------|------| +| `--export FMT`（默认） | 今天（本地时区）的所有文章 | +| `--export FMT --date X` | 指定日期 X 的所有文章 | +| `--export FMT --limit N` | 忽略日期，取**全库最新 N 篇** | +| `--export FMT --date X --limit N` | 日期 X 的最新 N 篇 | + +**参数速查** + +| 参数 | 说明 | +|------|------| +| `--export {txt,markdown,html}` | 触发导出模式并指定格式 | +| `--date YYYY-MM-DD` | 目标日期（本地时区，默认今天） | +| `--limit N` | 文章数上限；单独使用时忽略日期 | + +**输出去向** + +- 正文 → **stdout**（可直接 `> file.md` 重定向） +- 进度 → **stderr**（`📊 N 篇文章 → FMT`） + +**格式细节** + +- `txt`：分频道分组，纯文本无标记，适合终端查看或管道 +- `markdown`：`## 【频道】` + `- [title](link)` + 描述引用块 +- `html`：完整 `` 单文件，内嵌简洁 CSS，标题/URL/描述全部经 `html.escape` 转义（防 XSS） + ## 添加订阅示例 ### YouTube 频道 @@ -182,13 +194,12 @@ python3 ~/.hermes/skills/custom/blogwatcher-daily/scripts/blogwatcher-daily.py \ | 配置项 | 默认值 | |--------|--------| | RSSHub 地址 | `http://192.168.3.45:1200`（可设置 `RSSHUB_URL` 环境变量覆盖） | -| 笔记输出目录 | `~/Workspace/nexus/ishenwei/blogwatcher/` | | 数据库 | `~/.hermes/skills/custom/blogwatcher-daily/blogwatcher.db` | | 订阅列表 | `~/.hermes/skills/custom/blogwatcher-daily/subscriptions.txt` | ## Cron Job 设置 -推荐每天早上 6:00 自动执行： +推荐每天早上 6:00 自动扫描： ```bash cronjob --create \ @@ -200,13 +211,22 @@ cronjob --create \ 执行步骤： 1. 加载 blogwatcher-daily 技能 -2. 运行：python3 ~/.hermes/skills/custom/blogwatcher-daily/scripts/blogwatcher-daily.py -3. 检查输出，确认写入的笔记文件路径 -4. 如果有新文章，简短汇总发给我" +2. 运行扫描：python3 ~/.hermes/skills/custom/blogwatcher-daily/scripts/blogwatcher-daily.py +3. 汇报本次新增文章数量" +``` + +如需**扫描 + 落盘 HTML 摘要**（例如放进 Obsidian vault 或自建 dashboard），可用原生 crontab 串起来： + +```cron +0 6 * * * python3 ~/.hermes/skills/custom/blogwatcher-daily/scripts/blogwatcher-daily.py && \ + python3 ~/.hermes/skills/custom/blogwatcher-daily/scripts/blogwatcher-daily.py \ + --export html > ~/blogwatcher/$(date +\%Y-\%m-\%d).html ``` ## 数据流程 +### 扫描入库（默认命令） + 1. **加载订阅**：从 `subscriptions.txt` 读取所有 name|URL 对 2. **URL 路由**：`build_fetch_url()` 自动判断： - YouTube → 转为 `http://192.168.3.45:1200/youtube/channel/{id}` @@ -216,9 +236,14 @@ cronjob --create \ 4. **解析**：feedparser 提取 title、link、description、pub_date 5. **去重**：按 link 去重，已存在则跳过 6. **存储**：新文章存入 SQLite -7. **输出**：**追加写入** Markdown 笔记（同一文件多次扫描不会覆盖，仅追加新链接） - - 读取已有文件，用正则提取现有链接 → 去重 - - 新文章追加 `## 📦 新增 N 篇` 分隔块 +7. **输出**：控制台打印每个订阅新增数量与总计（不生成文件） + +### 导出（`--export`） + +1. **查询**：`fetch_articles()` 按 `date(fetched_at, 'localtime')` 过滤 + 可选 `LIMIT N` +2. **分组**：`_group_by_channel()` 按 `channel_title` 分桶 +3. **格式化**：`FORMATTERS[fmt]` 分发到 `format_txt` / `format_markdown` / `format_html` +4. **输出**：正文写 stdout，`📊 N 篇文章 → FMT` 进度写 stderr ## 已知问题 @@ -236,4 +261,5 @@ cronjob --create \ - 每次最多取每源 10 篇（TED Talks 等除外，取 20 篇） - 数据库自动创建，无需手动初始化 - OPML 文件可用 Python 解析后批量导入（参考 /wiki-ingest 流程中的 OPML 解析代码） -- **Markdown 追加模式**：同一天多次扫描不会覆盖旧内容，仅追加新链接（去重基于 URL） +- 导出的"今天"按**本地时区**（SQLite `date(fetched_at, 'localtime')`），不是 UTC +- HTML 导出使用 `html.escape` 转义标题/链接/描述，可安全嵌入网页；CSS 内嵌，单文件独立可用 diff --git a/blogwatcher-daily/scripts/__pycache__/blogwatcher-daily.cpython-311.pyc b/blogwatcher-daily/scripts/__pycache__/blogwatcher-daily.cpython-311.pyc deleted file mode 100644 index 4cb6ed423a0a350c8a8b157a99747619bf69b10b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 23285 zcmeHvdvFui-RQ2gl2$9p^4rEX_F@}j0k(NR%qupw!8}ZY?GPMN)p$iP_(4_@5Lc!m zA4vptiXcuZow@h- zJFC~qCQaIJ?)T4K9sTy~p5J-w*>is9_d5HI&Bjr1jMVXFpD{vR2;bwdVZnXM znE)$=EPxAzav}Q+-Bl&zz||rl7htteA>_k7hfn}li-kgfON4m3Sdi0l5W8?g(x>~^Pce(4_ zM|=DF+;yVM?dleL4%Zz_7F8o)QFZPfvGF$2kRnPJKvFAO92M1R2 zvkCw-UUO9N(`VnCK7W4ttbcm!?EUwD{2=t=pKp##z4U{r7q5Qy@ww?sSMOaLz5o8i z-zv4HrmxrSiIrW!PX#Vbz5I*RQvY@V-GacEKlt&52d7`13cga#mt(he?%h?+^U%zX zKAt)kRvY-Yi*omL57Z0}K<7i&sq;TkAM&#suc!|;{rbzFef;6yE~dX*JX#~Vx&_jL z@XOE->LWn`_udPCcHmPFua6q&<=qM|6dt9F0eiz2Q11@ODF_%LZHSKjh z=XJR~QIl8f8$9M3fO{-FTeBz*ic$0Sj*iDkbSNX`oz)5lhqBz>gX&xF9UO4idQN(V zs!rAI8x)Vak9GIB>IPgVz3#eR_hHw-30H6Gz=(3tf7?47olwnC;bD*GSbbgH^7Sif zm#iXp?mMt5a|NNlW<2~p)P}?)uU)OWEdtkuTS6BV?ffF^)EO892)FdB* zLF}$O-Yd8U-3Mw<_V*pw63gN26*eU@6=^bg-7YavU{7M|Km->HJ^a7;-vBOAo&?#Y zs3VEE2h!rlkpzJZW4>93hfUtnk~x9Crqhxon(|nZ*hh@<_@m1hrhHV`6vs^6r2Cqp9<+Pro)X{mT1Ozj*cjr$3th{?(3mVJNvy!t8Pm9__w> zG$uZUx9)9j=xF9U8X9*s^Q|p>+nx@-`Ki|Sj&{CV^z`=hx!inp0vmJn3Vdr@NAtGk zz5L^QTX#3?-OulA-p@C5cI;_wgW`5Kw{voc99Dnwv z=9Y%eT^;-~Z95OSkZudkZjaiSINH|Ey?dc!oE@#Zo7+1Yc0ZozAqRO+AA^hf9w28a#Pn;dpi$$qE{{_B5!4*0v>0Q}SxV?49-HQlJ` zlKrCns!7hQRx+z6R!Ph{nOUbW>mV{|$sXBqWvy(fP%IVxrU=8nyz7NsGn6%({hYEI z%wHmaO8`Zb!K@~-n8;!xfEfa^DQl*G=NC{d)svugaKXTzP|9yZ*D)=_6c2hqM0n;+H zq`i4p^VSYNKCxQ%?%AEBIs3LZ?``JEq~64DX>;hrTqrNflKC|_;Bqh`lDjB_C{bgN zS9HTFs22xMxbZ=eP>IMWM>C-O7$tKKxK4=3Z#Ki1mR0ZY1r;HtylsJFbq1Ter3C zfma37eoylCX>hCKRAecyA@fUlLe-a&*SwTZA_+FNgnp<2$sQM33!;W&V();* z!K$MQhMBr#MQxfEM-8OKB985-C8fbpHs0VU?Q$avfz8E39fekR_i=gteZxp0mzSOulu7rR)VETi81;Oziwsp=8_fxtTHVqQ9gN0t!NoGTY8u z&RNddMr;A{`|G47cZQ-3HAyhZ+N6vn<1_#WGP_h^mr8Smzrr!2hrDohV<)|^Y!&qZ z*HEngWp+cp{@3{ih|WzPZAEnJV46BJ)mRjx) zK{4Z_!zttoo;-F4Y406K7!WECQ5C5u{|{@m##=>E|3Iz!Z}zO%%P@Zb3)DRw{yFgQ zD82Ice)jRykG?nc(!WlukBzy9;7o9F+0^D-f{HT?cTZwE+Var#x{@ zy2#)E==#0Up8-WjRl;`f!B|H1B7t@9haZ5v@YyGy-Wwf{VNAXF;b$Me&SM#B>izdW zeQ@dhKi>@8`}no{pS~O`kFOKm?z&htAST}%$JYM&=2=qA!>qcPK$nmYAK(Cen31?o zn*6RQ%DFu-E!`)2fxENB;N~#7TsYN+-4VqD`d7xVXOQN5fh8)nd)83_F znX90dQ*P#jPL)1rYClE8PcRedCdJ0#L8&Bmc#7gNiaMkRxrYb2XMXaYz|Af92iP{~P(Uu`ljl0B}hG$&^o*~EX z!6DGIch{}1UB*{~FafHFUQm8);2V?r_PXV@%N(F`FdYW%FX$Oig^Oxk(3KL^gvSMn zScBJdsAgT%B)X3Efi4!%jV{#Zy4$?Sgo7Y+WrUA4DkjA^>`}ej-51SQH7QQPb*S6h z=W)V(#Ui5_pnvK)>O4H?_7KI0c`FEN&bB>m%~1=a7Jggex@E3g;kx}>XBeZQaMG3=Y!5G* zSSi`o%C@zNZS4%DGcUifE|On*ssCdCg~8FmJNZj*=Pw=KcVoYlzf{iOrsQu6Y@5s} z3GEB_$~nuFoMkfx$U`zF!RJ`cmk5&g=8Bb)IBd0;q<qZ)$DIuzC+v5ts6C%89oGGI^sh!AIh%qYY)LQnqMp53^L`t^eKjf?cZDY8QR zH;b6YC8pmj(PO-p#&{ixuPWHOME_e$-qtGpZ+RNyRR(|>YKWtNP{RnoM=9HVguw?I zm8L95B~nuPE*7U=&COs7dLfO@4S-E|SpO_F%s}o0{S*v>@kL$IRM|uY>1RU22Ehc> zDMdl=GYDzaOSJn8NwvMmrd|yj2TId&r%^F61Mm}C7txe@HEaY*o1nbVZ;1n*iqlfV zW3<2xo1nF9T7792Q}H0wkf7e;)1I6}q12ItkR(_IY)Jx7y9PiVs3D0RJ~mBUV;@nb zO~?RTc%I~H5>DZP;^n2UBk*SIsc&YO9e6J788fwgA$CLC6DywB^@#SU&zQi|w(Q^g z8R1Qu)1ERb?J05GWyE`pjJw3uuzA+F%TCLU<1Vq(j%8qGkFLs(T-d%a)ikRu&x30;QUj&cY(+^J$ik5X?FaS9 z^FyYMPk(FkGfOrP=?}PqrlM9}Zyf)-`)R#Xp4daw$>qSWhh!vfvy?)G>@wScFvsQp-#PISRa>pdtU zp@|yYfPG9Y{{Z4VgLjSH$G{RRIJl?w!M2)0ML98U%!6eb|jvBDi=c8uS zT06;d%@K{F8aJaxQH>IPA|kkPVuKzrb`W)fEC3~yc|q9q))5kk6wV*x z0aO3U5jdI~;rEv2N&#&}1JW63xvaP#i+hWDG7#{;F@+x(YsnL|0 z;GSIApcFRzomIhuLI1I8|1FU%Rf?qwD_JHKemAf^uwCuHR@qji*Z_wn zIGacWafSoi@3J=35=ZjNBDwP;#g&mle#UIb=VmAXpMzP*@+AVeG{f7&SO#UC7g{J; z$|R;tmHlDJBo17?$clat#?U1SR#r+QuK**a9$`qIeRz;{j2+sQBuI-UPl^TSPqvsSDQ%V%&D)C`zTo(d@p#Zo#QB%8RD~>9_EA55Uj?9f`%}Z<;ij z&zwM2$sh8oWs^fOIXh9-K_mi341%5YDd4sqrtdb~SQIBy2bJ56BgK^SBr1;v+! zE)HGrjr!msrvT_-RugE^-?9JLKB1TSRSLgKy%#Cr-`H_^N7(&?J!5-<&65R1q16|L zMu);rT>NhEyOFZW@DpPv-#B&olw4M)l+}fpNNGj5VeIf51D6Nn(psgoHl&~2^w_WR ze%^Jn>(+u_eCx)y!VAagx2oT%{c){azCtNqG0}6oe7ywWiO!HIQjE&bc}+A{yz}zT ziQF4KQgtH)xn!$SvQ;YC`q!Bp@+^9MxEU%5KPT}oQ|_du&W;l?Dw$n_kWb9sZgk1~ zWNT4VrT%x7OjEV#cS~r18Yw10^UC4%Xr-9hR0gP01u<*W$rg#=#NCqEzoWv7i3}o66^@8|3;EN{#L=7iIV4NT^F&9N2Cb|#|18`%nk{PKQTC<3uHFZ9UF9C*v zs9yr`Qy^(uOJz%$VkrYJ3wsVM!3LOZ00Mf@kO#aY^`V^GmJ-QQGHK0{vR96G0DvG{ zS1Hz2l66%ir!crWxO&DwS?B+SrMQegWbmNTPp}U0iR>WFA*f~%@reaU^dniW0tt;a3uf;GUHGfhr~D{7JFb7 znH0a~N*Y$^f4zcfSY!J2dKzF4bVuSqaS;de7wCudm6HM-9Nbx&!9#18N!6iMgP~6y z{OKxA7|hI-w51k2>rxhMP$?L+%M47*^#@>a7^a7HnwpGca+rKNflW?Wn3l69izb&D z@VQic!7{r=BlYO0>#+U>Y0PZzR4;N4GjF{5Q-1pL3)7ckj;5*}kAwSf@>8F@I(7CH zb&BxSUb2>acH<|GdX8x0hx9!E^pNfuo(%ROqq@$hnzl1gHpn@+vZzz!I}nSST?1ZN z4nYWV=*hf@(!FGAxbam&esxY{qhD>>8udLu_#^z?9{>R1!7@KFB`jcr)*0+X?UWp@ z5jG=WB%^Hg)rzq2rsFM##I06?%&bwEH4?LCGP^j`6kZ@_S1Q@Sz3U8%2smpD(1Det zz^Y(_lu-pCloNV#tXPf9>>`C-B(aMK%EF*Kw0d+%%2=od)vPA5RS-Z47b{ccWsyIK zABiEH33t&U!C&=t^!0Odw&^_NFzhTKLkor~a-V|XA1K?XoiH!~Ojqh*p4>5tZ412l&?)yu-r`Q6}zA%Hf5 zS1n>~57~wLYZms`EEIc9mjc*)pV*wOkeu&!KDxCdC0lFe&O!=h3!Ln zC&vvLYHDf#6Kx_+)%hJr!Sw66UV^)~lLLLY>L2Yj0!l>%|0jR&FSv&N$$x?49_nlO zr-?S>q3)S#qL6r~14g$B5FYAy{=w@QVQOn@IgyYAgNQRdst46dlm!jNA+#_Mu2wt( z3E(sdrUf^c2f%X@l_g-x6QjC*Avnm!8*AB564(SZf~ zPJ9E<{|tZkDV5DN*(FoKWKQwu#tWNAH{Z!wd^>0Hc)gsnM#)(dVDFljg{vj=GBx-c zBaBhvTCQv#+x3ICv9>!UD{q&qoY*0kY*9+KK!O@%riEYz9H;mA8}C}Ok(^{lSPtk0 zkcEIQV$TYgLBh_=_HU150$a=&fE@)8U?Mg!35+-a$6^aUC0XV}h!oEcY!B`|yDP$G z2D3-3ci7_FERZajU7)ZFBogixmA_GYxmGS(tQ3J_z=W!J0B17-reGeadQtF%l(9$+ zK)RrYTwov=Ggfad2sA~k8B%tAhzac*vq1#Hc<%V2Yb9zNplq!VG@|}~w_-=`*|sr~ znJwjS2(1d+l;YZP&xC&5qZF@#TWXLqHw1N1Y_Rg|jzA;iWw}7_S$jYi(4lnE`W3`b zVkqim`@cOoOkJOos`yl29YLRNri4`;M}6!_D&FT7&Qr@Gq}KorjX>IyTby9+PZ8I( z1nQjq5p~Ti=B5d_;+qeM#*Oxgtv)C2VcuqeO+0481^1Kwk}#mO?v2JDAu!FmS7jixk{GbpajqT*u( znLApt!6;(GyZ{v*RE|Rl$IG=dRSOoIiHyryAleHxhh{j4_nNp&t09ma1JaK_dopvv`W+CZxa8wkq4(g zoqF+=`)|K=?}x90m7-021u}MsIB_59Yx@Q9IfxN8(~uQ6IJvjL7N-O4OCJ0mIEEO< z(hc<@I%xL58kabLq*7>m`=T1aYc%Rjebh4+9jC8bee?z0h`pV9mnQ4^ zyWkri)zvN&{|WL&Gl<@*ci>qkS_IrNzTHpw_J`a?WZcocVDr#IBG-~FeBe2HY*6&n zZtNTE>F#rHhDaQ(7Fy|E1O@-qPf0m#Y6vYD@18JT>%DPGs@)9%IB;?XNY)K6nu|DK zMLc>Cg346ldyZ$m?Eq));hx87PBOC&eTPjq}MGQiccmcse1Q!vY^R4(Q z0%TjnQ3S6cK<0(8E#%o9IfQo+FX7z~0;0}C0u$xH)9>-l4rf#AURA%x92x8#0BelP zBc4Fi=Mn5cARxZ!Q15^MPPI`z>;M!|HcIk9;)`(U$c$RR`yaZFFhk;H%*7t+?Q`KS z5~4|nW+vW6%w-{}2mGj6>^`BsohaQGHDW2iKUwh{cOMqVu*NG0euN;wNQv*jr5pQK z73LD~u~cnE>RS~*1Y|qDMi+2)@EXA-_^a3|bz=z^?vR+pGP77=7DHsxo;7mfyziXv z%8v1MQgtlI_VtQ=y?;9>K=S8Z=p60zwsmYkCyO@n-q|Eaqwona` z+=5FxFYXN6rR5EBZljXh2v(taB|pd;%L`;gvWk$&LLSdt9%#6m4eA@(Rve;3$+D=>7lp-?IYj|@0R>;(Zk+|I}ZlUjUo!4wO92Y{T@rsTBwcilBXi!uylATCSU zy;pn2m&#RZm8!LH0pXLAH+?@pb@SAnEl=Lw@}#t{Ti$X|*>X@#1Sqq;0b?X5FTh4} zq30RYCjkJKGAl3GJ6eWZV^*k8vM+!TDO(U^L)jOs5o>O6@5rt@*0S5yvM?iC7b@0; z5($ygs-$9fhBBJr4Ffpa7T6Z-j97u`ToyI~P=jn;7H9!W9aL%qL;#>{&disJ8pBL@ z-&;0_K$ys#IP^h@8V4w6HU^>1uwOyRLG>5tr6ajTQt8(4s&SiAv1Y<^LqFkBDmKC` zHORSJgDp^QsPaNa&=@pAzuU_Gk^zdez~PWyHkT;o64K*gI<7}Ep0$sFqYLWHfE|wm z)R`5c&Mf6#GBp{C>C9$f!7I-7?{Qbcl>5!%ysZ}Se_^)TO~1{i0p41l-&(ByeKFHo zX8L_O4X~v{qen*n2b8|PXaV`{ZF`#7Wx`u)ZlPNV;|*Ep>KYtS5G8Sq0P> zxE-3ZXRFduYi3fN_+f!J%vN*>MsT`MQNL+xJeL7hLOu96Fq&G~RBN9vjh%*i%(Lt9 z>41@WFs#v}fCryJuq4;exiso&z7;8qn6}Ro`U#UiQ`1(P77rM+CiK4XQ6jRvMj<1W z2^s*h15B$OnATZob1|{-OivASsa~6B!Cym2X(jk!aH$#)xJqF+;VMPn0arQ6-gq0{ z%6o}71AE5wbBj*E7#e$NA9r=uNCKuWPi5mkucF47k^P$9$tSv;0?qqOWANd97W!C# z^RA?&Kwtikvl(B<$24us7p1=wO`D2+4Ach9XgZcpFW!bNCK9~6YpK3M>9*Tpizhce zgJ5fTLU5bgiz(I5RG6RYVVd;2(awswMF62x+g^BvGI$29_-6G~BwOaQ3cNNQm}#5O z7M`1#o+VVp4J<@0U^z`C5f))FMbjD>i-OV<)jhPMRr@HI7${Gvr05n8+`hQzE3 zH~mxZd{6B<-ivHB551sT8&F~BMfQ5;jnP+P-|MJX?F_034@Vc&{>Sq_#ch+Yi|1cq zXAk!PeUCJf57HhY`8)=Zgyd5+OsBrvvpTIDq#cdd3yDNj+)jyaK_L#BY{>-!22LoI ziAL?IMidgq;m%Nr8#)u;|6HQaA>^5t0~+wm((03HxZ(LjXOAQtmW~WagI?vxak=w^ z(s=?BCxiqN5rXD&AyfJIS}0Ss)j1b&}UP2u%-rh=H!CAp))VCghLOC%jc^Iakoa zs(z>hL}D^;Y5 z26uBOB(SI%uX+Z(;9lp#LXf8;x@=YM>m7j2UEuj=BW`$PPYZ0wRY%(o5%oF(G$@mO zn{GSAU_PWAF^P?hk3sRbfU*q--FA2q7<7N(C<7O$KE3D69{-+6G{=_9OohT!NK6G> z##7QQZTgs{ZKcevQrJ}zyNVdWKr=37w5UO5w<+v4iQT5!;4E2Z*NvpWvgViBw!1A?^-egn}|KUTDCeAt3$FnK<>DkUl6cFs_LY|lOaQ> zCnSb?#<&odm|GC}mG@Trt%JAPe>Vgd^6Hby>XQN6-CXbwWX;6{1ezwTg`wP#JG^>q zXnfO+${Qzc*4)ayb?85rfZt1Kw^FcpylJ9HS-SCaJ#E_z48LtNF!#32cdaFNtQEJd z6_Lz&p-nS-{lbj9+4(aDIDby*ZT2q_fN389OW6uT%VldxU}wad7s>(&2pkrtaC>xT z2myqj!ehT%bgf9e`W5V$#=8(?_DO|(QX=6l?6%$U!j99s&g}B<0#Q2coi~kxS+q$F zGSjRu%@Wgm*94NJ)!+#2Ag2g!(>9xzPi2)$6)WYeRZ7;XK;vY=ywMYvd>4IJcSy^e z|Ka2ZzMq}?;MAS9ZMWC9-5QeDb}4JSAVCds!MBxyZ-d7m$c2Yt8$?zm@!6zpCF3-V zh)br6rjYJ}Wz-V1fXrk=aViEtX0nxF7pB}xrY2*m%rpyY3SA$3VLAJIi))t9` zse)?Bv0g5yR|@K(VTe~3ehL5t5G)nzGKqwU4dhGO;2=YAk<3;rY_-Hz-?bD=Ofhj) zxK*3qnyLSNCexa0`h5Wnum|Q^;xIxCiN_bO1Jp`g$xr@}kM+RpOiEzn(>uU<%|39S z0q!BVW_p`}9Pn2U@=b=hbFi?d3BG7{&HA--*r0|Cr~%rX^r8QX8tj0d`3>;1glr+_ zTJF~_KTpU9Yz-&+6|E_N))da!8hjJ84lQqtdeNkb2>jJkuDZdSDQO52GgEi5X-fe` zG3*)xY_#)}L)?{mMWj6)G0V(p;t}R)IZn4V)xZRtd67>K{I4E-KvQ?LrSrRtX8(jP z-58T-Wny=P*yurxXzb8F6Ue@}^2ZLJLEA!jABHflM6(WUDIiHLNTLYE>7~T_EnzAX z3q1>y_|S*Q(UC(axi&w2OknOu;+M{i?=9$ySVBKUn8yL<4HO;P5stM3>Z*-_8yQPY^VhTY9kJ?wk+VyCA%mVv6> zFwiYN3r_z1E>HLIZc+S?SevD$#vT8@oBGw0k6)bg18<0l25<5SH`Na=<+BorueV|I z(Yy}L7rZF0PmjC|yWi0+G4=*uy&sO6-Med=nqWhq+5)zw2DZE%_8uhr{O-U1%X^=^ zpV+KBHZgU@PoA=`&wCV#n^Vl0iiGf8*AkA zWGd%pghEudPgT(~{oTC-VifLv51Bm+P)t8HY0Fb=mD2J~$@Zjdds4AI2`XWRyTcUR zW(q=VxN_VlH9jdZ1v0ZwVfIPPzGSK?{M?fk15P!67yIx#WW7zo1r%L>LB@Z_?F%g z-IoNyO^DyMWZkipfG4DEDOD_`{wC0Uv(_`;!PUDwl3z4>G+?=FvYdGynaVY=;Tu>* zg)Ncd;jY;>0xH)W!`8rP_ZwBvdoUjJYLGKh7blf zOFRd*PUS5gUn%F+DS34PZjzlBstj{7yF_7^Nb!Khmx2p0RBg{YdvfGtfSF|Tf`ZHz zD{QgE7UM@$xO3cD>xeaACEUgenO&){E2Vgd6~SQZGefJ!)=d~B7Cx-9Mq$@LL=C{> znX^bDBeAPxcD2H;hKL%7?1<73juNNSLBkgu#mz8S9J-bc2YiFf=@bTgKujXGEddfl z6t%_wir`xaaQ(zN*Wah^Gs=L^PxSR3tc6{)__fEB9lF+73fUl>vMsk6i@{|=M3F9P z>OKhHbp)f<$9To5-tcHii<;Yaw(i6JV_fO)Dmf~5TTYzbA<@S z`N@x%jYjsJcb;>G_Kxj)!+F^`{**j_jWU0Yx+B@ofqT!$w)1<=?FsFWt&0@vB4{vY z9a(gK>A9uB=cV!$GPhFUR{Bkmyn@kvmkwM!5Ozp4kI8urN?wCRWk+oGk$rb;Ww&i* z62D!xwJNq&KW?i6W8jhV1Lp=p2c#v>Wz>4nLT~is;N4Dgo9pG#PXGV_ diff --git a/blogwatcher-daily/scripts/blogwatcher-daily.py b/blogwatcher-daily/scripts/blogwatcher-daily.py index 7cd8d66..752ec25 100644 --- a/blogwatcher-daily/scripts/blogwatcher-daily.py +++ b/blogwatcher-daily/scripts/blogwatcher-daily.py @@ -4,20 +4,23 @@ Blogwatcher Daily - RSS Feed 监控脚本 放在 ~/.hermes/skills/research/blogwatcher-daily/scripts/blogwatcher-daily.py Usage: - python3 blogwatcher-daily.py # 扫描所有订阅，写入今日笔记 - python3 blogwatcher-daily.py --list # 列出所有订阅 + python3 blogwatcher-daily.py # 扫描所有订阅，新增文章存入数据库 + python3 blogwatcher-daily.py --list # 列出所有订阅 python3 blogwatcher-daily.py --add "频道名" "RSS URL" # 添加订阅 - python3 blogwatcher-daily.py --scan-only # 只扫描不写入文件 - python3 blogwatcher-daily.py --mark-read # 标记所有文章为已读 + python3 blogwatcher-daily.py --mark-read # 标记所有文章为已读 + python3 blogwatcher-daily.py --export markdown # 把今天的文章导出为 Markdown + python3 blogwatcher-daily.py --export html --date 2026-08-28 + python3 blogwatcher-daily.py --export txt --limit 20 """ import os import re +import sys import sqlite3 import argparse import feedparser -from datetime import datetime -from html import unescape +from datetime import date +from html import escape, unescape from urllib.request import Request, urlopen from urllib.error import URLError, HTTPError import ssl @@ -27,7 +30,6 @@ SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) SKILL_DIR = os.path.dirname(SCRIPT_DIR) # .../research/blogwatcher-daily DB_PATH = os.path.join(SKILL_DIR, "blogwatcher.db") SUBSCRIPTIONS_FILE = os.path.join(SKILL_DIR, "subscriptions.txt") -OUTPUT_DIR = os.path.expanduser("~/Workspace/nexus/ishenwei/blogwatcher") # RSSHub 基础地址（可配置） RSSHUB_BASE = os.environ.get("RSSHUB_URL", "http://192.168.3.45:1200") @@ -235,99 +237,10 @@ def list_subscriptions(): print(f" [{i}] {sub['name']}") print(f" {sub['url']}\n") -# ========== Markdown 输出 ========== -def generate_markdown(results, date_str=None): - """生成 Markdown 格式""" - if date_str is None: - date_str = datetime.now().strftime("%Y-%m-%d") - - md = f"# Blogwatcher Daily - {date_str}\n\n" - md += f"_生成时间: {datetime.now().strftime('%H:%M:%S')}_\n\n" - md += "---\n\n" - - if not results: - md += "_今日无新文章_\n" - return md - - # 按频道分组 - channels = {} - for item in results: - ch = item['channel'] - if ch not in channels: - channels[ch] = [] - channels[ch].append(item) - - for channel, items in channels.items(): - md += f"## 【{channel}】\n\n" - for item in items: - md += f"- [{item['title']}]({item['link']})\n" - if item['description']: - md += f" {item['description'][:150]}...\n" - md += "\n" - md += "---\n\n" - - return md - -def save_daily_report(new_results, date_str=None): - """保存每日报告（追加模式，避免覆盖）""" - if date_str is None: - date_str = datetime.now().strftime("%Y-%m-%d") - - os.makedirs(OUTPUT_DIR, exist_ok=True) - filepath = os.path.join(OUTPUT_DIR, f"{date_str}.md") - - # 读取已有内容，提取已有链接避免重复 - existing_links = set() - if os.path.exists(filepath): - with open(filepath, 'r') as f: - existing_content = f.read() - # 简单提取已有链接（用于去重），去掉末尾的 ) 等标点 - import re as _re - raw_links = _re.findall(r'https?://\S+', existing_content) - existing_links = set() - for l in raw_links: - # 去掉末尾的 ) 等非 URL 字符 - while l and l[-1] in '),;:': - l = l[:-1] - existing_links.add(l) - else: - existing_content = "" - - # 如果没有新结果，且文件已存在，直接返回 - if not new_results and existing_content: - return filepath - - # 生成新内容的 header - new_md = "" - new_articles = [a for a in new_results if a['link'] not in existing_links] - - if new_articles: - new_md += f"\n## 📦 新增 {len(new_articles)} 篇 ({datetime.now().strftime('%H:%M:%S')})\n\n" - channels = {} - for item in new_articles: - ch = item['channel'] - if ch not in channels: - channels[ch] = [] - channels[ch].append(item) - for channel, items in channels.items(): - new_md += f"### 【{channel}】\n\n" - for item in items: - new_md += f"- [{item['title']}]({item['link']})\n" - if item['description']: - new_md += f" {item['description'][:150]}...\n" - new_md += "\n" - - # 追加写入 - with open(filepath, 'a') as f: - f.write(new_md) - - return filepath - # ========== 主流程 ========== -def scan_all(force_all=False, write_file=True): +def scan_all(force_all=False): """扫描所有订阅 force_all: True 则忽略已读状态，每个频道强制抓10篇 - write_file: True 则写入文件 """ print("=" * 50) print("Blogwatcher Daily Scan") @@ -375,32 +288,149 @@ def scan_all(force_all=False, write_file=True): print("-" * 50) print(f"📊 扫描完成: 共发现 {new_count} 篇新文章\n") - if all_new_articles and write_file: - if force_all: - # --all 模式：写入独立文件（不追加日常报告） - md = generate_markdown(all_new_articles) - filepath = os.path.join(OUTPUT_DIR, f"all-{datetime.now().strftime('%Y-%m-%d')}.md") - with open(filepath, 'w') as f: - f.write(md) - print(f"📝 已写入（force-all 模式）: {filepath}") - else: - filepath = save_daily_report(all_new_articles) - print(f"📝 已写入: {filepath}") - elif not all_new_articles: + if not all_new_articles: print("📭 今日无新文章") return all_new_articles +# ========== 导出 ========== +def fetch_articles(target_date=None, limit=None): + """从 articles 表读取；可选按本地日期过滤（date(fetched_at,'localtime')）+ 可选限制数量""" + if not os.path.exists(DB_PATH): + print(f"❌ 数据库不存在: {DB_PATH}", file=sys.stderr) + return [] + + conn = sqlite3.connect(DB_PATH) + conn.row_factory = sqlite3.Row + + where = "" + params = [] + if target_date: + where = "WHERE date(fetched_at, 'localtime') = ?" + params.append(target_date) + + query = f""" + SELECT channel_title, title, link, description, pub_date, fetched_at + FROM articles + {where} + ORDER BY fetched_at DESC, channel_title + """ + if limit: + query += " LIMIT ?" + params.append(limit) + + cursor = conn.execute(query, params) + rows = [dict(r) for r in cursor.fetchall()] + conn.close() + return rows + +def _group_by_channel(articles): + grouped = {} + for a in articles: + ch = a["channel_title"] or "Unknown" + grouped.setdefault(ch, []).append(a) + return grouped + +def format_txt(articles, label): + lines = [f"Blogwatcher Daily — {label}", "=" * 50, ""] + if not articles: + lines.append("(无文章)") + return "\n".join(lines) + "\n" + lines.append(f"共 {len(articles)} 篇文章") + lines.append("") + for channel, items in _group_by_channel(articles).items(): + lines.append(f"[{channel}]") + for it in items: + title = (it["title"] or "Untitled").strip() + link = (it["link"] or "").strip() + lines.append(f" - {title}") + if link: + lines.append(f" {link}") + desc = (it["description"] or "").strip() + if desc: + lines.append(f" {desc[:200]}") + lines.append("") + return "\n".join(lines) + "\n" + +def format_markdown(articles, label): + md = f"# Blogwatcher Daily — {label}\n\n" + if not articles: + return md + "_无文章_\n" + md += f"共 **{len(articles)}** 篇文章。\n\n---\n\n" + for channel, items in _group_by_channel(articles).items(): + md += f"## 【{channel}】\n\n" + for it in items: + title = (it["title"] or "Untitled").strip() + link = (it["link"] or "").strip() + md += f"- [{title}]({link})\n" + desc = (it["description"] or "").strip() + if desc: + md += f" > {desc[:200]}\n" + md += "\n" + return md + +def format_html(articles, label): + parts = [ + "", + '', + f"Blogwatcher Daily — {escape(label)}", + "", + "", + f"

Blogwatcher Daily — {escape(label)}

", + ] + if not articles: + parts.append("

无文章

") + return "\n".join(parts) + "\n" + parts.append(f"

共 {len(articles)} 篇文章。

") + for channel, items in _group_by_channel(articles).items(): + parts.append(f"

【{escape(channel)}】

\n
    ") + for it in items: + title = (it["title"] or "Untitled").strip() + link = (it["link"] or "").strip() + parts.append(f'
  • {escape(title)}') + desc = (it["description"] or "").strip() + if desc: + parts.append(f"
    {escape(desc[:200])}
    ") + parts.append("
    • ") + parts.append("
") + parts.append("") + return "\n".join(parts) + "\n" + +FORMATTERS = { + "txt": format_txt, + "markdown": format_markdown, + "html": format_html, +} + +def export_articles(fmt, target_date, limit): + articles = fetch_articles(target_date, limit) + if target_date: + label = target_date + elif limit: + label = f"最新 {limit} 篇" + else: + label = "全部" + print(f"📊 {len(articles)} 篇文章 → {fmt}", file=sys.stderr) + sys.stdout.write(FORMATTERS[fmt](articles, label)) + # ========== CLI ========== def main(): parser = argparse.ArgumentParser(description='Blogwatcher Daily RSS 监控脚本') parser.add_argument('--list', '-l', action='store_true', help='列出所有订阅') parser.add_argument('--add', nargs=2, metavar=('NAME', 'URL'), help='添加订阅') - parser.add_argument('--scan-only', action='store_true', help='仅扫描不写入文件') parser.add_argument('--mark-read', action='store_true', help='标记所有为已读') - parser.add_argument('--date', '-d', help='指定日期 (YYYY-MM-DD)') parser.add_argument('--rsshub', help='设置 RSSHub 地址') - parser.add_argument('--all', action='store_true', help='忽略已读状态，强制抓取每频道最新10篇（写入 all-YYYY-MM-DD.md，不追加日常报告）') + parser.add_argument('--all', action='store_true', help='忽略已读状态，强制抓取每频道最新10篇') + parser.add_argument('--export', choices=['txt', 'markdown', 'html'], + help='导出文章为指定格式，写到 stdout（可搭配 --date/--limit）') + parser.add_argument('--date', help='导出的目标日期 YYYY-MM-DD（默认今天，仅 --export 有效）') + parser.add_argument('--limit', type=int, + help='导出的文章数上限；单独使用时忽略日期取全库最新 N 篇（仅 --export 有效）') args = parser.parse_args() @@ -426,11 +456,17 @@ def main(): conn.close() print("✅ 已标记所有文章为已读") - elif args.scan_only: - scan_all(force_all=args.all, write_file=False) + elif args.export: + if args.date: + target_date = args.date + elif args.limit: + target_date = None + else: + target_date = date.today().isoformat() + export_articles(args.export, target_date, args.limit) else: - scan_all(force_all=args.all, write_file=True) + scan_all(force_all=args.all) if __name__ == "__main__": main()