{ "schemaVersion": "1.0", "item": { "slug": "common-fetcher", "name": "Common-Fetcher", "source": "tencent", "type": "skill", "category": "开发工具", "sourceUrl": "https://clawhub.ai/lq707904686/common-fetcher", "canonicalUrl": "https://clawhub.ai/lq707904686/common-fetcher", "targetPlatform": "OpenClaw" }, "install": { "downloadMode": "redirect", "downloadUrl": "/downloads/common-fetcher", "sourceDownloadUrl": "https://wry-manatee-359.convex.site/api/v1/download?slug=common-fetcher", "sourcePlatform": "tencent", "targetPlatform": "OpenClaw", "installMethod": "Manual import", "extraction": "Extract archive", "prerequisites": [ "OpenClaw" ], "packageFormat": "ZIP package", "includedAssets": [ "SKILL.md" ], "primaryDoc": "SKILL.md", "quickSetup": [ "Download the package from Yavira.", "Extract the archive and review SKILL.md first.", "Import or place the package into your OpenClaw setup." ], "agentAssist": { "summary": "Hand the extracted package to your coding agent with a concrete install brief instead of figuring it out manually.", "steps": [ "Download the package from Yavira.", "Extract it into a folder your agent can access.", "Paste one of the prompts below and point your agent at the extracted folder." ], "prompts": [ { "label": "New install", "body": "I downloaded a skill package from Yavira. Read SKILL.md from the extracted folder and install it by following the included instructions. Tell me what you changed and call out any manual steps you could not complete." }, { "label": "Upgrade existing", "body": "I downloaded an updated skill package from Yavira. Read SKILL.md from the extracted folder, compare it with my current installation, and upgrade it while preserving any custom configuration unless the package docs explicitly say otherwise. Summarize what changed and any follow-up checks I should run." } ] }, "sourceHealth": { "source": "tencent", "status": "healthy", "reason": "direct_download_ok", "recommendedAction": "download", "checkedAt": "2026-04-30T16:55:25.780Z", "expiresAt": "2026-05-07T16:55:25.780Z", "httpStatus": 200, "finalUrl": "https://wry-manatee-359.convex.site/api/v1/download?slug=network", "contentType": "application/zip", "probeMethod": "head", "details": { "probeUrl": "https://wry-manatee-359.convex.site/api/v1/download?slug=network", "contentDisposition": "attachment; filename=\"network-1.0.0.zip\"", "redirectLocation": null, "bodySnippet": null }, "scope": "source", "summary": "Source download looks usable.", "detail": "Yavira can redirect you to the upstream package for this source.", "primaryActionLabel": "Download for OpenClaw", "primaryActionHref": "/downloads/common-fetcher" }, "validation": { "installChecklist": [ "Use the Yavira download entry.", "Review SKILL.md after the package is downloaded.", "Confirm the extracted package contains the expected setup assets." ], "postInstallChecks": [ "Confirm the extracted package includes the expected docs or setup files.", "Validate the skill or prompts are available in your target agent workspace.", "Capture any manual follow-up steps the agent could not complete." ] }, "downloadPageUrl": "https://openagent3.xyz/downloads/common-fetcher", "agentPageUrl": "https://openagent3.xyz/skills/common-fetcher/agent", "manifestUrl": "https://openagent3.xyz/skills/common-fetcher/agent.json", "briefUrl": "https://openagent3.xyz/skills/common-fetcher/agent.md" }, "agentAssist": { "summary": "Hand the extracted package to your coding agent with a concrete install brief instead of figuring it out manually.", "steps": [ "Download the package from Yavira.", "Extract it into a folder your agent can access.", "Paste one of the prompts below and point your agent at the extracted folder." ], "prompts": [ { "label": "New install", "body": "I downloaded a skill package from Yavira. Read SKILL.md from the extracted folder and install it by following the included instructions. Tell me what you changed and call out any manual steps you could not complete." }, { "label": "Upgrade existing", "body": "I downloaded an updated skill package from Yavira. Read SKILL.md from the extracted folder, compare it with my current installation, and upgrade it while preserving any custom configuration unless the package docs explicitly say otherwise. Summarize what changed and any follow-up checks I should run." } ] }, "documentation": { "source": "clawhub", "primaryDoc": "SKILL.md", "sections": [ { "title": "Common-Fetcher", "body": "统一采集框架，为 AI Agent 提供强大的信息采集能力。" }, { "title": "功能特性", "body": "🕸️ 多源支持: RSS、网页抓取、API 集成\n📊 大规模: 207+ 预配置采集源\n🤖 AI 处理: 自动评分、分类、摘要生成\n⚡ 高性能: <600ms/30 篇文章\n✅ 高可靠: 100% 成功率（已验证解析器）" }, { "title": "🏭 煤炭行业（27 个采集源）", "body": "国家级：发改委、能源局等 6 个\n省级：4 个\n市级：3 个\n数据平台：4 个\n企业自媒体：10 个" }, { "title": "🏠 房地产行业（23 个采集源）", "body": "国家级：住建部、央行等 5 个\n省级：1 个\n市级：3 个\n数据平台：4 个\n企业自媒体：10 个" }, { "title": "🤖 AI 技术（129 个采集源）", "body": "RSS 源：90 个（Hacker News, MIT Tech Review 等）\n网站/自媒体：39 个" }, { "title": "CLI 方式", "body": "# 抓取煤炭行业数据\ncommon-fetcher --industry coal --output daily.md\n\n# 抓取房地产行业数据\ncommon-fetcher --industry realestate --output daily.md\n\n# 抓取 AI 技术数据\ncommon-fetcher --industry ai --output daily.md\n\n# 自定义采集源\ncommon-fetcher --config custom-sources.json --output daily.md" }, { "title": "Node.js API", "body": "import { CommonFetcher } from 'common-fetcher';\n\nconst fetcher = new CommonFetcher({\n industry: 'coal',\n maxArticles: 50,\n timeout: 15000,\n});\n\nconst result = await fetcher.fetch();\nconsole.log(`成功抓取 ${result.totalArticles} 篇文章`);" }, { "title": "OpenClaw 集成", "body": "在 openclaw.json 中配置：\n\n{\n \"skills\": {\n \"common-fetcher\": {\n \"enabled\": true,\n \"industry\": \"coal\",\n \"schedule\": \"0 8 * * *\"\n }\n }\n}" }, { "title": "架构设计", "body": "┌─────────────────────────────────────────┐\n│ Common-Fetcher │\n├─────────────────────────────────────────┤\n│ Source Layer (采集源层) │\n│ ├─ RSS 源 │\n│ ├─ 网页源 │\n│ └─ API 源 │\n├─────────────────────────────────────────┤\n│ Fetcher Layer (抓取层) │\n│ ├─ RSS Fetcher (并发 + 超时) │\n│ ├─ Web Scraper (cheerio) │\n│ └─ Cache Manager │\n├─────────────────────────────────────────┤\n│ Processor Layer (处理层) │\n│ ├─ 去重 (标题/URL 哈希) │\n│ ├─ 时间过滤 │\n│ ├─ AI 评分/分类 │\n│ └─ AI 摘要 │\n├─────────────────────────────────────────┤\n│ Output Layer (输出层) │\n│ ├─ Markdown 报告 │\n│ ├─ JSON 数据 │\n│ └─ 多渠道推送 │\n└─────────────────────────────────────────┘" }, { "title": "性能指标", "body": "解析器文章数/次耗时成功率观点地产网30 篇605ms100%煤炭资源网30 篇455ms100%房天下17 篇579ms100%MIT Tech Review9 篇393ms100%总计86 篇/次~2s100%" }, { "title": "采集源配置", "body": "在 config/ 目录下管理采集源：\n\ncoal-sources.json - 煤炭行业采集源\nrealestate-sources.json - 房地产行业采集源\nai-sources.json - AI 技术采集源" }, { "title": "解析器开发", "body": "自定义解析器参考 src/parsers/ 目录：\n\nexport function parseGuandian(html: string, baseUrl: string): Article[] {\n // 解析逻辑\n}" }, { "title": "已实现 ✅", "body": "4 层架构设计\n6 个解析器（4 个生产就绪）\n207 个采集源配置\nCLI 工具\nNode.js API" }, { "title": "进行中 🔄", "body": "浏览器控制（Playwright）\nAI 验证挑战自动解决\n缓存机制" }, { "title": "计划中 ⏳", "body": "更多行业支持\n分布式抓取\n实时监控告警" }, { "title": "贡献指南", "body": "欢迎提交 Issue 和 PR！\n\nFork 项目\n创建特性分支\n提交改动\n推送到分支\n创建 Pull Request" }, { "title": "许可证", "body": "MIT License" }, { "title": "联系方式", "body": "GitHub: [你的 GitHub]\nMoltbook: ClawdOpenClaw20260223\nEmail: [你的邮箱]\n\nCommon-Fetcher - 为 AI Agent 提供强大的信息采集能力 🕸️" } ], "body": "Common-Fetcher\n\n统一采集框架，为 AI Agent 提供强大的信息采集能力。\n\n功能特性\n🕸️ 多源支持: RSS、网页抓取、API 集成\n📊 大规模: 207+ 预配置采集源\n🤖 AI 处理: 自动评分、分类、摘要生成\n⚡ 高性能: <600ms/30 篇文章\n✅ 高可靠: 100% 成功率（已验证解析器）\n支持的行业\n🏭 煤炭行业（27 个采集源）\n国家级：发改委、能源局等 6 个\n省级：4 个\n市级：3 个\n数据平台：4 个\n企业自媒体：10 个\n🏠 房地产行业（23 个采集源）\n国家级：住建部、央行等 5 个\n省级：1 个\n市级：3 个\n数据平台：4 个\n企业自媒体：10 个\n🤖 AI 技术（129 个采集源）\nRSS 源：90 个（Hacker News, MIT Tech Review 等）\n网站/自媒体：39 个\n使用方法\nCLI 方式\n# 抓取煤炭行业数据\ncommon-fetcher --industry coal --output daily.md\n\n# 抓取房地产行业数据\ncommon-fetcher --industry realestate --output daily.md\n\n# 抓取 AI 技术数据\ncommon-fetcher --industry ai --output daily.md\n\n# 自定义采集源\ncommon-fetcher --config custom-sources.json --output daily.md\n\nNode.js API\nimport { CommonFetcher } from 'common-fetcher';\n\nconst fetcher = new CommonFetcher({\n industry: 'coal',\n maxArticles: 50,\n timeout: 15000,\n});\n\nconst result = await fetcher.fetch();\nconsole.log(`成功抓取 ${result.totalArticles} 篇文章`);\n\nOpenClaw 集成\n\n在 openclaw.json 中配置：\n\n{\n \"skills\": {\n \"common-fetcher\": {\n \"enabled\": true,\n \"industry\": \"coal\",\n \"schedule\": \"0 8 * * *\"\n }\n }\n}\n\n架构设计\n┌─────────────────────────────────────────┐\n│ Common-Fetcher │\n├─────────────────────────────────────────┤\n│ Source Layer (采集源层) │\n│ ├─ RSS 源 │\n│ ├─ 网页源 │\n│ └─ API 源 │\n├─────────────────────────────────────────┤\n│ Fetcher Layer (抓取层) │\n│ ├─ RSS Fetcher (并发 + 超时) │\n│ ├─ Web Scraper (cheerio) │\n│ └─ Cache Manager │\n├─────────────────────────────────────────┤\n│ Processor Layer (处理层) │\n│ ├─ 去重 (标题/URL 哈希) │\n│ ├─ 时间过滤 │\n│ ├─ AI 评分/分类 │\n│ └─ AI 摘要 │\n├─────────────────────────────────────────┤\n│ Output Layer (输出层) │\n│ ├─ Markdown 报告 │\n│ ├─ JSON 数据 │\n│ └─ 多渠道推送 │\n└─────────────────────────────────────────┘\n\n性能指标\n解析器\t文章数/次\t耗时\t成功率\n观点地产网\t30 篇\t605ms\t100%\n煤炭资源网\t30 篇\t455ms\t100%\n房天下\t17 篇\t579ms\t100%\nMIT Tech Review\t9 篇\t393ms\t100%\n总计\t86 篇/次\t~2s\t100%\n配置说明\n采集源配置\n\n在 config/ 目录下管理采集源：\n\ncoal-sources.json - 煤炭行业采集源\nrealestate-sources.json - 房地产行业采集源\nai-sources.json - AI 技术采集源\n解析器开发\n\n自定义解析器参考 src/parsers/ 目录：\n\nexport function parseGuandian(html: string, baseUrl: string): Article[] {\n // 解析逻辑\n}\n\n开发计划\n已实现 ✅\n4 层架构设计\n6 个解析器（4 个生产就绪）\n207 个采集源配置\nCLI 工具\nNode.js API\n进行中 🔄\n浏览器控制（Playwright）\nAI 验证挑战自动解决\n缓存机制\n计划中 ⏳\n更多行业支持\n分布式抓取\n实时监控告警\n贡献指南\n\n欢迎提交 Issue 和 PR！\n\nFork 项目\n创建特性分支\n提交改动\n推送到分支\n创建 Pull Request\n许可证\n\nMIT License\n\n联系方式\nGitHub: [你的 GitHub]\nMoltbook: ClawdOpenClaw20260223\nEmail: [你的邮箱]\n\nCommon-Fetcher - 为 AI Agent 提供强大的信息采集能力 🕸️" }, "trust": { "sourceLabel": "tencent", "provenanceUrl": "https://clawhub.ai/lq707904686/common-fetcher", "publisherUrl": "https://clawhub.ai/lq707904686/common-fetcher", "owner": "lq707904686", "version": "1.0.0", "license": null, "verificationStatus": "Indexed source record" }, "links": { "detailUrl": "https://openagent3.xyz/skills/common-fetcher", "downloadUrl": "https://openagent3.xyz/downloads/common-fetcher", "agentUrl": "https://openagent3.xyz/skills/common-fetcher/agent", "manifestUrl": "https://openagent3.xyz/skills/common-fetcher/agent.json", "briefUrl": "https://openagent3.xyz/skills/common-fetcher/agent.md" } }