zjs105910/workbuddy-market / bundled/novel-writing-suite/skills/story-long-scan
story-long-scan
Verified长篇网文扫榜。分析起点、番茄、晋江等平台排行榜数据,提炼市场趋势与热门题材。触发方式:/story-long-scan、/长篇扫榜、「长篇什么火」「起点排行」。
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 10.6 KB |
| references/genre-trends.md | markdown | 7.0 KB |
| references/platform-ciweimao.md | markdown | 1002 B |
| references/platform-fanqie.md | markdown | 4.1 KB |
| references/platform-jjwxc.md | markdown | 1.6 KB |
| references/platform-qidian.md | markdown | 2.2 KB |
| references/platform-qimao.md | markdown | 1.3 KB |
| references/publishing-guide.md | markdown | 5.1 KB |
| references/reader-profiling.md | markdown | 3.9 KB |
| references/topic-decision.md | markdown | 4.4 KB |
| scripts/aggregate-rank.js | script | 25.8 KB |
| scripts/cdp-utils.js | script | 8.4 KB |
| scripts/ciweimao-rank-scraper.js | script | 8.8 KB |
| scripts/fanqie-rank-scraper.js | script | 25.6 KB |
| scripts/jjwxc-rank-scraper.js | script | 16.1 KB |
| scripts/qidian-rank-scraper.js | script | 22.0 KB |
| scripts/qimao-rank-scraper.js | script | 16.4 KB |
scripts/jjwxc-rank-scraper.js
View on GitHub#!/usr/bin/env node
/**
* 晋江文学城排行榜采集脚本
*
* 配合 browser-cdp skill 使用。先启动 Chrome CDP 环境,再运行本脚本。
* 采集策略:
* 1) topten.php 列表页(纯文本,频道名直接出现,书名/作者交替行)解出频道分组。
* 2) 从书名 anchor 取 novelid,逐本进 onebook.php 详情页补采核心指标
* (收藏数/营养液/积分/字数/状态),满足规范对晋江的硬性要求。
* 晋江页面为 gb18030 编码:详情页用 fetch+arrayBuffer+TextDecoder('gb18030') 解码
* (同步 XHR 的 responseText 会按 UTF-8 解码导致中文乱码)。
* 详情采集默认开启但有上限(每频道前 N + 总量上限),用 --list-only 可只采列表。
*
* 用法:
* node jjwxc-rank-scraper.js --type 12 # 收入金榜(默认含详情)
* node jjwxc-rank-scraper.js --type 12 --top 15 # 每频道补采前 15 本
* node jjwxc-rank-scraper.js --type 12 --detail-limit 60 # 详情总量上限 60
* node jjwxc-rank-scraper.js --type 12 --list-only # 只采列表(快,无核心指标)
* node jjwxc-rank-scraper.js --type all # 全部榜单
*
* 前置:
* node {SKILL_DIR}/browser-cdp/scripts/setup-cdp-chrome.js 9222
*/
const fs = require("fs");
const path = require("path");
const { ab, sleep, evalJSONBase64, getArg, localDateStamp, runCli } = require("./cdp-utils");
const BASE_URL = "https://www.jjwxc.net/topten.php";
const RANK_TYPES = [
{ id: "12", label: "收入金榜" },
{ id: "7", label: "月榜" },
{ id: "8", label: "季度榜" },
{ id: "14", label: "完结金榜" },
{ id: "15", label: "新手金榜" },
{ id: "17", label: "千字金榜" },
];
// 详情请求批大小(async fetch 并发,整批控制在 ab() 20s 超时内)
const DETAIL_CHUNK = 6;
/** 连通性 + 页面就绪自检 */
function probePage(port) {
return evalJSONBase64(
port,
"JSON.stringify({host:location.host,len:(document.body&&document.body.innerText||'').length})"
);
}
// ---------------------------------------------------------------------------
// 列表页提取
// ---------------------------------------------------------------------------
/**
* 提取晋江榜单数据(频道分组 + 书名/作者交替),并从书名 anchor 附上 novelid。
*/
function extractRankData(port) {
const js =
"JSON.stringify((function(){" +
"var result={channels:[]};" +
"var text=document.body.innerText||'';" +
"var lines=text.split(/\\n/).map(function(l){return l.trim()}).filter(Boolean);" +
// 书名 anchor → novelid(排除霸王票"X向《书名》投了Y"这类记录)
"var idMap={};" +
"Array.from(document.querySelectorAll('a')).forEach(function(a){" +
" var hm=(a.getAttribute('href')||'').match(/novelid=([0-9]+)/);if(!hm)return;" +
" var t=(a.innerText||a.textContent||'').trim();" +
" if(!t||t.indexOf('向《')>-1||t.indexOf('投')>-1||t.length>30)return;" +
" if(!idMap[t])idMap[t]=hm[1];" +
"});" +
"var channels=['古代言情','现代言情','古代穿越','现代都市纯爱','现代幻想纯爱','古代纯爱','衍生纯爱','幻想现言','奇幻言情','未来游戏悬疑','百合','无CP','二次元言情','衍生言情','衍生无cp','未来幻想纯爱','原创轻小说','多元'];" +
"var channelSet={};channels.forEach(function(c){channelSet[c]=true});" +
"var curChannel='';" +
"var channelBooks={};" +
"var expectTitle=true;" +
"var pendingTitle='';" +
"for(var i=0;i<lines.length;i++){" +
" var line=lines[i];" +
" if(/上榜天数记录|榜单说明/.test(line)){break}" +
" if(/^(免费强推|vip强推|新晋作者|月榜|季榜|半年榜|长生殿|总分榜|字数榜|收入金榜|霸王票|霸王总榜|勤奋指数|完结金榜|新手金榜|栽培月榜|驻站|完结高分|千字金榜|完结全订榜)$/.test(line)){continue}" +
" if(line.length>30&&line.indexOf('·')>0)continue;" +
" if(channelSet[line]){" +
" if(curChannel&&channelBooks[curChannel])channelBooks[curChannel]._finished=true;" +
" curChannel=line;" +
" if(!channelBooks[curChannel])channelBooks[curChannel]={books:[]};" +
" expectTitle=true;pendingTitle='';continue" +
" }" +
" if(!curChannel)continue;" +
" if(expectTitle){" +
" pendingTitle=line;expectTitle=false" +
" }else{" +
" if(pendingTitle){" +
" channelBooks[curChannel].books.push({title:pendingTitle,author:line,novelid:idMap[pendingTitle]||''})" +
" }" +
" expectTitle=true;pendingTitle=''" +
" }" +
"}" +
"for(var name in channelBooks){" +
" if(channelBooks[name].books.length>0){" +
" result.channels.push({name:name,books:channelBooks[name].books})" +
" }" +
"}" +
"return result" +
"})())";
return evalJSONBase64(port, js);
}
// ---------------------------------------------------------------------------
// 详情页提取(gb18030 + itemprop 微数据)
// ---------------------------------------------------------------------------
/** 构建:一批 novelid 的详情解码 JS(async fetch + TextDecoder,返回 JSON 字符串) */
function buildDetailJS(ids) {
return `Promise.all(${JSON.stringify(ids)}.map(function(id){
return fetch('/onebook.php?novelid='+id)
.then(function(r){return r.arrayBuffer()})
.then(function(b){
var h=new TextDecoder('gb18030').decode(new Uint8Array(b));
function prop(n){var m=h.match(new RegExp('itemprop="'+n+'"[^>]*>([^<]*)<'));return m?m[1].trim():'';}
var status=(h.match(/itemprop="updataStatus"[^>]*>\\s*([^<\\s]{1,6})/)||[,''])[1]
||(h.match(/(连载中|已完结|完结)/)||[,''])[1]||'';
return {id:id,collect:prop('collectedCount'),nutrition:prop('nutritionCount'),
score:prop('scoreCount'),review:prop('reviewCount'),words:prop('wordCount'),status:status};
})
.catch(function(e){return {id:id,err:String(e&&e.message||e)}});
})).then(function(arr){var map={};arr.forEach(function(o){map[o.id]=o});return JSON.stringify(map);})`;
}
/**
* 分批解码详情,合并结果。
* 每批单独 try/catch:整批的并发 fetch 贴着 ab() 的 20s 超时线,一次瞬时超时(或
* 返回非 JSON)只该丢这 6 本,不能连坐后面几十本,更不能把已解析好的列表带走。
*/
function fetchDetails(port, ids) {
const map = {};
let failedChunks = 0;
for (let i = 0; i < ids.length; i += DETAIL_CHUNK) {
const chunk = ids.slice(i, i + DETAIL_CHUNK);
try {
const part = evalJSONBase64(port, buildDetailJS(chunk)) || {};
Object.assign(map, part);
} catch (chunkErr) {
failedChunks++;
console.error(
` ⚠ 详情批次 ${Math.floor(i / DETAIL_CHUNK) + 1}(${chunk.length} 本)获取失败,跳过: ${chunkErr.message}`
);
}
sleep(400);
}
if (failedChunks > 0) {
console.error(` ⚠ 共 ${failedChunks} 个详情批次失败,这部分书只有列表数据。`);
}
return { map, failedChunks };
}
// ---------------------------------------------------------------------------
// 格式化
// ---------------------------------------------------------------------------
function fmtWan(s, unit) {
if (s == null || s === "") return "";
const n = parseInt(String(s).replace(/[^0-9]/g, ""), 10);
if (isNaN(n)) return "";
if (n >= 10000) return (n / 10000).toFixed(1) + "万" + (unit || "");
return n + (unit || "");
}
// ---------------------------------------------------------------------------
// 主流程
// ---------------------------------------------------------------------------
const args = process.argv.slice(2);
const PORT = parseInt(getArg(args, "--port") || "9222", 10);
const OUTDIR = getArg(args, "--outdir") || ".";
const RANKTYPE = getArg(args, "--type") || "12";
const CHANNEL = getArg(args, "--channel") || "0";
const TOP = parseInt(getArg(args, "--top") || "10", 10);
const DETAIL_LIMIT = parseInt(getArg(args, "--detail-limit") || "100", 10);
const LIST_ONLY = args.includes("--list-only");
function scrapeRank(port, rankTypeId, channelId) {
const rt = RANK_TYPES.find((r) => r.id === rankTypeId);
if (!rt) {
console.log(` ⚠ 未知榜单类型: ${rankTypeId}`);
return null;
}
const url = `${BASE_URL}?orderstr=${rankTypeId}&t=${channelId}`;
const chLabel = channelId === "0" ? "全站" : `频道${channelId}`;
console.log(`\n→ 采集 晋江${rt.label}(${chLabel})...`);
console.log(` URL: ${url}`);
let data;
try {
ab(port, "open", url);
sleep(4000);
// 连通性自检:CDP 未起/被重定向时给可操作报错,而非误报"结构已变"
const probe = probePage(port);
if (!probe) {
console.error(
` ✗ CDP 无响应。请确认已用 browser-cdp 启动 Chrome(端口 ${port}),且 agent-browser 可用。`
);
return null;
}
if (probe.host && probe.host.indexOf("jjwxc") === -1) {
console.error(` ✗ 当前页面非晋江(host=${probe.host}),可能被重定向,已跳过。`);
return null;
}
data = extractRankData(port);
if (!data?.channels?.length) {
console.error(`[jjwxc] 采集失败:未解析到榜单(页面结构可能变动或未加载)。请人工打开 ${url} 确认。`);
return null;
}
} catch (err) {
console.error(`[jjwxc] ${rt.label} 页面加载或提取出错: ${err.message}`);
return null;
}
let totalBooks = 0;
data.channels.forEach((ch) => {
totalBooks += ch.books.length;
const authors = new Set(ch.books.map((b) => b.author));
if (ch.books.length >= 5 && authors.size / ch.books.length < 0.2) {
console.log(` ⚠ ${ch.name}:${ch.books.length} 本只有 ${authors.size} 个唯一作者,可能提取有误`);
}
});
console.log(` ✓ 列表:${data.channels.length} 个频道,共 ${totalBooks} 本`);
// 选取每频道前 TOP 本(有 novelid 的)补采详情,受 DETAIL_LIMIT 总量约束
let detailMap = {};
let detailPlanned = 0;
let detailOk = 0;
let detailFailedChunks = 0;
if (!LIST_ONLY) {
const picked = [];
for (const ch of data.channels) {
let n = 0;
for (const b of ch.books) {
if (picked.length >= DETAIL_LIMIT) break;
if (n >= TOP) break;
if (b.novelid) { picked.push(b.novelid); n++; }
}
if (picked.length >= DETAIL_LIMIT) break;
}
detailPlanned = picked.length;
if (picked.length) {
console.log(` → 补采详情 ${picked.length} 本(每频道前 ${TOP},上限 ${DETAIL_LIMIT})...`);
// 详情是列表的增补,不是前提:整段失败也要保住已解析好的列表落盘
// (下面的质量门会把 detailOk===0 标成 [详情解析异常/登录态缺失])
try {
const detailResult = fetchDetails(port, picked);
detailMap = detailResult.map;
detailFailedChunks = detailResult.failedChunks;
} catch (detailErr) {
detailMap = {};
detailFailedChunks = Math.max(1, Math.ceil(picked.length / DETAIL_CHUNK));
console.error(` ⚠ 详情补采整体失败,仅保留列表数据: ${detailErr.message}`);
}
detailOk = Object.values(detailMap).filter((d) => d && d.collect).length;
console.log(` ✓ 详情命中收藏数 ${detailOk}/${picked.length}`);
}
}
// 质量状态:详情开启时,收藏数命中率是核心信号
let quality = "[OK]";
const detailPartial =
!LIST_ONLY &&
detailPlanned > 0 &&
(detailFailedChunks > 0 || detailOk < detailPlanned);
if (!LIST_ONLY && detailPlanned > 0 && detailOk === 0) {
quality = "[详情解析异常/登录态缺失]";
console.error(` ⚠ 详情全部无收藏数:可能页面结构变动或需登录,已在文件头标注。`);
} else if (detailPartial) {
quality = "[部分详情缺失]";
console.error(` ⚠ 详情仅命中 ${detailOk}/${detailPlanned},已按部分结果标注。`);
} else if (LIST_ONLY) {
quality = "[仅列表-无核心指标]";
}
const now = new Date().toISOString();
const lines = [
`# 晋江 · ${rt.label}`,
"",
`- 来源:${url}`,
`- 抓取时间:${now}`,
`- 频道数:${data.channels.length}`,
`- 总条目数:${totalBooks}`,
`- 详情采集:${detailOk} / ${detailPlanned}(每频道前 ${TOP},上限 ${DETAIL_LIMIT})`,
`- 数据质量:${quality}`,
"",
"---",
"",
];
for (const ch of data.channels) {
try {
lines.push(`## ${ch.name} — ${ch.books.length} 本`, "");
for (let i = 0; i < ch.books.length; i++) {
try {
const b = ch.books[i];
lines.push(`### #${i + 1} ${b.title}`);
const d = b.novelid ? detailMap[b.novelid] : null;
const seg = [b.author || ""];
if (d) {
if (d.collect) seg.push("收藏 " + fmtWan(d.collect));
if (d.nutrition) seg.push("营养液 " + fmtWan(d.nutrition));
if (d.score) seg.push("积分 " + d.score);
if (d.words) seg.push("字数 " + fmtWan(d.words, "字"));
if (d.status) seg.push(d.status);
}
const meta = seg.filter(Boolean).join(" · ");
if (meta) lines.push(`*${meta}*`);
if (b.novelid) lines.push(`[作品页](https://www.jjwxc.net/onebook.php?novelid=${b.novelid})`);
lines.push("");
} catch (bookErr) {
console.error(`[jjwxc] ${rt.label} ${ch.name} 第${i + 1}条处理出错: ${bookErr.message}`);
lines.push("");
}
}
lines.push("---", "");
} catch (chErr) {
console.error(`[jjwxc] ${rt.label} 频道「${ch.name}」处理出错,跳过: ${chErr.message}`);
}
}
return {
content: lines.join("\n"),
partial: detailPartial,
partialReason: detailPartial
? `${rt.label}: detail ${detailOk}/${detailPlanned}, failed chunks ${detailFailedChunks}`
: "",
};
}
function main() {
if (RANKTYPE !== "all" && !RANK_TYPES.some((rank) => rank.id === RANKTYPE)) {
throw new Error(`未知 --type: ${RANKTYPE}`);
}
// 当前脚本只实现全站榜(t=0);不能把任意数字静默标成“频道 N”。
// 若后续支持分频道,先从页面提取并维护明确 ID 白名单再开放。
if (CHANNEL !== "0") {
throw new Error(`未知 --channel: ${CHANNEL}(当前仅支持 0=全站)`);
}
const rankTypes = RANKTYPE === "all" ? RANK_TYPES.map((r) => r.id) : [RANKTYPE];
const channels = [CHANNEL]; // 晋江频道 ID 需从页面获取,默认全站
let written = 0;
let failed = 0;
let partial = false;
const partialReasons = [];
for (const rt of rankTypes) {
for (const ch of channels) {
// per-榜单隔离:一个榜单出错不该掐掉 --type all 后面的榜单(与番茄/刺猬猫一致)
try {
const result = scrapeRank(PORT, rt, ch);
if (!result) {
failed++;
const rtInfo = RANK_TYPES.find((r) => r.id === rt);
partialReasons.push(`${rtInfo ? rtInfo.label : rt}: no usable data`);
continue;
}
if (result.partial) {
partial = true;
if (result.partialReason) partialReasons.push(result.partialReason);
}
const rtInfo = RANK_TYPES.find((r) => r.id === rt);
const chLabel = ch === "0" ? "全站" : `频道${ch}`;
const filename = `晋江${rtInfo.label}_${chLabel}_${localDateStamp()}.md`;
fs.mkdirSync(OUTDIR, { recursive: true });
const filepath = path.join(OUTDIR, filename);
fs.writeFileSync(filepath, result.content, "utf-8");
written++;
console.log(` ✓ 已保存: ${filepath}`);
} catch (rankErr) {
failed++;
const rtInfo = RANK_TYPES.find((r) => r.id === rt);
const message = rankErr && rankErr.message ? rankErr.message : String(rankErr);
partialReasons.push(`${rtInfo ? rtInfo.label : rt}: ${message}`);
console.error(
`[jjwxc] ${rtInfo ? rtInfo.label : rt} 采集失败,跳过: ${message}`
);
}
}
}
return {
planned: rankTypes.length * channels.length,
written,
failed,
partial: partial || failed > 0,
partialReasons,
};
}
if (require.main === module) {
runCli(main, "晋江采集");
}
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack." | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |
| ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |