ShenyuanNext/wepr-growth-skills / skills/dbs-content-system
dbs-content-system
Verified把本地文稿、推文、选题、案例和课程稿整理成可持续生长、可重组复用的内容工程。用户要求结构化内容资产、整理旧素材或建立主题地图时使用。
tools/generate-duplicate-candidates.js
View on GitHub#!/usr/bin/env node
const fs = require("fs");
const path = require("path");
const root = path.resolve(process.cwd());
const unitRoot = path.join(root, "02-内容单元库");
const outputDir = path.join(root, "03-处理状态");
const csvOutput = path.join(outputDir, "去重候选索引.csv");
const summaryOutput = path.join(outputDir, "去重与冲突总览.md");
const auditOutput = path.join(outputDir, "去重与冲突审计.csv");
const manualCandidateInput = path.join(outputDir, "人工去重候选.csv");
const today = new Intl.DateTimeFormat("en-CA", {
timeZone: "Asia/Shanghai",
year: "numeric",
month: "2-digit",
day: "2-digit",
}).format(new Date());
function walkFiles(dir) {
if (!fs.existsSync(dir)) return [];
const files = [];
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
const full = path.join(dir, entry.name);
if (entry.isDirectory()) files.push(...walkFiles(full));
else if (entry.isFile() && path.extname(entry.name).toLowerCase() === ".md") files.push(full);
}
return files;
}
function extractFrontmatter(content) {
const match = content.match(/^---\n([\s\S]*?)\n---\n?/);
return match ? match[1] : "";
}
function getScalar(frontmatter, field) {
const escaped = field.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
const match = frontmatter.match(new RegExp(`^${escaped}:\\s*(.+)$`, "m"));
return match ? match[1].trim() : "";
}
function getList(frontmatter, field) {
const lines = frontmatter.split("\n");
const start = lines.findIndex((line) => line.trim() === `${field}:`);
if (start === -1) return [];
const items = [];
for (let i = start + 1; i < lines.length; i += 1) {
const line = lines[i];
if (!line.startsWith(" - ")) break;
items.push(line.replace(" - ", "").trim());
}
return items;
}
function getRelationships(frontmatter) {
const lines = frontmatter.split("\n");
const relationships = [];
let inRelationships = false;
let current = null;
for (const line of lines) {
if (!inRelationships) {
if (line.trim() === "relationships:") inRelationships = true;
continue;
}
if (!line.startsWith(" ")) break;
const trimmed = line.trim();
if (trimmed === "[]") break;
const typeMatch = trimmed.match(/^- type:\s*(.+)$/);
if (typeMatch) {
current = { type: typeMatch[1].trim(), target: "", note: "" };
relationships.push(current);
continue;
}
if (!current) continue;
const targetMatch = trimmed.match(/^target:\s*(.+)$/);
if (targetMatch) {
current.target = targetMatch[1].trim();
continue;
}
const noteMatch = trimmed.match(/^note:\s*(.+)$/);
if (noteMatch) current.note = noteMatch[1].trim();
}
return relationships;
}
function uniqueIntersection(a, b) {
const bSet = new Set(b);
return [...new Set(a.filter((item) => bSet.has(item)))];
}
function normalizeText(text) {
return text.replace(/[^\p{Script=Han}A-Za-z0-9]+/gu, "");
}
function uniqueChars(text) {
return [...new Set(normalizeText(text).split(""))].filter(Boolean);
}
function titleSimilarity(a, b) {
const aChars = uniqueChars(a);
const bChars = uniqueChars(b);
if (aChars.length === 0 || bChars.length === 0) return 0;
const shared = uniqueIntersection(aChars, bChars).length;
return shared / Math.max(aChars.length, bChars.length);
}
function escapeCsv(value) {
return `"${String(value ?? "").replaceAll("\"", "\"\"")}"`;
}
function parseCsvLine(line) {
const cells = [];
let current = "";
let inQuotes = false;
for (let i = 0; i < line.length; i += 1) {
const ch = line[i];
if (ch === "\"") {
if (inQuotes && line[i + 1] === "\"") {
current += "\"";
i += 1;
} else {
inQuotes = !inQuotes;
}
} else if (ch === "," && !inQuotes) {
cells.push(current);
current = "";
} else {
current += ch;
}
}
cells.push(current);
return cells;
}
function pairKey(a, b) {
return [a, b].sort().join("::");
}
function loadExistingReviews() {
const reviews = new Map();
if (!fs.existsSync(csvOutput)) return reviews;
const lines = fs.readFileSync(csvOutput, "utf8").trim().split("\n");
if (lines.length <= 1) return reviews;
const header = parseCsvLine(lines[0]);
const indexByName = Object.fromEntries(header.map((name, index) => [name, index]));
for (const line of lines.slice(1)) {
if (!line.trim()) continue;
const cells = parseCsvLine(line);
const unitA = cells[indexByName.unit_a_id];
const unitB = cells[indexByName.unit_b_id];
const status = cells[indexByName.status] || "待人工判断";
const note = cells[indexByName.note] || "";
reviews.set(pairKey(unitA, unitB), { status, note });
}
return reviews;
}
function loadManualCandidates() {
const manual = new Map();
if (!fs.existsSync(manualCandidateInput)) return manual;
const lines = fs.readFileSync(manualCandidateInput, "utf8").trim().split("\n");
if (lines.length <= 1) return manual;
const header = parseCsvLine(lines[0]);
const indexByName = Object.fromEntries(header.map((name, index) => [name, index]));
for (const line of lines.slice(1)) {
if (!line.trim()) continue;
const cells = parseCsvLine(line);
const unitA = cells[indexByName.unit_a_id];
const unitB = cells[indexByName.unit_b_id];
if (!unitA || !unitB) continue;
manual.set(pairKey(unitA, unitB), {
reason: cells[indexByName.reason] || "人工补充候选",
status: cells[indexByName.status] || "待人工判断",
note: cells[indexByName.note] || "",
});
}
return manual;
}
const units = walkFiles(unitRoot).map((file) => {
const content = fs.readFileSync(file, "utf8");
const frontmatter = extractFrontmatter(content);
return {
id: getScalar(frontmatter, "id"),
type: getScalar(frontmatter, "type"),
title: getScalar(frontmatter, "title"),
relPath: path.relative(root, file).replaceAll(path.sep, "/"),
themes: getList(frontmatter, "themes"),
keywords: getList(frontmatter, "keywords"),
sourceDocuments: getList(frontmatter, "source_documents"),
relationships: getRelationships(frontmatter),
};
}).filter((unit) => unit.id && unit.type && unit.title);
const unitById = new Map(units.map((unit) => [unit.id, unit]));
const candidates = [];
const existingReviews = loadExistingReviews();
const manualCandidates = loadManualCandidates();
const seenPairs = new Set();
for (let i = 0; i < units.length; i += 1) {
for (let j = i + 1; j < units.length; j += 1) {
const a = units[i];
const b = units[j];
if (a.type !== b.type) continue;
const sharedThemes = uniqueIntersection(a.themes, b.themes);
const sharedKeywords = uniqueIntersection(a.keywords, b.keywords);
const sharedSources = uniqueIntersection(a.sourceDocuments, b.sourceDocuments);
const similarity = titleSimilarity(a.title, b.title);
const score = sharedThemes.length * 3 + sharedKeywords.length * 2 + sharedSources.length * 2 + Math.round(similarity * 10);
const isCandidate =
(sharedThemes.length >= 1 && sharedKeywords.length >= 1) ||
(sharedSources.length >= 1 && (sharedThemes.length >= 1 || sharedKeywords.length >= 1)) ||
(similarity >= 0.35 && (sharedThemes.length >= 1 || sharedKeywords.length >= 1 || sharedSources.length >= 1));
if (!isCandidate) continue;
let candidateType = "近似重复候选";
if (sharedSources.length > 0 && sharedThemes.length > 0 && (sharedKeywords.length >= 1 || similarity >= 0.45)) {
candidateType = "同义重复候选";
}
const review = existingReviews.get(pairKey(a.id, b.id)) || { status: "待人工判断", note: "" };
const key = pairKey(a.id, b.id);
seenPairs.add(key);
candidates.push({
unit_a_id: a.id,
unit_a_type: a.type,
unit_a_title: a.title,
unit_b_id: b.id,
unit_b_type: b.type,
unit_b_title: b.title,
shared_themes: sharedThemes.join(" | "),
shared_keywords: sharedKeywords.join(" | "),
shared_sources: sharedSources.join(" | "),
title_similarity: similarity.toFixed(2),
score,
candidate_type: candidateType,
status: review.status,
note: review.note,
candidate_reason: "自动识别",
unit_a_file: a.relPath,
unit_b_file: b.relPath,
});
}
}
for (const [key, manual] of manualCandidates.entries()) {
if (seenPairs.has(key)) continue;
const [unitAId, unitBId] = key.split("::");
const a = unitById.get(unitAId);
const b = unitById.get(unitBId);
if (!a || !b) continue;
const sharedThemes = uniqueIntersection(a.themes, b.themes);
const sharedKeywords = uniqueIntersection(a.keywords, b.keywords);
const sharedSources = uniqueIntersection(a.sourceDocuments, b.sourceDocuments);
const similarity = titleSimilarity(a.title, b.title);
candidates.push({
unit_a_id: a.id,
unit_a_type: a.type,
unit_a_title: a.title,
unit_b_id: b.id,
unit_b_type: b.type,
unit_b_title: b.title,
shared_themes: sharedThemes.join(" | "),
shared_keywords: sharedKeywords.join(" | "),
shared_sources: sharedSources.join(" | "),
title_similarity: similarity.toFixed(2),
score: -1,
candidate_type: "人工补充候选",
status: manual.status,
note: manual.note,
candidate_reason: manual.reason,
unit_a_file: a.relPath,
unit_b_file: b.relPath,
});
}
candidates.sort((a, b) => {
if (b.score !== a.score) return b.score - a.score;
const byType = a.unit_a_type.localeCompare(b.unit_a_type, "zh-Hans-CN");
if (byType !== 0) return byType;
return a.unit_a_id.localeCompare(b.unit_a_id, "zh-Hans-CN");
});
const rows = [
[
"unit_a_id",
"unit_a_type",
"unit_a_title",
"unit_b_id",
"unit_b_type",
"unit_b_title",
"shared_themes",
"shared_keywords",
"shared_sources",
"title_similarity",
"score",
"candidate_type",
"candidate_reason",
"status",
"note",
"unit_a_file",
"unit_b_file",
],
...candidates.map((row) => [
row.unit_a_id,
row.unit_a_type,
row.unit_a_title,
row.unit_b_id,
row.unit_b_type,
row.unit_b_title,
row.shared_themes,
row.shared_keywords,
row.shared_sources,
row.title_similarity,
row.score,
row.candidate_type,
row.candidate_reason,
row.status,
row.note,
row.unit_a_file,
row.unit_b_file,
]),
];
fs.writeFileSync(csvOutput, rows.map((row) => row.map(escapeCsv).join(",")).join("\n") + "\n");
const typeCounts = candidates.reduce((acc, row) => {
acc[row.candidate_type] = (acc[row.candidate_type] || 0) + 1;
return acc;
}, {});
const statusCounts = candidates.reduce((acc, row) => {
acc[row.status] = (acc[row.status] || 0) + 1;
return acc;
}, {});
function getRelationshipBetween(unit, targetId, type = null) {
return unit.relationships.find((relationship) => {
if (relationship.target !== targetId) return false;
if (type && relationship.type !== type) return false;
return true;
});
}
const auditRows = [];
const auditCounts = {
conflict_backlinked_ok: 0,
conflict_backlinked_missing: 0,
layered_link_ok: 0,
layered_link_not_required: 0,
layered_link_missing: 0,
};
for (const candidate of candidates) {
const unitA = unitById.get(candidate.unit_a_id);
const unitB = unitById.get(candidate.unit_b_id);
if (!unitA || !unitB) continue;
if (candidate.status === "已判断:不合并,建立冲突关系") {
const aConflict = getRelationshipBetween(unitA, unitB.id, "冲突");
const bConflict = getRelationshipBetween(unitB, unitA.id, "冲突");
const ok = Boolean(aConflict?.note && bConflict?.note);
auditCounts[ok ? "conflict_backlinked_ok" : "conflict_backlinked_missing"] += 1;
auditRows.push({
candidate_pair: `${unitA.id} <-> ${unitB.id}`,
candidate_status: candidate.status,
audit_type: "冲突回写审计",
audit_result: ok ? "通过" : "缺失",
audit_note: ok
? "双方内容单元均已写入带 note 的冲突关系"
: "候选已判断为建立冲突关系,但至少一侧未回写冲突关系或缺少 note",
});
continue;
}
if (candidate.status === "已判断:不合并,分层保留") {
const aToB = getRelationshipBetween(unitA, unitB.id);
const bToA = getRelationshipBetween(unitB, unitA.id);
const hasExplicitLink = Boolean(aToB || bToA);
const isConceptToJudgmentOrSolution =
(unitA.type === "概念单元" && (unitB.type === "观点单元" || unitB.type === "方案单元")) ||
(unitB.type === "概念单元" && (unitA.type === "观点单元" || unitA.type === "方案单元"));
const isQuestionPair = unitA.type === "问题单元" && unitB.type === "问题单元";
const isViewToSolution =
(unitA.type === "观点单元" && unitB.type === "方案单元") ||
(unitA.type === "方案单元" && unitB.type === "观点单元");
let auditResult = "缺失";
let auditNote = "候选已判断为分层保留,但对应内容单元之间缺少显式关系";
if (hasExplicitLink) {
auditResult = "通过";
auditNote = "至少一侧内容单元已写入显式关系,能支撑分层保留";
auditCounts.layered_link_ok += 1;
} else if (isQuestionPair || isViewToSolution) {
auditResult = "无需";
auditNote = "当前分层保留组合不强制要求显式关系";
auditCounts.layered_link_not_required += 1;
} else if (isConceptToJudgmentOrSolution) {
auditResult = "缺失";
auditNote = "概念单元分层保留时,应至少通过 `解释` 关系显式支撑对应观点或方案";
auditCounts.layered_link_missing += 1;
} else {
auditResult = "无需";
auditNote = "当前分层保留组合不强制要求显式关系";
auditCounts.layered_link_not_required += 1;
}
auditRows.push({
candidate_pair: `${unitA.id} <-> ${unitB.id}`,
candidate_status: candidate.status,
audit_type: "分层保留关系审计",
audit_result: auditResult,
audit_note: auditNote,
});
}
}
const auditCsvRows = [
["candidate_pair", "candidate_status", "audit_type", "audit_result", "audit_note"],
...auditRows.map((row) => [row.candidate_pair, row.candidate_status, row.audit_type, row.audit_result, row.audit_note]),
];
fs.writeFileSync(auditOutput, auditCsvRows.map((row) => row.map(escapeCsv).join(",")).join("\n") + "\n");
const summaryLines = [
"# 去重与冲突总览",
"",
`最后更新:${today}`,
"",
"## 当前统计",
"",
`- 内容单元总数:${units.length}`,
`- 去重候选总数:${candidates.length}`,
"",
"## 候选类型分布",
"",
];
for (const type of Object.keys(typeCounts).sort((a, b) => a.localeCompare(b, "zh-Hans-CN"))) {
summaryLines.push(`- ${type}:${typeCounts[type]}`);
}
if (Object.keys(typeCounts).length === 0) summaryLines.push("- 暂无候选");
summaryLines.push("", "## 处理状态分布", "");
for (const status of Object.keys(statusCounts).sort((a, b) => a.localeCompare(b, "zh-Hans-CN"))) {
summaryLines.push(`- ${status}:${statusCounts[status]}`);
}
if (Object.keys(statusCounts).length === 0) summaryLines.push("- 暂无状态");
summaryLines.push(
"",
"## 当前说明",
"",
"- 本文件先给出去重候选,不自动合并",
"- 自动识别候选与人工补充候选会同时进入索引",
"- 已判断为 `建立冲突关系` 的候选,会审计双方内容单元是否都已回写 `冲突` 关系",
"- 已判断为 `分层保留` 的候选,会审计对应内容单元之间是否存在显式关系支撑",
"",
"## 关系审计",
"",
`- 冲突回写已通过:${auditCounts.conflict_backlinked_ok}`,
`- 冲突回写待补:${auditCounts.conflict_backlinked_missing}`,
`- 分层保留关系已通过:${auditCounts.layered_link_ok}`,
`- 分层保留关系无需直连:${auditCounts.layered_link_not_required}`,
`- 分层保留关系待补:${auditCounts.layered_link_missing}`,
"",
"## 权威文件",
"",
`- 候选明细:\`${path.relative(root, csvOutput).replaceAll(path.sep, "/")}\``,
`- 审计明细:\`${path.relative(root, auditOutput).replaceAll(path.sep, "/")}\``,
`- 人工补充候选:\`${path.relative(root, manualCandidateInput).replaceAll(path.sep, "/")}\``,
"- 去重规则:`00-规则与索引/内容单元去重与版本规则.md`",
"- 关系规则:`00-规则与索引/内容单元关系规则.md`",
);
fs.writeFileSync(summaryOutput, `${summaryLines.join("\n")}\n`);
console.log(JSON.stringify({
csvOutput,
summaryOutput,
auditOutput,
unitCount: units.length,
candidateCount: candidates.length,
}, null, 2));
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack." | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |
| ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |