ishenwei/agent-skills
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(llm-wiki): 迁入 llm-wiki 技能包及依赖（85 文件，含 baoyu-url-to-markdown 适配器）

Browse Source
This commit is contained in:
weishen
2026-09-15 07:04:50 +08:00
parent a9c4518b06
commit 4cb834de0d
85 changed files with 73954 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

305
llm-wiki/scripts/lib/wikilink-parser.js Normal file
View File

@@ -0,0 +1,305 @@
#!/usr/bin/env node
"use strict";
const crypto = require("node:crypto");
const path = require("node:path");
function sha256(buffer) {
return crypto.createHash("sha256").update(buffer).digest("hex");
}
function countCodePoints(value) {
return Array.from(value).length;
}
function extractLineAnnotations(lineText) {
const confidenceMatch = lineText.match(/<!--\s*confidence:\s*([A-Z]+)\s*-->/);
const relationTypeMatch = lineText.match(/<!--\s*relation(?:_type)?:\s*([^>]+?)\s*-->/);
return {
confidence: confidenceMatch ? confidenceMatch[1] : null,
relation_type: relationTypeMatch ? relationTypeMatch[1].trim() : null
};
}
function parseFenceCandidate(lineText) {
const match = lineText.match(/^( {0,3})(`{3,}|~{3,})(.*)$/);
if (!match) return null;
return {
marker: match[2][0],
length: match[2].length,
rest: match[3]
};
}
function collectInlineBacktickRuns(text) {
const positionsByLength = new Map();
let fence = null;
let lineStartIndex = 0;
while (lineStartIndex <= text.length) {
const nextNewlineIndex = text.indexOf("\n", lineStartIndex);
const lineEndIndex = nextNewlineIndex === -1 ? text.length : nextNewlineIndex;
const lineText = text.slice(lineStartIndex, lineEndIndex);
const fenceCandidate = parseFenceCandidate(lineText);
let handledAsFence = false;
if (fence) {
handledAsFence = true;
if (
fenceCandidate
&& fence.marker === fenceCandidate.marker
&& fenceCandidate.length >= fence.length
&& /^[ \t]*$/.test(fenceCandidate.rest)
) {
fence = null;
}
} else if (
fenceCandidate
&& !(fenceCandidate.marker === "`" && fenceCandidate.rest.includes("`"))
) {
fence = { marker: fenceCandidate.marker, length: fenceCandidate.length };
handledAsFence = true;
}
if (!handledAsFence) {
for (let index = 0; index < lineText.length;) {
if (lineText[index] !== "`") {
index += String.fromCodePoint(lineText.codePointAt(index)).length;
continue;
}
let runLength = 1;
while (lineText[index + runLength] === "`") {
runLength += 1;
}
const positions = positionsByLength.get(runLength) || [];
positions.push(lineStartIndex + index);
positionsByLength.set(runLength, positions);
index += runLength;
}
}
if (nextNewlineIndex === -1) break;
lineStartIndex = nextNewlineIndex + 1;
}
return { positionsByLength, cursorsByLength: new Map() };
}
function consumeInlineBacktickRun(runIndex, runLength, absoluteIndex) {
const positions = runIndex.positionsByLength.get(runLength) || [];
let cursor = runIndex.cursorsByLength.get(runLength) || 0;
while (cursor < positions.length && positions[cursor] <= absoluteIndex) {
cursor += 1;
}
runIndex.cursorsByLength.set(runLength, cursor);
return cursor < positions.length;
}
function parseOccurrence(rawLink, sourcePath, fileSha256, line, column, startByte, annotations) {
let innerStart = 0;
let innerEnd = rawLink.length;
let embedded = false;
let pending = false;
if (rawLink.startsWith("[待创建: [[")) {
innerStart = "[待创建: [[".length;
innerEnd = rawLink.length - "]]]".length;
pending = true;
} else if (rawLink.startsWith("[To create: [[")) {
innerStart = "[To create: [[".length;
innerEnd = rawLink.length - "]]]".length;
pending = true;
} else if (rawLink.startsWith("![[")) {
innerStart = "![[".length;
innerEnd = rawLink.length - "]]".length;
embedded = true;
} else {
innerStart = "[[".length;
innerEnd = rawLink.length - "]]".length;
}
const inner = rawLink.slice(innerStart, innerEnd);
const pipeIndex = inner.indexOf("|");
const targetAndAnchorRaw = pipeIndex >= 0 ? inner.slice(0, pipeIndex) : inner;
const displayRaw = pipeIndex >= 0 ? inner.slice(pipeIndex + 1) : null;
const anchorIndex = targetAndAnchorRaw.indexOf("#");
const targetRaw = anchorIndex >= 0 ? targetAndAnchorRaw.slice(0, anchorIndex) : targetAndAnchorRaw;
const anchor = anchorIndex >= 0 ? targetAndAnchorRaw.slice(anchorIndex + 1).trim() : null;
const display = displayRaw === null ? null : displayRaw.trim();
const leadingWhitespace = (targetRaw.match(/^\s*/) || [""])[0].length;
const trailingWhitespace = (targetRaw.match(/\s*$/) || [""])[0].length;
const targetStartInRaw = innerStart + leadingWhitespace;
const targetEndInRaw = innerStart + targetRaw.length - trailingWhitespace;
const pageTarget = targetRaw.trim();
const extension = path.posix.extname(pageTarget);
let linkKind = "page_wikilink";
if (pageTarget === "" && anchor) {
linkKind = "same_page_anchor";
} else if (extension && extension.toLowerCase() !== ".md") {
linkKind = "attachment_wikilink";
}
return {
occurrence_id: `${sourcePath}\0${fileSha256}\0${startByte}\0${startByte + Buffer.byteLength(rawLink, "utf8")}\0${rawLink}`,
source_path: sourcePath,
file_sha256: fileSha256,
raw_link: rawLink,
line,
column,
start_byte: startByte,
end_byte: startByte + Buffer.byteLength(rawLink, "utf8"),
link_kind: linkKind,
embedded,
pending,
page_target: pageTarget,
anchor,
display,
confidence: annotations.confidence,
relation_type: annotations.relation_type,
target_start_in_raw: targetStartInRaw,
target_end_in_raw: targetEndInRaw
};
}
function parseWikilinks(buffer, sourcePath) {
const text = buffer.toString("utf8");
const fileSha256 = sha256(buffer);
const occurrences = [];
const inlineBacktickRuns = collectInlineBacktickRuns(text);
let fence = null;
let inlineDelimiter = 0;
let lineNumber = 1;
let lineStartIndex = 0;
let positionBytesAdvanced = 0;
while (lineStartIndex <= text.length) {
const nextNewlineIndex = text.indexOf("\n", lineStartIndex);
const lineEndIndex = nextNewlineIndex === -1 ? text.length : nextNewlineIndex;
const lineText = text.slice(lineStartIndex, lineEndIndex);
const annotations = extractLineAnnotations(lineText);
const fenceCandidate = parseFenceCandidate(lineText);
let handledAsFence = false;
if (fence) {
handledAsFence = true;
if (
fenceCandidate
&& fence.marker === fenceCandidate.marker
&& fenceCandidate.length >= fence.length
&& /^[ \t]*$/.test(fenceCandidate.rest)
) {
fence = null;
}
} else if (
inlineDelimiter === 0
&& fenceCandidate
&& !(fenceCandidate.marker === "`" && fenceCandidate.rest.includes("`"))
) {
fence = { marker: fenceCandidate.marker, length: fenceCandidate.length };
handledAsFence = true;
}
if (handledAsFence) {
positionBytesAdvanced += Buffer.byteLength(lineText, "utf8");
} else {
let column = 1;
for (let index = 0; index < lineText.length;) {
if (lineText[index] === "`") {
let runLength = 1;
while (lineText[index + runLength] === "`") {
runLength += 1;
}
const hasEqualLengthCloser = consumeInlineBacktickRun(
inlineBacktickRuns,
runLength,
lineStartIndex + index
);
if (inlineDelimiter === 0) {
if (hasEqualLengthCloser) {
inlineDelimiter = runLength;
}
} else if (runLength === inlineDelimiter) {
inlineDelimiter = 0;
}
index += runLength;
column += runLength;
positionBytesAdvanced += runLength;
continue;
}
if (inlineDelimiter > 0) {
const symbol = String.fromCodePoint(lineText.codePointAt(index));
const symbolBytes = Buffer.byteLength(symbol, "utf8");
index += symbol.length;
column += 1;
positionBytesAdvanced += symbolBytes;
continue;
}
let rawLink = null;
let endIndex = null;
if (lineText.startsWith("[待创建: [[", index) || lineText.startsWith("[To create: [[", index)) {
const wrapperPrefix = lineText.startsWith("[待创建: [[", index) ? "[待创建: [[" : "[To create: [[";
const closeInner = lineText.indexOf("]]]", index + wrapperPrefix.length);
if (closeInner >= 0) {
rawLink = lineText.slice(index, closeInner + 3);
endIndex = closeInner + 3;
}
} else if (lineText.startsWith("![[", index) || lineText.startsWith("[[", index)) {
const closeInner = lineText.indexOf("]]", index + 2);
if (closeInner >= 0) {
rawLink = lineText.slice(index, closeInner + 2);
endIndex = closeInner + 2;
}
}
if (rawLink && endIndex !== null) {
const startByte = positionBytesAdvanced;
const rawLinkBytes = Buffer.byteLength(rawLink, "utf8");
occurrences.push(parseOccurrence(rawLink, sourcePath, fileSha256, lineNumber, column, startByte, annotations));
index = endIndex;
column += countCodePoints(rawLink);
positionBytesAdvanced += rawLinkBytes;
continue;
}
const symbol = String.fromCodePoint(lineText.codePointAt(index));
const symbolBytes = Buffer.byteLength(symbol, "utf8");
index += symbol.length;
column += 1;
positionBytesAdvanced += symbolBytes;
}
}
if (nextNewlineIndex === -1) {
break;
}
lineStartIndex = nextNewlineIndex + 1;
positionBytesAdvanced += 1;
lineNumber += 1;
}
return {
source_path: sourcePath,
file_sha256: fileSha256,
occurrences,
metrics: {
utf8_bytes_scanned: buffer.length,
position_bytes_advanced: positionBytesAdvanced
}
};
}
function renderWikilinkReplacement(occurrence, replacementTarget) {
return `${occurrence.raw_link.slice(0, occurrence.target_start_in_raw)}${replacementTarget}${occurrence.raw_link.slice(occurrence.target_end_in_raw)}`;
}
module.exports = { parseWikilinks, renderWikilinkReplacement };