feat(llm-wiki): 迁入 llm-wiki 技能包及依赖（85 文件，含 baoyu-url-to-markdown 适配器）
This commit is contained in:
424
llm-wiki/scripts/adapter-state.sh
Normal file
424
llm-wiki/scripts/adapter-state.sh
Normal file
@@ -0,0 +1,424 @@
|
||||
#!/bin/bash
|
||||
# 外挂状态检测脚本：统一判断可选外挂的安装/环境/运行状态
|
||||
# 五种状态：not_installed / env_unavailable（仅 uv 依赖的来源） / runtime_failed / unsupported / empty_result
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
SOURCE_REGISTRY_SCRIPT="$SCRIPT_DIR/source-registry.sh"
|
||||
# 微信工具 URL 从共享配置读取，与 install.sh 保持一致
|
||||
source "$SCRIPT_DIR/shared-config.sh"
|
||||
source "$SCRIPT_DIR/runtime-context.sh"
|
||||
SKILL_ROOT_OVERRIDE=""
|
||||
LAYOUT_MODE_OVERRIDE=""
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
用法：
|
||||
bash scripts/adapter-state.sh [--skill-root <path>] [--layout-mode <source_checkout|installed_skill|upgrade_target>] check <source_id>
|
||||
bash scripts/adapter-state.sh [--skill-root <path>] [--layout-mode <source_checkout|installed_skill|upgrade_target>] summary
|
||||
bash scripts/adapter-state.sh [--skill-root <path>] [--layout-mode <source_checkout|installed_skill|upgrade_target>] summary-human
|
||||
bash scripts/adapter-state.sh [--skill-root <path>] [--layout-mode <source_checkout|installed_skill|upgrade_target>] classify-run <source_id> <exit_code> <output_path>
|
||||
EOF
|
||||
}
|
||||
|
||||
resolve_optional_root() {
|
||||
resolve_optional_adapter_root "$PROJECT_ROOT" "$SKILL_ROOT_OVERRIDE" "$LAYOUT_MODE_OVERRIDE"
|
||||
}
|
||||
|
||||
dependency_installed() {
|
||||
local dependency_name="$1"
|
||||
local dependency_type="$2"
|
||||
local optional_root
|
||||
|
||||
case "$dependency_type" in
|
||||
bundled)
|
||||
optional_root="$(resolve_optional_root)"
|
||||
if [ -d "$optional_root/$dependency_name" ]; then
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
;;
|
||||
install_time)
|
||||
command -v "$dependency_name" >/dev/null 2>&1
|
||||
;;
|
||||
none)
|
||||
return 0
|
||||
;;
|
||||
*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
has_uv() {
|
||||
command -v uv >/dev/null 2>&1
|
||||
}
|
||||
|
||||
chrome_debug_ready() {
|
||||
if command -v lsof >/dev/null 2>&1; then
|
||||
lsof -i :9222 -sTCP:LISTEN >/dev/null 2>&1
|
||||
return $?
|
||||
fi
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
print_header() {
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||||
"source_id" \
|
||||
"source_label" \
|
||||
"state" \
|
||||
"state_label" \
|
||||
"detail" \
|
||||
"recovery_action" \
|
||||
"install_hint" \
|
||||
"fallback_hint"
|
||||
}
|
||||
|
||||
state_label() {
|
||||
case "$1" in
|
||||
available)
|
||||
printf '%s\n' "可用"
|
||||
;;
|
||||
not_installed)
|
||||
printf '%s\n' "未安装"
|
||||
;;
|
||||
env_unavailable)
|
||||
printf '%s\n' "环境不满足"
|
||||
;;
|
||||
runtime_failed)
|
||||
printf '%s\n' "运行失败"
|
||||
;;
|
||||
unsupported)
|
||||
printf '%s\n' "不支持自动提取"
|
||||
;;
|
||||
empty_result)
|
||||
printf '%s\n' "结果为空"
|
||||
;;
|
||||
*)
|
||||
printf '%s\n' "$1"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
default_install_hint() {
|
||||
local source_id="$1"
|
||||
local adapter_name="$2"
|
||||
|
||||
case "$source_id" in
|
||||
web_article|x_twitter|zhihu_article)
|
||||
printf '%s\n' "重新运行当前平台的 llm-wiki 安装命令，并追加 --with-optional-adapters，确认 ${adapter_name} 已准备到技能目录"
|
||||
;;
|
||||
wechat_article)
|
||||
printf '%s\n' "先安装 uv，再执行：uv tool install ${WECHAT_TOOL_URL}"
|
||||
;;
|
||||
youtube_video)
|
||||
printf '%s\n' "重新运行当前平台的 llm-wiki 安装命令，并追加 --with-optional-adapters，确认 ${adapter_name} 已准备到技能目录"
|
||||
;;
|
||||
*)
|
||||
printf '%s\n' "-"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
optional_hint() {
|
||||
local source_id="$1"
|
||||
|
||||
case "$source_id" in
|
||||
web_article|x_twitter|zhihu_article)
|
||||
printf '%s\n' '如需复用已登录的浏览器会话，可执行：open -na "Google Chrome" --args --remote-debugging-port=9222'
|
||||
;;
|
||||
wechat_article|youtube_video)
|
||||
printf '%s\n' "先安装 uv：brew install uv"
|
||||
;;
|
||||
*)
|
||||
printf '%s\n' "-"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
emit_state_row() {
|
||||
local source_id="$1"
|
||||
local source_label="$2"
|
||||
local state="$3"
|
||||
local detail="$4"
|
||||
local recovery_action="$5"
|
||||
local install_hint="$6"
|
||||
local fallback_hint="$7"
|
||||
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||||
"$source_id" \
|
||||
"$source_label" \
|
||||
"$state" \
|
||||
"$(state_label "$state")" \
|
||||
"$detail" \
|
||||
"$recovery_action" \
|
||||
"$install_hint" \
|
||||
"$fallback_hint"
|
||||
}
|
||||
|
||||
resolve_preflight_state() {
|
||||
local source_id="$1"
|
||||
local record
|
||||
local source_label source_category input_mode match_rule raw_dir adapter_name dependency_name dependency_type fallback_hint
|
||||
local state detail recovery_action install_hint
|
||||
|
||||
record="$(bash "$SOURCE_REGISTRY_SCRIPT" get "$source_id")" || {
|
||||
echo "未知来源：$source_id" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
IFS=$'\t' read -r source_id source_label source_category input_mode match_rule raw_dir adapter_name dependency_name dependency_type fallback_hint <<EOF
|
||||
$record
|
||||
EOF
|
||||
|
||||
case "$source_category" in
|
||||
core_builtin)
|
||||
state="available"
|
||||
detail="核心主线可直接进入，不依赖外挂"
|
||||
recovery_action="直接继续主线"
|
||||
install_hint="-"
|
||||
;;
|
||||
manual_only)
|
||||
state="unsupported"
|
||||
detail="该来源当前只支持手动进入主线"
|
||||
recovery_action="直接走手动入口"
|
||||
install_hint="-"
|
||||
;;
|
||||
optional_adapter)
|
||||
case "$source_id" in
|
||||
wechat_article)
|
||||
if ! has_uv; then
|
||||
state="env_unavailable"
|
||||
detail="缺少 uv，当前无法准备微信公众号自动提取环境"
|
||||
recovery_action="先补环境；现在也可以直接走手动入口"
|
||||
install_hint="$(optional_hint "$source_id")"
|
||||
elif ! dependency_installed "$dependency_name" "$dependency_type"; then
|
||||
state="not_installed"
|
||||
detail="未找到 ${adapter_name}"
|
||||
recovery_action="先补安装；现在也可以直接走手动入口"
|
||||
install_hint="$(default_install_hint "$source_id" "$adapter_name")"
|
||||
else
|
||||
state="available"
|
||||
detail="${adapter_name} 已可用"
|
||||
recovery_action="继续自动提取"
|
||||
install_hint="-"
|
||||
fi
|
||||
;;
|
||||
web_article|x_twitter|zhihu_article)
|
||||
if ! dependency_installed "$dependency_name" "$dependency_type"; then
|
||||
state="not_installed"
|
||||
detail="未找到 ${adapter_name}"
|
||||
recovery_action="先补安装；现在也可以直接走手动入口"
|
||||
install_hint="$(default_install_hint "$source_id" "$adapter_name")"
|
||||
else
|
||||
state="available"
|
||||
if chrome_debug_ready; then
|
||||
detail="${adapter_name} 已可用，且已检测到可复用的 Chrome 调试会话"
|
||||
recovery_action="继续自动提取"
|
||||
install_hint="-"
|
||||
else
|
||||
detail="${adapter_name} 已可用；未检测到 9222，将在需要时自动拉起临时浏览器"
|
||||
recovery_action="继续自动提取；如需复用已登录会话，可先开启 Chrome 调试端口 9222"
|
||||
install_hint="$(optional_hint "$source_id")"
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
youtube_video)
|
||||
if ! dependency_installed "$dependency_name" "$dependency_type"; then
|
||||
state="not_installed"
|
||||
detail="未找到 ${adapter_name}"
|
||||
recovery_action="先补安装；现在也可以直接走手动入口"
|
||||
install_hint="$(default_install_hint "$source_id" "$adapter_name")"
|
||||
elif ! has_uv; then
|
||||
state="env_unavailable"
|
||||
detail="缺少 uv，当前无法运行 YouTube 字幕提取"
|
||||
recovery_action="先补环境；现在也可以直接走手动入口"
|
||||
install_hint="$(optional_hint "$source_id")"
|
||||
else
|
||||
state="available"
|
||||
detail="${adapter_name} 已可用"
|
||||
recovery_action="继续自动提取"
|
||||
install_hint="-"
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
if ! dependency_installed "$dependency_name" "$dependency_type"; then
|
||||
state="not_installed"
|
||||
detail="未找到 ${adapter_name}"
|
||||
recovery_action="先补安装；现在也可以直接走手动入口"
|
||||
install_hint="$(default_install_hint "$source_id" "$adapter_name")"
|
||||
else
|
||||
state="available"
|
||||
detail="${adapter_name} 已可用"
|
||||
recovery_action="继续自动提取"
|
||||
install_hint="-"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
*)
|
||||
echo "未知来源分类：$source_category" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
emit_state_row \
|
||||
"$source_id" \
|
||||
"$source_label" \
|
||||
"$state" \
|
||||
"$detail" \
|
||||
"$recovery_action" \
|
||||
"$install_hint" \
|
||||
"$fallback_hint"
|
||||
}
|
||||
|
||||
classify_run_state() {
|
||||
local source_id="$1"
|
||||
local exit_code="$2"
|
||||
local output_path="$3"
|
||||
|
||||
# 校验 exit_code 为整数，防止 set -e 下非数字参数导致脚本崩溃
|
||||
case "$exit_code" in
|
||||
''|*[!0-9-]*) echo "exit_code 必须是整数，收到：$exit_code" >&2; exit 1 ;;
|
||||
esac
|
||||
|
||||
local row
|
||||
local source_label state state_label_value detail recovery_action install_hint fallback_hint
|
||||
|
||||
row="$(resolve_preflight_state "$source_id")"
|
||||
IFS=$'\t' read -r _ source_label state state_label_value detail recovery_action install_hint fallback_hint <<EOF
|
||||
$row
|
||||
EOF
|
||||
|
||||
if [ "$state" != "available" ]; then
|
||||
emit_state_row \
|
||||
"$source_id" \
|
||||
"$source_label" \
|
||||
"$state" \
|
||||
"$detail" \
|
||||
"$recovery_action" \
|
||||
"$install_hint" \
|
||||
"$fallback_hint"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ "$exit_code" -ne 0 ]; then
|
||||
emit_state_row \
|
||||
"$source_id" \
|
||||
"$source_label" \
|
||||
"runtime_failed" \
|
||||
"自动提取执行失败" \
|
||||
"可以先重试一次；如果还不行，就改走手动入口" \
|
||||
"-" \
|
||||
"$fallback_hint"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ ! -f "$output_path" ] || ! grep -q '[^[:space:]]' "$output_path" 2>/dev/null; then
|
||||
emit_state_row \
|
||||
"$source_id" \
|
||||
"$source_label" \
|
||||
"empty_result" \
|
||||
"自动提取完成，但没有拿到有效正文" \
|
||||
"请手动补全文本后继续主线" \
|
||||
"-" \
|
||||
"$fallback_hint"
|
||||
return 0
|
||||
fi
|
||||
|
||||
emit_state_row \
|
||||
"$source_id" \
|
||||
"$source_label" \
|
||||
"available" \
|
||||
"自动提取已拿到有效正文" \
|
||||
"继续进入主线" \
|
||||
"-" \
|
||||
"$fallback_hint"
|
||||
}
|
||||
|
||||
print_summary() {
|
||||
local source_id
|
||||
|
||||
print_header
|
||||
|
||||
while IFS=$'\t' read -r source_id _; do
|
||||
[ -n "$source_id" ] || continue
|
||||
resolve_preflight_state "$source_id"
|
||||
done <<EOF
|
||||
$(bash "$SOURCE_REGISTRY_SCRIPT" list | awk -F '\t' 'NR > 1 && ($3 == "optional_adapter" || $3 == "manual_only") { print $1 "\t" $2 }')
|
||||
EOF
|
||||
}
|
||||
|
||||
print_summary_human() {
|
||||
local row
|
||||
local source_id source_label state state_label_value detail recovery_action install_hint fallback_hint
|
||||
|
||||
while IFS= read -r row; do
|
||||
[ -n "$row" ] || continue
|
||||
|
||||
IFS=$'\t' read -r source_id source_label state state_label_value detail recovery_action install_hint fallback_hint <<EOF
|
||||
$row
|
||||
EOF
|
||||
|
||||
printf '%s\n' "- ${source_label}：${state_label_value}。${detail}。"
|
||||
printf '%s\n' " 下一步：${recovery_action}。"
|
||||
if [ "$install_hint" != "-" ]; then
|
||||
if [ "$state" = "available" ]; then
|
||||
printf '%s\n' " 补充说明：${install_hint}。"
|
||||
else
|
||||
printf '%s\n' " 安装提示：${install_hint}。"
|
||||
fi
|
||||
fi
|
||||
printf '%s\n' " 回退方式：${fallback_hint}。"
|
||||
done <<EOF
|
||||
$(print_summary | tail -n +2)
|
||||
EOF
|
||||
}
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--skill-root)
|
||||
[ $# -ge 2 ] || { usage; exit 1; }
|
||||
SKILL_ROOT_OVERRIDE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--layout-mode)
|
||||
[ $# -ge 2 ] || { usage; exit 1; }
|
||||
LAYOUT_MODE_OVERRIDE="$2"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
break
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
command_name="${1:-}"
|
||||
|
||||
case "$command_name" in
|
||||
check)
|
||||
[ "$#" -eq 2 ] || { usage; exit 1; }
|
||||
print_header
|
||||
resolve_preflight_state "$2"
|
||||
;;
|
||||
summary)
|
||||
[ "$#" -eq 1 ] || { usage; exit 1; }
|
||||
print_summary
|
||||
;;
|
||||
summary-human)
|
||||
[ "$#" -eq 1 ] || { usage; exit 1; }
|
||||
print_summary_human
|
||||
;;
|
||||
classify-run)
|
||||
[ "$#" -eq 4 ] || { usage; exit 1; }
|
||||
print_header
|
||||
classify_run_state "$2" "$3" "$4"
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
234
llm-wiki/scripts/build-graph-data.sh
Executable file
234
llm-wiki/scripts/build-graph-data.sh
Executable file
@@ -0,0 +1,234 @@
|
||||
#!/bin/bash
|
||||
# build-graph-data.sh — build a path-identity graph and its sibling warning sidecar.
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/build-graph-data.sh <kb-root> [<kb-internal-dir>/graph-data.json]
|
||||
#
|
||||
# A custom output must remain inside the knowledge base, keep the graph-data.json
|
||||
# basename, and owns graph-warnings.json in the same existing directory.
|
||||
# Exit codes: 0 = valid graph produced (including data warnings); 1 = tool/input failure.
|
||||
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="${BASH_SOURCE[0]%/*}"
|
||||
[ "$SCRIPT_DIR" = "${BASH_SOURCE[0]}" ] && SCRIPT_DIR="."
|
||||
SCRIPT_DIR="$(cd "$SCRIPT_DIR" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "$SCRIPT_DIR/shared-config.sh"
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage:
|
||||
bash scripts/build-graph-data.sh <kb-root> [<kb-internal-dir>/graph-data.json]
|
||||
|
||||
Custom output rules:
|
||||
- the destination directory must already exist inside the knowledge base
|
||||
- the graph filename must remain graph-data.json
|
||||
- graph-warnings.json is committed beside it as the unique paired warning file
|
||||
USAGE
|
||||
}
|
||||
|
||||
[ "$#" -ge 1 ] && [ "$#" -le 2 ] || {
|
||||
usage >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
command -v jq >/dev/null 2>&1 || {
|
||||
echo "ERROR: jq is not installed. Install it via:" >&2
|
||||
print_install_hint jq
|
||||
exit 1
|
||||
}
|
||||
command -v node >/dev/null 2>&1 || {
|
||||
echo "ERROR: node is not installed. Install it via:" >&2
|
||||
print_install_hint node
|
||||
exit 1
|
||||
}
|
||||
|
||||
WIKI_ROOT_INPUT="$1"
|
||||
[ -d "$WIKI_ROOT_INPUT" ] || {
|
||||
echo "ERROR: knowledge base does not exist or is not readable: $WIKI_ROOT_INPUT" >&2
|
||||
exit 1
|
||||
}
|
||||
WIKI_ROOT="$(cd "$WIKI_ROOT_INPUT" && pwd)"
|
||||
WIKI_DIR="$WIKI_ROOT/wiki"
|
||||
[ -d "$WIKI_DIR" ] || {
|
||||
echo "ERROR: wiki directory does not exist: $WIKI_DIR" >&2
|
||||
echo " Run init-wiki.sh before building the graph." >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
if [ "$#" -eq 2 ]; then
|
||||
OUTPUT_INPUT="$2"
|
||||
OUTPUT_PARENT="$(dirname "$OUTPUT_INPUT")"
|
||||
[ -d "$OUTPUT_PARENT" ] || {
|
||||
echo "ERROR: custom graph output directory does not exist: $OUTPUT_PARENT" >&2
|
||||
exit 1
|
||||
}
|
||||
OUTPUT="$(cd "$OUTPUT_PARENT" && pwd)/$(basename "$OUTPUT_INPUT")"
|
||||
else
|
||||
OUTPUT="$WIKI_ROOT/wiki/graph-data.json"
|
||||
fi
|
||||
|
||||
SKILL_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
HELPER="$SCRIPT_DIR/graph-analysis.js"
|
||||
CLI="$SCRIPT_DIR/wiki-link-cli.js"
|
||||
BUNDLE="$SCRIPT_DIR/lib/graph-warning-bundle.js"
|
||||
[ -f "$HELPER" ] || {
|
||||
echo "ERROR: 找不到图谱分析 helper：$HELPER" >&2
|
||||
echo " Reinstall the skill and retry." >&2
|
||||
exit 1
|
||||
}
|
||||
[ -f "$CLI" ] || {
|
||||
echo "ERROR: shared wikilink resolver is missing: $CLI" >&2
|
||||
echo " Reinstall the skill and retry." >&2
|
||||
exit 1
|
||||
}
|
||||
[ -f "$BUNDLE" ] || {
|
||||
echo "ERROR: graph warning bundle helper is missing: $BUNDLE" >&2
|
||||
echo " Reinstall the skill and retry." >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
MAX_CONTENT_BYTES=$((2 * 1024 * 1024))
|
||||
MAX_CONTENT_LINES=500
|
||||
MAX_INSIGHT_NODES=250
|
||||
MAX_INSIGHT_EDGES=1000
|
||||
|
||||
TMPDIR="$(mktemp -d -t llm-wiki-graph.XXXXXX)"
|
||||
trap 'rm -rf "$TMPDIR"' EXIT
|
||||
SCAN_DIR="$TMPDIR/scan"
|
||||
mkdir -p "$SCAN_DIR"
|
||||
|
||||
GRAPH_FLAGS=""
|
||||
if [ "${LLM_WIKI_TEST_MODE:-0}" = "1" ]; then
|
||||
BUILD_DATE="2026-01-01T00:00:00Z"
|
||||
GRAPH_FLAGS="--test-mode"
|
||||
else
|
||||
BUILD_DATE="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
fi
|
||||
|
||||
if [ -n "$GRAPH_FLAGS" ]; then
|
||||
node "$CLI" graph "$WIKI_ROOT" "$SCAN_DIR" "$GRAPH_FLAGS" > "$TMPDIR/scan.out"
|
||||
else
|
||||
node "$CLI" graph "$WIKI_ROOT" "$SCAN_DIR" > "$TMPDIR/scan.out"
|
||||
fi
|
||||
|
||||
for SCAN_FILE in nodes.json edges.json warning-groups.json candidate-sets.json scan-metrics.json; do
|
||||
[ -f "$SCAN_DIR/$SCAN_FILE" ] || {
|
||||
echo "ERROR: graph scan did not produce $SCAN_FILE" >&2
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
|
||||
WIKI_TITLE=""
|
||||
if [ -f "$WIKI_ROOT/purpose.md" ]; then
|
||||
WIKI_TITLE="$(awk '/^# / { sub(/^# +/, ""); print; exit }' "$WIKI_ROOT/purpose.md")"
|
||||
fi
|
||||
[ -n "$WIKI_TITLE" ] || WIKI_TITLE="$(basename "$WIKI_ROOT")"
|
||||
|
||||
TOTAL_SIZE="$(jq -r '.graph_source_bytes // 0' "$SCAN_DIR/scan-metrics.json")"
|
||||
DEGRADE=0
|
||||
if [ "$TOTAL_SIZE" -gt "$MAX_CONTENT_BYTES" ]; then
|
||||
DEGRADE=1
|
||||
fi
|
||||
|
||||
ANALYSIS_JSON="$TMPDIR/analysis.json"
|
||||
if ! node "$HELPER" \
|
||||
"$SCAN_DIR/nodes.json" \
|
||||
"$SCAN_DIR/edges.json" \
|
||||
"$ANALYSIS_JSON" \
|
||||
"$DEGRADE" \
|
||||
"$MAX_CONTENT_LINES" \
|
||||
"$MAX_INSIGHT_NODES" \
|
||||
"$MAX_INSIGHT_EDGES"; then
|
||||
echo "ERROR: graph analysis helper failed: $HELPER" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
jq -e '
|
||||
(.nodes | type) == "array" and
|
||||
(.edges | type) == "array" and
|
||||
(.insights | type) == "object" and
|
||||
(.learning | type) == "object"
|
||||
' "$ANALYSIS_JSON" > /dev/null 2>&1 || {
|
||||
echo "ERROR: graph analysis helper returned invalid JSON: $ANALYSIS_JSON" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
if [ "${LLM_WIKI_TEST_MODE:-0}" = "1" ]; then
|
||||
jq '.nodes | sort_by(.id)' "$ANALYSIS_JSON" > "$TMPDIR/nodes.sorted.json"
|
||||
jq '.edges | sort_by(.from, .to, .relation_type, .type)
|
||||
| to_entries
|
||||
| map(.value + {id: ("e" + ((.key + 1) | tostring))})' \
|
||||
"$ANALYSIS_JSON" > "$TMPDIR/edges.sorted.json"
|
||||
else
|
||||
jq '.nodes' "$ANALYSIS_JSON" > "$TMPDIR/nodes.sorted.json"
|
||||
jq '.edges' "$ANALYSIS_JSON" > "$TMPDIR/edges.sorted.json"
|
||||
fi
|
||||
|
||||
INITIAL_VIEW="$TMPDIR/initial-view.json"
|
||||
jq --slurpfile nodes "$TMPDIR/nodes.sorted.json" '
|
||||
. as $edges
|
||||
| ($nodes[0]) as $node_rows
|
||||
| (reduce $edges[] as $edge ({};
|
||||
.[$edge.from] = (.[$edge.from] // 0) + 1
|
||||
| .[$edge.to] = (.[$edge.to] // 0) + 1
|
||||
)) as $degree
|
||||
| ($node_rows | group_by(.community // "_")) as $groups
|
||||
| ([ $groups[] | max_by(($degree[.id] // 0)) | .id ]) as $representatives
|
||||
| ($node_rows
|
||||
| sort_by(-($degree[.id] // 0), .id)
|
||||
| map(.id)
|
||||
| map(select(. as $id | $representatives | index($id) | not))) as $rest
|
||||
| ($representatives + $rest)[0:30]
|
||||
' "$TMPDIR/edges.sorted.json" > "$INITIAL_VIEW"
|
||||
|
||||
NODE_COUNT="$(jq 'length' "$TMPDIR/nodes.sorted.json")"
|
||||
EDGE_COUNT="$(jq 'length' "$TMPDIR/edges.sorted.json")"
|
||||
INSIGHTS_DEGRADED="$(jq '.insights.meta.degraded == true' "$ANALYSIS_JSON")"
|
||||
|
||||
GRAPH_INPUT="$TMPDIR/graph-data.json"
|
||||
jq -n \
|
||||
--arg build_date "$BUILD_DATE" \
|
||||
--arg wiki_title "$WIKI_TITLE" \
|
||||
--argjson total_nodes "$NODE_COUNT" \
|
||||
--argjson total_edges "$EDGE_COUNT" \
|
||||
--slurpfile initial_view "$INITIAL_VIEW" \
|
||||
--slurpfile nodes "$TMPDIR/nodes.sorted.json" \
|
||||
--slurpfile edges "$TMPDIR/edges.sorted.json" \
|
||||
--slurpfile analysis "$ANALYSIS_JSON" \
|
||||
--argjson degraded "$DEGRADE" \
|
||||
--argjson insights_degraded "$INSIGHTS_DEGRADED" \
|
||||
'{
|
||||
meta: {
|
||||
build_date: $build_date,
|
||||
wiki_title: $wiki_title,
|
||||
total_nodes: $total_nodes,
|
||||
total_edges: $total_edges,
|
||||
initial_view: $initial_view[0],
|
||||
degraded: ($degraded == 1),
|
||||
insights_degraded: $insights_degraded
|
||||
},
|
||||
nodes: $nodes[0],
|
||||
edges: $edges[0],
|
||||
insights: $analysis[0].insights,
|
||||
learning: $analysis[0].learning
|
||||
}' > "$GRAPH_INPUT"
|
||||
|
||||
if ! node "$CLI" commit-pair \
|
||||
"$WIKI_ROOT" \
|
||||
"$GRAPH_INPUT" \
|
||||
"$SCAN_DIR/warning-groups.json" \
|
||||
"$SCAN_DIR/candidate-sets.json" \
|
||||
"$OUTPUT" > "$TMPDIR/commit.out"; then
|
||||
echo "ERROR: graph artifact pair commit failed for $OUTPUT" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Graph data generated: $OUTPUT"
|
||||
echo " Nodes: $NODE_COUNT"
|
||||
echo " Edges: $EDGE_COUNT"
|
||||
echo " Warning sidecar: $(dirname "$OUTPUT")/graph-warnings.json"
|
||||
[ "$DEGRADE" = "1" ] && echo " Warning: content embedding degraded above 2 MiB"
|
||||
[ "$INSIGHTS_DEGRADED" = "true" ] && echo " Warning: insights degraded for graph size"
|
||||
exit 0
|
||||
651
llm-wiki/scripts/build-graph-html.sh
Executable file
651
llm-wiki/scripts/build-graph-html.sh
Executable file
@@ -0,0 +1,651 @@
|
||||
#!/bin/bash
|
||||
# build-graph-html.sh — 生成共享 graph-engine 驱动的离线知识图谱 HTML
|
||||
#
|
||||
# 用法：
|
||||
# bash scripts/build-graph-html.sh <wiki_root>
|
||||
#
|
||||
# 前置：需要先运行 build-graph-data.sh 生成 wiki/graph-data.json
|
||||
#
|
||||
# 行为：
|
||||
# 1. 读取 packages/graph-engine/dist/engine.iife.js
|
||||
# 2. 验证并内嵌配对告警、graph-data.json 与可选 .wiki-graph-layout.json 钉位
|
||||
# 3. 注入离线启动脚本：创建 graph engine，持久化钉位到 localStorage
|
||||
# 4. 生成单文件 knowledge-graph.html
|
||||
#
|
||||
# 退出码：0 成功；1 依赖/文件缺失/参数错误
|
||||
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="${BASH_SOURCE[0]%/*}"
|
||||
[ "$SCRIPT_DIR" = "${BASH_SOURCE[0]}" ] && SCRIPT_DIR="."
|
||||
SCRIPT_DIR="$(cd "$SCRIPT_DIR" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "$SCRIPT_DIR/shared-config.sh"
|
||||
|
||||
print_usage() {
|
||||
cat <<'USAGE'
|
||||
用法：
|
||||
bash scripts/build-graph-html.sh <wiki_root>
|
||||
|
||||
示例：
|
||||
bash scripts/build-graph-html.sh /path/to/wiki-root
|
||||
USAGE
|
||||
}
|
||||
|
||||
die() {
|
||||
echo "ERROR: $1" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
ensure_file() {
|
||||
local file="$1"
|
||||
local label="${2:-文件}"
|
||||
[ -f "$file" ] || {
|
||||
echo "ERROR: 找不到${label} $file" >&2
|
||||
echo " 请先运行 npm run build -w @llm-wiki/graph-engine，或重装 skill。" >&2
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
json_for_script() {
|
||||
node - "$1" <<'NODE'
|
||||
const fs = require("node:fs");
|
||||
const text = fs.readFileSync(process.argv[2], "utf8");
|
||||
const escaped = text.replace(/[<>&\u2028\u2029]/g, (character) => (
|
||||
`\\u${character.codePointAt(0).toString(16).padStart(4, "0")}`
|
||||
));
|
||||
process.stdout.write(escaped);
|
||||
NODE
|
||||
}
|
||||
|
||||
script_for_inline() {
|
||||
perl -pe 's|//# sourceMappingURL=.*$||' "$1"
|
||||
}
|
||||
|
||||
html_escape_text() {
|
||||
printf '%s' "$1" | perl -pe 's/&/&/g; s/</</g; s/>/>/g; s/"/"/g; s/'"'"'/'/g'
|
||||
}
|
||||
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
-h|--help)
|
||||
print_usage
|
||||
exit 0
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
break
|
||||
;;
|
||||
-*)
|
||||
die "未知选项: $1"
|
||||
;;
|
||||
*)
|
||||
break
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
[ "$#" -eq 1 ] || {
|
||||
print_usage >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
WIKI_ROOT="$1"
|
||||
|
||||
command -v jq >/dev/null 2>&1 || {
|
||||
echo "ERROR: jq is not installed. Install it via:" >&2
|
||||
print_install_hint jq
|
||||
exit 1
|
||||
}
|
||||
|
||||
SKILL_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
DATA="$WIKI_ROOT/wiki/graph-data.json"
|
||||
WARNINGS="$WIKI_ROOT/wiki/graph-warnings.json"
|
||||
LAYOUT="$WIKI_ROOT/.wiki-graph-layout.json"
|
||||
ENGINE="$SKILL_DIR/packages/graph-engine/dist/engine.iife.js"
|
||||
MARKED="$SKILL_DIR/deps/marked.min.js"
|
||||
PURIFY="$SKILL_DIR/deps/purify.min.js"
|
||||
OUTPUT="$WIKI_ROOT/wiki/knowledge-graph.html"
|
||||
WARNING_CLI="$SCRIPT_DIR/wiki-link-cli.js"
|
||||
|
||||
[ -f "$DATA" ] || {
|
||||
echo "ERROR: 未找到 $DATA" >&2
|
||||
echo " 请先运行 build-graph-data.sh 生成图谱数据" >&2
|
||||
exit 1
|
||||
}
|
||||
ensure_file "$ENGINE" "graph-engine IIFE 产物"
|
||||
ensure_file "$MARKED" "marked vendor"
|
||||
ensure_file "$PURIFY" "purify vendor"
|
||||
ensure_file "$WARNING_CLI" "warning verifier"
|
||||
|
||||
WIKI_TITLE=$(jq -r '.meta.wiki_title // "知识库"' "$DATA")
|
||||
NODE_COUNT=$(jq -r '.meta.total_nodes // 0' "$DATA")
|
||||
EDGE_COUNT=$(jq -r '.meta.total_edges // 0' "$DATA")
|
||||
BUILD_DATE=$(jq -r '.meta.build_date // ""' "$DATA")
|
||||
BUILD_DATE_SHORT="${BUILD_DATE:0:10}"
|
||||
[ -n "$BUILD_DATE_SHORT" ] || BUILD_DATE_SHORT="未知"
|
||||
WIKI_TITLE_HTML=$(html_escape_text "$WIKI_TITLE")
|
||||
NODE_COUNT_HTML=$(html_escape_text "$NODE_COUNT")
|
||||
EDGE_COUNT_HTML=$(html_escape_text "$EDGE_COUNT")
|
||||
BUILD_DATE_SHORT_HTML=$(html_escape_text "$BUILD_DATE_SHORT")
|
||||
|
||||
layout_json='{"version":2,"pins":{},"updatedAt":""}'
|
||||
if [ -f "$LAYOUT" ]; then
|
||||
if layout_json_candidate=$(jq -c '{version:(.version // 1), pins:(.pins // {}), updatedAt:(.updatedAt // "")}' "$LAYOUT" 2>/dev/null); then
|
||||
layout_json="$layout_json_candidate"
|
||||
else
|
||||
echo "WARN: 忽略损坏的钉位文件：$LAYOUT" >&2
|
||||
fi
|
||||
fi
|
||||
|
||||
output_dir="$(dirname "$OUTPUT")"
|
||||
mkdir -p "$output_dir"
|
||||
output_tmp="$OUTPUT.partial"
|
||||
output_next="$OUTPUT.next"
|
||||
warning_data_tmp="$(mktemp -t llm-wiki-warning.XXXXXX)"
|
||||
trap 'rm -f "$warning_data_tmp" "$output_tmp" "$output_next"' EXIT
|
||||
rm -f "$output_tmp" "$output_next"
|
||||
|
||||
node "$WARNING_CLI" warning-embed "$WIKI_ROOT" "$DATA" "$WARNINGS" "$warning_data_tmp" \
|
||||
|| die "告警详情验证失败，无法生成安全的离线载荷"
|
||||
|
||||
cat > "$output_tmp" <<HTML_HEAD
|
||||
<!doctype html>
|
||||
<html lang="zh-Hans">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>知识图谱 · ${WIKI_TITLE_HTML}</title>
|
||||
<style>
|
||||
:root {
|
||||
color-scheme: light;
|
||||
--page-bg: #f7f1e5;
|
||||
--panel: rgba(255, 252, 244, .86);
|
||||
--ink: #2f2924;
|
||||
--muted: #766b5f;
|
||||
--rule: rgba(79, 64, 46, .2);
|
||||
--accent: #a83f35;
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
html, body { margin: 0; min-height: 100%; }
|
||||
body {
|
||||
min-height: 100vh;
|
||||
color: var(--ink);
|
||||
background: var(--page-bg);
|
||||
font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
|
||||
}
|
||||
.offline-shell {
|
||||
display: grid;
|
||||
grid-template-rows: auto auto minmax(0, 1fr);
|
||||
min-height: 100vh;
|
||||
}
|
||||
.offline-header {
|
||||
position: relative;
|
||||
z-index: 20;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 16px;
|
||||
min-height: 64px;
|
||||
padding: 12px 18px;
|
||||
border-bottom: 1px solid var(--rule);
|
||||
background: var(--panel);
|
||||
backdrop-filter: blur(10px);
|
||||
}
|
||||
.offline-title { min-width: 0; }
|
||||
.offline-title h1 {
|
||||
margin: 0;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
font-family: Georgia, "Times New Roman", serif;
|
||||
font-size: 20px;
|
||||
letter-spacing: 0;
|
||||
}
|
||||
.offline-title p {
|
||||
margin: 4px 0 0;
|
||||
color: var(--muted);
|
||||
font-size: 12px;
|
||||
}
|
||||
.offline-badges {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
flex-wrap: wrap;
|
||||
justify-content: flex-end;
|
||||
color: var(--muted);
|
||||
font-size: 12px;
|
||||
}
|
||||
.offline-badges span {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
min-height: 26px;
|
||||
padding: 4px 9px;
|
||||
border: 1px solid var(--rule);
|
||||
border-radius: 999px;
|
||||
background: rgba(255, 255, 255, .46);
|
||||
}
|
||||
.offline-theme-toggle {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
min-height: 26px;
|
||||
border: 1px solid var(--rule);
|
||||
border-radius: 999px;
|
||||
background: rgba(255, 255, 255, .52);
|
||||
color: var(--ink);
|
||||
padding: 4px 10px;
|
||||
font: inherit;
|
||||
cursor: pointer;
|
||||
}
|
||||
.offline-theme-toggle:hover {
|
||||
background: rgba(168, 63, 53, .08);
|
||||
}
|
||||
.offline-toolbar-host {
|
||||
position: relative;
|
||||
z-index: 5;
|
||||
flex: 1 1 320px;
|
||||
min-width: 240px;
|
||||
min-height: 38px;
|
||||
}
|
||||
.offline-toolbar-host .graph-toolbar {
|
||||
position: static;
|
||||
inset: auto;
|
||||
justify-items: center;
|
||||
}
|
||||
.offline-toolbar-host .graph-toolbar-panel {
|
||||
position: absolute;
|
||||
top: 38px;
|
||||
left: 50%;
|
||||
transform: translateX(-50%);
|
||||
}
|
||||
.offline-main {
|
||||
position: relative;
|
||||
z-index: 1;
|
||||
grid-row: 3;
|
||||
min-height: 0;
|
||||
padding: 0;
|
||||
}
|
||||
#graph-root {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
min-height: 560px;
|
||||
}
|
||||
.offline-error {
|
||||
margin: 24px;
|
||||
padding: 16px;
|
||||
border: 1px solid rgba(168, 63, 53, .35);
|
||||
border-radius: 8px;
|
||||
background: rgba(168, 63, 53, .08);
|
||||
color: #7b2b24;
|
||||
font-size: 14px;
|
||||
line-height: 1.6;
|
||||
}
|
||||
.offline-storage-warning {
|
||||
margin: 12px 18px 0;
|
||||
}
|
||||
.offline-warning-banner {
|
||||
position: relative;
|
||||
z-index: 15;
|
||||
margin: 10px 18px 0;
|
||||
padding: 12px 14px;
|
||||
border: 1px solid rgba(168, 63, 53, .3);
|
||||
border-radius: 10px;
|
||||
background: rgba(255, 249, 237, .94);
|
||||
color: var(--ink);
|
||||
font-size: 13px;
|
||||
line-height: 1.55;
|
||||
}
|
||||
.offline-warning-banner[hidden] { display: none; }
|
||||
.offline-warning-summary { font-weight: 650; }
|
||||
.offline-warning-notice { margin-top: 6px; color: #7b2b24; }
|
||||
.offline-warning-details { margin-top: 8px; }
|
||||
.offline-warning-details > summary { cursor: pointer; color: var(--accent); }
|
||||
.offline-warning-group { margin: 10px 0 0 14px; }
|
||||
.offline-warning-group h3 { margin: 0; font-size: 13px; }
|
||||
.offline-warning-group ul { margin: 4px 0 0; padding-left: 20px; }
|
||||
@media (max-width: 720px) {
|
||||
.offline-header { align-items: flex-start; flex-direction: column; }
|
||||
.offline-toolbar-host { width: 100%; flex-basis: auto; }
|
||||
.offline-toolbar-host .graph-toolbar { justify-items: start; }
|
||||
.offline-toolbar-host .graph-toolbar-panel { left: 0; transform: none; }
|
||||
.offline-badges { justify-content: flex-start; }
|
||||
#graph-root { min-height: 520px; }
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="offline-shell" data-llm-wiki-offline-graph="engine">
|
||||
<header class="offline-header">
|
||||
<div class="offline-title">
|
||||
<h1>${WIKI_TITLE_HTML} 知识舆图</h1>
|
||||
<p>国风知识库·数字山水图</p>
|
||||
</div>
|
||||
<div class="offline-toolbar-host" data-testid="offline-toolbar-host"></div>
|
||||
<div class="offline-badges" aria-label="图谱统计">
|
||||
<span>${NODE_COUNT_HTML} 节点</span>
|
||||
<span>${EDGE_COUNT_HTML} 关联</span>
|
||||
<span>${BUILD_DATE_SHORT_HTML}</span>
|
||||
<button class="offline-theme-toggle" type="button" data-testid="offline-theme-toggle" aria-label="切换墨夜主题">墨夜</button>
|
||||
</div>
|
||||
</header>
|
||||
<section class="offline-warning-banner" data-testid="offline-warning-banner" aria-label="图谱告警" hidden>
|
||||
<div class="offline-warning-summary" data-testid="offline-warning-summary"></div>
|
||||
<div class="offline-warning-notice" data-testid="offline-warning-unavailable" hidden></div>
|
||||
<div class="offline-warning-notice" data-testid="offline-warning-truncated" hidden></div>
|
||||
<details class="offline-warning-details" data-testid="offline-warning-details">
|
||||
<summary>查看告警详情</summary>
|
||||
<div data-testid="offline-warning-groups"></div>
|
||||
</details>
|
||||
</section>
|
||||
<main class="offline-main">
|
||||
<div id="graph-root" data-testid="offline-graph-root"></div>
|
||||
</main>
|
||||
</div>
|
||||
<script id="graph-data" type="application/json">
|
||||
HTML_HEAD
|
||||
json_for_script "$DATA" >> "$output_tmp"
|
||||
cat >> "$output_tmp" <<'HTML_MID'
|
||||
</script>
|
||||
<script id="graph-warning-data" type="application/json">
|
||||
HTML_MID
|
||||
cat "$warning_data_tmp" >> "$output_tmp"
|
||||
cat >> "$output_tmp" <<'HTML_WARNING_END'
|
||||
</script>
|
||||
<script id="graph-layout" type="application/json">
|
||||
HTML_WARNING_END
|
||||
printf '%s\n' "$layout_json" | perl -pe 's|</script>|<\/script>|gi' >> "$output_tmp"
|
||||
cat >> "$output_tmp" <<'HTML_ENGINE'
|
||||
</script>
|
||||
<script>
|
||||
HTML_ENGINE
|
||||
script_for_inline "$MARKED" >> "$output_tmp"
|
||||
printf '\n' >> "$output_tmp"
|
||||
script_for_inline "$PURIFY" >> "$output_tmp"
|
||||
printf '\n' >> "$output_tmp"
|
||||
script_for_inline "$ENGINE" >> "$output_tmp"
|
||||
cat >> "$output_tmp" <<'HTML_BOOT'
|
||||
</script>
|
||||
<script>
|
||||
(function () {
|
||||
var root = document.getElementById("graph-root");
|
||||
var toolbarHost = document.querySelector("[data-testid='offline-toolbar-host']");
|
||||
var dataEl = document.getElementById("graph-data");
|
||||
var warningDataEl = document.getElementById("graph-warning-data");
|
||||
var layoutEl = document.getElementById("graph-layout");
|
||||
var storageAvailable = true;
|
||||
function showError(message) {
|
||||
if (!root) return;
|
||||
root.innerHTML = "";
|
||||
if (toolbarHost) toolbarHost.innerHTML = "";
|
||||
var storageWarning = document.querySelector(".offline-storage-warning");
|
||||
if (storageWarning) storageWarning.remove();
|
||||
try {
|
||||
if (window.__LLM_WIKI_GRAPH_ENGINE__) window.__LLM_WIKI_GRAPH_ENGINE__.destroy();
|
||||
} catch (_) {}
|
||||
window.__LLM_WIKI_GRAPH_ENGINE__ = undefined;
|
||||
var box = document.createElement("div");
|
||||
box.className = "offline-error";
|
||||
box.textContent = message;
|
||||
root.appendChild(box);
|
||||
}
|
||||
function showStorageRecoveryHint() {
|
||||
if (document.querySelector(".offline-storage-warning")) return;
|
||||
var main = document.querySelector(".offline-main");
|
||||
if (!main || !root) return;
|
||||
var box = document.createElement("div");
|
||||
box.className = "offline-error offline-storage-warning";
|
||||
box.setAttribute("role", "status");
|
||||
box.textContent = "浏览器存储不可用。图谱仍可浏览，但刷新后主题与固定位置会恢复默认。";
|
||||
main.insertBefore(box, root);
|
||||
}
|
||||
function parseJson(el, fallback) {
|
||||
try { return el && el.textContent ? JSON.parse(el.textContent) : fallback; }
|
||||
catch (err) { return fallback; }
|
||||
}
|
||||
function safeRelativePath(value) {
|
||||
var text = String(value == null ? "" : value);
|
||||
if (!text || text.charAt(0) === "/" || text.indexOf("\\") >= 0 || text === ".." || text.indexOf("../") === 0 || text.indexOf("/../") >= 0) {
|
||||
return "（路径不可用）";
|
||||
}
|
||||
return text;
|
||||
}
|
||||
function appendTextList(parent, values) {
|
||||
if (!values.length) return;
|
||||
var list = document.createElement("ul");
|
||||
for (var i = 0; i < values.length; i++) {
|
||||
var item = document.createElement("li");
|
||||
item.textContent = values[i];
|
||||
list.appendChild(item);
|
||||
}
|
||||
parent.appendChild(list);
|
||||
}
|
||||
function renderWarnings(payload, modelWarnings) {
|
||||
var banner = document.querySelector("[data-testid='offline-warning-banner']");
|
||||
var summaryBox = document.querySelector("[data-testid='offline-warning-summary']");
|
||||
var unavailable = document.querySelector("[data-testid='offline-warning-unavailable']");
|
||||
var truncated = document.querySelector("[data-testid='offline-warning-truncated']");
|
||||
var details = document.querySelector("[data-testid='offline-warning-details']");
|
||||
var groupsBox = document.querySelector("[data-testid='offline-warning-groups']");
|
||||
if (!banner || !summaryBox || !payload) return;
|
||||
var summary = payload.summary || {};
|
||||
var warningById = {};
|
||||
for (var warningIndex = 0; warningIndex < (modelWarnings || []).length; warningIndex++) {
|
||||
var warning = modelWarnings[warningIndex];
|
||||
if (warning && warning.warning_id && !warningById[warning.warning_id]) warningById[warning.warning_id] = warning;
|
||||
}
|
||||
var warnings = Object.keys(warningById).sort().map(function (warningId) { return warningById[warningId]; });
|
||||
var hasWarningSummary = typeof summary.total_groups === "number"
|
||||
|| typeof summary.total_occurrences === "number";
|
||||
if (!hasWarningSummary && warnings.length === 0) return;
|
||||
if (!hasWarningSummary) {
|
||||
summary = { total_groups: warnings.length, total_occurrences: 0, error_occurrences: 0, warning_occurrences: 0, by_code: {} };
|
||||
for (var summaryIndex = 0; summaryIndex < warnings.length; summaryIndex++) {
|
||||
var summaryWarning = warnings[summaryIndex];
|
||||
var count = Number(summaryWarning.occurrence_count || 0);
|
||||
summary.total_occurrences += count;
|
||||
if (summaryWarning.severity === "error") summary.error_occurrences += count;
|
||||
else summary.warning_occurrences += count;
|
||||
summary.by_code[summaryWarning.code] = (summary.by_code[summaryWarning.code] || 0) + count;
|
||||
}
|
||||
}
|
||||
var codes = Object.keys(summary.by_code || {}).sort().map(function (code) {
|
||||
return code + ": " + summary.by_code[code];
|
||||
});
|
||||
summaryBox.textContent = "图谱告警 " + (summary.total_groups || 0) + " 组 · "
|
||||
+ (summary.total_occurrences || 0) + " 处 · 错误 " + (summary.error_occurrences || 0)
|
||||
+ " · 提示 " + (summary.warning_occurrences || 0)
|
||||
+ (codes.length ? " · " + codes.join(" · ") : "");
|
||||
banner.hidden = false;
|
||||
|
||||
if (payload.details_status !== "available") {
|
||||
if (unavailable) {
|
||||
unavailable.hidden = false;
|
||||
unavailable.textContent = "告警详情暂不可用，请重新构建图谱";
|
||||
}
|
||||
}
|
||||
if (payload.warning_details_truncated && truncated) {
|
||||
truncated.hidden = false;
|
||||
truncated.textContent = "详情过大，已精简；运行 check 查看完整报告"
|
||||
+ "（省略 " + (payload.omitted_group_count || 0) + " 组、"
|
||||
+ (payload.omitted_candidate_set_count || 0) + " 个候选集合）";
|
||||
}
|
||||
|
||||
var bundle = payload.bundle || { candidate_sets: [], groups: [] };
|
||||
var candidateSets = {};
|
||||
for (var setIndex = 0; setIndex < (bundle.candidate_sets || []).length; setIndex++) {
|
||||
var candidateSet = bundle.candidate_sets[setIndex];
|
||||
candidateSets[candidateSet.candidate_set_id] = candidateSet;
|
||||
}
|
||||
if (groupsBox) groupsBox.innerHTML = "";
|
||||
for (var groupIndex = 0; groupsBox && groupIndex < warnings.length; groupIndex++) {
|
||||
var group = warnings[groupIndex];
|
||||
var groupBox = document.createElement("section");
|
||||
groupBox.className = "offline-warning-group";
|
||||
groupBox.setAttribute("data-warning-id", group.warning_id);
|
||||
var title = document.createElement("h3");
|
||||
title.textContent = group.code + " · " + group.occurrence_count + " 处";
|
||||
groupBox.appendChild(title);
|
||||
if (group.message) {
|
||||
var message = document.createElement("div");
|
||||
message.textContent = group.message;
|
||||
groupBox.appendChild(message);
|
||||
}
|
||||
var set = candidateSets[group.candidate_set_id];
|
||||
appendTextList(groupBox, set ? (set.candidates || []).map(safeRelativePath) : []);
|
||||
appendTextList(groupBox, (group.occurrences || []).map(function (occurrence) {
|
||||
return safeRelativePath(occurrence.source_path) + ":" + occurrence.line + ":" + occurrence.column + " " + occurrence.raw_link;
|
||||
}));
|
||||
groupsBox.appendChild(groupBox);
|
||||
}
|
||||
if (details) details.hidden = warnings.length === 0;
|
||||
}
|
||||
function normalizeStorageSegment(value) {
|
||||
return String(value == null ? "" : value).trim().toLowerCase()
|
||||
.replace(/[^a-z0-9一-鿿]+/g, "-")
|
||||
.replace(/^-+|-+$/g, "")
|
||||
.slice(0, 48);
|
||||
}
|
||||
function hashString(value) {
|
||||
var input = String(value == null ? "" : value);
|
||||
var hash = 0;
|
||||
for (var i = 0; i < input.length; i++) {
|
||||
hash = ((hash << 5) - hash + input.charCodeAt(i)) >>> 0;
|
||||
}
|
||||
return hash.toString(36);
|
||||
}
|
||||
function storageNamespace(meta, pathname) {
|
||||
var title = normalizeStorageSegment(meta && meta.wiki_title ? meta.wiki_title : "");
|
||||
var basis = typeof pathname === "string" && pathname ? pathname : (meta && meta.wiki_title) || title || "default";
|
||||
return "llm-wiki:" + (title || "default") + ":" + hashString(basis);
|
||||
}
|
||||
function readStoredPins(key) {
|
||||
try {
|
||||
var raw = window.localStorage && window.localStorage.getItem(key);
|
||||
var parsed = raw ? JSON.parse(raw) : null;
|
||||
return parsed && typeof parsed === "object" ? parsed : {};
|
||||
} catch (_) {
|
||||
storageAvailable = false;
|
||||
return {};
|
||||
}
|
||||
}
|
||||
function writeStoredPins(key, pins) {
|
||||
try {
|
||||
if (window.localStorage) window.localStorage.setItem(key, JSON.stringify(pins || {}));
|
||||
} catch (_) {
|
||||
storageAvailable = false;
|
||||
showStorageRecoveryHint();
|
||||
}
|
||||
}
|
||||
function normalizeBakedPins(layout) {
|
||||
return window.LlmWikiGraphEngine.normalizeGraphLayoutFile(layout).pins;
|
||||
}
|
||||
function normalizeStoredPins(rawPins) {
|
||||
return window.LlmWikiGraphEngine.normalizeGraphPinMap(rawPins);
|
||||
}
|
||||
if (!root || !dataEl || !window.LlmWikiGraphEngine || !window.LlmWikiGraphEngine.createGraphEngine || !window.LlmWikiGraphEngine.projectGraphInput) {
|
||||
showError("图谱引擎加载失败。请确认 HTML 文件完整生成。");
|
||||
return;
|
||||
}
|
||||
var graphData = parseJson(dataEl, null);
|
||||
if (!graphData || !Array.isArray(graphData.nodes) || !Array.isArray(graphData.edges)) {
|
||||
showError("图谱数据格式不完整。请重新运行 build-graph-data.sh 与 build-graph-html.sh。");
|
||||
return;
|
||||
}
|
||||
var warningPayload = parseJson(warningDataEl, { details_status: "unavailable", summary: {} });
|
||||
var inputWarningGroups = warningPayload.details_status === "available" && warningPayload.bundle
|
||||
? (warningPayload.bundle.groups || [])
|
||||
: [];
|
||||
var projection = window.LlmWikiGraphEngine.projectGraphInput(graphData, inputWarningGroups);
|
||||
graphData = projection.data;
|
||||
renderWarnings(warningPayload, projection.warnings || []);
|
||||
var bakedLayout = parseJson(layoutEl, { pins: {} });
|
||||
var key = storageNamespace(graphData.meta || {}, window.location && window.location.pathname) + ":graph-pins";
|
||||
var themeKey = storageNamespace(graphData.meta || {}, window.location && window.location.pathname) + ":graph-theme";
|
||||
var pins = Object.assign({}, normalizeBakedPins(bakedLayout), normalizeStoredPins(readStoredPins(key)));
|
||||
var themeToggle = document.querySelector("[data-testid='offline-theme-toggle']");
|
||||
function readStoredTheme() {
|
||||
try {
|
||||
var value = window.localStorage && window.localStorage.getItem(themeKey);
|
||||
return value === "mo-ye" ? "mo-ye" : "shan-shui";
|
||||
} catch (_) {
|
||||
storageAvailable = false;
|
||||
return "shan-shui";
|
||||
}
|
||||
}
|
||||
function writeStoredTheme(theme) {
|
||||
try {
|
||||
if (window.localStorage) window.localStorage.setItem(themeKey, theme);
|
||||
} catch (_) {
|
||||
storageAvailable = false;
|
||||
showStorageRecoveryHint();
|
||||
}
|
||||
}
|
||||
function syncThemeToggle(theme) {
|
||||
if (!themeToggle) return;
|
||||
var next = theme === "mo-ye" ? "shan-shui" : "mo-ye";
|
||||
themeToggle.textContent = theme === "mo-ye" ? "山水" : "墨夜";
|
||||
themeToggle.setAttribute("aria-label", next === "mo-ye" ? "切换墨夜主题" : "切换山水主题");
|
||||
}
|
||||
var currentTheme = readStoredTheme();
|
||||
var engine = null;
|
||||
try {
|
||||
engine = window.LlmWikiGraphEngine.createGraphEngine(root, {
|
||||
data: graphData,
|
||||
pins: pins,
|
||||
theme: currentTheme,
|
||||
toolbarContainer: toolbarHost,
|
||||
capabilities: window.LlmWikiGraphEngine.createGraphOfflineCapabilities({
|
||||
persistPins: function (nextPins) {
|
||||
writeStoredPins(key, nextPins || {});
|
||||
return Promise.resolve();
|
||||
}
|
||||
}).capabilities
|
||||
});
|
||||
syncThemeToggle(currentTheme);
|
||||
if (themeToggle) {
|
||||
themeToggle.addEventListener("click", function () {
|
||||
currentTheme = currentTheme === "mo-ye" ? "shan-shui" : "mo-ye";
|
||||
engine.setTheme(currentTheme);
|
||||
writeStoredTheme(currentTheme);
|
||||
syncThemeToggle(currentTheme);
|
||||
});
|
||||
}
|
||||
window.__LLM_WIKI_GRAPH_ENGINE__ = engine;
|
||||
window.__LLM_WIKI_GRAPH_PINS_KEY__ = key;
|
||||
window.__LLM_WIKI_GRAPH_THEME_KEY__ = themeKey;
|
||||
if (!storageAvailable) showStorageRecoveryHint();
|
||||
} catch (_) {
|
||||
try { if (engine) engine.destroy(); } catch (_) {}
|
||||
showError("图谱引擎加载失败。请确认 HTML 文件完整生成。");
|
||||
return;
|
||||
}
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
HTML_BOOT
|
||||
|
||||
mv "$output_tmp" "$output_next"
|
||||
mv "$output_next" "$OUTPUT"
|
||||
|
||||
rm -f \
|
||||
"$output_dir/d3.min.js" \
|
||||
"$output_dir/rough.min.js" \
|
||||
"$output_dir/marked.min.js" \
|
||||
"$output_dir/purify.min.js" \
|
||||
"$output_dir/graph-wash.js" \
|
||||
"$output_dir/graph-wash-helpers.js" \
|
||||
"$output_dir/LICENSE-d3.txt" \
|
||||
"$output_dir/LICENSE-roughjs.txt" \
|
||||
"$output_dir/LICENSE-marked.txt" \
|
||||
"$output_dir/LICENSE-purify.txt"
|
||||
|
||||
output_size=$(wc -c < "$OUTPUT" | tr -d ' ')
|
||||
output_kb=$((output_size / 1024))
|
||||
|
||||
echo "交互式图谱已生成："
|
||||
echo " - $OUTPUT (${output_kb} KB)"
|
||||
echo " 节点 $NODE_COUNT · 关联 $EDGE_COUNT"
|
||||
echo ""
|
||||
echo "查看方式："
|
||||
echo " 双击 $OUTPUT"
|
||||
352
llm-wiki/scripts/cache.sh
Executable file
352
llm-wiki/scripts/cache.sh
Executable file
@@ -0,0 +1,352 @@
|
||||
#!/bin/bash
|
||||
# llm-wiki 缓存脚本
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "$SCRIPT_DIR/shared-config.sh"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
用法：
|
||||
bash scripts/cache.sh check <file>
|
||||
bash scripts/cache.sh update <file> <source_page>
|
||||
bash scripts/cache.sh invalidate <file>
|
||||
EOF
|
||||
}
|
||||
|
||||
require_file() {
|
||||
local file_path="$1"
|
||||
|
||||
[ -n "$file_path" ] || {
|
||||
usage
|
||||
exit 1
|
||||
}
|
||||
|
||||
[ -f "$file_path" ] || {
|
||||
echo "文件不存在：$file_path" >&2
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
find_wiki_root() {
|
||||
local file_path="$1"
|
||||
local dir parent
|
||||
|
||||
dir="$(cd "$(dirname "$file_path")" && pwd)"
|
||||
|
||||
while true; do
|
||||
if [ -f "$dir/.wiki-cache.json" ] || [ -f "$dir/.wiki-schema.md" ]; then
|
||||
printf '%s\n' "$dir"
|
||||
return 0
|
||||
fi
|
||||
|
||||
parent="$(dirname "$dir")"
|
||||
[ "$parent" = "$dir" ] && return 1
|
||||
dir="$parent"
|
||||
done
|
||||
}
|
||||
|
||||
cache_file_path() {
|
||||
printf '%s/.wiki-cache.json\n' "$1"
|
||||
}
|
||||
|
||||
ensure_cache_file() {
|
||||
local cache_file="$1"
|
||||
|
||||
if [ ! -f "$cache_file" ]; then
|
||||
cat > "$cache_file" <<'EOF'
|
||||
{
|
||||
"version": 1,
|
||||
"entries": {}
|
||||
}
|
||||
EOF
|
||||
fi
|
||||
}
|
||||
|
||||
relative_path() {
|
||||
require_python_cmd
|
||||
|
||||
"$PYTHON_CMD" - "$1" "$2" <<'PY'
|
||||
import os
|
||||
import sys
|
||||
|
||||
print(os.path.relpath(os.path.realpath(sys.argv[2]), os.path.realpath(sys.argv[1])))
|
||||
PY
|
||||
}
|
||||
|
||||
normalized_source_page() {
|
||||
local wiki_root="$1"
|
||||
local source_page="$2"
|
||||
|
||||
if [ -z "$source_page" ]; then
|
||||
printf '%s\n' ""
|
||||
return 0
|
||||
fi
|
||||
|
||||
case "$source_page" in
|
||||
/*)
|
||||
require_python_cmd
|
||||
|
||||
"$PYTHON_CMD" - "$wiki_root" "$source_page" <<'PY'
|
||||
import os
|
||||
import sys
|
||||
|
||||
wiki_root = os.path.realpath(sys.argv[1])
|
||||
source_page = os.path.realpath(sys.argv[2])
|
||||
|
||||
try:
|
||||
common = os.path.commonpath([wiki_root, source_page])
|
||||
except ValueError:
|
||||
common = ""
|
||||
|
||||
if common == wiki_root:
|
||||
print(os.path.relpath(source_page, wiki_root))
|
||||
else:
|
||||
print(sys.argv[2])
|
||||
PY
|
||||
;;
|
||||
*)
|
||||
printf '%s\n' "$source_page"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
file_hash() {
|
||||
require_python_cmd
|
||||
|
||||
"$PYTHON_CMD" - "$1" "$2" <<'PY'
|
||||
import hashlib
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
relative_path = sys.argv[1].encode("utf-8")
|
||||
file_path = pathlib.Path(sys.argv[2])
|
||||
content = file_path.read_bytes()
|
||||
|
||||
digest = hashlib.sha256(relative_path + b"\0" + content).hexdigest()
|
||||
print(f"sha256:{digest}")
|
||||
PY
|
||||
}
|
||||
|
||||
cache_check() {
|
||||
local file_path="$1"
|
||||
local wiki_root cache_file relative_path_value current_hash result
|
||||
|
||||
require_file "$file_path"
|
||||
wiki_root="$(find_wiki_root "$file_path")" || {
|
||||
echo "未找到知识库根目录：$file_path" >&2
|
||||
exit 1
|
||||
}
|
||||
cache_file="$(cache_file_path "$wiki_root")"
|
||||
|
||||
if [ ! -f "$cache_file" ]; then
|
||||
printf 'MISS\n'
|
||||
return 0
|
||||
fi
|
||||
|
||||
require_python_cmd
|
||||
|
||||
relative_path_value="$(relative_path "$wiki_root" "$file_path")"
|
||||
current_hash="$(file_hash "$relative_path_value" "$file_path")"
|
||||
|
||||
result="$(
|
||||
"$PYTHON_CMD" - "$cache_file" "$wiki_root" "$relative_path_value" "$current_hash" <<'PY'
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
cache_file, wiki_root, relative_path, current_hash = sys.argv[1:5]
|
||||
|
||||
with open(cache_file, "r", encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
|
||||
entry = data.get("entries", {}).get(relative_path)
|
||||
|
||||
# 无 cache entry → 尝试自愈（exact filename stem match + source_path 验证）
|
||||
if not entry:
|
||||
raw_stem = pathlib.Path(relative_path).stem
|
||||
sources_dir = os.path.join(wiki_root, "wiki", "sources")
|
||||
if os.path.isdir(sources_dir):
|
||||
for f in os.listdir(sources_dir):
|
||||
if pathlib.Path(f).stem == raw_stem and f.endswith(".md"):
|
||||
source_page = os.path.join("wiki", "sources", f)
|
||||
source_abs = os.path.join(wiki_root, source_page)
|
||||
# 验证 source 页面的 source_path frontmatter 是否指向当前 raw 文件
|
||||
source_path_match = False
|
||||
try:
|
||||
with open(source_abs, "r", encoding="utf-8") as sf:
|
||||
in_frontmatter = False
|
||||
for line in sf:
|
||||
stripped = line.strip()
|
||||
if stripped == "---":
|
||||
if in_frontmatter:
|
||||
break # end of frontmatter
|
||||
in_frontmatter = True
|
||||
continue
|
||||
if in_frontmatter and stripped.startswith("source_path:"):
|
||||
fm_value = stripped.split(":", 1)[1].strip()
|
||||
# 匹配相对路径的末尾部分
|
||||
if relative_path.endswith(fm_value) or fm_value.endswith(relative_path) or fm_value == relative_path:
|
||||
source_path_match = True
|
||||
break
|
||||
except (OSError, UnicodeDecodeError):
|
||||
pass
|
||||
if not source_path_match:
|
||||
# stem 匹配但 source_path 不一致 → 不信任，需要验证
|
||||
print("MISS:repaired_needs_verify")
|
||||
raise SystemExit(0)
|
||||
# stem + source_path 都匹配 → 安全自愈
|
||||
timestamp = __import__("datetime").datetime.now(__import__("datetime").timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
entries = data.setdefault("entries", {})
|
||||
entries[relative_path] = {
|
||||
"hash": current_hash,
|
||||
"ingested_at": timestamp,
|
||||
"source_page": source_page,
|
||||
}
|
||||
tmp_file = cache_file + ".tmp"
|
||||
with open(tmp_file, "w", encoding="utf-8") as fh2:
|
||||
json.dump(data, fh2, ensure_ascii=False, indent=2)
|
||||
fh2.write("\n")
|
||||
os.replace(tmp_file, cache_file)
|
||||
print("HIT(repaired)")
|
||||
raise SystemExit(0)
|
||||
print("MISS:no_entry")
|
||||
raise SystemExit(0)
|
||||
|
||||
if entry.get("hash") != current_hash:
|
||||
print("MISS:hash_changed")
|
||||
raise SystemExit(0)
|
||||
|
||||
source_page = entry.get("source_page")
|
||||
if not source_page:
|
||||
print("MISS:no_entry")
|
||||
raise SystemExit(0)
|
||||
|
||||
source_path = source_page
|
||||
if not os.path.isabs(source_path):
|
||||
source_path = os.path.join(wiki_root, source_path)
|
||||
|
||||
if not os.path.isfile(source_path):
|
||||
print("MISS:no_source")
|
||||
else:
|
||||
print("HIT")
|
||||
PY
|
||||
)"
|
||||
|
||||
printf '%s\n' "$result"
|
||||
}
|
||||
|
||||
cache_update() {
|
||||
local file_path="$1"
|
||||
local source_page="$2"
|
||||
local wiki_root cache_file relative_path_value current_hash normalized_source timestamp
|
||||
|
||||
require_file "$file_path"
|
||||
wiki_root="$(find_wiki_root "$file_path")" || {
|
||||
echo "未找到知识库根目录：$file_path" >&2
|
||||
exit 1
|
||||
}
|
||||
cache_file="$(cache_file_path "$wiki_root")"
|
||||
ensure_cache_file "$cache_file"
|
||||
|
||||
require_python_cmd
|
||||
|
||||
relative_path_value="$(relative_path "$wiki_root" "$file_path")"
|
||||
current_hash="$(file_hash "$relative_path_value" "$file_path")"
|
||||
normalized_source="$(normalized_source_page "$wiki_root" "$source_page")"
|
||||
timestamp="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
|
||||
|
||||
"$PYTHON_CMD" - "$cache_file" "$relative_path_value" "$current_hash" "$timestamp" "$normalized_source" <<'PY'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
cache_file, relative_path, file_hash_value, timestamp, source_page = sys.argv[1:6]
|
||||
|
||||
with open(cache_file, "r", encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
|
||||
entries = data.setdefault("entries", {})
|
||||
entries[relative_path] = {
|
||||
"hash": file_hash_value,
|
||||
"ingested_at": timestamp,
|
||||
"source_page": source_page,
|
||||
}
|
||||
|
||||
tmp_file = cache_file + ".tmp"
|
||||
with open(tmp_file, "w", encoding="utf-8") as fh:
|
||||
json.dump(data, fh, ensure_ascii=False, indent=2)
|
||||
fh.write("\n")
|
||||
os.replace(tmp_file, cache_file)
|
||||
PY
|
||||
|
||||
printf 'UPDATED\n'
|
||||
}
|
||||
|
||||
cache_invalidate() {
|
||||
local file_path="$1"
|
||||
local wiki_root cache_file relative_path_value
|
||||
|
||||
# 不调用 require_file：文件可能已被删除（级联删除场景）
|
||||
# 直接通过路径查找缓存条目
|
||||
wiki_root="$(find_wiki_root "$file_path")" || {
|
||||
echo "未找到知识库根目录：$file_path" >&2
|
||||
exit 1
|
||||
}
|
||||
cache_file="$(cache_file_path "$wiki_root")"
|
||||
|
||||
if [ ! -f "$cache_file" ]; then
|
||||
printf 'INVALIDATED\n'
|
||||
return 0
|
||||
fi
|
||||
|
||||
require_python_cmd
|
||||
|
||||
relative_path_value="$(relative_path "$wiki_root" "$file_path")"
|
||||
|
||||
"$PYTHON_CMD" - "$cache_file" "$relative_path_value" <<'PY'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
cache_file, relative_path = sys.argv[1:3]
|
||||
|
||||
with open(cache_file, "r", encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
|
||||
data.setdefault("entries", {}).pop(relative_path, None)
|
||||
|
||||
tmp_file = cache_file + ".tmp"
|
||||
with open(tmp_file, "w", encoding="utf-8") as fh:
|
||||
json.dump(data, fh, ensure_ascii=False, indent=2)
|
||||
fh.write("\n")
|
||||
os.replace(tmp_file, cache_file)
|
||||
PY
|
||||
|
||||
printf 'INVALIDATED\n'
|
||||
}
|
||||
|
||||
command_name="${1:-}"
|
||||
|
||||
case "$command_name" in
|
||||
check)
|
||||
[ "$#" -eq 2 ] || { usage; exit 1; }
|
||||
cache_check "$2"
|
||||
;;
|
||||
update)
|
||||
[ "$#" -eq 3 ] || { usage; exit 1; }
|
||||
cache_update "$2" "$3"
|
||||
;;
|
||||
invalidate)
|
||||
[ "$#" -eq 2 ] || { usage; exit 1; }
|
||||
cache_invalidate "$2"
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
100
llm-wiki/scripts/create-source-page.sh
Executable file
100
llm-wiki/scripts/create-source-page.sh
Executable file
@@ -0,0 +1,100 @@
|
||||
#!/bin/bash
|
||||
# llm-wiki source 页面写入脚本
|
||||
# 原子写入 source 页面 + 自动更新缓存，绑定为一项操作
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
用法：
|
||||
bash scripts/create-source-page.sh <raw_file> <output_path> <content_file>
|
||||
|
||||
参数：
|
||||
raw_file : 原始素材文件路径（绝对或相对路径）
|
||||
output_path : 目标页面路径（相对于知识库根目录，如 wiki/sources/2026-04-16-rlhf.md）
|
||||
content_file : 包含待写入内容的临时文件路径
|
||||
EOF
|
||||
}
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
|
||||
# 参数校验
|
||||
if [ "$#" -ne 3 ]; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
raw_file="$1"
|
||||
output_path="$2"
|
||||
content_file="$3"
|
||||
|
||||
# raw_file 和 content_file 必须存在
|
||||
if [ ! -f "$raw_file" ]; then
|
||||
echo "ERROR: 原始素材文件不存在：$raw_file" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "$content_file" ]; then
|
||||
echo "ERROR: 内容文件不存在：$content_file" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 通过 cache.sh 的 find_wiki_root 逻辑找到知识库根目录
|
||||
# 复用 cache.sh 里的函数
|
||||
source_cache_helpers() {
|
||||
# 内联 find_wiki_root（与 cache.sh 保持一致）
|
||||
find_wiki_root() {
|
||||
local file_path="$1"
|
||||
local dir parent
|
||||
|
||||
dir="$(cd "$(dirname "$file_path")" && pwd)"
|
||||
|
||||
while true; do
|
||||
if [ -f "$dir/.wiki-cache.json" ] || [ -f "$dir/.wiki-schema.md" ]; then
|
||||
printf '%s\n' "$dir"
|
||||
return 0
|
||||
fi
|
||||
|
||||
parent="$(dirname "$dir")"
|
||||
[ "$parent" = "$dir" ] && return 1
|
||||
dir="$parent"
|
||||
done
|
||||
}
|
||||
}
|
||||
|
||||
source_cache_helpers
|
||||
|
||||
wiki_root="$(find_wiki_root "$raw_file")" || {
|
||||
echo "ERROR: 未找到知识库根目录：$raw_file" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# 拼接完整目标路径
|
||||
full_output="$wiki_root/$output_path"
|
||||
|
||||
# 确保目标目录存在
|
||||
mkdir -p "$(dirname "$full_output")"
|
||||
|
||||
# 第一步：原子写入（临时文件 + rename，防止写一半崩溃）
|
||||
tmp_output="${full_output}.tmp.$$"
|
||||
if ! cp "$content_file" "$tmp_output"; then
|
||||
rm -f "$tmp_output" 2>/dev/null || true
|
||||
echo "ERROR: 写入临时文件失败" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! mv "$tmp_output" "$full_output"; then
|
||||
rm -f "$tmp_output" 2>/dev/null || true
|
||||
echo "ERROR: 原子重命名失败" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 第二步：更新缓存
|
||||
if ! bash "$SCRIPT_DIR/cache.sh" update "$raw_file" "$output_path"; then
|
||||
# 缓存更新失败 → 回滚：删除已写入的文件
|
||||
rm -f "$full_output"
|
||||
echo "ERROR: 缓存更新失败，已回滚写入" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "SUCCESS"
|
||||
68
llm-wiki/scripts/delete-helper.sh
Executable file
68
llm-wiki/scripts/delete-helper.sh
Executable file
@@ -0,0 +1,68 @@
|
||||
#!/bin/bash
|
||||
# llm-wiki 删除辅助脚本
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "$SCRIPT_DIR/shared-config.sh"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
用法：
|
||||
bash scripts/delete-helper.sh scan-refs <wiki_root> <素材文件名>
|
||||
EOF
|
||||
}
|
||||
|
||||
scan_refs() {
|
||||
local wiki_root="$1"
|
||||
local needle="$2"
|
||||
local wiki_dir="$wiki_root/wiki"
|
||||
|
||||
[ -n "$needle" ] || {
|
||||
echo "素材文件名不能为空" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
[ -d "$wiki_dir" ] || {
|
||||
echo "知识库目录不存在：$wiki_dir" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
require_python_cmd
|
||||
|
||||
{
|
||||
grep -rlF --include='*.md' -- "$needle" "$wiki_dir" 2>/dev/null || true
|
||||
} | "$PYTHON_CMD" -c '
|
||||
import os
|
||||
import sys
|
||||
|
||||
wiki_root = os.path.realpath(sys.argv[1])
|
||||
seen = []
|
||||
|
||||
for line in sys.stdin:
|
||||
path = line.strip()
|
||||
if not path:
|
||||
continue
|
||||
real_path = os.path.realpath(path)
|
||||
if real_path in seen:
|
||||
continue
|
||||
seen.append(real_path)
|
||||
|
||||
for path in sorted(seen):
|
||||
print(os.path.relpath(path, wiki_root))
|
||||
' "$wiki_root"
|
||||
}
|
||||
|
||||
command_name="${1:-}"
|
||||
|
||||
case "$command_name" in
|
||||
scan-refs)
|
||||
[ "$#" -eq 3 ] || { usage; exit 1; }
|
||||
scan_refs "$2" "$3"
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
737
llm-wiki/scripts/graph-analysis.js
Normal file
737
llm-wiki/scripts/graph-analysis.js
Normal file
@@ -0,0 +1,737 @@
|
||||
#!/usr/bin/env node
|
||||
"use strict";
|
||||
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
const { extractFrontmatter, parseSourcesFrontmatter, sortedUnique } = require("./lib/source-signal-eligibility");
|
||||
|
||||
function readJson(filePath) {
|
||||
return JSON.parse(fs.readFileSync(filePath, "utf8"));
|
||||
}
|
||||
|
||||
function writeJson(filePath, value) {
|
||||
fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
|
||||
}
|
||||
|
||||
function roundNumber(value, digits = 3) {
|
||||
const factor = 10 ** digits;
|
||||
return Math.round(value * factor) / factor;
|
||||
}
|
||||
|
||||
function clamp01(value) {
|
||||
return Math.max(0, Math.min(1, value));
|
||||
}
|
||||
|
||||
function sortedPairKey(a, b) {
|
||||
return a < b ? `${a}\t${b}` : `${b}\t${a}`;
|
||||
}
|
||||
|
||||
function normalizeBody(text, degraded, maxLines) {
|
||||
const { body } = extractFrontmatter(text);
|
||||
const normalized = body.replace(/^\s+/, "").replace(/\s+$/, "");
|
||||
if (!degraded) return normalized;
|
||||
return normalized.split(/\r?\n/).slice(0, maxLines).join("\n").replace(/\s+$/, "");
|
||||
}
|
||||
|
||||
function loadNodeDetails(nodes, degraded, maxLines) {
|
||||
const byId = {};
|
||||
|
||||
for (const node of nodes) {
|
||||
const hasPreloadedContent = Object.prototype.hasOwnProperty.call(node, "_content");
|
||||
const raw = hasPreloadedContent
|
||||
? String(node._content == null ? "" : node._content)
|
||||
: fs.readFileSync(node._file_path || node.source_path, "utf8");
|
||||
const frontmatter = extractFrontmatter(raw);
|
||||
const parsedSources = parseSourcesFrontmatter(frontmatter.frontmatter);
|
||||
const preloadedSignals = node._signals && typeof node._signals === "object"
|
||||
? node._signals
|
||||
: null;
|
||||
const normalizedNode = {
|
||||
...node,
|
||||
content: normalizeBody(raw, degraded, maxLines),
|
||||
_signals: preloadedSignals || {
|
||||
sources: parsedSources.sources,
|
||||
sourceSignalAvailable: parsedSources.signalAvailable,
|
||||
sourceFieldPresent: parsedSources.hasField,
|
||||
sourceFieldParsed: parsedSources.parsed
|
||||
}
|
||||
};
|
||||
byId[node.id] = normalizedNode;
|
||||
}
|
||||
|
||||
return byId;
|
||||
}
|
||||
|
||||
function buildInlinks(edges) {
|
||||
const inlinks = new Map();
|
||||
|
||||
for (const edge of edges) {
|
||||
if (!inlinks.has(edge.to)) inlinks.set(edge.to, new Set());
|
||||
inlinks.get(edge.to).add(edge.from);
|
||||
}
|
||||
|
||||
return inlinks;
|
||||
}
|
||||
|
||||
function intersectionCount(setA, setB) {
|
||||
if (!setA || !setB) return 0;
|
||||
let small = setA;
|
||||
let large = setB;
|
||||
if (setB.size < setA.size) {
|
||||
small = setB;
|
||||
large = setA;
|
||||
}
|
||||
|
||||
let count = 0;
|
||||
for (const value of small) {
|
||||
if (large.has(value)) count += 1;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
function typeAffinity(typeA, typeB) {
|
||||
const pair = [typeA || "other", typeB || "other"].sort().join(":");
|
||||
switch (pair) {
|
||||
case "entity:entity":
|
||||
case "entity:topic":
|
||||
return 1;
|
||||
case "topic:topic":
|
||||
return 0.8;
|
||||
case "entity:source":
|
||||
return 0.6;
|
||||
case "source:source":
|
||||
return 0.3;
|
||||
default:
|
||||
return 0.5;
|
||||
}
|
||||
}
|
||||
|
||||
function computePairMetrics(nodesById, edges) {
|
||||
const inlinks = buildInlinks(edges);
|
||||
const pairMetrics = new Map();
|
||||
|
||||
for (const edge of edges) {
|
||||
const pairKey = sortedPairKey(edge.from, edge.to);
|
||||
if (pairMetrics.has(pairKey)) continue;
|
||||
|
||||
const fromNode = nodesById[edge.from];
|
||||
const toNode = nodesById[edge.to];
|
||||
if (!fromNode || !toNode) continue;
|
||||
|
||||
const fromInlinks = inlinks.get(edge.from) || new Set();
|
||||
const toInlinks = inlinks.get(edge.to) || new Set();
|
||||
const sharedInlinks = intersectionCount(fromInlinks, toInlinks);
|
||||
const coCitation = sharedInlinks / Math.max(fromInlinks.size, toInlinks.size, 1);
|
||||
const affinity = typeAffinity(fromNode.type, toNode.type);
|
||||
|
||||
const signals = [coCitation, affinity];
|
||||
let sourceOverlap = null;
|
||||
const sourceSignalAvailable = Boolean(
|
||||
fromNode._signals.sourceSignalAvailable && toNode._signals.sourceSignalAvailable
|
||||
);
|
||||
|
||||
if (sourceSignalAvailable) {
|
||||
const fromSources = new Set(fromNode._signals.sources);
|
||||
const toSources = new Set(toNode._signals.sources);
|
||||
const overlap = intersectionCount(fromSources, toSources);
|
||||
const minSize = Math.min(fromSources.size, toSources.size);
|
||||
sourceOverlap = minSize > 0 ? overlap / minSize : 0;
|
||||
signals.push(sourceOverlap);
|
||||
}
|
||||
|
||||
const weight = clamp01(signals.reduce((sum, value) => sum + value, 0) / signals.length);
|
||||
|
||||
pairMetrics.set(pairKey, {
|
||||
weight: roundNumber(weight),
|
||||
signals: {
|
||||
co_citation: roundNumber(coCitation),
|
||||
source_overlap: sourceOverlap == null ? null : roundNumber(sourceOverlap),
|
||||
type_affinity: roundNumber(affinity)
|
||||
},
|
||||
source_signal_available: sourceSignalAvailable
|
||||
});
|
||||
}
|
||||
|
||||
return pairMetrics;
|
||||
}
|
||||
|
||||
function buildUndirectedGraph(nodeIds, pairMetrics) {
|
||||
const adjacency = new Map();
|
||||
const degrees = new Map();
|
||||
|
||||
for (const nodeId of nodeIds) {
|
||||
adjacency.set(nodeId, new Map());
|
||||
degrees.set(nodeId, 0);
|
||||
}
|
||||
|
||||
for (const [pairKey, metrics] of pairMetrics.entries()) {
|
||||
const [left, right] = pairKey.split("\t");
|
||||
if (!adjacency.has(left) || !adjacency.has(right)) continue;
|
||||
const weight = metrics.weight;
|
||||
adjacency.get(left).set(right, weight);
|
||||
adjacency.get(right).set(left, weight);
|
||||
degrees.set(left, degrees.get(left) + weight);
|
||||
degrees.set(right, degrees.get(right) + weight);
|
||||
}
|
||||
|
||||
return { adjacency, degrees };
|
||||
}
|
||||
|
||||
function runLocalMove(graph) {
|
||||
const nodes = Array.from(graph.nodes.keys()).sort();
|
||||
const communities = new Map();
|
||||
const totals = new Map();
|
||||
let moved = false;
|
||||
|
||||
for (const nodeId of nodes) {
|
||||
communities.set(nodeId, nodeId);
|
||||
totals.set(nodeId, graph.degrees.get(nodeId) || 0);
|
||||
}
|
||||
|
||||
if (graph.m2 === 0) {
|
||||
return { communities, changed: false };
|
||||
}
|
||||
|
||||
let changedInPass = true;
|
||||
let passCount = 0;
|
||||
while (changedInPass && passCount < 50) {
|
||||
passCount++;
|
||||
changedInPass = false;
|
||||
|
||||
for (const nodeId of nodes) {
|
||||
const degree = graph.degrees.get(nodeId) || 0;
|
||||
const currentCommunity = communities.get(nodeId);
|
||||
const neighborCommunities = new Map();
|
||||
|
||||
for (const [neighborId, weight] of graph.nodes.get(nodeId).entries()) {
|
||||
const communityId = communities.get(neighborId);
|
||||
neighborCommunities.set(communityId, (neighborCommunities.get(communityId) || 0) + weight);
|
||||
}
|
||||
|
||||
totals.set(currentCommunity, (totals.get(currentCommunity) || 0) - degree);
|
||||
if ((neighborCommunities.get(currentCommunity) || 0) === 0) {
|
||||
neighborCommunities.set(currentCommunity, 0);
|
||||
}
|
||||
|
||||
let bestCommunity = currentCommunity;
|
||||
let bestGain = 0;
|
||||
|
||||
const candidates = Array.from(neighborCommunities.keys()).sort();
|
||||
for (const communityId of candidates) {
|
||||
const inWeight = neighborCommunities.get(communityId) || 0;
|
||||
const gain = inWeight - ((totals.get(communityId) || 0) * degree) / graph.m2;
|
||||
if (gain > bestGain + 1e-9) {
|
||||
bestGain = gain;
|
||||
bestCommunity = communityId;
|
||||
}
|
||||
}
|
||||
|
||||
communities.set(nodeId, bestCommunity);
|
||||
totals.set(bestCommunity, (totals.get(bestCommunity) || 0) + degree);
|
||||
|
||||
if (bestCommunity !== currentCommunity) {
|
||||
changedInPass = true;
|
||||
moved = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { communities, changed: moved };
|
||||
}
|
||||
|
||||
function aggregateGraph(graph, communities) {
|
||||
const communityIds = sortedUnique(Array.from(communities.values()));
|
||||
const aggregatedNodes = new Map();
|
||||
const aggregatedDegrees = new Map();
|
||||
const members = new Map();
|
||||
|
||||
for (const communityId of communityIds) {
|
||||
aggregatedNodes.set(communityId, new Map());
|
||||
aggregatedDegrees.set(communityId, 0);
|
||||
members.set(communityId, []);
|
||||
}
|
||||
|
||||
for (const [nodeId, communityId] of communities.entries()) {
|
||||
members.get(communityId).push(...(graph.members.get(nodeId) || [nodeId]));
|
||||
}
|
||||
|
||||
for (const [nodeId, neighbors] of graph.nodes.entries()) {
|
||||
const sourceCommunity = communities.get(nodeId);
|
||||
for (const [neighborId, weight] of neighbors.entries()) {
|
||||
if (nodeId > neighborId) continue;
|
||||
const targetCommunity = communities.get(neighborId);
|
||||
const current = aggregatedNodes.get(sourceCommunity).get(targetCommunity) || 0;
|
||||
aggregatedNodes.get(sourceCommunity).set(targetCommunity, current + weight);
|
||||
if (sourceCommunity !== targetCommunity) {
|
||||
const mirrored = aggregatedNodes.get(targetCommunity).get(sourceCommunity) || 0;
|
||||
aggregatedNodes.get(targetCommunity).set(sourceCommunity, mirrored + weight);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const [communityId, neighbors] of aggregatedNodes.entries()) {
|
||||
let degree = 0;
|
||||
for (const [neighborId, weight] of neighbors.entries()) {
|
||||
degree += neighborId === communityId ? weight * 2 : weight;
|
||||
}
|
||||
aggregatedDegrees.set(communityId, degree);
|
||||
}
|
||||
|
||||
return {
|
||||
nodes: aggregatedNodes,
|
||||
degrees: aggregatedDegrees,
|
||||
members,
|
||||
m2: Array.from(aggregatedDegrees.values()).reduce((sum, value) => sum + value, 0)
|
||||
};
|
||||
}
|
||||
|
||||
function runLouvain(nodeIds, pairMetrics) {
|
||||
const baseGraph = buildUndirectedGraph(nodeIds, pairMetrics);
|
||||
let graph = {
|
||||
nodes: baseGraph.adjacency,
|
||||
degrees: baseGraph.degrees,
|
||||
members: new Map(nodeIds.map((nodeId) => [nodeId, [nodeId]])),
|
||||
m2: Array.from(baseGraph.degrees.values()).reduce((sum, value) => sum + value, 0)
|
||||
};
|
||||
|
||||
let bestMembers = graph.members;
|
||||
|
||||
while (true) {
|
||||
const phase = runLocalMove(graph);
|
||||
const nextGraph = aggregateGraph(graph, phase.communities);
|
||||
bestMembers = nextGraph.members;
|
||||
|
||||
if (!phase.changed || nextGraph.nodes.size === graph.nodes.size) {
|
||||
break;
|
||||
}
|
||||
|
||||
graph = nextGraph;
|
||||
}
|
||||
|
||||
const finalCommunities = new Map();
|
||||
for (const [communityId, members] of bestMembers.entries()) {
|
||||
for (const nodeId of members) {
|
||||
finalCommunities.set(nodeId, communityId);
|
||||
}
|
||||
}
|
||||
|
||||
return finalCommunities;
|
||||
}
|
||||
|
||||
function buildDirectedDegree(edges) {
|
||||
const degree = new Map();
|
||||
for (const edge of edges) {
|
||||
degree.set(edge.from, (degree.get(edge.from) || 0) + 1);
|
||||
degree.set(edge.to, (degree.get(edge.to) || 0) + 1);
|
||||
}
|
||||
return degree;
|
||||
}
|
||||
|
||||
function chooseCommunityLabels(nodeIds, communityAssignments, nodesById, edges) {
|
||||
const groups = new Map();
|
||||
const degree = buildDirectedDegree(edges);
|
||||
|
||||
for (const nodeId of nodeIds) {
|
||||
const communityId = communityAssignments.get(nodeId) || nodeId;
|
||||
if (!groups.has(communityId)) groups.set(communityId, []);
|
||||
groups.get(communityId).push(nodeId);
|
||||
}
|
||||
|
||||
const labeledAssignments = new Map();
|
||||
|
||||
for (const members of groups.values()) {
|
||||
members.sort();
|
||||
if (members.length === 1) {
|
||||
labeledAssignments.set(members[0], null);
|
||||
continue;
|
||||
}
|
||||
|
||||
const memberNodes = members.map((memberId) => nodesById[memberId]);
|
||||
const topics = memberNodes.filter((node) => node && node.type === "topic");
|
||||
const candidates = topics.length ? topics : memberNodes;
|
||||
candidates.sort((left, right) => {
|
||||
const degreeDiff = (degree.get(right.id) || 0) - (degree.get(left.id) || 0);
|
||||
if (degreeDiff !== 0) return degreeDiff;
|
||||
return left.id.localeCompare(right.id);
|
||||
});
|
||||
|
||||
const label = candidates[0] ? candidates[0].id : members[0];
|
||||
for (const memberId of members) {
|
||||
labeledAssignments.set(memberId, label);
|
||||
}
|
||||
}
|
||||
|
||||
return labeledAssignments;
|
||||
}
|
||||
|
||||
function buildInsights(nodesById, edges, pairMetrics, communityAssignments, options) {
|
||||
const directedDegree = buildDirectedDegree(edges);
|
||||
const undirectedPairs = new Map();
|
||||
const adjacency = new Map();
|
||||
|
||||
for (const nodeId of Object.keys(nodesById)) {
|
||||
adjacency.set(nodeId, new Set());
|
||||
}
|
||||
|
||||
for (const edge of edges) {
|
||||
const pairKey = sortedPairKey(edge.from, edge.to);
|
||||
if (!undirectedPairs.has(pairKey)) {
|
||||
undirectedPairs.set(pairKey, {
|
||||
from: pairKey.split("\t")[0],
|
||||
to: pairKey.split("\t")[1],
|
||||
weight: pairMetrics.get(pairKey)?.weight || 0
|
||||
});
|
||||
}
|
||||
adjacency.get(edge.from)?.add(edge.to);
|
||||
adjacency.get(edge.to)?.add(edge.from);
|
||||
}
|
||||
|
||||
const isolatedNodes = Object.values(nodesById)
|
||||
.filter((node) => (directedDegree.get(node.id) || 0) <= 1)
|
||||
.sort((left, right) => left.id.localeCompare(right.id))
|
||||
.map((node) => ({
|
||||
id: node.id,
|
||||
label: node.label,
|
||||
degree: directedDegree.get(node.id) || 0,
|
||||
community: communityAssignments.get(node.id) || null
|
||||
}));
|
||||
|
||||
const bridgeNodes = [];
|
||||
for (const node of Object.values(nodesById).sort((left, right) => left.id.localeCompare(right.id))) {
|
||||
const ownCommunity = communityAssignments.get(node.id) || null;
|
||||
const connectedCommunities = sortedUnique(
|
||||
Array.from(adjacency.get(node.id) || [])
|
||||
.map((neighborId) => communityAssignments.get(neighborId) || null)
|
||||
.filter((c) => c && c !== ownCommunity)
|
||||
);
|
||||
|
||||
if (connectedCommunities.length >= 2) {
|
||||
bridgeNodes.push({
|
||||
id: node.id,
|
||||
label: node.label,
|
||||
community: ownCommunity,
|
||||
connected_communities: connectedCommunities,
|
||||
community_count: connectedCommunities.length
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const communityMembers = new Map();
|
||||
for (const node of Object.values(nodesById)) {
|
||||
const communityId = communityAssignments.get(node.id) || null;
|
||||
if (!communityId) continue;
|
||||
if (!communityMembers.has(communityId)) communityMembers.set(communityId, []);
|
||||
communityMembers.get(communityId).push(node.id);
|
||||
}
|
||||
|
||||
const sparseCommunities = [];
|
||||
for (const [communityId, members] of Array.from(communityMembers.entries()).sort((left, right) => left[0].localeCompare(right[0]))) {
|
||||
if (members.length < 3) continue;
|
||||
|
||||
const memberSet = new Set(members);
|
||||
let internalEdges = 0;
|
||||
for (const pair of undirectedPairs.values()) {
|
||||
if (memberSet.has(pair.from) && memberSet.has(pair.to)) internalEdges += 1;
|
||||
}
|
||||
|
||||
const possibleEdges = (members.length * (members.length - 1)) / 2;
|
||||
const density = possibleEdges === 0 ? 0 : internalEdges / possibleEdges;
|
||||
if (density < 0.15) {
|
||||
sparseCommunities.push({
|
||||
id: communityId,
|
||||
label: nodesById[communityId]?.label || communityId,
|
||||
node_count: members.length,
|
||||
density: roundNumber(density),
|
||||
members: members.sort(),
|
||||
internal_edges: internalEdges
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const surprisingConnections = Array.from(undirectedPairs.values())
|
||||
.filter((pair) => {
|
||||
const fromCommunity = communityAssignments.get(pair.from) || null;
|
||||
const toCommunity = communityAssignments.get(pair.to) || null;
|
||||
return fromCommunity && toCommunity && fromCommunity !== toCommunity && pair.weight >= 0.75;
|
||||
})
|
||||
.sort((left, right) => {
|
||||
if (right.weight !== left.weight) return right.weight - left.weight;
|
||||
if (left.from !== right.from) return left.from.localeCompare(right.from);
|
||||
return left.to.localeCompare(right.to);
|
||||
})
|
||||
.slice(0, 8)
|
||||
.map((pair) => ({
|
||||
from: pair.from,
|
||||
to: pair.to,
|
||||
weight: pair.weight,
|
||||
from_community: communityAssignments.get(pair.from) || null,
|
||||
to_community: communityAssignments.get(pair.to) || null
|
||||
}));
|
||||
|
||||
const degraded = options.nodeCount > options.maxInsightNodes || options.edgeCount > options.maxInsightEdges;
|
||||
if (degraded) {
|
||||
return {
|
||||
surprising_connections: [],
|
||||
isolated_nodes: isolatedNodes,
|
||||
bridge_nodes: [],
|
||||
sparse_communities: [],
|
||||
meta: {
|
||||
degraded: true,
|
||||
node_count: options.nodeCount,
|
||||
edge_count: options.edgeCount,
|
||||
max_insight_nodes: options.maxInsightNodes,
|
||||
max_insight_edges: options.maxInsightEdges
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
surprising_connections: surprisingConnections,
|
||||
isolated_nodes: isolatedNodes,
|
||||
bridge_nodes: bridgeNodes,
|
||||
sparse_communities: sparseCommunities,
|
||||
meta: {
|
||||
degraded: false,
|
||||
node_count: options.nodeCount,
|
||||
edge_count: options.edgeCount,
|
||||
max_insight_nodes: options.maxInsightNodes,
|
||||
max_insight_edges: options.maxInsightEdges
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function buildLearning(analyzedNodes, analyzedEdges) {
|
||||
const degreeMap = new Map();
|
||||
for (const edge of analyzedEdges) {
|
||||
degreeMap.set(edge.from, (degreeMap.get(edge.from) || 0) + 1);
|
||||
degreeMap.set(edge.to, (degreeMap.get(edge.to) || 0) + 1);
|
||||
}
|
||||
|
||||
const communityGroups = new Map();
|
||||
for (const node of analyzedNodes) {
|
||||
if (node.community == null) continue;
|
||||
if (!communityGroups.has(node.community)) communityGroups.set(node.community, []);
|
||||
communityGroups.get(node.community).push(node);
|
||||
}
|
||||
|
||||
const communities = [];
|
||||
for (const [cid, members] of communityGroups.entries()) {
|
||||
const memberIds = new Set(members.map(n => n.id));
|
||||
let totalWeight = 0;
|
||||
for (const edge of analyzedEdges) {
|
||||
if (memberIds.has(edge.from) && memberIds.has(edge.to)) totalWeight += edge.weight;
|
||||
}
|
||||
const isWeak = members.length < 3;
|
||||
const startNode = members.slice().sort((a, b) => {
|
||||
const degDiff = (degreeMap.get(b.id) || 0) - (degreeMap.get(a.id) || 0);
|
||||
if (degDiff !== 0) return degDiff;
|
||||
return a.id.localeCompare(b.id);
|
||||
})[0];
|
||||
|
||||
communities.push({
|
||||
id: cid,
|
||||
label: (members.find(n => n.id === cid) || members[0]).label,
|
||||
node_count: members.length,
|
||||
source_count: members.filter(n => n.type === "source").length,
|
||||
internal_edge_weight: roundNumber(totalWeight),
|
||||
is_primary: false,
|
||||
is_weak: isWeak,
|
||||
recommended_start_node_id: startNode.id
|
||||
});
|
||||
}
|
||||
|
||||
communities.sort((a, b) => {
|
||||
if (b.node_count !== a.node_count) return b.node_count - a.node_count;
|
||||
if (b.internal_edge_weight !== a.internal_edge_weight) return b.internal_edge_weight - a.internal_edge_weight;
|
||||
return a.id.localeCompare(b.id);
|
||||
});
|
||||
|
||||
if (communities.length > 0) communities[0].is_primary = true;
|
||||
|
||||
const primary = communities.length > 0 ? communities[0] : null;
|
||||
const startNodeId = primary ? primary.recommended_start_node_id : null;
|
||||
|
||||
let pathNodeIds = [];
|
||||
let pathDegraded = false;
|
||||
if (primary && !primary.is_weak && startNodeId) {
|
||||
const primaryMemberIds = new Set(communityGroups.get(primary.id).map(n => n.id));
|
||||
const neighbors = analyzedEdges
|
||||
.filter(e => (e.from === startNodeId && primaryMemberIds.has(e.to)) ||
|
||||
(e.to === startNodeId && primaryMemberIds.has(e.from)))
|
||||
.map(e => e.from === startNodeId ? e.to : e.from);
|
||||
pathNodeIds = [startNodeId, ...sortedUnique(neighbors).filter(id => id !== startNodeId)];
|
||||
if (pathNodeIds.length < 2) pathDegraded = true;
|
||||
} else {
|
||||
pathDegraded = true;
|
||||
}
|
||||
|
||||
let communityNodeIds = [];
|
||||
let communityDegraded = false;
|
||||
if (primary && !primary.is_weak) {
|
||||
communityNodeIds = communityGroups.get(primary.id).map(n => n.id).sort();
|
||||
} else {
|
||||
communityDegraded = true;
|
||||
}
|
||||
|
||||
const globalNodeIds = analyzedNodes.slice().sort((a, b) => {
|
||||
const degDiff = (degreeMap.get(b.id) || 0) - (degreeMap.get(a.id) || 0);
|
||||
if (degDiff !== 0) return degDiff;
|
||||
return a.id.localeCompare(b.id);
|
||||
}).map(n => n.id);
|
||||
|
||||
const defaultMode = "global";
|
||||
|
||||
return {
|
||||
version: 1,
|
||||
entry: {
|
||||
recommended_start_node_id: startNodeId,
|
||||
recommended_start_reason: startNodeId ? "community_hub" : null,
|
||||
default_mode: defaultMode
|
||||
},
|
||||
views: {
|
||||
path: {
|
||||
enabled: !pathDegraded,
|
||||
start_node_id: pathDegraded ? null : startNodeId,
|
||||
node_ids: pathDegraded ? [] : pathNodeIds,
|
||||
degraded: pathDegraded
|
||||
},
|
||||
community: {
|
||||
enabled: !communityDegraded,
|
||||
community_id: primary && !communityDegraded ? primary.id : null,
|
||||
label: primary && !communityDegraded ? primary.label : null,
|
||||
node_ids: communityDegraded ? [] : communityNodeIds,
|
||||
is_weak: primary ? primary.is_weak : false,
|
||||
degraded: communityDegraded
|
||||
},
|
||||
global: {
|
||||
enabled: true,
|
||||
node_ids: globalNodeIds,
|
||||
degraded: false
|
||||
}
|
||||
},
|
||||
communities,
|
||||
degraded: {
|
||||
path_to_community: pathDegraded,
|
||||
community_to_global: communityDegraded
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function analyzeGraph(nodes, edges, options = {}) {
|
||||
const degraded = options.degraded === true;
|
||||
const maxLines = options.maxLines || 500;
|
||||
const maxInsightNodes = options.maxInsightNodes || 250;
|
||||
const maxInsightEdges = options.maxInsightEdges || 1000;
|
||||
|
||||
const nodesById = loadNodeDetails(nodes, degraded, maxLines);
|
||||
const pairMetrics = computePairMetrics(nodesById, edges);
|
||||
const nodeIds = nodes.map((node) => node.id);
|
||||
const communityAssignments = chooseCommunityLabels(
|
||||
nodeIds,
|
||||
runLouvain(nodeIds, pairMetrics),
|
||||
nodesById,
|
||||
edges
|
||||
);
|
||||
|
||||
const analyzedNodes = nodes.map((node) => ({
|
||||
id: node.id,
|
||||
label: node.label,
|
||||
type: node.type,
|
||||
source_path: node.source_path,
|
||||
community: communityAssignments.get(node.id) || null,
|
||||
content: nodesById[node.id].content
|
||||
}));
|
||||
|
||||
const analyzedEdges = edges.map((edge) => {
|
||||
const pairKey = sortedPairKey(edge.from, edge.to);
|
||||
const metrics = pairMetrics.get(pairKey) || {
|
||||
weight: 0,
|
||||
signals: { co_citation: 0, source_overlap: null, type_affinity: 0.5 },
|
||||
source_signal_available: false
|
||||
};
|
||||
|
||||
return {
|
||||
id: edge.id,
|
||||
from: edge.from,
|
||||
to: edge.to,
|
||||
type: edge.type,
|
||||
confidence: edge.confidence || edge.type,
|
||||
relation_type: edge.relation_type || "依赖",
|
||||
weight: metrics.weight,
|
||||
source_signal_available: metrics.source_signal_available,
|
||||
signals: metrics.signals
|
||||
};
|
||||
});
|
||||
|
||||
const insights = buildInsights(nodesById, analyzedEdges, pairMetrics, communityAssignments, {
|
||||
nodeCount: analyzedNodes.length,
|
||||
edgeCount: analyzedEdges.length,
|
||||
maxInsightNodes,
|
||||
maxInsightEdges
|
||||
});
|
||||
|
||||
const learning = buildLearning(analyzedNodes, analyzedEdges);
|
||||
|
||||
return { nodes: analyzedNodes, edges: analyzedEdges, insights, learning };
|
||||
}
|
||||
|
||||
function main(argv) {
|
||||
if (argv.length < 7) {
|
||||
console.error("Usage: node graph-analysis.js <nodes.json> <edges.json> <output.json> <degraded:0|1> <max-lines> <max-insight-nodes> <max-insight-edges>");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const timer = setTimeout(() => {
|
||||
console.error("ERROR: graph analysis timed out (120s)");
|
||||
process.exit(2);
|
||||
}, 120_000);
|
||||
timer.unref();
|
||||
|
||||
const [, , nodesPath, edgesPath, outputPath, degradedRaw, maxLinesRaw, maxInsightNodesRaw, maxInsightEdgesRaw] = argv;
|
||||
|
||||
for (const p of [nodesPath, edgesPath]) {
|
||||
if (!fs.existsSync(p)) {
|
||||
console.error(`ERROR: File not found: ${p}`);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
const analyzed = analyzeGraph(readJson(nodesPath), readJson(edgesPath), {
|
||||
degraded: degradedRaw === "1",
|
||||
maxLines: Number(maxLinesRaw) || 500,
|
||||
maxInsightNodes: Number(maxInsightNodesRaw) || 250,
|
||||
maxInsightEdges: Number(maxInsightEdgesRaw) || 1000
|
||||
});
|
||||
|
||||
writeJson(outputPath, analyzed);
|
||||
clearTimeout(timer);
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
try {
|
||||
main(process.argv);
|
||||
} catch (error) {
|
||||
const code = error && error.code;
|
||||
if (code === "ENOENT") {
|
||||
console.error(`ERROR: File not found: ${error.path || "(unknown)"}`);
|
||||
} else if (error instanceof SyntaxError) {
|
||||
console.error(`ERROR: Invalid JSON in input: ${error.message}`);
|
||||
} else {
|
||||
console.error(`ERROR: ${error && error.message ? error.message : String(error)}`);
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
analyzeGraph,
|
||||
buildInsights,
|
||||
buildLearning,
|
||||
chooseCommunityLabels,
|
||||
computePairMetrics,
|
||||
extractFrontmatter,
|
||||
normalizeBody,
|
||||
parseSourcesFrontmatter,
|
||||
runLouvain,
|
||||
typeAffinity
|
||||
};
|
||||
46
llm-wiki/scripts/hook-session-start.sh
Executable file
46
llm-wiki/scripts/hook-session-start.sh
Executable file
@@ -0,0 +1,46 @@
|
||||
#!/bin/bash
|
||||
# SessionStart hook: 会话开始时注入 wiki 上下文（只触发一次）
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "$SCRIPT_DIR/shared-config.sh"
|
||||
|
||||
WIKI_PATH=""
|
||||
|
||||
if [ -f "$HOME/.llm-wiki-path" ]; then
|
||||
WIKI_PATH="$(cat "$HOME/.llm-wiki-path")"
|
||||
fi
|
||||
|
||||
if [ -z "$WIKI_PATH" ] && [ -f .wiki-schema.md ]; then
|
||||
WIKI_PATH="$(pwd)"
|
||||
fi
|
||||
|
||||
if [ -z "$WIKI_PATH" ] || [ ! -f "$WIKI_PATH/.wiki-schema.md" ]; then
|
||||
printf '{}\n'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
require_python_cmd
|
||||
|
||||
"$PYTHON_CMD" - "$WIKI_PATH" <<'PY'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
# 防御性：即使上游 shared-config.sh 未设置 PYTHONIOENCODING，
|
||||
# 此处也强制 stdout 为 UTF-8，避免 Agent 接到 gbk 字节
|
||||
if hasattr(sys.stdout, "reconfigure"):
|
||||
sys.stdout.reconfigure(encoding="utf-8")
|
||||
|
||||
wiki_path = os.path.realpath(sys.argv[1])
|
||||
message = f"[llm-wiki] 检测到知识库: {wiki_path}/index.md，回答问题时优先查阅 wiki 内容获取上下文"
|
||||
|
||||
print(json.dumps({
|
||||
"hookSpecificOutput": {
|
||||
"hookEventName": "SessionStart",
|
||||
"additionalContext": message,
|
||||
}
|
||||
}, ensure_ascii=False))
|
||||
PY
|
||||
95
llm-wiki/scripts/init-wiki.sh
Executable file
95
llm-wiki/scripts/init-wiki.sh
Executable file
@@ -0,0 +1,95 @@
|
||||
#!/bin/bash
|
||||
# llm-wiki 初始化脚本
|
||||
# 自动创建知识库的目录结构
|
||||
# 用法：bash init-wiki.sh <知识库路径> <主题>
|
||||
|
||||
set -e
|
||||
|
||||
WIKI_ROOT="${1:-$HOME/Documents/我的知识库}"
|
||||
TOPIC="${2:-我的知识库}"
|
||||
LANGUAGE="${3:-中文}"
|
||||
DATE=$(date +%Y-%m-%d)
|
||||
SKILL_DIR="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
|
||||
# 安全的模板变量替换函数（用 perl 替代 sed，避免中文/空格/特殊字符问题）
|
||||
replace_vars() {
|
||||
local input_file="$1"
|
||||
local output_file="$2"
|
||||
TOPIC_VALUE="$TOPIC" \
|
||||
DATE_VALUE="$DATE" \
|
||||
WIKI_ROOT_VALUE="$WIKI_ROOT" \
|
||||
LANGUAGE_VALUE="$LANGUAGE" \
|
||||
perl -pe '
|
||||
s/\{\{TOPIC\}\}/$ENV{TOPIC_VALUE}/g;
|
||||
s/\{\{DATE\}\}/$ENV{DATE_VALUE}/g;
|
||||
s/\{\{WIKI_ROOT\}\}/$ENV{WIKI_ROOT_VALUE}/g;
|
||||
s/\{\{LANGUAGE\}\}/$ENV{LANGUAGE_VALUE}/g;
|
||||
' "$input_file" > "$output_file"
|
||||
}
|
||||
|
||||
echo "正在创建知识库..."
|
||||
echo " 路径：$WIKI_ROOT"
|
||||
echo " 主题：$TOPIC"
|
||||
echo " 语言：$LANGUAGE"
|
||||
echo ""
|
||||
|
||||
# 创建目录结构（包含小红书和知乎）
|
||||
mkdir -p "$WIKI_ROOT"/raw/{articles,tweets,wechat,xiaohongshu,zhihu,pdfs,notes,assets}
|
||||
mkdir -p "$WIKI_ROOT"/wiki/{entities,topics,sources,comparisons,synthesis,synthesis/sessions,queries}
|
||||
|
||||
cat > "$WIKI_ROOT/.gitignore" <<'EOF'
|
||||
.wiki-tmp/
|
||||
EOF
|
||||
|
||||
echo "[完成] 目录结构已创建"
|
||||
|
||||
# 从模板生成文件
|
||||
replace_vars "$SKILL_DIR/templates/schema-template.md" "$WIKI_ROOT/.wiki-schema.md"
|
||||
echo "[完成] Schema 文件已生成"
|
||||
|
||||
replace_vars "$SKILL_DIR/templates/index-template.md" "$WIKI_ROOT/index.md"
|
||||
echo "[完成] 索引文件已生成"
|
||||
|
||||
replace_vars "$SKILL_DIR/templates/log-template.md" "$WIKI_ROOT/log.md"
|
||||
echo "[完成] 日志文件已生成"
|
||||
|
||||
replace_vars "$SKILL_DIR/templates/overview-template.md" "$WIKI_ROOT/wiki/overview.md"
|
||||
echo "[完成] 总览文件已生成"
|
||||
|
||||
if [ "$LANGUAGE" = "English" ]; then
|
||||
replace_vars "$SKILL_DIR/templates/purpose-en-template.md" "$WIKI_ROOT/purpose.md"
|
||||
else
|
||||
replace_vars "$SKILL_DIR/templates/purpose-template.md" "$WIKI_ROOT/purpose.md"
|
||||
fi
|
||||
echo "[完成] 研究方向文件已生成"
|
||||
|
||||
cat > "$WIKI_ROOT/.wiki-cache.json" <<'EOF'
|
||||
{
|
||||
"version": 1,
|
||||
"entries": {}
|
||||
}
|
||||
EOF
|
||||
echo "[完成] 缓存文件已生成"
|
||||
|
||||
echo ""
|
||||
echo "知识库创建完成！"
|
||||
echo ""
|
||||
echo "目录结构："
|
||||
echo " $WIKI_ROOT/"
|
||||
echo " ├── raw/ （原始素材）"
|
||||
echo " │ ├── articles/ 网页文章"
|
||||
echo " │ ├── tweets/ X/Twitter"
|
||||
echo " │ ├── wechat/ 微信公众号"
|
||||
echo " │ ├── xiaohongshu/ 小红书"
|
||||
echo " │ ├── zhihu/ 知乎"
|
||||
echo " │ ├── pdfs/ PDF"
|
||||
echo " │ ├── notes/ 笔记"
|
||||
echo " │ └── assets/ 图片等附件"
|
||||
echo " ├── wiki/ （知识库）"
|
||||
echo " ├── index.md （索引）"
|
||||
echo " ├── log.md （日志）"
|
||||
echo " ├── purpose.md （研究方向）"
|
||||
echo " ├── .wiki-cache.json （缓存）"
|
||||
echo " └── .wiki-schema.md （配置）"
|
||||
echo ""
|
||||
echo "下一步：给 agent 一个链接或文件，开始构建知识库！"
|
||||
625
llm-wiki/scripts/lib/graph-warning-bundle.js
Normal file
625
llm-wiki/scripts/lib/graph-warning-bundle.js
Normal file
@@ -0,0 +1,625 @@
|
||||
#!/usr/bin/env node
|
||||
"use strict";
|
||||
|
||||
const crypto = require("node:crypto");
|
||||
const fs = require("node:fs");
|
||||
const fsp = require("node:fs/promises");
|
||||
const path = require("node:path");
|
||||
const zlib = require("node:zlib");
|
||||
const { normalizeRelativePosixPath } = require("./wiki-file-discovery");
|
||||
|
||||
const DEFAULT_WARNING_DETAILS_REF = "wiki/graph-warnings.json";
|
||||
const OFFLINE_WARNING_LIMIT_BYTES = 2 * 1024 * 1024;
|
||||
function sha256(bytes) {
|
||||
return crypto.createHash("sha256").update(bytes).digest("hex");
|
||||
}
|
||||
|
||||
function canonicalize(value) {
|
||||
if (Array.isArray(value)) return value.map(canonicalize);
|
||||
if (!value || typeof value !== "object") return value;
|
||||
const result = {};
|
||||
for (const key of Object.keys(value).sort()) {
|
||||
if (value[key] !== undefined) result[key] = canonicalize(value[key]);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function canonicalBytes(value) {
|
||||
return Buffer.from(JSON.stringify(canonicalize(value)), "utf8");
|
||||
}
|
||||
|
||||
function serializeJsonForHtmlScript(value) {
|
||||
const json = JSON.stringify(canonicalize(value));
|
||||
return Buffer.from(json.replace(/[<>&\u2028\u2029]/g, (character) => (
|
||||
`\\u${character.codePointAt(0).toString(16).padStart(4, "0")}`
|
||||
)), "utf8");
|
||||
}
|
||||
|
||||
function compareText(left, right) {
|
||||
return String(left).localeCompare(String(right), "en");
|
||||
}
|
||||
|
||||
function assertRelativeContentPath(value, fieldName) {
|
||||
if (typeof value !== "string" || !value || value.includes("\\")) {
|
||||
throw new Error(`${fieldName} must be a POSIX knowledge-base-relative path`);
|
||||
}
|
||||
let normalized;
|
||||
try {
|
||||
normalized = normalizeRelativePosixPath(value);
|
||||
} catch (_) {
|
||||
throw new Error(`${fieldName} must be a POSIX knowledge-base-relative path`);
|
||||
}
|
||||
if (normalized !== value || path.posix.isAbsolute(value)) {
|
||||
throw new Error(`${fieldName} must be a POSIX knowledge-base-relative path`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function validateDetailsRef(detailsRef) {
|
||||
assertRelativeContentPath(detailsRef, "details_ref");
|
||||
if (path.posix.basename(detailsRef) !== "graph-warnings.json") {
|
||||
throw new Error("details_ref must name graph-warnings.json");
|
||||
}
|
||||
return detailsRef;
|
||||
}
|
||||
|
||||
function normalizeOccurrence(occurrence) {
|
||||
if (!occurrence || typeof occurrence !== "object") {
|
||||
throw new Error("warning occurrence must be an object");
|
||||
}
|
||||
assertRelativeContentPath(occurrence.source_path, "source_path");
|
||||
return canonicalize(occurrence);
|
||||
}
|
||||
|
||||
function normalizeCandidateSets(candidateSets) {
|
||||
if (!Array.isArray(candidateSets)) throw new Error("candidateSets must be an array");
|
||||
const seen = new Set();
|
||||
return candidateSets.map((candidateSet) => {
|
||||
if (!candidateSet || typeof candidateSet !== "object" || !candidateSet.candidate_set_id) {
|
||||
throw new Error("candidate_set_id is required");
|
||||
}
|
||||
if (seen.has(candidateSet.candidate_set_id)) {
|
||||
throw new Error(`duplicate candidate_set_id: ${candidateSet.candidate_set_id}`);
|
||||
}
|
||||
seen.add(candidateSet.candidate_set_id);
|
||||
const candidates = Array.from(new Set((candidateSet.candidates || []).map((candidate) => (
|
||||
assertRelativeContentPath(candidate, "candidate path")
|
||||
)))).sort(compareText);
|
||||
if (candidateSet.candidate_count !== candidates.length) {
|
||||
throw new Error(`candidate_count does not match candidates for ${candidateSet.candidate_set_id}`);
|
||||
}
|
||||
return canonicalize({ ...candidateSet, candidates });
|
||||
}).sort((left, right) => compareText(left.candidate_set_id, right.candidate_set_id));
|
||||
}
|
||||
|
||||
function normalizeGroups(groups, candidateSetIds) {
|
||||
if (!Array.isArray(groups)) throw new Error("groups must be an array");
|
||||
const seen = new Set();
|
||||
const seenOccurrences = new Set();
|
||||
return groups.map((group) => {
|
||||
if (!group || typeof group !== "object" || !group.warning_id) {
|
||||
throw new Error("warning_id is required");
|
||||
}
|
||||
if (seen.has(group.warning_id)) throw new Error(`duplicate warning_id: ${group.warning_id}`);
|
||||
seen.add(group.warning_id);
|
||||
if (group.candidate_set_id && !candidateSetIds.has(group.candidate_set_id)) {
|
||||
throw new Error(`warning references missing candidate set: ${group.candidate_set_id}`);
|
||||
}
|
||||
if (!Number.isSafeInteger(group.occurrence_count) || group.occurrence_count < 0) {
|
||||
throw new Error(`invalid occurrence_count for ${group.warning_id}`);
|
||||
}
|
||||
const occurrences = (group.occurrences || []).map(normalizeOccurrence)
|
||||
.sort((left, right) => compareText(left.occurrence_id, right.occurrence_id));
|
||||
for (const occurrence of occurrences) {
|
||||
if (seenOccurrences.has(occurrence.occurrence_id)) {
|
||||
throw new Error(`duplicate occurrence_id: ${occurrence.occurrence_id}`);
|
||||
}
|
||||
seenOccurrences.add(occurrence.occurrence_id);
|
||||
}
|
||||
if (group.occurrence_count !== occurrences.length) {
|
||||
throw new Error(`occurrence_count does not match occurrences for ${group.warning_id}`);
|
||||
}
|
||||
return canonicalize({ ...group, occurrences });
|
||||
}).sort((left, right) => compareText(left.warning_id, right.warning_id));
|
||||
}
|
||||
|
||||
function sortGraphCollections(graphData) {
|
||||
const graph = structuredClone(graphData || {});
|
||||
if (graph.meta && typeof graph.meta === "object") delete graph.meta.warning_summary;
|
||||
if (Array.isArray(graph.nodes)) graph.nodes.sort((left, right) => compareText(left.id, right.id));
|
||||
if (Array.isArray(graph.edges)) {
|
||||
graph.edges.sort((left, right) => compareText(
|
||||
`${left.id || ""}\0${left.from || ""}\0${left.to || ""}`,
|
||||
`${right.id || ""}\0${right.from || ""}\0${right.to || ""}`
|
||||
));
|
||||
}
|
||||
if (graph.learning && Array.isArray(graph.learning.communities)) {
|
||||
graph.learning.communities.sort((left, right) => compareText(left.id, right.id));
|
||||
}
|
||||
return canonicalize(graph);
|
||||
}
|
||||
|
||||
function graphBuildIdentityProjection(graphData) {
|
||||
const graph = sortGraphCollections(graphData);
|
||||
if (graph.meta && typeof graph.meta === "object") delete graph.meta.build_date;
|
||||
return canonicalize(graph);
|
||||
}
|
||||
|
||||
function canonicalWarningDetailBytes(bundle) {
|
||||
return canonicalBytes({
|
||||
version: bundle.version,
|
||||
build_id: bundle.build_id,
|
||||
candidate_sets: bundle.candidate_sets,
|
||||
groups: bundle.groups
|
||||
});
|
||||
}
|
||||
|
||||
function summarizeWarningGroups(groups) {
|
||||
const byCode = {};
|
||||
let errorOccurrences = 0;
|
||||
let warningOccurrences = 0;
|
||||
for (const group of groups) {
|
||||
byCode[group.code] = (byCode[group.code] || 0) + group.occurrence_count;
|
||||
if (group.severity === "error") errorOccurrences += group.occurrence_count;
|
||||
else warningOccurrences += group.occurrence_count;
|
||||
}
|
||||
return canonicalize({
|
||||
total_groups: groups.length,
|
||||
total_occurrences: errorOccurrences + warningOccurrences,
|
||||
error_occurrences: errorOccurrences,
|
||||
warning_occurrences: warningOccurrences,
|
||||
by_code: byCode
|
||||
});
|
||||
}
|
||||
|
||||
function summaryCountsMatch(summary, groups) {
|
||||
const actual = canonicalize({
|
||||
total_groups: summary.total_groups,
|
||||
total_occurrences: summary.total_occurrences,
|
||||
error_occurrences: summary.error_occurrences,
|
||||
warning_occurrences: summary.warning_occurrences,
|
||||
by_code: summary.by_code
|
||||
});
|
||||
return canonicalBytes(actual).equals(canonicalBytes(summarizeWarningGroups(groups)));
|
||||
}
|
||||
|
||||
function assembleGraphArtifactPair({
|
||||
graphData,
|
||||
groups,
|
||||
candidateSets,
|
||||
detailsRef = DEFAULT_WARNING_DETAILS_REF
|
||||
}) {
|
||||
const validatedDetailsRef = validateDetailsRef(detailsRef);
|
||||
const candidate_sets = normalizeCandidateSets(candidateSets);
|
||||
const normalizedGroups = normalizeGroups(groups, new Set(candidate_sets.map((item) => item.candidate_set_id)));
|
||||
const graphWithoutSummary = sortGraphCollections(graphData);
|
||||
const build_id = sha256(canonicalBytes({
|
||||
graph_without_warning_summary: graphBuildIdentityProjection(graphWithoutSummary),
|
||||
warning_details: { candidate_sets, groups: normalizedGroups }
|
||||
}));
|
||||
const detailProjection = {
|
||||
version: 1,
|
||||
build_id,
|
||||
candidate_sets,
|
||||
groups: normalizedGroups
|
||||
};
|
||||
const details_sha256 = sha256(canonicalBytes(detailProjection));
|
||||
const counts = summarizeWarningGroups(normalizedGroups);
|
||||
const summary = canonicalize({
|
||||
build_id,
|
||||
...counts,
|
||||
details_ref: validatedDetailsRef,
|
||||
details_sha256
|
||||
});
|
||||
const normalizedGraph = canonicalize({
|
||||
...graphWithoutSummary,
|
||||
meta: { ...(graphWithoutSummary.meta || {}), warning_summary: summary }
|
||||
});
|
||||
const warningBundle = canonicalize({
|
||||
version: 1,
|
||||
build_id,
|
||||
summary,
|
||||
candidate_sets,
|
||||
groups: normalizedGroups
|
||||
});
|
||||
return { graphData: normalizedGraph, warningBundle };
|
||||
}
|
||||
|
||||
function summariesMatch(left, right) {
|
||||
return Boolean(left && right && canonicalBytes(left).equals(canonicalBytes(right)));
|
||||
}
|
||||
|
||||
function graphWithoutWarningSummary(graphData) {
|
||||
return sortGraphCollections(graphData);
|
||||
}
|
||||
|
||||
function recalculateBuildId(graphData, warningBundle) {
|
||||
return sha256(canonicalBytes({
|
||||
graph_without_warning_summary: graphBuildIdentityProjection(graphData),
|
||||
warning_details: {
|
||||
candidate_sets: warningBundle.candidate_sets,
|
||||
groups: warningBundle.groups
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
function parseArtifactBytes(bytes) {
|
||||
return JSON.parse(Buffer.isBuffer(bytes) ? bytes.toString("utf8") : String(bytes));
|
||||
}
|
||||
|
||||
function validateArtifactObjects({ graphData, warningBundle, expectedDetailsRef }) {
|
||||
const summary = graphData && graphData.meta && graphData.meta.warning_summary;
|
||||
if (!summary || typeof summary !== "object") {
|
||||
return { status: "unavailable", reason: "invalid", summary: summary || null };
|
||||
}
|
||||
|
||||
try {
|
||||
validateDetailsRef(summary.details_ref);
|
||||
} catch (_) {
|
||||
return { status: "unavailable", reason: "invalid", summary };
|
||||
}
|
||||
if (summary.details_ref !== expectedDetailsRef) {
|
||||
return { status: "unavailable", reason: "details_ref_mismatch", summary };
|
||||
}
|
||||
if (!warningBundle || typeof warningBundle !== "object" || warningBundle.version !== 1) {
|
||||
return { status: "unavailable", reason: "invalid", summary };
|
||||
}
|
||||
if (summary.build_id !== warningBundle.build_id || !summariesMatch(summary, warningBundle.summary)) {
|
||||
return { status: "unavailable", reason: "build_id_mismatch", summary };
|
||||
}
|
||||
|
||||
let normalizedSets;
|
||||
let normalizedGroups;
|
||||
try {
|
||||
normalizedSets = normalizeCandidateSets(warningBundle.candidate_sets);
|
||||
normalizedGroups = normalizeGroups(
|
||||
warningBundle.groups,
|
||||
new Set(normalizedSets.map((item) => item.candidate_set_id))
|
||||
);
|
||||
} catch (_) {
|
||||
return { status: "unavailable", reason: "invalid", summary };
|
||||
}
|
||||
const canonicalBundle = canonicalize({
|
||||
...warningBundle,
|
||||
candidate_sets: normalizedSets,
|
||||
groups: normalizedGroups
|
||||
});
|
||||
if (!summaryCountsMatch(summary, normalizedGroups)) {
|
||||
return { status: "unavailable", reason: "invalid", summary };
|
||||
}
|
||||
const actualDetailsSha256 = sha256(canonicalWarningDetailBytes(canonicalBundle));
|
||||
if (summary.details_sha256 !== actualDetailsSha256) {
|
||||
return { status: "unavailable", reason: "details_sha256_mismatch", summary };
|
||||
}
|
||||
if (recalculateBuildId(graphData, canonicalBundle) !== summary.build_id) {
|
||||
return { status: "unavailable", reason: "build_id_mismatch", summary };
|
||||
}
|
||||
|
||||
return { status: "available", graphData, warningBundle: canonicalBundle };
|
||||
}
|
||||
|
||||
function isWithinRoot(rootPath, candidatePath) {
|
||||
return candidatePath === rootPath || candidatePath.startsWith(`${rootPath}${path.sep}`);
|
||||
}
|
||||
|
||||
async function validateArtifactDestinations({ kbRoot, graphPath, warningPath, detailsRef }) {
|
||||
const rootReal = await fsp.realpath(kbRoot);
|
||||
const graphAbsolute = path.resolve(graphPath);
|
||||
const warningAbsolute = path.resolve(warningPath);
|
||||
if (path.basename(graphAbsolute) !== "graph-data.json") {
|
||||
throw new Error("graph destination basename must be graph-data.json");
|
||||
}
|
||||
if (path.basename(warningAbsolute) !== "graph-warnings.json") {
|
||||
throw new Error("warning destination basename must be graph-warnings.json");
|
||||
}
|
||||
|
||||
const graphParentReal = await fsp.realpath(path.dirname(graphAbsolute));
|
||||
const warningParentReal = await fsp.realpath(path.dirname(warningAbsolute));
|
||||
if (!isWithinRoot(rootReal, graphParentReal) || !isWithinRoot(rootReal, warningParentReal)) {
|
||||
throw new Error("artifact destination must remain inside the knowledge base");
|
||||
}
|
||||
if (graphParentReal !== warningParentReal) {
|
||||
throw new Error("graph-data.json and graph-warnings.json must be sibling artifacts");
|
||||
}
|
||||
const graphFinal = path.join(graphParentReal, "graph-data.json");
|
||||
const warningFinal = path.join(warningParentReal, "graph-warnings.json");
|
||||
for (const finalPath of [graphFinal, warningFinal]) {
|
||||
try {
|
||||
const stat = await fsp.lstat(finalPath);
|
||||
if (stat.isSymbolicLink() || !stat.isFile()) {
|
||||
throw new Error(`artifact destination is not a regular file: ${finalPath}`);
|
||||
}
|
||||
} catch (error) {
|
||||
if (error.code !== "ENOENT") throw error;
|
||||
}
|
||||
}
|
||||
|
||||
const normalizedDetailsRef = validateDetailsRef(detailsRef);
|
||||
const detailsAbsolute = path.resolve(rootReal, ...normalizedDetailsRef.split("/"));
|
||||
const detailsParentReal = await fsp.realpath(path.dirname(detailsAbsolute));
|
||||
const resolvedDetails = path.join(detailsParentReal, path.basename(detailsAbsolute));
|
||||
if (!isWithinRoot(rootReal, detailsParentReal) || resolvedDetails !== warningFinal) {
|
||||
throw new Error("details_ref does not resolve to the final sibling graph-warnings.json");
|
||||
}
|
||||
const expectedDetailsRef = path.relative(rootReal, warningFinal).split(path.sep).join("/");
|
||||
if (normalizedDetailsRef !== expectedDetailsRef) {
|
||||
throw new Error("details_ref does not match the final warning destination");
|
||||
}
|
||||
return { rootReal, graphFinal, warningFinal, outputParent: graphParentReal, expectedDetailsRef };
|
||||
}
|
||||
|
||||
async function writeSyncedFile(filePath, bytes) {
|
||||
const handle = await fsp.open(filePath, "wx", 0o600);
|
||||
try {
|
||||
await handle.writeFile(bytes);
|
||||
await handle.sync();
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function fsyncDirectory(directoryPath) {
|
||||
let handle;
|
||||
try {
|
||||
handle = await fsp.open(directoryPath, fs.constants.O_RDONLY);
|
||||
await handle.sync();
|
||||
} catch (error) {
|
||||
if (!["EINVAL", "ENOTSUP", "EBADF", "EPERM", "EISDIR"].includes(error.code)) throw error;
|
||||
} finally {
|
||||
if (handle) await handle.close();
|
||||
}
|
||||
}
|
||||
|
||||
function operationDirectoryName(name) {
|
||||
return /^[a-f0-9]{64}-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(name);
|
||||
}
|
||||
|
||||
async function pruneOldOperationDirectories(buildRoot, currentDirectory, now) {
|
||||
const entries = await fsp.readdir(buildRoot, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory() || !operationDirectoryName(entry.name)) continue;
|
||||
const candidate = path.join(buildRoot, entry.name);
|
||||
if (candidate === currentDirectory) continue;
|
||||
const stat = await fsp.stat(candidate);
|
||||
if (now - stat.mtimeMs > 24 * 60 * 60 * 1000) {
|
||||
await fsp.rm(candidate, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function commitGraphArtifactPair({ kbRoot, graphPath, warningPath, pair, hooks = {} }) {
|
||||
if (!pair || !pair.graphData || !pair.warningBundle) throw new Error("artifact pair is required");
|
||||
const summary = pair.graphData.meta && pair.graphData.meta.warning_summary;
|
||||
if (!summary) throw new Error("graph warning summary is required");
|
||||
const destinations = await validateArtifactDestinations({
|
||||
kbRoot,
|
||||
graphPath,
|
||||
warningPath,
|
||||
detailsRef: summary.details_ref
|
||||
});
|
||||
const stat = hooks.stat || ((target) => fsp.stat(target));
|
||||
const tempParent = path.join(destinations.rootReal, ".wiki-tmp");
|
||||
await fsp.mkdir(tempParent, { recursive: true, mode: 0o700 });
|
||||
const tempParentReal = await fsp.realpath(tempParent);
|
||||
if (!isWithinRoot(destinations.rootReal, tempParentReal)) {
|
||||
throw new Error("temporary graph build directory escapes knowledge base");
|
||||
}
|
||||
const buildRoot = path.join(tempParentReal, "graph-build");
|
||||
await fsp.mkdir(buildRoot, { recursive: true, mode: 0o700 });
|
||||
|
||||
const [tempDevice, graphDevice, warningDevice] = await Promise.all([
|
||||
stat(tempParentReal),
|
||||
stat(path.dirname(destinations.graphFinal)),
|
||||
stat(path.dirname(destinations.warningFinal))
|
||||
]);
|
||||
if (tempDevice.dev !== graphDevice.dev || tempDevice.dev !== warningDevice.dev) {
|
||||
throw new Error("graph artifact destinations must use the same filesystem device as .wiki-tmp");
|
||||
}
|
||||
|
||||
const graphBytes = Buffer.from(`${JSON.stringify(pair.graphData, null, 2)}\n`, "utf8");
|
||||
const warningBytes = Buffer.from(`${JSON.stringify(pair.warningBundle, null, 2)}\n`, "utf8");
|
||||
let graphObject;
|
||||
let warningObject;
|
||||
try {
|
||||
graphObject = parseArtifactBytes(graphBytes);
|
||||
warningObject = parseArtifactBytes(warningBytes);
|
||||
} catch (error) {
|
||||
throw new Error(`invalid artifact pair JSON: ${error.message}`);
|
||||
}
|
||||
const preflight = validateArtifactObjects({
|
||||
graphData: graphObject,
|
||||
warningBundle: warningObject,
|
||||
expectedDetailsRef: destinations.expectedDetailsRef
|
||||
});
|
||||
if (preflight.status !== "available") {
|
||||
throw new Error(`invalid artifact pair: ${preflight.reason}`);
|
||||
}
|
||||
|
||||
const operationDirectory = path.join(
|
||||
buildRoot,
|
||||
`${summary.build_id}-${crypto.randomUUID()}`
|
||||
);
|
||||
await fsp.mkdir(operationDirectory, { recursive: false, mode: 0o700 });
|
||||
const tempGraph = path.join(operationDirectory, "graph-data.json");
|
||||
const tempWarning = path.join(operationDirectory, "graph-warnings.json");
|
||||
|
||||
await writeSyncedFile(tempGraph, graphBytes);
|
||||
await writeSyncedFile(tempWarning, warningBytes);
|
||||
const verifiedTemporary = validateArtifactObjects({
|
||||
graphData: parseArtifactBytes(await fsp.readFile(tempGraph)),
|
||||
warningBundle: parseArtifactBytes(await fsp.readFile(tempWarning)),
|
||||
expectedDetailsRef: destinations.expectedDetailsRef
|
||||
});
|
||||
if (verifiedTemporary.status !== "available") {
|
||||
throw new Error(`temporary artifact verification failed: ${verifiedTemporary.reason}`);
|
||||
}
|
||||
|
||||
await fsp.rename(tempWarning, destinations.warningFinal);
|
||||
await fsyncDirectory(destinations.outputParent);
|
||||
if (hooks.afterWarningReplace) await hooks.afterWarningReplace();
|
||||
await fsp.rename(tempGraph, destinations.graphFinal);
|
||||
await fsyncDirectory(destinations.outputParent);
|
||||
await fsp.rm(operationDirectory, { recursive: true, force: true });
|
||||
await pruneOldOperationDirectories(
|
||||
buildRoot,
|
||||
operationDirectory,
|
||||
hooks.now ? hooks.now() : Date.now()
|
||||
);
|
||||
}
|
||||
|
||||
async function verifyGraphArtifactPair({ kbRoot, graphPath, warningPath }) {
|
||||
let graphData;
|
||||
try {
|
||||
graphData = parseArtifactBytes(await fsp.readFile(graphPath));
|
||||
} catch (_) {
|
||||
return { status: "unavailable", reason: "invalid", summary: null };
|
||||
}
|
||||
const summary = graphData && graphData.meta && graphData.meta.warning_summary;
|
||||
if (!summary || typeof summary !== "object") {
|
||||
return { status: "unavailable", reason: "invalid", summary: summary || null };
|
||||
}
|
||||
|
||||
let destinations;
|
||||
try {
|
||||
destinations = await validateArtifactDestinations({
|
||||
kbRoot,
|
||||
graphPath,
|
||||
warningPath,
|
||||
detailsRef: summary.details_ref
|
||||
});
|
||||
} catch (error) {
|
||||
return {
|
||||
status: "unavailable",
|
||||
reason: error.message.includes("details_ref") ? "details_ref_mismatch" : "invalid",
|
||||
summary
|
||||
};
|
||||
}
|
||||
|
||||
let warningBytes;
|
||||
try {
|
||||
warningBytes = await fsp.readFile(destinations.warningFinal);
|
||||
} catch (error) {
|
||||
return {
|
||||
status: "unavailable",
|
||||
reason: error.code === "ENOENT" ? "missing" : "invalid",
|
||||
summary
|
||||
};
|
||||
}
|
||||
let warningBundle;
|
||||
try {
|
||||
warningBundle = parseArtifactBytes(warningBytes);
|
||||
} catch (_) {
|
||||
return { status: "unavailable", reason: "invalid", summary };
|
||||
}
|
||||
return validateArtifactObjects({
|
||||
graphData,
|
||||
warningBundle,
|
||||
expectedDetailsRef: destinations.expectedDetailsRef
|
||||
});
|
||||
}
|
||||
|
||||
function canonicalOfflineBundle(bundle) {
|
||||
const candidate_sets = (bundle.candidate_sets || []).map((candidateSet) => canonicalize({
|
||||
...candidateSet,
|
||||
candidates: (candidateSet.candidates || []).slice().sort(compareText)
|
||||
})).sort((left, right) => compareText(left.candidate_set_id, right.candidate_set_id));
|
||||
const groups = (bundle.groups || []).map((group) => canonicalize({
|
||||
...group,
|
||||
occurrences: (group.occurrences || []).slice()
|
||||
.sort((left, right) => compareText(left.occurrence_id, right.occurrence_id))
|
||||
})).sort((left, right) => compareText(left.warning_id, right.warning_id));
|
||||
return canonicalize({ ...bundle, candidate_sets, groups });
|
||||
}
|
||||
|
||||
function offlinePayload(summary, bundle, truncated, omittedGroupCount, omittedCandidateSetCount) {
|
||||
return canonicalize({
|
||||
summary,
|
||||
details_status: "available",
|
||||
details_unavailable_reason: null,
|
||||
warning_details_truncated: truncated,
|
||||
omitted_group_count: omittedGroupCount,
|
||||
omitted_candidate_set_count: omittedCandidateSetCount,
|
||||
bundle
|
||||
});
|
||||
}
|
||||
|
||||
function compressedPayloadBytes(payload) {
|
||||
const scriptBytes = serializeJsonForHtmlScript(payload);
|
||||
return {
|
||||
scriptBytes,
|
||||
compressedBytes: zlib.gzipSync(scriptBytes, { level: 9 }).length
|
||||
};
|
||||
}
|
||||
|
||||
function prepareOfflineWarningPayload({
|
||||
summary,
|
||||
bundle,
|
||||
maxCompressedBytes = OFFLINE_WARNING_LIMIT_BYTES
|
||||
}) {
|
||||
if (!Number.isSafeInteger(maxCompressedBytes) || maxCompressedBytes <= 0) {
|
||||
throw new Error("maxCompressedBytes must be a positive integer");
|
||||
}
|
||||
const completeBundle = canonicalOfflineBundle(bundle);
|
||||
let payload = offlinePayload(summary, completeBundle, false, 0, 0);
|
||||
let { scriptBytes, compressedBytes } = compressedPayloadBytes(payload);
|
||||
if (compressedBytes <= maxCompressedBytes) return { payload, scriptBytes, compressedBytes };
|
||||
|
||||
const compactBundle = canonicalOfflineBundle({
|
||||
...completeBundle,
|
||||
groups: completeBundle.groups.map((group) => ({
|
||||
...group,
|
||||
occurrences: group.occurrences.slice(0, 20)
|
||||
})),
|
||||
candidate_sets: completeBundle.candidate_sets.map((candidateSet) => ({
|
||||
...candidateSet,
|
||||
candidates: candidateSet.candidates.slice(0, 20)
|
||||
}))
|
||||
});
|
||||
let omittedGroupCount = 0;
|
||||
let omittedCandidateSetCount = 0;
|
||||
const refresh = () => {
|
||||
payload = offlinePayload(summary, compactBundle, true, omittedGroupCount, omittedCandidateSetCount);
|
||||
({ scriptBytes, compressedBytes } = compressedPayloadBytes(payload));
|
||||
return compressedBytes <= maxCompressedBytes;
|
||||
};
|
||||
if (refresh()) return { payload, scriptBytes, compressedBytes };
|
||||
|
||||
for (let index = compactBundle.groups.length - 1; index >= 0; index -= 1) {
|
||||
while (compactBundle.groups[index].occurrences.length > 0) {
|
||||
compactBundle.groups[index].occurrences.pop();
|
||||
if (refresh()) return { payload, scriptBytes, compressedBytes };
|
||||
}
|
||||
}
|
||||
for (let index = compactBundle.candidate_sets.length - 1; index >= 0; index -= 1) {
|
||||
while (compactBundle.candidate_sets[index].candidates.length > 0) {
|
||||
compactBundle.candidate_sets[index].candidates.pop();
|
||||
if (refresh()) return { payload, scriptBytes, compressedBytes };
|
||||
}
|
||||
}
|
||||
|
||||
while (compactBundle.groups.length > 0) {
|
||||
compactBundle.groups.pop();
|
||||
omittedGroupCount += 1;
|
||||
if (refresh()) return { payload, scriptBytes, compressedBytes };
|
||||
}
|
||||
while (compactBundle.candidate_sets.length > 0) {
|
||||
compactBundle.candidate_sets.pop();
|
||||
omittedCandidateSetCount += 1;
|
||||
if (refresh()) return { payload, scriptBytes, compressedBytes };
|
||||
}
|
||||
if (!refresh()) {
|
||||
throw new Error("offline warning summary exceeds the compressed payload limit");
|
||||
}
|
||||
return { payload, scriptBytes, compressedBytes };
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
DEFAULT_WARNING_DETAILS_REF,
|
||||
OFFLINE_WARNING_LIMIT_BYTES,
|
||||
assembleGraphArtifactPair,
|
||||
canonicalWarningDetailBytes,
|
||||
commitGraphArtifactPair,
|
||||
prepareOfflineWarningPayload,
|
||||
serializeJsonForHtmlScript,
|
||||
verifyGraphArtifactPair
|
||||
};
|
||||
158
llm-wiki/scripts/lib/source-signal-eligibility.js
Normal file
158
llm-wiki/scripts/lib/source-signal-eligibility.js
Normal file
@@ -0,0 +1,158 @@
|
||||
#!/usr/bin/env node
|
||||
"use strict";
|
||||
|
||||
const SCAN_KINDS = [
|
||||
{ subdir: "entities", pageType: "entity", applicable: true },
|
||||
{ subdir: "topics", pageType: "topic", applicable: true },
|
||||
{ subdir: "sources", pageType: "source", applicable: true },
|
||||
{ subdir: "comparisons", pageType: "comparison", applicable: true },
|
||||
{ subdir: "queries", pageType: "query", applicable: false },
|
||||
{ subdir: "synthesis", pageType: "synthesis", applicable: false }
|
||||
];
|
||||
|
||||
function sortedUnique(values) {
|
||||
return Array.from(new Set(values)).sort();
|
||||
}
|
||||
|
||||
function extractFrontmatter(text) {
|
||||
if (!text.startsWith("---\n") && !text.startsWith("---\r\n")) {
|
||||
return { hasFrontmatter: false, frontmatter: "", body: text };
|
||||
}
|
||||
|
||||
const match = text.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)([\s\S]*)$/);
|
||||
if (!match) {
|
||||
return { hasFrontmatter: false, frontmatter: "", body: text };
|
||||
}
|
||||
|
||||
return {
|
||||
hasFrontmatter: true,
|
||||
frontmatter: match[1],
|
||||
body: match[2]
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeSourceToken(token) {
|
||||
const trimmed = String(token || "").trim();
|
||||
if (!trimmed) return null;
|
||||
|
||||
let value = trimmed;
|
||||
if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
|
||||
value = value.slice(1, -1).trim();
|
||||
}
|
||||
|
||||
return value || null;
|
||||
}
|
||||
|
||||
function parseInlineSources(raw) {
|
||||
const trimmed = raw.trim();
|
||||
if (trimmed === "[]") return { ok: true, values: [] };
|
||||
if (!(trimmed.startsWith("[") && trimmed.endsWith("]"))) {
|
||||
return { ok: false, values: [] };
|
||||
}
|
||||
|
||||
const inner = trimmed.slice(1, -1).trim();
|
||||
if (!inner) return { ok: true, values: [] };
|
||||
|
||||
const values = inner
|
||||
.split(",")
|
||||
.map(normalizeSourceToken)
|
||||
.filter(Boolean);
|
||||
|
||||
return { ok: true, values };
|
||||
}
|
||||
|
||||
function parseSourcesFrontmatter(frontmatter) {
|
||||
if (!frontmatter) {
|
||||
return { hasField: false, parsed: false, sources: [], signalAvailable: false };
|
||||
}
|
||||
|
||||
const lines = frontmatter.split(/\r?\n/);
|
||||
|
||||
for (let index = 0; index < lines.length; index += 1) {
|
||||
const match = lines[index].match(/^sources:\s*(.*)$/);
|
||||
if (!match) continue;
|
||||
|
||||
const rest = match[1].trim();
|
||||
if (rest) {
|
||||
if (!rest.startsWith("[")) {
|
||||
const single = normalizeSourceToken(rest);
|
||||
return {
|
||||
hasField: true,
|
||||
parsed: Boolean(single),
|
||||
sources: single ? [single] : [],
|
||||
signalAvailable: Boolean(single)
|
||||
};
|
||||
}
|
||||
|
||||
const parsedInline = parseInlineSources(rest);
|
||||
return {
|
||||
hasField: true,
|
||||
parsed: parsedInline.ok,
|
||||
sources: parsedInline.ok ? sortedUnique(parsedInline.values) : [],
|
||||
signalAvailable: parsedInline.ok && parsedInline.values.length > 0
|
||||
};
|
||||
}
|
||||
|
||||
const collected = [];
|
||||
let parsed = true;
|
||||
let consumed = 0;
|
||||
|
||||
for (let cursor = index + 1; cursor < lines.length; cursor += 1) {
|
||||
const line = lines[cursor];
|
||||
if (!line.trim()) {
|
||||
consumed += 1;
|
||||
continue;
|
||||
}
|
||||
if (/^[^\s-]/.test(line)) break;
|
||||
const itemMatch = line.match(/^\s*-\s*(.+)$/);
|
||||
if (!itemMatch) {
|
||||
parsed = false;
|
||||
consumed += 1;
|
||||
continue;
|
||||
}
|
||||
const token = normalizeSourceToken(itemMatch[1]);
|
||||
if (token) collected.push(token);
|
||||
consumed += 1;
|
||||
}
|
||||
|
||||
index += consumed;
|
||||
return {
|
||||
hasField: true,
|
||||
parsed,
|
||||
sources: parsed ? sortedUnique(collected) : [],
|
||||
signalAvailable: parsed && collected.length > 0
|
||||
};
|
||||
}
|
||||
|
||||
return { hasField: false, parsed: false, sources: [], signalAvailable: false };
|
||||
}
|
||||
|
||||
function evaluateSourceSignalEligibility({ pageType, frontmatter }) {
|
||||
const kind = SCAN_KINDS.find((k) => k.pageType === pageType);
|
||||
if (!kind || !kind.applicable) {
|
||||
return { eligible: false, reason: "not_applicable", sources: [] };
|
||||
}
|
||||
|
||||
const parsed = parseSourcesFrontmatter(frontmatter);
|
||||
|
||||
if (!parsed.hasField) {
|
||||
return { eligible: false, reason: "missing_sources", sources: [] };
|
||||
}
|
||||
if (!parsed.parsed) {
|
||||
return { eligible: false, reason: "invalid_sources", sources: [] };
|
||||
}
|
||||
if (parsed.sources.length === 0) {
|
||||
return { eligible: false, reason: "empty_sources", sources: [] };
|
||||
}
|
||||
|
||||
return { eligible: true, reason: "ok", sources: parsed.sources };
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
SCAN_KINDS,
|
||||
extractFrontmatter,
|
||||
evaluateSourceSignalEligibility,
|
||||
normalizeSourceToken,
|
||||
parseSourcesFrontmatter,
|
||||
sortedUnique
|
||||
};
|
||||
68
llm-wiki/scripts/lib/unicode-case-folding.js
Normal file
68
llm-wiki/scripts/lib/unicode-case-folding.js
Normal file
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env node
|
||||
"use strict";
|
||||
|
||||
const crypto = require("node:crypto");
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
const { loadUnicode17NfcNormalizer } = require("./unicode-normalization");
|
||||
|
||||
const TABLE_PATH = path.join(__dirname, "../../deps/unicode/CaseFolding-17.0.0.txt");
|
||||
const EXPECTED_HASH = "ff8d8fefbf123574205085d6714c36149eb946d717a0c585c27f0f4ef58c4183";
|
||||
|
||||
let cached = null;
|
||||
|
||||
function sha256(buffer) {
|
||||
return crypto.createHash("sha256").update(buffer).digest("hex");
|
||||
}
|
||||
|
||||
function parseUnicode17CaseFolding(text, normalizeNfc = loadUnicode17NfcNormalizer()) {
|
||||
const mappings = new Map();
|
||||
|
||||
for (const rawLine of text.split(/\r?\n/)) {
|
||||
const line = rawLine.replace(/#.*/, "").trim();
|
||||
if (!line) continue;
|
||||
|
||||
const [sourceHex, status, targetHex] = line.split(";").map((part) => part.trim());
|
||||
if (status !== "C" && status !== "F") continue;
|
||||
|
||||
mappings.set(
|
||||
Number.parseInt(sourceHex, 16),
|
||||
targetHex
|
||||
.split(/\s+/)
|
||||
.filter(Boolean)
|
||||
.map((value) => String.fromCodePoint(Number.parseInt(value, 16)))
|
||||
.join("")
|
||||
);
|
||||
}
|
||||
|
||||
return (value) => {
|
||||
let folded = "";
|
||||
for (const character of normalizeNfc(String(value))) {
|
||||
folded += mappings.get(character.codePointAt(0)) || character;
|
||||
}
|
||||
return normalizeNfc(folded);
|
||||
};
|
||||
}
|
||||
|
||||
function loadUnicode17CaseFolder() {
|
||||
if (cached) return cached;
|
||||
|
||||
const content = fs.readFileSync(TABLE_PATH);
|
||||
if (sha256(content) !== EXPECTED_HASH) {
|
||||
throw new Error(`Unicode runtime data hash mismatch for ${path.basename(TABLE_PATH)}`);
|
||||
}
|
||||
|
||||
cached = parseUnicode17CaseFolding(content.toString("utf8"));
|
||||
return cached;
|
||||
}
|
||||
|
||||
function defaultCaseFoldUnicode17(value) {
|
||||
return loadUnicode17CaseFolder()(value);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
TABLE_PATH,
|
||||
defaultCaseFoldUnicode17,
|
||||
loadUnicode17CaseFolder,
|
||||
parseUnicode17CaseFolding
|
||||
};
|
||||
362
llm-wiki/scripts/lib/unicode-normalization.js
Normal file
362
llm-wiki/scripts/lib/unicode-normalization.js
Normal file
@@ -0,0 +1,362 @@
|
||||
#!/usr/bin/env node
|
||||
"use strict";
|
||||
|
||||
const crypto = require("node:crypto");
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
|
||||
const UNICODE_DATA_PATH = path.join(__dirname, "../../deps/unicode/UnicodeData-17.0.0.txt");
|
||||
const DERIVED_NORMALIZATION_PROPS_PATH = path.join(
|
||||
__dirname,
|
||||
"../../deps/unicode/DerivedNormalizationProps-17.0.0.txt"
|
||||
);
|
||||
|
||||
const EXPECTED_HASHES = {
|
||||
[UNICODE_DATA_PATH]: "2e1efc1dcb59c575eedf5ccae60f95229f706ee6d031835247d843c11d96470c",
|
||||
[DERIVED_NORMALIZATION_PROPS_PATH]: "71fd6a206a2c0cdd41feb6b7f656aa31091db45e9cedc926985d718397f9e488"
|
||||
};
|
||||
|
||||
const HANGUL = {
|
||||
SBase: 0xac00,
|
||||
LBase: 0x1100,
|
||||
VBase: 0x1161,
|
||||
TBase: 0x11a7,
|
||||
LCount: 19,
|
||||
VCount: 21,
|
||||
TCount: 28
|
||||
};
|
||||
HANGUL.NCount = HANGUL.VCount * HANGUL.TCount;
|
||||
HANGUL.SCount = HANGUL.LCount * HANGUL.NCount;
|
||||
|
||||
let cachedTables = null;
|
||||
let cachedNormalizer = null;
|
||||
|
||||
function sha256(buffer) {
|
||||
return crypto.createHash("sha256").update(buffer).digest("hex");
|
||||
}
|
||||
|
||||
function verifyRuntimeFile(filePath) {
|
||||
const actualHash = sha256(fs.readFileSync(filePath));
|
||||
const expectedHash = EXPECTED_HASHES[filePath];
|
||||
|
||||
if (actualHash !== expectedHash) {
|
||||
throw new Error(`Unicode runtime data hash mismatch for ${path.basename(filePath)}`);
|
||||
}
|
||||
}
|
||||
|
||||
function parseCodePointRange(rangeText) {
|
||||
const [startHex, endHex] = rangeText.split("..");
|
||||
return {
|
||||
start: Number.parseInt(startHex, 16),
|
||||
end: Number.parseInt(endHex || startHex, 16)
|
||||
};
|
||||
}
|
||||
|
||||
function expandRange(start, end, callback) {
|
||||
for (let codePoint = start; codePoint <= end; codePoint += 1) {
|
||||
callback(codePoint);
|
||||
}
|
||||
}
|
||||
|
||||
function parseDecomposition(rawField) {
|
||||
if (!rawField) return null;
|
||||
if (rawField.startsWith("<")) return null;
|
||||
return rawField.split(/\s+/).filter(Boolean).map((value) => Number.parseInt(value, 16));
|
||||
}
|
||||
|
||||
function pairKey(left, right) {
|
||||
return `${left}:${right}`;
|
||||
}
|
||||
|
||||
function lookupRangeValue(ranges, codePoint) {
|
||||
let low = 0;
|
||||
let high = ranges.length - 1;
|
||||
|
||||
while (low <= high) {
|
||||
const middle = Math.floor((low + high) / 2);
|
||||
const entry = ranges[middle];
|
||||
|
||||
if (codePoint < entry.start) {
|
||||
high = middle - 1;
|
||||
} else if (codePoint > entry.end) {
|
||||
low = middle + 1;
|
||||
} else {
|
||||
return entry.value;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
function getCanonicalCombiningClass(tables, codePoint) {
|
||||
return tables.combiningClasses.get(codePoint) || lookupRangeValue(tables.combiningClassRanges, codePoint);
|
||||
}
|
||||
|
||||
function isHangulSyllable(codePoint) {
|
||||
return codePoint >= HANGUL.SBase && codePoint < HANGUL.SBase + HANGUL.SCount;
|
||||
}
|
||||
|
||||
function isHangulL(codePoint) {
|
||||
return codePoint >= HANGUL.LBase && codePoint < HANGUL.LBase + HANGUL.LCount;
|
||||
}
|
||||
|
||||
function isHangulV(codePoint) {
|
||||
return codePoint >= HANGUL.VBase && codePoint < HANGUL.VBase + HANGUL.VCount;
|
||||
}
|
||||
|
||||
function isHangulT(codePoint) {
|
||||
return codePoint > HANGUL.TBase && codePoint < HANGUL.TBase + HANGUL.TCount;
|
||||
}
|
||||
|
||||
function decomposeHangul(codePoint) {
|
||||
const sIndex = codePoint - HANGUL.SBase;
|
||||
const lIndex = Math.floor(sIndex / HANGUL.NCount);
|
||||
const vIndex = Math.floor((sIndex % HANGUL.NCount) / HANGUL.TCount);
|
||||
const tIndex = sIndex % HANGUL.TCount;
|
||||
|
||||
const result = [
|
||||
HANGUL.LBase + lIndex,
|
||||
HANGUL.VBase + vIndex
|
||||
];
|
||||
|
||||
if (tIndex !== 0) {
|
||||
result.push(HANGUL.TBase + tIndex);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function composeHangul(left, right) {
|
||||
if (isHangulL(left) && isHangulV(right)) {
|
||||
const lIndex = left - HANGUL.LBase;
|
||||
const vIndex = right - HANGUL.VBase;
|
||||
return HANGUL.SBase + (lIndex * HANGUL.NCount) + (vIndex * HANGUL.TCount);
|
||||
}
|
||||
|
||||
if (
|
||||
isHangulSyllable(left)
|
||||
&& ((left - HANGUL.SBase) % HANGUL.TCount === 0)
|
||||
&& isHangulT(right)
|
||||
) {
|
||||
return left + (right - HANGUL.TBase);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function parseUnicode17NormalizationData(unicodeDataText, derivedPropsText) {
|
||||
const combiningClasses = new Map();
|
||||
const combiningClassRanges = [];
|
||||
const canonicalDecompositions = new Map();
|
||||
const compositionExclusions = new Set();
|
||||
const compositionMap = new Map();
|
||||
|
||||
let pendingRange = null;
|
||||
|
||||
for (const rawLine of unicodeDataText.split(/\r?\n/)) {
|
||||
if (!rawLine) continue;
|
||||
const fields = rawLine.split(";");
|
||||
if (fields.length < 6) continue;
|
||||
|
||||
const codePoint = Number.parseInt(fields[0], 16);
|
||||
const name = fields[1];
|
||||
const canonicalCombiningClass = Number.parseInt(fields[3], 10) || 0;
|
||||
const decomposition = parseDecomposition(fields[5]);
|
||||
|
||||
if (name.endsWith(", First>")) {
|
||||
pendingRange = {
|
||||
start: codePoint,
|
||||
combiningClass: canonicalCombiningClass,
|
||||
decomposition
|
||||
};
|
||||
continue;
|
||||
}
|
||||
|
||||
if (name.endsWith(", Last>") && pendingRange) {
|
||||
if (pendingRange.combiningClass !== 0) {
|
||||
combiningClassRanges.push({
|
||||
start: pendingRange.start,
|
||||
end: codePoint,
|
||||
value: pendingRange.combiningClass
|
||||
});
|
||||
}
|
||||
|
||||
if (pendingRange.decomposition) {
|
||||
expandRange(pendingRange.start, codePoint, (rangeCodePoint) => {
|
||||
canonicalDecompositions.set(rangeCodePoint, pendingRange.decomposition);
|
||||
});
|
||||
}
|
||||
|
||||
pendingRange = null;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (canonicalCombiningClass !== 0) {
|
||||
combiningClasses.set(codePoint, canonicalCombiningClass);
|
||||
}
|
||||
if (decomposition) {
|
||||
canonicalDecompositions.set(codePoint, decomposition);
|
||||
}
|
||||
}
|
||||
|
||||
combiningClassRanges.sort((left, right) => left.start - right.start);
|
||||
|
||||
for (const rawLine of derivedPropsText.split(/\r?\n/)) {
|
||||
const line = rawLine.replace(/#.*/, "").trim();
|
||||
if (!line) continue;
|
||||
|
||||
const [rangeText, property] = line.split(";").map((part) => part.trim());
|
||||
if (property !== "Full_Composition_Exclusion") continue;
|
||||
|
||||
const { start, end } = parseCodePointRange(rangeText);
|
||||
expandRange(start, end, (codePoint) => {
|
||||
compositionExclusions.add(codePoint);
|
||||
});
|
||||
}
|
||||
|
||||
for (const [composite, decomposition] of canonicalDecompositions.entries()) {
|
||||
if (compositionExclusions.has(composite)) continue;
|
||||
if (decomposition.length !== 2) continue;
|
||||
compositionMap.set(pairKey(decomposition[0], decomposition[1]), composite);
|
||||
}
|
||||
|
||||
return Object.freeze({
|
||||
combiningClasses,
|
||||
combiningClassRanges,
|
||||
canonicalDecompositions,
|
||||
compositionExclusions,
|
||||
compositionMap
|
||||
});
|
||||
}
|
||||
|
||||
function recursivelyDecompose(codePoint, tables, output) {
|
||||
if (isHangulSyllable(codePoint)) {
|
||||
for (const part of decomposeHangul(codePoint)) {
|
||||
recursivelyDecompose(part, tables, output);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const decomposition = tables.canonicalDecompositions.get(codePoint);
|
||||
if (!decomposition) {
|
||||
output.push(codePoint);
|
||||
return;
|
||||
}
|
||||
|
||||
for (const part of decomposition) {
|
||||
recursivelyDecompose(part, tables, output);
|
||||
}
|
||||
}
|
||||
|
||||
function reorderSegment(segment, combiningClasses) {
|
||||
if (segment.length <= 1) return segment;
|
||||
|
||||
const starterCount = combiningClasses[0] === 0 ? 1 : 0;
|
||||
const head = segment.slice(0, starterCount);
|
||||
const marks = segment.slice(starterCount).map((codePoint, index) => ({
|
||||
codePoint,
|
||||
ccc: combiningClasses[starterCount + index],
|
||||
index
|
||||
}));
|
||||
|
||||
marks.sort((left, right) => {
|
||||
if (left.ccc !== right.ccc) return left.ccc - right.ccc;
|
||||
return left.index - right.index;
|
||||
});
|
||||
|
||||
return head.concat(marks.map((item) => item.codePoint));
|
||||
}
|
||||
|
||||
function canonicalOrder(codePoints, tables) {
|
||||
const ordered = [];
|
||||
let segment = [];
|
||||
let classes = [];
|
||||
|
||||
function flush() {
|
||||
if (segment.length === 0) return;
|
||||
ordered.push(...reorderSegment(segment, classes));
|
||||
segment = [];
|
||||
classes = [];
|
||||
}
|
||||
|
||||
for (const codePoint of codePoints) {
|
||||
const ccc = getCanonicalCombiningClass(tables, codePoint);
|
||||
if (ccc === 0 && segment.length > 0) {
|
||||
flush();
|
||||
}
|
||||
segment.push(codePoint);
|
||||
classes.push(ccc);
|
||||
}
|
||||
|
||||
flush();
|
||||
return ordered;
|
||||
}
|
||||
|
||||
function recompose(codePoints, tables) {
|
||||
if (codePoints.length === 0) return [];
|
||||
|
||||
const result = [codePoints[0]];
|
||||
let starterIndex = getCanonicalCombiningClass(tables, codePoints[0]) === 0 ? 0 : -1;
|
||||
let starter = starterIndex === 0 ? codePoints[0] : null;
|
||||
let lastCombiningClass = getCanonicalCombiningClass(tables, codePoints[0]);
|
||||
|
||||
for (let index = 1; index < codePoints.length; index += 1) {
|
||||
const codePoint = codePoints[index];
|
||||
const combiningClass = getCanonicalCombiningClass(tables, codePoint);
|
||||
let composite = null;
|
||||
|
||||
if (starter !== null) {
|
||||
composite = composeHangul(starter, codePoint) || tables.compositionMap.get(pairKey(starter, codePoint)) || null;
|
||||
}
|
||||
|
||||
if (composite !== null && (lastCombiningClass < combiningClass || lastCombiningClass === 0)) {
|
||||
result[starterIndex] = composite;
|
||||
starter = composite;
|
||||
continue;
|
||||
}
|
||||
|
||||
result.push(codePoint);
|
||||
lastCombiningClass = combiningClass;
|
||||
|
||||
if (combiningClass === 0) {
|
||||
starterIndex = result.length - 1;
|
||||
starter = codePoint;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function normalizeNfcUnicode17(value, tables) {
|
||||
const input = String(value);
|
||||
const decomposed = [];
|
||||
|
||||
for (const character of input) {
|
||||
recursivelyDecompose(character.codePointAt(0), tables, decomposed);
|
||||
}
|
||||
|
||||
const ordered = canonicalOrder(decomposed, tables);
|
||||
return String.fromCodePoint(...recompose(ordered, tables));
|
||||
}
|
||||
|
||||
function loadUnicode17NfcNormalizer() {
|
||||
if (cachedNormalizer) return cachedNormalizer;
|
||||
|
||||
verifyRuntimeFile(UNICODE_DATA_PATH);
|
||||
verifyRuntimeFile(DERIVED_NORMALIZATION_PROPS_PATH);
|
||||
|
||||
cachedTables ||= parseUnicode17NormalizationData(
|
||||
fs.readFileSync(UNICODE_DATA_PATH, "utf8"),
|
||||
fs.readFileSync(DERIVED_NORMALIZATION_PROPS_PATH, "utf8")
|
||||
);
|
||||
cachedNormalizer = (value) => normalizeNfcUnicode17(value, cachedTables);
|
||||
return cachedNormalizer;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
DERIVED_NORMALIZATION_PROPS_PATH,
|
||||
UNICODE_DATA_PATH,
|
||||
loadUnicode17NfcNormalizer,
|
||||
normalizeNfcUnicode17,
|
||||
parseUnicode17NormalizationData
|
||||
};
|
||||
204
llm-wiki/scripts/lib/wiki-file-discovery.js
Normal file
204
llm-wiki/scripts/lib/wiki-file-discovery.js
Normal file
@@ -0,0 +1,204 @@
|
||||
#!/usr/bin/env node
|
||||
"use strict";
|
||||
|
||||
const crypto = require("node:crypto");
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
|
||||
const GRAPH_PAGE_TYPES = Object.freeze({
|
||||
entities: "entity",
|
||||
topics: "topic",
|
||||
sources: "source",
|
||||
comparisons: "comparison",
|
||||
synthesis: "synthesis",
|
||||
queries: "query"
|
||||
});
|
||||
|
||||
const ROOT_EDITABLE_MARKDOWN = new Set(["index.md", "log.md", "purpose.md"]);
|
||||
const EXCLUDED_DIRECTORY_NAMES = new Set([".obsidian", ".git", ".wiki-tmp", "node_modules"]);
|
||||
const EXCLUDED_BASENAMES = new Set(["graph-data.json", "graph-warnings.json"]);
|
||||
|
||||
function sha256(text) {
|
||||
return crypto.createHash("sha256").update(text).digest("hex");
|
||||
}
|
||||
|
||||
function normalizeRelativePosixPath(pathValue) {
|
||||
const value = String(pathValue || "").replaceAll("\\", "/");
|
||||
if (!value) {
|
||||
throw new Error("Path must not be empty");
|
||||
}
|
||||
if (value.startsWith("/")) {
|
||||
throw new Error("Path must be relative");
|
||||
}
|
||||
|
||||
const normalized = path.posix.normalize(value);
|
||||
if (
|
||||
normalized === "."
|
||||
|| normalized === ".."
|
||||
|| normalized.startsWith("../")
|
||||
|| normalized.includes("/../")
|
||||
) {
|
||||
throw new Error("Path escapes knowledge-base root");
|
||||
}
|
||||
|
||||
return normalized.replace(/^\.\//, "");
|
||||
}
|
||||
|
||||
function isWithinRoot(rootRealPath, candidateAbsolutePath) {
|
||||
return candidateAbsolutePath === rootRealPath || candidateAbsolutePath.startsWith(`${rootRealPath}${path.sep}`);
|
||||
}
|
||||
|
||||
function resolveInsideKnowledgeBase(kbRoot, relativePath) {
|
||||
const rootRealPath = fs.realpathSync.native(kbRoot);
|
||||
const normalized = normalizeRelativePosixPath(relativePath);
|
||||
const absolutePath = path.resolve(rootRealPath, ...normalized.split("/"));
|
||||
|
||||
if (!isWithinRoot(rootRealPath, absolutePath)) {
|
||||
throw new Error(`Path escapes knowledge-base root: ${relativePath}`);
|
||||
}
|
||||
|
||||
return absolutePath;
|
||||
}
|
||||
|
||||
function isGeneratedArtifact(relativePath) {
|
||||
const basename = path.posix.basename(relativePath);
|
||||
if (EXCLUDED_BASENAMES.has(basename)) return true;
|
||||
return /^knowledge-graph(?:[^/]*)\.html$/i.test(basename);
|
||||
}
|
||||
|
||||
function isRenameStagingFile(relativePath) {
|
||||
return path.posix.basename(relativePath).startsWith(".llm-wiki-rename-");
|
||||
}
|
||||
|
||||
function isMarkdown(relativePath) {
|
||||
return relativePath.toLowerCase().endsWith(".md");
|
||||
}
|
||||
|
||||
function isAttachment(relativePath) {
|
||||
return !isMarkdown(relativePath);
|
||||
}
|
||||
|
||||
function graphTypeFor(relativePath) {
|
||||
const parts = relativePath.split("/");
|
||||
return parts[0] === "wiki" ? (GRAPH_PAGE_TYPES[parts[1]] || null) : null;
|
||||
}
|
||||
|
||||
function isGraphSource(relativePath) {
|
||||
return isMarkdown(relativePath) && graphTypeFor(relativePath) !== null;
|
||||
}
|
||||
|
||||
function isLintSource(relativePath) {
|
||||
return relativePath === "index.md" || (relativePath.startsWith("wiki/") && isMarkdown(relativePath));
|
||||
}
|
||||
|
||||
function isRenameEditableSource(relativePath) {
|
||||
return ROOT_EDITABLE_MARKDOWN.has(relativePath) || (relativePath.startsWith("wiki/") && isMarkdown(relativePath));
|
||||
}
|
||||
|
||||
function isRenameReadOnlySource(relativePath) {
|
||||
return isMarkdown(relativePath) && !isRenameEditableSource(relativePath);
|
||||
}
|
||||
|
||||
function fileSetSignature(items) {
|
||||
return sha256(items.map((item) => `${item.path}\0${item.kind}\0${item.size}\0${item.mtimeNs}`).join("\n"));
|
||||
}
|
||||
|
||||
function fileKindFor(relativePath) {
|
||||
return isMarkdown(relativePath) ? "markdown" : "attachment";
|
||||
}
|
||||
|
||||
function walkKnowledgeBase(rootRealPath, directoryAbsolutePath, results) {
|
||||
const entries = fs.readdirSync(directoryAbsolutePath, { withFileTypes: true })
|
||||
.sort((left, right) => left.name.localeCompare(right.name, "en"));
|
||||
|
||||
for (const entry of entries) {
|
||||
const absolutePath = path.join(directoryAbsolutePath, entry.name);
|
||||
const relativePath = path.relative(rootRealPath, absolutePath).split(path.sep).join("/");
|
||||
const topLevelName = relativePath.split("/")[0];
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
if (
|
||||
entry.name.startsWith(".")
|
||||
|| EXCLUDED_DIRECTORY_NAMES.has(entry.name)
|
||||
|| EXCLUDED_DIRECTORY_NAMES.has(topLevelName)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
if (entry.isSymbolicLink && entry.isSymbolicLink()) {
|
||||
continue;
|
||||
}
|
||||
const resolvedDirectoryPath = fs.realpathSync.native(absolutePath);
|
||||
if (!isWithinRoot(rootRealPath, resolvedDirectoryPath)) {
|
||||
continue;
|
||||
}
|
||||
walkKnowledgeBase(rootRealPath, absolutePath, results);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (entry.isSymbolicLink && entry.isSymbolicLink()) {
|
||||
continue;
|
||||
}
|
||||
if (!entry.isFile()) {
|
||||
continue;
|
||||
}
|
||||
if (EXCLUDED_DIRECTORY_NAMES.has(topLevelName)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const normalizedPath = normalizeRelativePosixPath(relativePath);
|
||||
if (entry.name.startsWith(".") && normalizedPath !== ".wiki-schema.md") {
|
||||
continue;
|
||||
}
|
||||
if (isGeneratedArtifact(normalizedPath) || isRenameStagingFile(normalizedPath)) {
|
||||
continue;
|
||||
}
|
||||
if (!isMarkdown(normalizedPath) && !isAttachment(normalizedPath)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const resolvedFilePath = fs.realpathSync.native(absolutePath);
|
||||
if (!isWithinRoot(rootRealPath, resolvedFilePath)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const stat = fs.statSync(absolutePath, { bigint: true });
|
||||
results.push({
|
||||
path: normalizedPath,
|
||||
absolutePath,
|
||||
kind: fileKindFor(normalizedPath),
|
||||
editable: isRenameEditableSource(normalizedPath),
|
||||
graphType: graphTypeFor(normalizedPath),
|
||||
size: Number(stat.size),
|
||||
mtimeNs: String(stat.mtimeNs)
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function discoverKnowledgeBaseFiles(kbRoot) {
|
||||
const rootRealPath = fs.realpathSync.native(kbRoot);
|
||||
const inventory = [];
|
||||
walkKnowledgeBase(rootRealPath, rootRealPath, inventory);
|
||||
inventory.sort((left, right) => left.path.localeCompare(right.path, "en"));
|
||||
|
||||
const graphSources = inventory.filter((item) => item.kind === "markdown" && isGraphSource(item.path));
|
||||
const lintSources = inventory.filter((item) => item.kind === "markdown" && isLintSource(item.path));
|
||||
const renameEditableSources = inventory.filter((item) => item.kind === "markdown" && isRenameEditableSource(item.path));
|
||||
const renameReadOnlySources = inventory.filter((item) => item.kind === "markdown" && isRenameReadOnlySource(item.path));
|
||||
const targets = inventory.map(({ size, mtimeNs, ...item }) => item);
|
||||
|
||||
return {
|
||||
graphSources,
|
||||
lintSources,
|
||||
renameEditableSources,
|
||||
renameReadOnlySources,
|
||||
targets,
|
||||
fileSetSha256: fileSetSignature(inventory)
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
GRAPH_PAGE_TYPES,
|
||||
discoverKnowledgeBaseFiles,
|
||||
normalizeRelativePosixPath,
|
||||
resolveInsideKnowledgeBase
|
||||
};
|
||||
452
llm-wiki/scripts/lib/wiki-link-index.js
Normal file
452
llm-wiki/scripts/lib/wiki-link-index.js
Normal file
@@ -0,0 +1,452 @@
|
||||
#!/usr/bin/env node
|
||||
"use strict";
|
||||
|
||||
const crypto = require("node:crypto");
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
const { loadUnicode17CaseFolder } = require("./unicode-case-folding");
|
||||
const {
|
||||
validateGraphRenameFilenameSyntax
|
||||
} = require("../../packages/workbench-contracts/src/graph-rename-filename.js");
|
||||
const {
|
||||
discoverKnowledgeBaseFiles,
|
||||
normalizeRelativePosixPath
|
||||
} = require("./wiki-file-discovery");
|
||||
const { extractFrontmatter, parseSourcesFrontmatter } = require("./source-signal-eligibility");
|
||||
const { parseWikilinks, renderWikilinkReplacement } = require("./wikilink-parser");
|
||||
|
||||
function sha256(text) {
|
||||
return crypto.createHash("sha256").update(text).digest("hex");
|
||||
}
|
||||
|
||||
function stableId(prefix, value) {
|
||||
return `${prefix}-${sha256(value).slice(0, 16)}`;
|
||||
}
|
||||
|
||||
function portablePathKey(pathValue, fold = loadUnicode17CaseFolder()) {
|
||||
return fold(normalizeRelativePosixPath(pathValue));
|
||||
}
|
||||
|
||||
function buildWikiTargetIndex(inventory) {
|
||||
const exactPaths = new Map();
|
||||
const portablePaths = new Map();
|
||||
const portableBasenames = new Map();
|
||||
|
||||
for (const item of inventory) {
|
||||
exactPaths.set(item.path, item);
|
||||
|
||||
const pathKey = portablePathKey(item.path);
|
||||
const pathItems = portablePaths.get(pathKey) || [];
|
||||
pathItems.push(item);
|
||||
portablePaths.set(pathKey, pathItems);
|
||||
|
||||
if (item.kind === "markdown") {
|
||||
const basenameKey = portablePathKey(path.posix.basename(item.path, ".md"));
|
||||
const basenameItems = portableBasenames.get(basenameKey) || [];
|
||||
basenameItems.push(item);
|
||||
portableBasenames.set(basenameKey, basenameItems);
|
||||
}
|
||||
}
|
||||
|
||||
const portableCollisions = Array.from(portablePaths.values())
|
||||
.filter((items) => items.length > 1)
|
||||
.map((items) => {
|
||||
const candidates = items.map((item) => item.path).sort();
|
||||
return {
|
||||
collision_id: stableId("portable-collision", candidates.join("\n")),
|
||||
candidate_set_id: stableId("candidate-set", candidates.join("\n")),
|
||||
candidates
|
||||
};
|
||||
})
|
||||
.sort((left, right) => left.candidates.join("\n").localeCompare(right.candidates.join("\n"), "en"));
|
||||
|
||||
return { exactPaths, portablePaths, portableBasenames, portableCollisions };
|
||||
}
|
||||
|
||||
function normalizeExplicitTarget(target) {
|
||||
const normalized = normalizeRelativePosixPath(target);
|
||||
const extension = path.posix.extname(normalized);
|
||||
if (!extension) {
|
||||
return `${normalized}.md`;
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function warningSeverity(code) {
|
||||
if (code === "pending_wikilink" || code === "noncanonical_wikilink") {
|
||||
return "warning";
|
||||
}
|
||||
return "error";
|
||||
}
|
||||
|
||||
function resolveWikilink(occurrence, sourcePath, index) {
|
||||
if (occurrence.link_kind === "same_page_anchor") {
|
||||
return {
|
||||
status: "resolved",
|
||||
target_path: sourcePath,
|
||||
creates_edge: false,
|
||||
warning_code: null,
|
||||
candidate_paths: [],
|
||||
target_key: sourcePath
|
||||
};
|
||||
}
|
||||
|
||||
const rawTarget = occurrence.page_target;
|
||||
const explicitPath = rawTarget.includes("/") || occurrence.link_kind === "attachment_wikilink" || rawTarget.endsWith(".md");
|
||||
const targetKey = rawTarget ? normalizeRelativePosixPath(rawTarget) : sourcePath;
|
||||
|
||||
let candidates = [];
|
||||
let warningCode = null;
|
||||
|
||||
if (explicitPath) {
|
||||
const normalizedTarget = normalizeExplicitTarget(rawTarget);
|
||||
const exactMatch = index.exactPaths.get(normalizedTarget);
|
||||
if (exactMatch) {
|
||||
candidates = [exactMatch];
|
||||
} else {
|
||||
candidates = index.portablePaths.get(portablePathKey(normalizedTarget)) || [];
|
||||
if (candidates.length === 1) {
|
||||
warningCode = "noncanonical_wikilink";
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const basename = rawTarget.replace(/\.md$/i, "");
|
||||
candidates = index.portableBasenames.get(portablePathKey(basename)) || [];
|
||||
}
|
||||
|
||||
if (candidates.length === 0) {
|
||||
return {
|
||||
status: "missing",
|
||||
target_path: null,
|
||||
creates_edge: false,
|
||||
warning_code: occurrence.pending ? "pending_wikilink" : (occurrence.link_kind === "attachment_wikilink" ? null : "broken_wikilink"),
|
||||
candidate_paths: [],
|
||||
target_key: targetKey
|
||||
};
|
||||
}
|
||||
|
||||
if (candidates.length > 1) {
|
||||
return {
|
||||
status: "ambiguous",
|
||||
target_path: null,
|
||||
creates_edge: false,
|
||||
warning_code: "ambiguous_wikilink",
|
||||
candidate_paths: candidates.map((item) => item.path).sort(),
|
||||
target_key: targetKey
|
||||
};
|
||||
}
|
||||
|
||||
const [target] = candidates;
|
||||
const createsEdge = Boolean(
|
||||
target.kind === "markdown"
|
||||
&& target.graphType
|
||||
&& target.path !== sourcePath
|
||||
&& occurrence.link_kind !== "attachment_wikilink"
|
||||
);
|
||||
|
||||
return {
|
||||
status: "resolved",
|
||||
target_path: target.path,
|
||||
creates_edge: createsEdge,
|
||||
warning_code: warningCode,
|
||||
candidate_paths: [],
|
||||
target_key: targetKey
|
||||
};
|
||||
}
|
||||
|
||||
function warningMessage(code, targetKey) {
|
||||
switch (code) {
|
||||
case "ambiguous_wikilink":
|
||||
return `Ambiguous wikilink: ${targetKey}`;
|
||||
case "broken_wikilink":
|
||||
return `Broken wikilink: ${targetKey}`;
|
||||
case "pending_wikilink":
|
||||
return `Pending wikilink: ${targetKey}`;
|
||||
case "noncanonical_wikilink":
|
||||
return `Noncanonical wikilink: ${targetKey}`;
|
||||
case "portable_path_collision":
|
||||
return "Portable path collision";
|
||||
default:
|
||||
return code;
|
||||
}
|
||||
}
|
||||
|
||||
function addWarningGroup(groupMap, candidateSetMap, code, resolution, occurrenceRecord) {
|
||||
const candidatePaths = resolution.candidate_paths || [];
|
||||
const candidateSetId = candidatePaths.length > 0
|
||||
? stableId("candidate-set", candidatePaths.join("\n"))
|
||||
: null;
|
||||
|
||||
if (candidateSetId && !candidateSetMap.has(candidateSetId)) {
|
||||
candidateSetMap.set(candidateSetId, {
|
||||
candidate_set_id: candidateSetId,
|
||||
candidate_count: candidatePaths.length,
|
||||
candidates: candidatePaths
|
||||
});
|
||||
}
|
||||
|
||||
const warningKey = `${code}\0${resolution.target_key || ""}\0${candidateSetId || ""}`;
|
||||
const warningId = stableId("warning", warningKey);
|
||||
if (!groupMap.has(warningId)) {
|
||||
groupMap.set(warningId, {
|
||||
warning_id: warningId,
|
||||
code,
|
||||
severity: warningSeverity(code),
|
||||
message: warningMessage(code, resolution.target_key || ""),
|
||||
target_key: resolution.target_key || undefined,
|
||||
candidate_set_id: candidateSetId || undefined,
|
||||
occurrence_count: 0,
|
||||
occurrences: []
|
||||
});
|
||||
}
|
||||
|
||||
const group = groupMap.get(warningId);
|
||||
group.occurrence_count += 1;
|
||||
group.occurrences.push(occurrenceRecord);
|
||||
}
|
||||
|
||||
function scanPolicySources(inventory, policy) {
|
||||
if (policy === "graph") return inventory.graphSources;
|
||||
if (policy === "lint") return inventory.lintSources;
|
||||
if (policy === "rename") {
|
||||
return inventory.renameEditableSources.concat(inventory.renameReadOnlySources)
|
||||
.sort((left, right) => left.path.localeCompare(right.path, "en"));
|
||||
}
|
||||
throw new Error(`Unknown scan policy: ${policy}`);
|
||||
}
|
||||
|
||||
function parseImagePaths(frontmatter) {
|
||||
if (!frontmatter) return [];
|
||||
const lines = frontmatter.split(/\r?\n/);
|
||||
for (let index = 0; index < lines.length; index += 1) {
|
||||
const match = lines[index].match(/^image_paths:\s*(.*)$/);
|
||||
if (!match) continue;
|
||||
const inline = match[1].trim();
|
||||
if (inline) {
|
||||
if (inline === "[]") return [];
|
||||
if (!inline.startsWith("[") || !inline.endsWith("]")) return [];
|
||||
return inline.slice(1, -1).split(",")
|
||||
.map((value) => value.trim().replace(/^['"]|['"]$/g, ""))
|
||||
.filter(Boolean);
|
||||
}
|
||||
const values = [];
|
||||
for (let cursor = index + 1; cursor < lines.length; cursor += 1) {
|
||||
if (!lines[cursor].trim()) continue;
|
||||
const item = lines[cursor].match(/^\s*-\s*(.+?)\s*$/);
|
||||
if (!item) break;
|
||||
values.push(item[1].trim().replace(/^['"]|['"]$/g, ""));
|
||||
}
|
||||
return values.filter(Boolean);
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
function scanKnowledgeBaseLinks(kbRoot, policy) {
|
||||
const inventory = discoverKnowledgeBaseFiles(kbRoot);
|
||||
const index = buildWikiTargetIndex(inventory.targets);
|
||||
const sources = scanPolicySources(inventory, policy);
|
||||
const candidateSetMap = new Map();
|
||||
const groupMap = new Map();
|
||||
const occurrences = [];
|
||||
const edges = [];
|
||||
const sourceDocuments = [];
|
||||
const stalePendingWrappers = [];
|
||||
const edgeByEndpoints = new Map();
|
||||
const metrics = {
|
||||
inventory_walks: 1,
|
||||
target_index_builds: 1,
|
||||
source_files_parsed: 0,
|
||||
files_read: 0,
|
||||
files_parsed: 0,
|
||||
graph_source_bytes: 0,
|
||||
utf8_bytes_scanned: 0,
|
||||
position_bytes_advanced: 0
|
||||
};
|
||||
|
||||
for (const collision of index.portableCollisions) {
|
||||
candidateSetMap.set(collision.candidate_set_id, {
|
||||
candidate_set_id: collision.candidate_set_id,
|
||||
candidate_count: collision.candidates.length,
|
||||
candidates: collision.candidates
|
||||
});
|
||||
groupMap.set(collision.collision_id, {
|
||||
warning_id: collision.collision_id,
|
||||
code: "portable_path_collision",
|
||||
severity: "error",
|
||||
message: "Portable path collision",
|
||||
id: collision.collision_id,
|
||||
candidate_set_id: collision.candidate_set_id,
|
||||
occurrence_count: 0,
|
||||
occurrences: []
|
||||
});
|
||||
}
|
||||
|
||||
for (const source of sources) {
|
||||
const buffer = fs.readFileSync(source.absolutePath);
|
||||
const rawContent = buffer.toString("utf8");
|
||||
const frontmatter = extractFrontmatter(rawContent);
|
||||
const parsedSources = parseSourcesFrontmatter(frontmatter.frontmatter);
|
||||
const heading = frontmatter.body.match(/^#\s+(.+?)\s*$/m);
|
||||
sourceDocuments.push({
|
||||
source_path: source.path,
|
||||
graph_type: source.graphType,
|
||||
label: heading ? heading[1].trim() : path.posix.basename(source.path, ".md"),
|
||||
_content: rawContent,
|
||||
_signals: {
|
||||
sources: parsedSources.sources,
|
||||
sourceSignalAvailable: parsedSources.signalAvailable,
|
||||
sourceFieldPresent: parsedSources.hasField,
|
||||
sourceFieldParsed: parsedSources.parsed,
|
||||
imagePaths: parseImagePaths(frontmatter.frontmatter)
|
||||
}
|
||||
});
|
||||
metrics.files_read += 1;
|
||||
metrics.files_parsed += 1;
|
||||
metrics.source_files_parsed += 1;
|
||||
if (source.graphType) metrics.graph_source_bytes += buffer.length;
|
||||
|
||||
const parsed = parseWikilinks(buffer, source.path);
|
||||
metrics.utf8_bytes_scanned += parsed.metrics.utf8_bytes_scanned;
|
||||
metrics.position_bytes_advanced += parsed.metrics.position_bytes_advanced;
|
||||
for (const occurrence of parsed.occurrences) {
|
||||
const resolution = resolveWikilink(occurrence, source.path, index);
|
||||
const resolutionCandidateSetId = resolution.candidate_paths.length > 0
|
||||
? stableId("candidate-set", resolution.candidate_paths.join("\n"))
|
||||
: null;
|
||||
const occurrenceRecord = {
|
||||
occurrence_id: stableId(
|
||||
"occurrence",
|
||||
`${occurrence.source_path}\0${occurrence.file_sha256}\0${occurrence.start_byte}\0${occurrence.end_byte}\0${occurrence.raw_link}`
|
||||
),
|
||||
source_path: occurrence.source_path,
|
||||
line: occurrence.line,
|
||||
column: occurrence.column,
|
||||
start_byte: occurrence.start_byte,
|
||||
end_byte: occurrence.end_byte,
|
||||
raw_link: occurrence.raw_link,
|
||||
file_sha256: occurrence.file_sha256,
|
||||
link_kind: occurrence.link_kind,
|
||||
read_only: source.editable === false
|
||||
};
|
||||
|
||||
occurrences.push({
|
||||
...occurrence,
|
||||
read_only: source.editable === false,
|
||||
resolution: {
|
||||
...resolution,
|
||||
candidate_paths: undefined,
|
||||
candidate_set_id: resolutionCandidateSetId || undefined
|
||||
}
|
||||
});
|
||||
|
||||
if (occurrence.pending && resolution.status === "resolved") {
|
||||
stalePendingWrappers.push({
|
||||
source_path: source.path,
|
||||
raw_link: occurrence.raw_link,
|
||||
replacement: renderWikilinkReplacement(occurrence, occurrence.page_target)
|
||||
});
|
||||
} else if (resolution.warning_code) {
|
||||
addWarningGroup(groupMap, candidateSetMap, resolution.warning_code, resolution, occurrenceRecord);
|
||||
}
|
||||
|
||||
if (resolution.creates_edge && resolution.target_path) {
|
||||
const edgeKey = `${source.path}\0${resolution.target_path}`;
|
||||
if (!edgeByEndpoints.has(edgeKey)) {
|
||||
const edge = {
|
||||
from: source.path,
|
||||
to: resolution.target_path,
|
||||
relation_type: occurrence.relation_type || "依赖",
|
||||
confidence: occurrence.confidence || "EXTRACTED",
|
||||
_relation_explicit: Boolean(occurrence.relation_type),
|
||||
_confidence_explicit: Boolean(occurrence.confidence)
|
||||
};
|
||||
edgeByEndpoints.set(edgeKey, edge);
|
||||
edges.push(edge);
|
||||
} else {
|
||||
const edge = edgeByEndpoints.get(edgeKey);
|
||||
if (!edge._confidence_explicit && occurrence.confidence) {
|
||||
edge.confidence = occurrence.confidence;
|
||||
edge._confidence_explicit = true;
|
||||
}
|
||||
if (!edge._relation_explicit && occurrence.relation_type) {
|
||||
edge.relation_type = occurrence.relation_type;
|
||||
edge._relation_explicit = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const candidate_sets = Array.from(candidateSetMap.values())
|
||||
.sort((left, right) => left.candidate_set_id.localeCompare(right.candidate_set_id, "en"));
|
||||
const groups = Array.from(groupMap.values())
|
||||
.map((group) => ({
|
||||
...group,
|
||||
occurrences: group.occurrences.slice().sort((left, right) => {
|
||||
if (left.source_path !== right.source_path) {
|
||||
return left.source_path.localeCompare(right.source_path, "en");
|
||||
}
|
||||
return left.start_byte - right.start_byte;
|
||||
})
|
||||
}))
|
||||
.sort((left, right) => left.warning_id.localeCompare(right.warning_id, "en"));
|
||||
|
||||
edges.sort((left, right) => {
|
||||
const leftKey = `${left.from}\0${left.to}\0${left.relation_type}`;
|
||||
const rightKey = `${right.from}\0${right.to}\0${right.relation_type}`;
|
||||
return leftKey.localeCompare(rightKey, "en");
|
||||
});
|
||||
for (const edge of edges) {
|
||||
delete edge._confidence_explicit;
|
||||
delete edge._relation_explicit;
|
||||
}
|
||||
|
||||
stalePendingWrappers.sort((left, right) => left.source_path.localeCompare(right.source_path, "en"));
|
||||
|
||||
return {
|
||||
inventory,
|
||||
edges,
|
||||
candidate_sets,
|
||||
groups,
|
||||
occurrences: policy === "graph" ? [] : occurrences,
|
||||
source_documents: sourceDocuments,
|
||||
stale_pending_wrappers: stalePendingWrappers,
|
||||
metrics
|
||||
};
|
||||
}
|
||||
|
||||
function validatePortableMarkdownFilename(sourcePath, newName, inventoryOrTargets) {
|
||||
const targets = Array.isArray(inventoryOrTargets)
|
||||
? inventoryOrTargets
|
||||
: inventoryOrTargets.targets;
|
||||
const syntax = validateGraphRenameFilenameSyntax(String(newName ?? ""));
|
||||
if (!syntax.ok) return syntax;
|
||||
const normalizedName = syntax.normalized_name;
|
||||
|
||||
const sourceDir = path.posix.dirname(sourcePath);
|
||||
const targetPath = sourceDir === "." ? normalizedName : `${sourceDir}/${normalizedName}`;
|
||||
const targetPortableKey = portablePathKey(targetPath);
|
||||
const collisions = targets
|
||||
.filter((item) => item.path !== sourcePath && portablePathKey(item.path) === targetPortableKey)
|
||||
.map((item) => item.path)
|
||||
.sort();
|
||||
|
||||
if (collisions.length > 0) {
|
||||
return { ok: false, reason: "portable_path_collision", collision_paths: collisions };
|
||||
}
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
normalized_name: normalizedName,
|
||||
target_path: targetPath,
|
||||
requires_transit: portablePathKey(sourcePath) === targetPortableKey && sourcePath !== targetPath
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
buildWikiTargetIndex,
|
||||
portablePathKey,
|
||||
resolveWikilink,
|
||||
scanKnowledgeBaseLinks,
|
||||
validatePortableMarkdownFilename
|
||||
};
|
||||
305
llm-wiki/scripts/lib/wikilink-parser.js
Normal file
305
llm-wiki/scripts/lib/wikilink-parser.js
Normal file
@@ -0,0 +1,305 @@
|
||||
#!/usr/bin/env node
|
||||
"use strict";
|
||||
|
||||
const crypto = require("node:crypto");
|
||||
const path = require("node:path");
|
||||
|
||||
function sha256(buffer) {
|
||||
return crypto.createHash("sha256").update(buffer).digest("hex");
|
||||
}
|
||||
|
||||
function countCodePoints(value) {
|
||||
return Array.from(value).length;
|
||||
}
|
||||
|
||||
function extractLineAnnotations(lineText) {
|
||||
const confidenceMatch = lineText.match(/<!--\s*confidence:\s*([A-Z]+)\s*-->/);
|
||||
const relationTypeMatch = lineText.match(/<!--\s*relation(?:_type)?:\s*([^>]+?)\s*-->/);
|
||||
|
||||
return {
|
||||
confidence: confidenceMatch ? confidenceMatch[1] : null,
|
||||
relation_type: relationTypeMatch ? relationTypeMatch[1].trim() : null
|
||||
};
|
||||
}
|
||||
|
||||
function parseFenceCandidate(lineText) {
|
||||
const match = lineText.match(/^( {0,3})(`{3,}|~{3,})(.*)$/);
|
||||
if (!match) return null;
|
||||
return {
|
||||
marker: match[2][0],
|
||||
length: match[2].length,
|
||||
rest: match[3]
|
||||
};
|
||||
}
|
||||
|
||||
function collectInlineBacktickRuns(text) {
|
||||
const positionsByLength = new Map();
|
||||
let fence = null;
|
||||
let lineStartIndex = 0;
|
||||
|
||||
while (lineStartIndex <= text.length) {
|
||||
const nextNewlineIndex = text.indexOf("\n", lineStartIndex);
|
||||
const lineEndIndex = nextNewlineIndex === -1 ? text.length : nextNewlineIndex;
|
||||
const lineText = text.slice(lineStartIndex, lineEndIndex);
|
||||
const fenceCandidate = parseFenceCandidate(lineText);
|
||||
let handledAsFence = false;
|
||||
|
||||
if (fence) {
|
||||
handledAsFence = true;
|
||||
if (
|
||||
fenceCandidate
|
||||
&& fence.marker === fenceCandidate.marker
|
||||
&& fenceCandidate.length >= fence.length
|
||||
&& /^[ \t]*$/.test(fenceCandidate.rest)
|
||||
) {
|
||||
fence = null;
|
||||
}
|
||||
} else if (
|
||||
fenceCandidate
|
||||
&& !(fenceCandidate.marker === "`" && fenceCandidate.rest.includes("`"))
|
||||
) {
|
||||
fence = { marker: fenceCandidate.marker, length: fenceCandidate.length };
|
||||
handledAsFence = true;
|
||||
}
|
||||
|
||||
if (!handledAsFence) {
|
||||
for (let index = 0; index < lineText.length;) {
|
||||
if (lineText[index] !== "`") {
|
||||
index += String.fromCodePoint(lineText.codePointAt(index)).length;
|
||||
continue;
|
||||
}
|
||||
|
||||
let runLength = 1;
|
||||
while (lineText[index + runLength] === "`") {
|
||||
runLength += 1;
|
||||
}
|
||||
const positions = positionsByLength.get(runLength) || [];
|
||||
positions.push(lineStartIndex + index);
|
||||
positionsByLength.set(runLength, positions);
|
||||
index += runLength;
|
||||
}
|
||||
}
|
||||
|
||||
if (nextNewlineIndex === -1) break;
|
||||
lineStartIndex = nextNewlineIndex + 1;
|
||||
}
|
||||
|
||||
return { positionsByLength, cursorsByLength: new Map() };
|
||||
}
|
||||
|
||||
function consumeInlineBacktickRun(runIndex, runLength, absoluteIndex) {
|
||||
const positions = runIndex.positionsByLength.get(runLength) || [];
|
||||
let cursor = runIndex.cursorsByLength.get(runLength) || 0;
|
||||
while (cursor < positions.length && positions[cursor] <= absoluteIndex) {
|
||||
cursor += 1;
|
||||
}
|
||||
runIndex.cursorsByLength.set(runLength, cursor);
|
||||
return cursor < positions.length;
|
||||
}
|
||||
|
||||
function parseOccurrence(rawLink, sourcePath, fileSha256, line, column, startByte, annotations) {
|
||||
let innerStart = 0;
|
||||
let innerEnd = rawLink.length;
|
||||
let embedded = false;
|
||||
let pending = false;
|
||||
|
||||
if (rawLink.startsWith("[待创建: [[")) {
|
||||
innerStart = "[待创建: [[".length;
|
||||
innerEnd = rawLink.length - "]]]".length;
|
||||
pending = true;
|
||||
} else if (rawLink.startsWith("[To create: [[")) {
|
||||
innerStart = "[To create: [[".length;
|
||||
innerEnd = rawLink.length - "]]]".length;
|
||||
pending = true;
|
||||
} else if (rawLink.startsWith("![[")) {
|
||||
innerStart = "![[".length;
|
||||
innerEnd = rawLink.length - "]]".length;
|
||||
embedded = true;
|
||||
} else {
|
||||
innerStart = "[[".length;
|
||||
innerEnd = rawLink.length - "]]".length;
|
||||
}
|
||||
|
||||
const inner = rawLink.slice(innerStart, innerEnd);
|
||||
const pipeIndex = inner.indexOf("|");
|
||||
const targetAndAnchorRaw = pipeIndex >= 0 ? inner.slice(0, pipeIndex) : inner;
|
||||
const displayRaw = pipeIndex >= 0 ? inner.slice(pipeIndex + 1) : null;
|
||||
const anchorIndex = targetAndAnchorRaw.indexOf("#");
|
||||
const targetRaw = anchorIndex >= 0 ? targetAndAnchorRaw.slice(0, anchorIndex) : targetAndAnchorRaw;
|
||||
const anchor = anchorIndex >= 0 ? targetAndAnchorRaw.slice(anchorIndex + 1).trim() : null;
|
||||
const display = displayRaw === null ? null : displayRaw.trim();
|
||||
|
||||
const leadingWhitespace = (targetRaw.match(/^\s*/) || [""])[0].length;
|
||||
const trailingWhitespace = (targetRaw.match(/\s*$/) || [""])[0].length;
|
||||
const targetStartInRaw = innerStart + leadingWhitespace;
|
||||
const targetEndInRaw = innerStart + targetRaw.length - trailingWhitespace;
|
||||
const pageTarget = targetRaw.trim();
|
||||
const extension = path.posix.extname(pageTarget);
|
||||
|
||||
let linkKind = "page_wikilink";
|
||||
if (pageTarget === "" && anchor) {
|
||||
linkKind = "same_page_anchor";
|
||||
} else if (extension && extension.toLowerCase() !== ".md") {
|
||||
linkKind = "attachment_wikilink";
|
||||
}
|
||||
|
||||
return {
|
||||
occurrence_id: `${sourcePath}\0${fileSha256}\0${startByte}\0${startByte + Buffer.byteLength(rawLink, "utf8")}\0${rawLink}`,
|
||||
source_path: sourcePath,
|
||||
file_sha256: fileSha256,
|
||||
raw_link: rawLink,
|
||||
line,
|
||||
column,
|
||||
start_byte: startByte,
|
||||
end_byte: startByte + Buffer.byteLength(rawLink, "utf8"),
|
||||
link_kind: linkKind,
|
||||
embedded,
|
||||
pending,
|
||||
page_target: pageTarget,
|
||||
anchor,
|
||||
display,
|
||||
confidence: annotations.confidence,
|
||||
relation_type: annotations.relation_type,
|
||||
target_start_in_raw: targetStartInRaw,
|
||||
target_end_in_raw: targetEndInRaw
|
||||
};
|
||||
}
|
||||
|
||||
function parseWikilinks(buffer, sourcePath) {
|
||||
const text = buffer.toString("utf8");
|
||||
const fileSha256 = sha256(buffer);
|
||||
const occurrences = [];
|
||||
const inlineBacktickRuns = collectInlineBacktickRuns(text);
|
||||
|
||||
let fence = null;
|
||||
let inlineDelimiter = 0;
|
||||
let lineNumber = 1;
|
||||
let lineStartIndex = 0;
|
||||
let positionBytesAdvanced = 0;
|
||||
|
||||
while (lineStartIndex <= text.length) {
|
||||
const nextNewlineIndex = text.indexOf("\n", lineStartIndex);
|
||||
const lineEndIndex = nextNewlineIndex === -1 ? text.length : nextNewlineIndex;
|
||||
const lineText = text.slice(lineStartIndex, lineEndIndex);
|
||||
const annotations = extractLineAnnotations(lineText);
|
||||
const fenceCandidate = parseFenceCandidate(lineText);
|
||||
let handledAsFence = false;
|
||||
|
||||
if (fence) {
|
||||
handledAsFence = true;
|
||||
if (
|
||||
fenceCandidate
|
||||
&& fence.marker === fenceCandidate.marker
|
||||
&& fenceCandidate.length >= fence.length
|
||||
&& /^[ \t]*$/.test(fenceCandidate.rest)
|
||||
) {
|
||||
fence = null;
|
||||
}
|
||||
} else if (
|
||||
inlineDelimiter === 0
|
||||
&& fenceCandidate
|
||||
&& !(fenceCandidate.marker === "`" && fenceCandidate.rest.includes("`"))
|
||||
) {
|
||||
fence = { marker: fenceCandidate.marker, length: fenceCandidate.length };
|
||||
handledAsFence = true;
|
||||
}
|
||||
|
||||
if (handledAsFence) {
|
||||
positionBytesAdvanced += Buffer.byteLength(lineText, "utf8");
|
||||
} else {
|
||||
let column = 1;
|
||||
|
||||
for (let index = 0; index < lineText.length;) {
|
||||
if (lineText[index] === "`") {
|
||||
let runLength = 1;
|
||||
while (lineText[index + runLength] === "`") {
|
||||
runLength += 1;
|
||||
}
|
||||
const hasEqualLengthCloser = consumeInlineBacktickRun(
|
||||
inlineBacktickRuns,
|
||||
runLength,
|
||||
lineStartIndex + index
|
||||
);
|
||||
if (inlineDelimiter === 0) {
|
||||
if (hasEqualLengthCloser) {
|
||||
inlineDelimiter = runLength;
|
||||
}
|
||||
} else if (runLength === inlineDelimiter) {
|
||||
inlineDelimiter = 0;
|
||||
}
|
||||
index += runLength;
|
||||
column += runLength;
|
||||
positionBytesAdvanced += runLength;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (inlineDelimiter > 0) {
|
||||
const symbol = String.fromCodePoint(lineText.codePointAt(index));
|
||||
const symbolBytes = Buffer.byteLength(symbol, "utf8");
|
||||
index += symbol.length;
|
||||
column += 1;
|
||||
positionBytesAdvanced += symbolBytes;
|
||||
continue;
|
||||
}
|
||||
|
||||
let rawLink = null;
|
||||
let endIndex = null;
|
||||
|
||||
if (lineText.startsWith("[待创建: [[", index) || lineText.startsWith("[To create: [[", index)) {
|
||||
const wrapperPrefix = lineText.startsWith("[待创建: [[", index) ? "[待创建: [[" : "[To create: [[";
|
||||
const closeInner = lineText.indexOf("]]]", index + wrapperPrefix.length);
|
||||
if (closeInner >= 0) {
|
||||
rawLink = lineText.slice(index, closeInner + 3);
|
||||
endIndex = closeInner + 3;
|
||||
}
|
||||
} else if (lineText.startsWith("![[", index) || lineText.startsWith("[[", index)) {
|
||||
const closeInner = lineText.indexOf("]]", index + 2);
|
||||
if (closeInner >= 0) {
|
||||
rawLink = lineText.slice(index, closeInner + 2);
|
||||
endIndex = closeInner + 2;
|
||||
}
|
||||
}
|
||||
|
||||
if (rawLink && endIndex !== null) {
|
||||
const startByte = positionBytesAdvanced;
|
||||
const rawLinkBytes = Buffer.byteLength(rawLink, "utf8");
|
||||
occurrences.push(parseOccurrence(rawLink, sourcePath, fileSha256, lineNumber, column, startByte, annotations));
|
||||
index = endIndex;
|
||||
column += countCodePoints(rawLink);
|
||||
positionBytesAdvanced += rawLinkBytes;
|
||||
continue;
|
||||
}
|
||||
|
||||
const symbol = String.fromCodePoint(lineText.codePointAt(index));
|
||||
const symbolBytes = Buffer.byteLength(symbol, "utf8");
|
||||
index += symbol.length;
|
||||
column += 1;
|
||||
positionBytesAdvanced += symbolBytes;
|
||||
}
|
||||
}
|
||||
|
||||
if (nextNewlineIndex === -1) {
|
||||
break;
|
||||
}
|
||||
|
||||
lineStartIndex = nextNewlineIndex + 1;
|
||||
positionBytesAdvanced += 1;
|
||||
lineNumber += 1;
|
||||
}
|
||||
|
||||
return {
|
||||
source_path: sourcePath,
|
||||
file_sha256: fileSha256,
|
||||
occurrences,
|
||||
metrics: {
|
||||
utf8_bytes_scanned: buffer.length,
|
||||
position_bytes_advanced: positionBytesAdvanced
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function renderWikilinkReplacement(occurrence, replacementTarget) {
|
||||
return `${occurrence.raw_link.slice(0, occurrence.target_start_in_raw)}${replacementTarget}${occurrence.raw_link.slice(occurrence.target_end_in_raw)}`;
|
||||
}
|
||||
|
||||
module.exports = { parseWikilinks, renderWikilinkReplacement };
|
||||
114
llm-wiki/scripts/lint-fix.sh
Executable file
114
llm-wiki/scripts/lint-fix.sh
Executable file
@@ -0,0 +1,114 @@
|
||||
#!/bin/bash
|
||||
# lint-fix.sh — 自动修复 lint 发现的低风险问题
|
||||
# 用法：bash scripts/lint-fix.sh <wiki_root> [--dry-run]
|
||||
# 修复范围：仅处理确定性修复（补 index 条目），不做高风险操作（删页面、改内容）
|
||||
# 退出码：0 = 完成，1 = 参数错误
|
||||
|
||||
set -u
|
||||
shopt -s nullglob
|
||||
|
||||
WIKI_ROOT="${1:-.}"
|
||||
DRY_RUN=false
|
||||
[ "${2:-}" = "--dry-run" ] && DRY_RUN=true
|
||||
|
||||
WIKI_DIR="$WIKI_ROOT/wiki"
|
||||
INDEX_FILE="$WIKI_ROOT/index.md"
|
||||
|
||||
if [ ! -d "$WIKI_DIR" ]; then
|
||||
echo "ERROR: wiki directory not found: $WIKI_DIR" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -f "$INDEX_FILE" ]; then
|
||||
echo "ERROR: index.md not found: $INDEX_FILE" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
FIXED=0
|
||||
|
||||
index_has_entry() {
|
||||
local entry="$1"
|
||||
grep -ohE "\[\[[^]]+\]\]" "$INDEX_FILE" 2>/dev/null | \
|
||||
sed -e 's/\[\[//g' -e 's/\]\]//g' -e 's/|.*//' | \
|
||||
grep -Fxq "$entry"
|
||||
}
|
||||
|
||||
# Insert a [[link]] entry after the matching section header in index.md.
|
||||
# If no matching section is found, appends to end of file as fallback.
|
||||
insert_under_section() {
|
||||
local index_file="$1"
|
||||
local section_pattern="$2"
|
||||
local entry="$3"
|
||||
|
||||
# Find the line number of the section header
|
||||
local line_num
|
||||
line_num=$(grep -n -i -E "^#.*($section_pattern)" "$index_file" 2>/dev/null | head -1 | cut -d: -f1)
|
||||
|
||||
if [ -n "$line_num" ]; then
|
||||
# Scan from section header to find insert point:
|
||||
# last "- [[" line before next "##" header or EOF
|
||||
local total_lines last_list_line offset
|
||||
total_lines=$(wc -l < "$index_file" | tr -d ' ')
|
||||
last_list_line="$line_num"
|
||||
offset=$((line_num + 1))
|
||||
while [ "$offset" -le "$total_lines" ]; do
|
||||
local cur_line
|
||||
cur_line=$(sed -n "${offset}p" "$index_file")
|
||||
case "$cur_line" in
|
||||
"##"*) break ;;
|
||||
"- [["*) last_list_line="$offset" ;;
|
||||
esac
|
||||
offset=$((offset + 1))
|
||||
done
|
||||
# Insert after the last list item
|
||||
local tmp_file
|
||||
tmp_file=$(mktemp "${index_file}.tmp.XXXXXX") || return 1
|
||||
awk -v insert_after="$last_list_line" -v entry="$entry" '
|
||||
{ print }
|
||||
NR == insert_after { print "- [[" entry "]]" }
|
||||
' "$index_file" > "$tmp_file" && mv "$tmp_file" "$index_file"
|
||||
else
|
||||
# Fallback: append to end of file
|
||||
printf '\n- [[%s]]\n' "$entry" >> "$index_file"
|
||||
fi
|
||||
}
|
||||
|
||||
echo "=== lint-fix: low-risk auto-repair ==="
|
||||
echo ""
|
||||
|
||||
# Fix 1: Add unlisted pages to index.md
|
||||
# Only adds pages that exist in wiki/ but are not referenced in index.md
|
||||
# Skips derived pages (queries/, sessions/)
|
||||
echo "--- Checking for unlisted pages ---"
|
||||
for _subdir in entities topics sources comparisons synthesis; do
|
||||
for f in "$WIKI_DIR"/$_subdir/*.md; do
|
||||
[ -f "$f" ] || continue
|
||||
BASENAME=$(basename "$f" .md)
|
||||
# Skip derived pages
|
||||
case "$f" in
|
||||
*/queries/*|*/sessions/*) continue ;;
|
||||
esac
|
||||
if ! index_has_entry "$BASENAME"; then
|
||||
SECTION_PATTERN=""
|
||||
case "$_subdir" in
|
||||
entities) SECTION_PATTERN="实体页|Entities" ;;
|
||||
topics) SECTION_PATTERN="主题页|Topics" ;;
|
||||
sources) SECTION_PATTERN="素材摘要|Sources" ;;
|
||||
comparisons) SECTION_PATTERN="对比分析|Comparisons" ;;
|
||||
synthesis) SECTION_PATTERN="综合分析|Synthesis" ;;
|
||||
esac
|
||||
if [ "$DRY_RUN" = true ]; then
|
||||
echo " [dry-run] Would add [[$BASENAME]] under $_subdir section"
|
||||
else
|
||||
insert_under_section "$INDEX_FILE" "$SECTION_PATTERN" "$BASENAME"
|
||||
echo " Fixed: added [[$BASENAME]] under $_subdir section"
|
||||
fi
|
||||
FIXED=$((FIXED + 1))
|
||||
fi
|
||||
done
|
||||
done
|
||||
[ "$FIXED" -eq 0 ] && echo " (all pages already listed)"
|
||||
echo ""
|
||||
|
||||
echo "=== lint-fix complete: $FIXED fix(es) applied ==="
|
||||
[ "$DRY_RUN" = true ] && echo "(dry-run mode — no files were modified)"
|
||||
exit 0
|
||||
250
llm-wiki/scripts/lint-runner.sh
Executable file
250
llm-wiki/scripts/lint-runner.sh
Executable file
@@ -0,0 +1,250 @@
|
||||
#!/bin/bash
|
||||
# lint-runner.sh — render the single shared path-aware wikilink report.
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/lint-runner.sh <kb-root>
|
||||
# bash scripts/lint-runner.sh <kb-root> --strict
|
||||
# bash scripts/lint-runner.sh <kb-root> --json
|
||||
# bash scripts/lint-runner.sh <kb-root> --strict --json
|
||||
#
|
||||
# Exit codes: 0 complete report; 1 tool/input failure; 2 strict report contains an error.
|
||||
|
||||
set -u
|
||||
|
||||
SCRIPT_DIR="${BASH_SOURCE[0]%/*}"
|
||||
[ "$SCRIPT_DIR" = "${BASH_SOURCE[0]}" ] && SCRIPT_DIR="."
|
||||
SCRIPT_DIR="$(cd "$SCRIPT_DIR" && pwd)"
|
||||
CLI="$SCRIPT_DIR/wiki-link-cli.js"
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage:
|
||||
bash scripts/lint-runner.sh <kb-root> [--strict] [--json]
|
||||
USAGE
|
||||
}
|
||||
|
||||
[ "$#" -ge 1 ] || {
|
||||
usage >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
WIKI_ROOT="$1"
|
||||
shift
|
||||
STRICT=0
|
||||
FORMAT=text
|
||||
SEEN_STRICT=0
|
||||
SEEN_JSON=0
|
||||
for FLAG in "$@"; do
|
||||
case "$FLAG" in
|
||||
--strict)
|
||||
[ "$SEEN_STRICT" -eq 0 ] || { echo "ERROR: duplicate argument: --strict" >&2; exit 1; }
|
||||
STRICT=1
|
||||
SEEN_STRICT=1
|
||||
;;
|
||||
--json)
|
||||
[ "$SEEN_JSON" -eq 0 ] || { echo "ERROR: duplicate argument: --json" >&2; exit 1; }
|
||||
FORMAT=json
|
||||
SEEN_JSON=1
|
||||
;;
|
||||
*)
|
||||
echo "ERROR: unknown argument: $FLAG" >&2
|
||||
usage >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
[ -d "$WIKI_ROOT/wiki" ] || {
|
||||
echo "ERROR: wiki directory does not exist: $WIKI_ROOT/wiki" >&2
|
||||
exit 1
|
||||
}
|
||||
[ -f "$WIKI_ROOT/index.md" ] || {
|
||||
echo "ERROR: index.md does not exist: $WIKI_ROOT/index.md" >&2
|
||||
exit 1
|
||||
}
|
||||
[ -f "$CLI" ] || {
|
||||
echo "ERROR: shared wikilink checker is missing: $CLI" >&2
|
||||
exit 1
|
||||
}
|
||||
command -v node >/dev/null 2>&1 || { echo "ERROR: node is required" >&2; exit 1; }
|
||||
|
||||
REPORT_FILE="$(mktemp -t llm-wiki-lint.XXXXXX)"
|
||||
trap 'rm -f "$REPORT_FILE"' EXIT
|
||||
if ! node "$CLI" check "$WIKI_ROOT" --json > "$REPORT_FILE"; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
node - "$REPORT_FILE" "$WIKI_ROOT" "$FORMAT" <<'NODE'
|
||||
"use strict";
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
|
||||
const [reportPath, kbRoot, format] = process.argv.slice(2);
|
||||
const report = JSON.parse(fs.readFileSync(reportPath, "utf8"));
|
||||
const targets = report.inventory.targets;
|
||||
const targetPaths = new Set(targets.map((item) => item.path));
|
||||
const metadata = new Map(report.source_metadata.map((item) => [item.source_path, item]));
|
||||
const resolvedOccurrences = report.occurrences.filter((item) => (
|
||||
item.resolution && item.resolution.status === "resolved" && item.resolution.target_path
|
||||
));
|
||||
const incoming = new Map();
|
||||
for (const occurrence of resolvedOccurrences) {
|
||||
const target = occurrence.resolution.target_path;
|
||||
if (target === occurrence.source_path) continue;
|
||||
if (!incoming.has(target)) incoming.set(target, new Set());
|
||||
incoming.get(target).add(occurrence.source_path);
|
||||
}
|
||||
|
||||
const orphanPages = report.inventory.lintSources
|
||||
.filter((item) => ["entity", "topic", "source"].includes(item.graphType))
|
||||
.map((item) => item.path)
|
||||
.filter((item) => !incoming.has(item))
|
||||
.sort();
|
||||
|
||||
const indexOccurrences = report.occurrences.filter((item) => item.source_path === "index.md");
|
||||
const indexResolvedTargetPaths = Array.from(new Set(indexOccurrences
|
||||
.filter((item) => item.resolution && item.resolution.status === "resolved")
|
||||
.map((item) => item.resolution.target_path)))
|
||||
.sort();
|
||||
const indexMissingTargets = Array.from(new Set(indexOccurrences
|
||||
.filter((item) => item.resolution && item.resolution.status === "missing" && item.resolution.warning_code)
|
||||
.map((item) => item.resolution.target_key)))
|
||||
.sort();
|
||||
const indexed = new Set(indexResolvedTargetPaths);
|
||||
const indexUnlistedPaths = report.inventory.lintSources
|
||||
.filter((item) => ["entity", "topic", "source", "comparison", "synthesis"].includes(item.graphType))
|
||||
.map((item) => item.path)
|
||||
.filter((item) => !item.includes("/sessions/") && !indexed.has(item))
|
||||
.sort();
|
||||
|
||||
const imageIssues = [];
|
||||
for (const [sourcePath, item] of metadata.entries()) {
|
||||
if (item.graph_type !== "source") continue;
|
||||
for (const imagePath of item._signals.imagePaths || []) {
|
||||
if (!targetPaths.has(imagePath)) imageIssues.push({ source_path: sourcePath, image_path: imagePath });
|
||||
}
|
||||
}
|
||||
imageIssues.sort((left, right) => `${left.source_path}\0${left.image_path}`.localeCompare(`${right.source_path}\0${right.image_path}`, "en"));
|
||||
|
||||
const sourceSignal = {
|
||||
applicable_total: 0,
|
||||
ok: 0,
|
||||
missing_sources: 0,
|
||||
empty_sources: 0,
|
||||
invalid_sources: 0,
|
||||
not_applicable: 0,
|
||||
pages: []
|
||||
};
|
||||
for (const item of report.source_metadata) {
|
||||
if (!item.graph_type) continue;
|
||||
const applicable = ["entity", "topic", "source", "comparison"].includes(item.graph_type);
|
||||
let reason;
|
||||
if (!applicable) reason = "not_applicable";
|
||||
else if (!item._signals.sourceFieldPresent) reason = "missing_sources";
|
||||
else if (!item._signals.sourceFieldParsed) reason = "invalid_sources";
|
||||
else if (!item._signals.sources.length) reason = "empty_sources";
|
||||
else reason = "ok";
|
||||
sourceSignal[reason] += 1;
|
||||
if (reason !== "not_applicable") sourceSignal.applicable_total += 1;
|
||||
sourceSignal.pages.push({ path: item.source_path, reason, source_count: item._signals.sources.length });
|
||||
}
|
||||
sourceSignal.pages.sort((left, right) => left.path.localeCompare(right.path, "en"));
|
||||
|
||||
const derived = {
|
||||
orphan_count: orphanPages.length,
|
||||
orphan_paths: orphanPages,
|
||||
broken_count: report.groups.filter((group) => group.code === "broken_wikilink")
|
||||
.reduce((sum, group) => sum + group.occurrence_count, 0),
|
||||
index_missing_count: indexMissingTargets.length,
|
||||
index_missing_targets: indexMissingTargets,
|
||||
index_unlisted_count: indexUnlistedPaths.length,
|
||||
index_unlisted_paths: indexUnlistedPaths,
|
||||
index_resolved_target_paths: indexResolvedTargetPaths,
|
||||
image_issue_count: imageIssues.length,
|
||||
image_issues: imageIssues,
|
||||
source_signal: sourceSignal
|
||||
};
|
||||
const output = {
|
||||
derived,
|
||||
stale_pending_wrappers: report.stale_pending_wrappers,
|
||||
warning_report: report
|
||||
};
|
||||
|
||||
if (format === "json") {
|
||||
process.stdout.write(`${JSON.stringify(output, null, 2)}\n`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
function displayPage(relativePath) {
|
||||
return relativePath.replace(/^wiki\//, "").replace(/\.md$/, "");
|
||||
}
|
||||
function section(title, rows, emptyText) {
|
||||
console.log(`--- ${title} ---`);
|
||||
if (rows.length === 0) console.log(` ${emptyText}`);
|
||||
else for (const row of rows) console.log(` ${row}`);
|
||||
console.log("");
|
||||
}
|
||||
function warningRows(code) {
|
||||
const candidateSets = new Map(report.candidate_sets.map((item) => [item.candidate_set_id, item]));
|
||||
const rows = [];
|
||||
for (const group of report.groups.filter((item) => item.code === code)) {
|
||||
rows.push(`${group.target_key || group.id || group.warning_id}（${group.occurrence_count} 处）`);
|
||||
const candidateSet = candidateSets.get(group.candidate_set_id);
|
||||
if (candidateSet) {
|
||||
for (const candidate of candidateSet.candidates) rows.push(` 候选: ${candidate}`);
|
||||
}
|
||||
for (const occurrence of group.occurrences) {
|
||||
rows.push(` ${occurrence.source_path}:${occurrence.line}:${occurrence.column} ${occurrence.raw_link}`);
|
||||
}
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
console.log("=== llm-wiki lint 报告 ===");
|
||||
const now = new Date();
|
||||
const parts = new Intl.DateTimeFormat("sv-SE", {
|
||||
year: "numeric", month: "2-digit", day: "2-digit", hour: "2-digit", minute: "2-digit", hour12: false
|
||||
}).formatToParts(now).reduce((result, item) => ({ ...result, [item.type]: item.value }), {});
|
||||
console.log(`时间：${parts.year}-${parts.month}-${parts.day} ${parts.hour}:${parts.minute}`);
|
||||
console.log(`检查路径：${path.join(kbRoot, "wiki")}`);
|
||||
console.log("");
|
||||
|
||||
section("孤立页面（没有被其他页面引用）", orphanPages.map((item) => `孤立: ${displayPage(item)}`), "（无孤立页面）");
|
||||
section("断链（被链接但不存在的页面）", warningRows("broken_wikilink").map((item) => item.startsWith(" ") ? item : `断链: [[${item.replace(/（.*$/, "")}]]`), "（无断链）");
|
||||
section("index 一致性（index.md 有记录但文件缺失）", indexMissingTargets.map((item) => `index 有但文件缺失: ${item}`), "（index 与文件一致）");
|
||||
section("反向 index 一致性（文件存在但 index.md 未收录）", indexUnlistedPaths.map((item) => `未收录: ${displayPage(item)}`), "（所有页面均已收录）");
|
||||
section("图片资产一致性（image_paths 声明但文件缺失）", imageIssues.map((item) => `缺失: ${path.posix.basename(item.source_path, ".md")} → ${item.image_path}`), "（无缺失图片）");
|
||||
|
||||
console.log("--- source-signal 覆盖情况 ---");
|
||||
console.log(` 已参与：${sourceSignal.ok}`);
|
||||
console.log(` 缺少 sources 字段：${sourceSignal.missing_sources}`);
|
||||
console.log(` sources 为空：${sourceSignal.empty_sources}`);
|
||||
console.log(` sources 格式无效：${sourceSignal.invalid_sources}`);
|
||||
console.log(` 当前不参与：${sourceSignal.not_applicable}`);
|
||||
for (const [reason, label] of [
|
||||
["missing_sources", "缺少 sources 字段"],
|
||||
["empty_sources", "sources 为空"],
|
||||
["invalid_sources", "sources 格式无效"]
|
||||
]) {
|
||||
const pages = sourceSignal.pages.filter((item) => item.reason === reason);
|
||||
if (!pages.length) continue;
|
||||
console.log("");
|
||||
console.log(` ${label}：`);
|
||||
for (const page of pages) console.log(` - ${page.path}`);
|
||||
}
|
||||
console.log("");
|
||||
|
||||
section("歧义链接（同名候选，未建边）", warningRows("ambiguous_wikilink"), "（无歧义链接）");
|
||||
section("待创建链接（尚未建边）", warningRows("pending_wikilink"), "（无待创建链接）");
|
||||
section("非规范路径链接（已按实际路径建边）", warningRows("noncanonical_wikilink"), "（无非规范路径链接）");
|
||||
section("可移植路径冲突", warningRows("portable_path_collision"), "（无可移植路径冲突）");
|
||||
section("待创建包装清理（目标现已存在）", report.stale_pending_wrappers.map((item) => `${item.source_path}: ${item.raw_link} → ${item.replacement}`), "（无待清理包装）");
|
||||
console.log("=== 机械检查完成。矛盾检测、交叉引用、置信度抽查由 AI 继续执行 ===");
|
||||
NODE
|
||||
RENDER_STATUS=$?
|
||||
[ "$RENDER_STATUS" -eq 0 ] || exit 1
|
||||
|
||||
if [ "$STRICT" -eq 1 ] && jq -e 'any(.groups[]; .severity == "error")' "$REPORT_FILE" > /dev/null 2>&1; then
|
||||
exit 2
|
||||
fi
|
||||
exit 0
|
||||
77
llm-wiki/scripts/runtime-context.sh
Normal file
77
llm-wiki/scripts/runtime-context.sh
Normal file
@@ -0,0 +1,77 @@
|
||||
#!/bin/bash
|
||||
# 共享运行场景解析：供 install.sh 和 adapter-state.sh 复用
|
||||
|
||||
resolve_platform_skill_root() {
|
||||
case "$1" in
|
||||
claude)
|
||||
printf '%s\n' "$HOME/.claude/skills"
|
||||
;;
|
||||
codex)
|
||||
if [ -d "$HOME/.codex/skills" ] || [ ! -d "$HOME/.Codex/skills" ]; then
|
||||
printf '%s\n' "$HOME/.codex/skills"
|
||||
else
|
||||
printf '%s\n' "$HOME/.Codex/skills"
|
||||
fi
|
||||
;;
|
||||
openclaw)
|
||||
printf '%s\n' "$HOME/.openclaw/skills"
|
||||
;;
|
||||
hermes)
|
||||
printf '%s\n' "$HOME/.hermes/skills"
|
||||
;;
|
||||
*)
|
||||
echo "不支持的平台：$1" >&2
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
detect_layout_mode() {
|
||||
local bundle_root="$1"
|
||||
|
||||
if [ -e "$bundle_root/.git" ]; then
|
||||
printf '%s\n' "source_checkout"
|
||||
return 0
|
||||
fi
|
||||
|
||||
printf '%s\n' "installed_skill"
|
||||
}
|
||||
|
||||
resolve_layout_mode() {
|
||||
local bundle_root="$1"
|
||||
local override_mode="${2:-}"
|
||||
|
||||
if [ -n "$override_mode" ]; then
|
||||
printf '%s\n' "$override_mode"
|
||||
return 0
|
||||
fi
|
||||
|
||||
detect_layout_mode "$bundle_root"
|
||||
}
|
||||
|
||||
resolve_optional_adapter_root() {
|
||||
local bundle_root="$1"
|
||||
local skill_root_override="${2:-}"
|
||||
local override_mode="${3:-}"
|
||||
local layout_mode
|
||||
|
||||
if [ -n "$skill_root_override" ]; then
|
||||
printf '%s\n' "$skill_root_override"
|
||||
return 0
|
||||
fi
|
||||
|
||||
layout_mode="$(resolve_layout_mode "$bundle_root" "$override_mode")"
|
||||
|
||||
case "$layout_mode" in
|
||||
source_checkout)
|
||||
printf '%s\n' "$bundle_root/deps"
|
||||
;;
|
||||
installed_skill|upgrade_target)
|
||||
printf '%s\n' "$(dirname "$bundle_root")"
|
||||
;;
|
||||
*)
|
||||
echo "未知运行模式：$layout_mode" >&2
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
83
llm-wiki/scripts/shared-config.sh
Normal file
83
llm-wiki/scripts/shared-config.sh
Normal file
@@ -0,0 +1,83 @@
|
||||
#!/bin/bash
|
||||
# 共享配置：被 install.sh / hook-session-start.sh / cache.sh / delete-helper.sh 等引用
|
||||
# 微信公众号提取工具的 Git 仓库地址
|
||||
WECHAT_TOOL_URL="git+https://github.com/jackwener/wechat-article-to-markdown.git"
|
||||
|
||||
# Python 命令检测：Windows 默认安装为 python.exe，不存在 python3 命令
|
||||
# （Microsoft Store 的 python3 是安装提示 stub，运行会失败）
|
||||
_python_version_check='import sys; sys.exit(0 if sys.version_info >= (3, 8) else 1)'
|
||||
|
||||
_python_cmd_is_valid() {
|
||||
local candidate="$1"
|
||||
|
||||
command -v "$candidate" >/dev/null 2>&1 && "$candidate" -c "$_python_version_check" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
_detect_python_cmd() {
|
||||
# 要求 Python 3.8+（见 README Windows 小节与下方错误消息）
|
||||
if _python_cmd_is_valid python3; then
|
||||
echo "python3"
|
||||
elif _python_cmd_is_valid python; then
|
||||
echo "python"
|
||||
else
|
||||
echo ""
|
||||
fi
|
||||
}
|
||||
|
||||
require_python_cmd() {
|
||||
local detected_cmd
|
||||
|
||||
if [ "${PYTHON_CMD_READY:-0}" = "1" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ -n "${PYTHON_CMD:-}" ] && _python_cmd_is_valid "$PYTHON_CMD"; then
|
||||
export PYTHON_CMD
|
||||
PYTHON_CMD_READY=1
|
||||
return 0
|
||||
fi
|
||||
|
||||
detected_cmd="$(_detect_python_cmd)"
|
||||
if [ -z "$detected_cmd" ]; then
|
||||
echo "[llm-wiki] 错误：找不到可用的 Python 3，请先安装 Python 3.8+ 并加入 PATH" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
PYTHON_CMD="$detected_cmd"
|
||||
export PYTHON_CMD
|
||||
PYTHON_CMD_READY=1
|
||||
}
|
||||
|
||||
# 统一 Python 子进程 stdout/stderr 编码为 UTF-8
|
||||
# Windows 中文环境下 Python 无 TTY 时 sys.stdout.encoding 默认 gbk (cp936)，
|
||||
# 会导致 Agent 通过 subprocess 读取的 JSON / 输出出现乱码 (issue #16)
|
||||
export PYTHONIOENCODING="${PYTHONIOENCODING:-utf-8}"
|
||||
|
||||
# 输出指定工具的跨平台安装提示，缩进 2 空格便于嵌套在 ERROR 消息下；
|
||||
# 输出走 stderr，与 ERROR 消息保持同一通道。
|
||||
print_install_hint() {
|
||||
local tool="$1"
|
||||
case "$tool" in
|
||||
jq)
|
||||
echo " macOS: brew install jq" >&2
|
||||
echo " Linux/WSL: sudo apt-get install jq (Debian/Ubuntu)" >&2
|
||||
echo " sudo dnf install jq (RHEL/Fedora)" >&2
|
||||
echo " Windows: winget install jqlang.jq (or choco install jq)" >&2
|
||||
;;
|
||||
node)
|
||||
echo " macOS: brew install node" >&2
|
||||
echo " Linux/WSL: sudo apt-get install nodejs npm" >&2
|
||||
echo " Windows: winget install OpenJS.NodeJS (or choco install nodejs)" >&2
|
||||
;;
|
||||
uv)
|
||||
echo " macOS/Linux: curl -LsSf https://astral.sh/uv/install.sh | sh (official)" >&2
|
||||
echo " brew install uv (alternative)" >&2
|
||||
echo " Windows: powershell -c \"irm https://astral.sh/uv/install.ps1 | iex\" (official)" >&2
|
||||
echo " winget install --id=astral-sh.uv -e (alternative)" >&2
|
||||
;;
|
||||
*)
|
||||
echo " unknown tool: $tool" >&2
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
10
llm-wiki/scripts/source-record-contract.tsv
Normal file
10
llm-wiki/scripts/source-record-contract.tsv
Normal file
@@ -0,0 +1,10 @@
|
||||
field_name requiredness filled_by value_rule
|
||||
source_id required router 必须匹配 source-registry.tsv 中的 source_id
|
||||
source_label required router 必须匹配 source-registry.tsv 中的 source_label
|
||||
source_category required router 必须匹配 source-registry.tsv 中的 source_category
|
||||
input_mode required caller_or_router 只能是 url / file / text / asset
|
||||
raw_dir required router 必须是 raw/ 下的相对目录
|
||||
original_ref required caller 保存原始 URL、文件路径或用户粘贴说明
|
||||
ingest_text required adapter_or_user 进入主线前必须是非空文本
|
||||
adapter_name required_may_be_empty router_or_adapter 核心主线和手动入口留空；外挂来源写实际 adapter 名称
|
||||
fallback_hint required router 必须给出用户可执行的手动回退提示
|
||||
|
349
llm-wiki/scripts/source-registry.sh
Executable file
349
llm-wiki/scripts/source-registry.sh
Executable file
@@ -0,0 +1,349 @@
|
||||
#!/bin/bash
|
||||
# 统一来源总表读取与验证脚本
|
||||
# 权威数据文件：source-registry.tsv（来源定义）、source-record-contract.tsv（字段契约）
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
CONTRACT_FILE="$SCRIPT_DIR/source-record-contract.tsv"
|
||||
REGISTRY_FILE="$SCRIPT_DIR/source-registry.tsv"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
用法：
|
||||
bash scripts/source-registry.sh fields
|
||||
bash scripts/source-registry.sh list
|
||||
bash scripts/source-registry.sh get <source_id>
|
||||
bash scripts/source-registry.sh match-url <url>
|
||||
bash scripts/source-registry.sh match-file <path>
|
||||
bash scripts/source-registry.sh list-by-category <core_builtin|optional_adapter|manual_only>
|
||||
bash scripts/source-registry.sh unique-dependencies <bundled|install_time|none>
|
||||
bash scripts/source-registry.sh validate
|
||||
EOF
|
||||
}
|
||||
|
||||
require_file() {
|
||||
local file="$1"
|
||||
|
||||
[ -f "$file" ] || {
|
||||
echo "缺少文件：$file" >&2
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
expect_header() {
|
||||
local file="$1"
|
||||
local expected="$2"
|
||||
local actual
|
||||
|
||||
actual="$(head -n 1 "$file")"
|
||||
[ "$actual" = "$expected" ] || {
|
||||
echo "表头不匹配：$file" >&2
|
||||
echo "期望：$expected" >&2
|
||||
echo "实际：$actual" >&2
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
validate_contract() {
|
||||
require_file "$CONTRACT_FILE"
|
||||
expect_header "$CONTRACT_FILE" $'field_name\trequiredness\tfilled_by\tvalue_rule'
|
||||
|
||||
awk -F '\t' '
|
||||
BEGIN {
|
||||
required["source_id"] = 1
|
||||
required["source_label"] = 1
|
||||
required["source_category"] = 1
|
||||
required["input_mode"] = 1
|
||||
required["raw_dir"] = 1
|
||||
required["original_ref"] = 1
|
||||
required["ingest_text"] = 1
|
||||
required["adapter_name"] = 1
|
||||
required["fallback_hint"] = 1
|
||||
}
|
||||
NR == 1 { next }
|
||||
{
|
||||
if ($1 == "" || $2 == "" || $3 == "" || $4 == "") {
|
||||
printf("source-record-contract.tsv 第 %d 行存在空字段\n", NR) > "/dev/stderr"
|
||||
failed = 1
|
||||
}
|
||||
|
||||
seen[$1] += 1
|
||||
}
|
||||
END {
|
||||
for (field in required) {
|
||||
if (seen[field] != 1) {
|
||||
printf("source-record-contract.tsv 缺少或重复字段：%s\n", field) > "/dev/stderr"
|
||||
failed = 1
|
||||
}
|
||||
}
|
||||
|
||||
exit failed ? 1 : 0
|
||||
}
|
||||
' "$CONTRACT_FILE"
|
||||
}
|
||||
|
||||
validate_registry() {
|
||||
require_file "$REGISTRY_FILE"
|
||||
expect_header "$REGISTRY_FILE" $'source_id\tsource_label\tsource_category\tinput_mode\tmatch_rule\traw_dir\tadapter_name\tdependency_name\tdependency_type\tfallback_hint'
|
||||
|
||||
awk -F '\t' '
|
||||
NR == 1 { next }
|
||||
{
|
||||
if ($1 == "" || $2 == "" || $3 == "" || $4 == "" || $5 == "" || $6 == "" || $10 == "") {
|
||||
printf("source-registry.tsv 第 %d 行存在空字段\n", NR) > "/dev/stderr"
|
||||
failed = 1
|
||||
}
|
||||
|
||||
if ($3 != "core_builtin" && $3 != "optional_adapter" && $3 != "manual_only") {
|
||||
printf("source-registry.tsv 第 %d 行存在未知分类：%s\n", NR, $3) > "/dev/stderr"
|
||||
failed = 1
|
||||
}
|
||||
|
||||
if ($4 != "url" && $4 != "file" && $4 != "text" && $4 != "asset") {
|
||||
printf("source-registry.tsv 第 %d 行存在未知输入模式：%s\n", NR, $4) > "/dev/stderr"
|
||||
failed = 1
|
||||
}
|
||||
|
||||
if ($4 == "url" && $5 !~ /^url_host:/) {
|
||||
printf("source-registry.tsv 第 %d 行 URL 来源必须声明 url_host 规则：%s\n", NR, $5) > "/dev/stderr"
|
||||
failed = 1
|
||||
}
|
||||
|
||||
if ($4 == "file" && $5 !~ /^file_ext:/) {
|
||||
printf("source-registry.tsv 第 %d 行文件来源必须声明 file_ext 规则：%s\n", NR, $5) > "/dev/stderr"
|
||||
failed = 1
|
||||
}
|
||||
|
||||
if ($4 == "text" && $5 !~ /^text:/) {
|
||||
printf("source-registry.tsv 第 %d 行文本来源必须声明 text 规则：%s\n", NR, $5) > "/dev/stderr"
|
||||
failed = 1
|
||||
}
|
||||
|
||||
if ($4 == "asset" && $5 !~ /^asset:/) {
|
||||
printf("source-registry.tsv 第 %d 行附件来源必须声明 asset 规则：%s\n", NR, $5) > "/dev/stderr"
|
||||
failed = 1
|
||||
}
|
||||
|
||||
if ($6 !~ /^raw\//) {
|
||||
printf("source-registry.tsv 第 %d 行 raw_dir 必须位于 raw/ 下：%s\n", NR, $6) > "/dev/stderr"
|
||||
failed = 1
|
||||
}
|
||||
|
||||
if (seen[$1]++) {
|
||||
printf("source-registry.tsv source_id 重复：%s\n", $1) > "/dev/stderr"
|
||||
failed = 1
|
||||
}
|
||||
|
||||
category_seen[$3] = 1
|
||||
|
||||
if ($3 == "optional_adapter") {
|
||||
if ($7 == "-" || $8 == "-" || $9 == "none") {
|
||||
printf("source-registry.tsv 第 %d 行 optional_adapter 缺少依赖信息\n", NR) > "/dev/stderr"
|
||||
failed = 1
|
||||
}
|
||||
} else if ($7 != "-" || $8 != "-" || $9 != "none") {
|
||||
printf("source-registry.tsv 第 %d 行非外挂来源不应声明依赖\n", NR) > "/dev/stderr"
|
||||
failed = 1
|
||||
}
|
||||
}
|
||||
END {
|
||||
if (!category_seen["core_builtin"]) {
|
||||
print "source-registry.tsv 缺少 core_builtin 来源" > "/dev/stderr"
|
||||
failed = 1
|
||||
}
|
||||
|
||||
if (!category_seen["optional_adapter"]) {
|
||||
print "source-registry.tsv 缺少 optional_adapter 来源" > "/dev/stderr"
|
||||
failed = 1
|
||||
}
|
||||
|
||||
if (!category_seen["manual_only"]) {
|
||||
print "source-registry.tsv 缺少 manual_only 来源" > "/dev/stderr"
|
||||
failed = 1
|
||||
}
|
||||
|
||||
exit failed ? 1 : 0
|
||||
}
|
||||
' "$REGISTRY_FILE"
|
||||
}
|
||||
|
||||
print_contract() {
|
||||
validate_contract
|
||||
cat "$CONTRACT_FILE"
|
||||
}
|
||||
|
||||
print_registry() {
|
||||
validate_registry
|
||||
cat "$REGISTRY_FILE"
|
||||
}
|
||||
|
||||
get_source() {
|
||||
local source_id="$1"
|
||||
|
||||
validate_registry
|
||||
|
||||
awk -F '\t' -v source_id="$source_id" '
|
||||
NR == 1 { next }
|
||||
$1 == source_id {
|
||||
print
|
||||
found = 1
|
||||
}
|
||||
END {
|
||||
exit found ? 0 : 1
|
||||
}
|
||||
' "$REGISTRY_FILE"
|
||||
}
|
||||
|
||||
extract_url_host() {
|
||||
local url="$1"
|
||||
local rest host
|
||||
|
||||
rest="${url#*://}"
|
||||
if [ "$rest" = "$url" ]; then
|
||||
rest="$url"
|
||||
fi
|
||||
|
||||
rest="${rest#*@}"
|
||||
host="${rest%%/*}"
|
||||
host="${host%%\?*}"
|
||||
host="${host%%#*}"
|
||||
host="${host%%:*}"
|
||||
|
||||
printf '%s\n' "$host" | tr '[:upper:]' '[:lower:]'
|
||||
}
|
||||
|
||||
host_matches_pattern() {
|
||||
local host="$1"
|
||||
local pattern="$2"
|
||||
|
||||
case "$host" in
|
||||
"$pattern"|*."$pattern")
|
||||
return 0
|
||||
;;
|
||||
*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
match_url() {
|
||||
local url="$1"
|
||||
local host row source_id source_label source_category input_mode match_rule raw_dir adapter_name dependency_name dependency_type fallback_hint
|
||||
local fallback_row=""
|
||||
local pattern pattern_list
|
||||
|
||||
validate_registry
|
||||
host="$(extract_url_host "$url")"
|
||||
|
||||
while IFS=$'\t' read -r source_id source_label source_category input_mode match_rule raw_dir adapter_name dependency_name dependency_type fallback_hint; do
|
||||
[ "$source_id" = "source_id" ] && continue
|
||||
[ "$input_mode" = "url" ] || continue
|
||||
|
||||
pattern_list="${match_rule#url_host:}"
|
||||
if [ "$pattern_list" = "*" ]; then
|
||||
fallback_row="$(printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$source_id" "$source_label" "$source_category" "$input_mode" "$match_rule" "$raw_dir" "$adapter_name" "$dependency_name" "$dependency_type" "$fallback_hint")"
|
||||
continue
|
||||
fi
|
||||
|
||||
for pattern in ${pattern_list//,/ }; do
|
||||
if host_matches_pattern "$host" "$pattern"; then
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$source_id" "$source_label" "$source_category" "$input_mode" "$match_rule" "$raw_dir" "$adapter_name" "$dependency_name" "$dependency_type" "$fallback_hint"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
done < "$REGISTRY_FILE"
|
||||
|
||||
[ -n "$fallback_row" ] || return 1
|
||||
printf '%s\n' "$fallback_row"
|
||||
}
|
||||
|
||||
match_file() {
|
||||
local path="$1"
|
||||
local lowered_path source_id source_label source_category input_mode match_rule raw_dir adapter_name dependency_name dependency_type fallback_hint
|
||||
local extension_list extension
|
||||
|
||||
validate_registry
|
||||
lowered_path="$(printf '%s\n' "$path" | tr '[:upper:]' '[:lower:]')"
|
||||
|
||||
while IFS=$'\t' read -r source_id source_label source_category input_mode match_rule raw_dir adapter_name dependency_name dependency_type fallback_hint; do
|
||||
[ "$source_id" = "source_id" ] && continue
|
||||
[ "$input_mode" = "file" ] || continue
|
||||
|
||||
extension_list="${match_rule#file_ext:}"
|
||||
for extension in ${extension_list//,/ }; do
|
||||
case "$lowered_path" in
|
||||
*"$extension")
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$source_id" "$source_label" "$source_category" "$input_mode" "$match_rule" "$raw_dir" "$adapter_name" "$dependency_name" "$dependency_type" "$fallback_hint"
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
done
|
||||
done < "$REGISTRY_FILE"
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
list_by_category() {
|
||||
local category="$1"
|
||||
|
||||
validate_registry
|
||||
|
||||
awk -F '\t' -v category="$category" '
|
||||
NR == 1 { next }
|
||||
$3 == category { print }
|
||||
' "$REGISTRY_FILE"
|
||||
}
|
||||
|
||||
list_unique_dependencies() {
|
||||
local dependency_type="$1"
|
||||
|
||||
validate_registry
|
||||
|
||||
awk -F '\t' -v dependency_type="$dependency_type" '
|
||||
NR == 1 { next }
|
||||
$9 == dependency_type && $8 != "-" { print $8 }
|
||||
' "$REGISTRY_FILE" | sort -u
|
||||
}
|
||||
|
||||
command_name="${1:-}"
|
||||
|
||||
case "$command_name" in
|
||||
fields)
|
||||
[ "$#" -eq 1 ] || { usage; exit 1; }
|
||||
print_contract
|
||||
;;
|
||||
list)
|
||||
[ "$#" -eq 1 ] || { usage; exit 1; }
|
||||
print_registry
|
||||
;;
|
||||
get)
|
||||
[ "$#" -eq 2 ] || { usage; exit 1; }
|
||||
get_source "$2"
|
||||
;;
|
||||
match-url)
|
||||
[ "$#" -eq 2 ] || { usage; exit 1; }
|
||||
match_url "$2"
|
||||
;;
|
||||
match-file)
|
||||
[ "$#" -eq 2 ] || { usage; exit 1; }
|
||||
match_file "$2"
|
||||
;;
|
||||
list-by-category)
|
||||
[ "$#" -eq 2 ] || { usage; exit 1; }
|
||||
list_by_category "$2"
|
||||
;;
|
||||
unique-dependencies)
|
||||
[ "$#" -eq 2 ] || { usage; exit 1; }
|
||||
list_unique_dependencies "$2"
|
||||
;;
|
||||
validate)
|
||||
[ "$#" -eq 1 ] || { usage; exit 1; }
|
||||
validate_contract
|
||||
validate_registry
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
10
llm-wiki/scripts/source-registry.tsv
Normal file
10
llm-wiki/scripts/source-registry.tsv
Normal file
@@ -0,0 +1,10 @@
|
||||
source_id source_label source_category input_mode match_rule raw_dir adapter_name dependency_name dependency_type fallback_hint
|
||||
local_pdf PDF / 本地 PDF core_builtin file file_ext:.pdf raw/pdfs - - none 直接提供文件路径即可进入主线
|
||||
local_document Markdown/文本/HTML core_builtin file file_ext:.md,.txt,.html raw/notes - - none 直接提供文件路径即可进入主线
|
||||
plain_text 纯文本粘贴 core_builtin text text:inline raw/notes - - none 直接粘贴正文即可进入主线
|
||||
x_twitter X/Twitter optional_adapter url url_host:x.com,twitter.com raw/tweets baoyu-url-to-markdown baoyu-url-to-markdown bundled 自动提取失败时，改为复制全文粘贴
|
||||
wechat_article 微信公众号 optional_adapter url url_host:mp.weixin.qq.com raw/wechat wechat-article-to-markdown wechat-article-to-markdown install_time 自动提取失败时，在浏览器打开后复制全文粘贴
|
||||
youtube_video YouTube optional_adapter url url_host:youtube.com,youtu.be raw/articles youtube-transcript youtube-transcript bundled 自动提取失败时，提供字幕文件或手动粘贴文本
|
||||
zhihu_article 知乎 optional_adapter url url_host:zhihu.com raw/zhihu baoyu-url-to-markdown baoyu-url-to-markdown bundled 自动提取失败时，改为复制全文粘贴
|
||||
xiaohongshu_post 小红书 manual_only url url_host:xiaohongshu.com,xhslink.com raw/xiaohongshu - - none 请先从 App 或网页复制内容，再粘贴进来
|
||||
web_article 网页文章 optional_adapter url url_host:* raw/articles baoyu-url-to-markdown baoyu-url-to-markdown bundled 自动提取失败时，改为复制全文或保存为本地文件后继续
|
||||
|
83
llm-wiki/scripts/source-signal-coverage.js
Normal file
83
llm-wiki/scripts/source-signal-coverage.js
Normal file
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env node
|
||||
"use strict";
|
||||
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
const {
|
||||
SCAN_KINDS,
|
||||
extractFrontmatter,
|
||||
evaluateSourceSignalEligibility
|
||||
} = require("./lib/source-signal-eligibility");
|
||||
|
||||
function scanWiki(wikiRoot) {
|
||||
const wikiDir = path.join(wikiRoot, "wiki");
|
||||
if (!fs.existsSync(wikiDir)) {
|
||||
console.error(`ERROR: wiki 目录不存在：${wikiDir}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const pages = [];
|
||||
const summary = {
|
||||
applicable_total: 0,
|
||||
ok: 0,
|
||||
missing_sources: 0,
|
||||
empty_sources: 0,
|
||||
invalid_sources: 0,
|
||||
not_applicable: 0
|
||||
};
|
||||
|
||||
for (const kind of SCAN_KINDS) {
|
||||
const dir = path.join(wikiDir, kind.subdir);
|
||||
if (!fs.existsSync(dir)) continue;
|
||||
|
||||
const files = fs.readdirSync(dir)
|
||||
.filter((f) => f.endsWith(".md"))
|
||||
.sort();
|
||||
|
||||
for (const file of files) {
|
||||
const id = path.basename(file, ".md");
|
||||
if (["index", "log", "purpose", ".wiki-schema", "README"].includes(id)) continue;
|
||||
|
||||
const filePath = path.join(dir, file);
|
||||
const raw = fs.readFileSync(filePath, "utf8");
|
||||
const { frontmatter } = extractFrontmatter(raw);
|
||||
const result = evaluateSourceSignalEligibility({
|
||||
pageType: kind.pageType,
|
||||
frontmatter
|
||||
});
|
||||
|
||||
pages.push({
|
||||
path: path.relative(wikiRoot, filePath),
|
||||
id,
|
||||
pageType: kind.pageType,
|
||||
eligible: result.eligible,
|
||||
reason: result.reason,
|
||||
sourceCount: result.sources.length
|
||||
});
|
||||
|
||||
summary[result.reason] += 1;
|
||||
if (result.reason !== "not_applicable") {
|
||||
summary.applicable_total += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { summary, pages };
|
||||
}
|
||||
|
||||
function main(argv) {
|
||||
if (argv.length < 3) {
|
||||
console.error("Usage: node scripts/source-signal-coverage.js <wiki_root>");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const wikiRoot = path.resolve(argv[2]);
|
||||
const result = scanWiki(wikiRoot);
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
main(process.argv);
|
||||
}
|
||||
|
||||
module.exports = { scanWiki };
|
||||
144
llm-wiki/scripts/validate-step1.sh
Executable file
144
llm-wiki/scripts/validate-step1.sh
Executable file
@@ -0,0 +1,144 @@
|
||||
#!/bin/bash
|
||||
# 验证 ingest Step 1 的 JSON 输出格式
|
||||
# 用法：bash validate-step1.sh <json_file>
|
||||
# 返回：0 = 格式正确，1 = 格式有问题（触发回退）
|
||||
|
||||
SCRIPT_DIR="${BASH_SOURCE[0]%/*}"
|
||||
[ "$SCRIPT_DIR" = "${BASH_SOURCE[0]}" ] && SCRIPT_DIR="."
|
||||
SCRIPT_DIR="$(cd "$SCRIPT_DIR" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "$SCRIPT_DIR/shared-config.sh"
|
||||
|
||||
JSON_FILE="$1"
|
||||
|
||||
# 参数检查
|
||||
[ -z "$1" ] && { echo "ERROR: usage: validate-step1.sh <json_file>"; exit 1; }
|
||||
|
||||
# 检查 jq 是否可用（必需依赖）
|
||||
command -v jq >/dev/null 2>&1 || {
|
||||
echo "ERROR: jq is not installed. Install it via:" >&2
|
||||
print_install_hint jq
|
||||
exit 1
|
||||
}
|
||||
|
||||
# 检查文件是否存在
|
||||
[ -f "$JSON_FILE" ] || { echo "ERROR: file not found: $JSON_FILE"; exit 1; }
|
||||
|
||||
# 检查是否是有效 JSON
|
||||
jq empty "$JSON_FILE" 2>/dev/null || { echo "ERROR: invalid JSON format"; exit 1; }
|
||||
|
||||
# 检查必需字段存在且类型正确
|
||||
jq -e '.entities | type == "array"' "$JSON_FILE" >/dev/null 2>&1 || { echo "ERROR: 'entities' must be an array"; exit 1; }
|
||||
jq -e '.topics | type == "array"' "$JSON_FILE" >/dev/null 2>&1 || { echo "ERROR: 'topics' must be an array"; exit 1; }
|
||||
jq -e '.connections | type == "array"' "$JSON_FILE" >/dev/null 2>&1 || { echo "ERROR: 'connections' must be an array"; exit 1; }
|
||||
jq -e '.contradictions | type == "array"' "$JSON_FILE" >/dev/null 2>&1 || { echo "ERROR: 'contradictions' must be an array"; exit 1; }
|
||||
jq -e '.new_vs_existing | type == "object"' "$JSON_FILE" >/dev/null 2>&1 || { echo "ERROR: 'new_vs_existing' must be an object"; exit 1; }
|
||||
|
||||
# 检查每个 entity 的必需子字段
|
||||
VALID_CONFIDENCE="EXTRACTED|INFERRED|AMBIGUOUS|UNVERIFIED"
|
||||
|
||||
ENTITY_COUNT=$(jq '.entities | length' "$JSON_FILE" 2>/dev/null)
|
||||
if [ "$ENTITY_COUNT" -gt 0 ] 2>/dev/null; then
|
||||
NON_OBJECT_ENTITY_COUNT=$(jq '[.entities[] | select(type != "object")] | length' "$JSON_FILE" 2>/dev/null)
|
||||
if [ "$NON_OBJECT_ENTITY_COUNT" -gt 0 ] 2>/dev/null; then
|
||||
echo "ERROR: $NON_OBJECT_ENTITY_COUNT entity/entities must be objects"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# name, type, confidence 必须存在且非空
|
||||
BAD_ENTITY_COUNT=$(jq '
|
||||
[.entities[] | select(
|
||||
(.name // "" | length) == 0 or
|
||||
(.type // "" | length) == 0 or
|
||||
(.confidence // "" | length) == 0
|
||||
)] | length
|
||||
' "$JSON_FILE" 2>/dev/null)
|
||||
if [ "$BAD_ENTITY_COUNT" -gt 0 ] 2>/dev/null; then
|
||||
echo "ERROR: $BAD_ENTITY_COUNT entity/entities missing required fields (name/type/confidence)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# confidence 值必须是四个有效值之一
|
||||
INVALID=$(jq -r '.entities[]? | (.confidence // "MISSING")' "$JSON_FILE" 2>/dev/null | \
|
||||
grep -v -E "^($VALID_CONFIDENCE)$" | head -3)
|
||||
if [ -n "$INVALID" ]; then
|
||||
echo "ERROR: invalid entity confidence value(s): $INVALID"
|
||||
echo " Valid values: EXTRACTED | INFERRED | AMBIGUOUS | UNVERIFIED"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# EXTRACTED 和 INFERRED 必须提供 evidence 字段
|
||||
NO_EVIDENCE_COUNT=$(jq '
|
||||
[.entities[] | select(
|
||||
(.confidence == "EXTRACTED" or .confidence == "INFERRED") and
|
||||
((.evidence // "" | length) == 0)
|
||||
)] | length
|
||||
' "$JSON_FILE" 2>/dev/null)
|
||||
if [ "$NO_EVIDENCE_COUNT" -gt 0 ] 2>/dev/null; then
|
||||
echo "WARN: $NO_EVIDENCE_COUNT entity/entities with EXTRACTED/INFERRED confidence missing 'evidence' field"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 检查每个 topic 的必需子字段
|
||||
TOPIC_COUNT=$(jq '.topics | length' "$JSON_FILE" 2>/dev/null)
|
||||
if [ "$TOPIC_COUNT" -gt 0 ] 2>/dev/null; then
|
||||
NON_OBJECT_TOPIC_COUNT=$(jq '[.topics[] | select(type != "object")] | length' "$JSON_FILE" 2>/dev/null)
|
||||
if [ "$NON_OBJECT_TOPIC_COUNT" -gt 0 ] 2>/dev/null; then
|
||||
echo "ERROR: $NON_OBJECT_TOPIC_COUNT topic(s) must be objects"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
BAD_TOPIC_COUNT=$(jq '
|
||||
[.topics[] | select(
|
||||
(.name // "" | length) == 0
|
||||
)] | length
|
||||
' "$JSON_FILE" 2>/dev/null)
|
||||
if [ "$BAD_TOPIC_COUNT" -gt 0 ] 2>/dev/null; then
|
||||
echo "ERROR: $BAD_TOPIC_COUNT topic(s) missing required 'name' field"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# 检查每个 connection 的必需子字段（from, to, confidence）
|
||||
CONN_COUNT=$(jq '.connections | length' "$JSON_FILE" 2>/dev/null)
|
||||
if [ "$CONN_COUNT" -gt 0 ] 2>/dev/null; then
|
||||
NON_OBJECT_CONN_COUNT=$(jq '[.connections[] | select(type != "object")] | length' "$JSON_FILE" 2>/dev/null)
|
||||
if [ "$NON_OBJECT_CONN_COUNT" -gt 0 ] 2>/dev/null; then
|
||||
echo "ERROR: $NON_OBJECT_CONN_COUNT connection(s) must be objects"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
BAD_CONN_COUNT=$(jq '
|
||||
[.connections[] | select(
|
||||
(.from // "" | length) == 0 or
|
||||
(.to // "" | length) == 0 or
|
||||
(.confidence // "" | length) == 0
|
||||
)] | length
|
||||
' "$JSON_FILE" 2>/dev/null)
|
||||
if [ "$BAD_CONN_COUNT" -gt 0 ] 2>/dev/null; then
|
||||
echo "ERROR: $BAD_CONN_COUNT connection(s) missing required fields (from/to/confidence)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
INVALID_CONN_CONF=$(jq -r '.connections[]? | (.confidence // "MISSING")' "$JSON_FILE" 2>/dev/null | \
|
||||
grep -v -E "^($VALID_CONFIDENCE)$" | head -3)
|
||||
if [ -n "$INVALID_CONN_CONF" ]; then
|
||||
echo "ERROR: invalid connection confidence value(s): $INVALID_CONN_CONF"
|
||||
echo " Valid values: EXTRACTED | INFERRED | AMBIGUOUS | UNVERIFIED"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# EXTRACTED 和 INFERRED connections 必须提供 evidence
|
||||
NO_CONN_EVIDENCE=$(jq '
|
||||
[.connections[] | select(
|
||||
(.confidence == "EXTRACTED" or .confidence == "INFERRED") and
|
||||
((.evidence // "" | length) == 0)
|
||||
)] | length
|
||||
' "$JSON_FILE" 2>/dev/null)
|
||||
if [ "$NO_CONN_EVIDENCE" -gt 0 ] 2>/dev/null; then
|
||||
echo "WARN: $NO_CONN_EVIDENCE connection(s) with EXTRACTED/INFERRED confidence missing 'evidence' field"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "OK: Step 1 JSON validation passed"
|
||||
exit 0
|
||||
267
llm-wiki/scripts/wiki-compat.sh
Executable file
267
llm-wiki/scripts/wiki-compat.sh
Executable file
@@ -0,0 +1,267 @@
|
||||
#!/bin/bash
|
||||
# 旧知识库兼容脚本：惰性默认、目录检查、按需创建
|
||||
# 原则：migration_required=no，只有确实无法兼容时才引入显式迁移
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
SOURCE_REGISTRY_SCRIPT="$SCRIPT_DIR/source-registry.sh"
|
||||
|
||||
LEGACY_REQUIRED_RAW_DIRS=(
|
||||
"raw/articles"
|
||||
"raw/tweets"
|
||||
"raw/wechat"
|
||||
"raw/pdfs"
|
||||
"raw/notes"
|
||||
"raw/assets"
|
||||
)
|
||||
|
||||
REQUIRED_PATHS=(
|
||||
".wiki-schema.md"
|
||||
"index.md"
|
||||
"log.md"
|
||||
"raw"
|
||||
"wiki"
|
||||
"wiki/entities"
|
||||
"wiki/topics"
|
||||
"wiki/sources"
|
||||
"wiki/comparisons"
|
||||
"wiki/synthesis"
|
||||
"wiki/overview.md"
|
||||
)
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
用法：
|
||||
bash scripts/wiki-compat.sh inspect <wiki_root>
|
||||
bash scripts/wiki-compat.sh validate <wiki_root>
|
||||
bash scripts/wiki-compat.sh ensure-source-dir <wiki_root> <source_id>
|
||||
EOF
|
||||
}
|
||||
|
||||
trim() {
|
||||
printf '%s' "$1" | awk '{ gsub(/^[[:space:]]+|[[:space:]]+$/, "", $0); printf "%s", $0 }'
|
||||
}
|
||||
|
||||
require_wiki_root() {
|
||||
local wiki_root="$1"
|
||||
|
||||
[ -n "$wiki_root" ] || {
|
||||
usage
|
||||
exit 1
|
||||
}
|
||||
|
||||
[ -d "$wiki_root" ] || {
|
||||
echo "知识库不存在：$wiki_root" >&2
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
schema_field_value() {
|
||||
local wiki_root="$1"
|
||||
local field_name="$2"
|
||||
local default_value="$3"
|
||||
local schema_path value
|
||||
|
||||
schema_path="$wiki_root/.wiki-schema.md"
|
||||
|
||||
if [ ! -f "$schema_path" ]; then
|
||||
printf '%s\n' "$default_value"
|
||||
return 0
|
||||
fi
|
||||
|
||||
value="$(
|
||||
awk -v field_name="$field_name" '
|
||||
$0 ~ "^-[[:space:]]*" field_name "[：:]" {
|
||||
line = $0
|
||||
sub("^-[[:space:]]*" field_name "[：:][[:space:]]*", "", line)
|
||||
print line
|
||||
exit
|
||||
}
|
||||
' "$schema_path"
|
||||
)"
|
||||
|
||||
value="$(trim "$value")"
|
||||
|
||||
if [ -n "$value" ]; then
|
||||
printf '%s\n' "$value"
|
||||
else
|
||||
printf '%s\n' "$default_value"
|
||||
fi
|
||||
}
|
||||
|
||||
resolved_language() {
|
||||
local wiki_root="$1"
|
||||
local raw_value
|
||||
|
||||
raw_value="$(schema_field_value "$wiki_root" "语言" "")"
|
||||
|
||||
case "$raw_value" in
|
||||
English|english|EN|en)
|
||||
printf 'en\n'
|
||||
;;
|
||||
*)
|
||||
printf 'zh\n'
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
resolved_schema_version() {
|
||||
local wiki_root="$1"
|
||||
|
||||
schema_field_value "$wiki_root" "版本" "1.0"
|
||||
}
|
||||
|
||||
is_legacy_required_raw_dir() {
|
||||
case "$1" in
|
||||
raw/articles|raw/tweets|raw/wechat|raw/pdfs|raw/notes|raw/assets)
|
||||
return 0
|
||||
;;
|
||||
*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
missing_optional_raw_dirs() {
|
||||
local wiki_root="$1"
|
||||
local raw_dir
|
||||
local missing=()
|
||||
|
||||
while IFS= read -r raw_dir; do
|
||||
[ -n "$raw_dir" ] || continue
|
||||
|
||||
if is_legacy_required_raw_dir "$raw_dir"; then
|
||||
continue
|
||||
fi
|
||||
|
||||
if [ ! -d "$wiki_root/$raw_dir" ]; then
|
||||
missing+=("$raw_dir")
|
||||
fi
|
||||
done < <(
|
||||
bash "$SOURCE_REGISTRY_SCRIPT" list | awk -F '\t' 'NR > 1 { print $6 }' | LC_ALL=C sort -u
|
||||
)
|
||||
|
||||
if [ "${#missing[@]}" -eq 0 ]; then
|
||||
printf '%s\n' '-'
|
||||
else
|
||||
local IFS=,
|
||||
printf '%s\n' "${missing[*]}"
|
||||
fi
|
||||
}
|
||||
|
||||
file_presence() {
|
||||
local wiki_root="$1"
|
||||
local relative_path="$2"
|
||||
|
||||
if [ -e "$wiki_root/$relative_path" ]; then
|
||||
printf 'present\n'
|
||||
else
|
||||
printf 'missing\n'
|
||||
fi
|
||||
}
|
||||
|
||||
validate_layout() {
|
||||
local wiki_root="$1"
|
||||
local failed=0
|
||||
local path
|
||||
|
||||
require_wiki_root "$wiki_root"
|
||||
|
||||
for path in "${REQUIRED_PATHS[@]}"; do
|
||||
if [ ! -e "$wiki_root/$path" ]; then
|
||||
echo "缺少必要路径：$path" >&2
|
||||
failed=1
|
||||
fi
|
||||
done
|
||||
|
||||
for path in "${LEGACY_REQUIRED_RAW_DIRS[@]}"; do
|
||||
if [ ! -d "$wiki_root/$path" ]; then
|
||||
echo "缺少必要旧目录：$path" >&2
|
||||
failed=1
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$failed" -ne 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
source_raw_dir() {
|
||||
local source_id="$1"
|
||||
local record raw_dir
|
||||
|
||||
record="$(
|
||||
bash "$SOURCE_REGISTRY_SCRIPT" get "$source_id" 2>/dev/null
|
||||
)" || {
|
||||
echo "未知来源：$source_id" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
IFS=$'\t' read -r _ _ _ _ _ raw_dir _ _ _ _ <<EOF
|
||||
$record
|
||||
EOF
|
||||
|
||||
printf '%s\n' "$raw_dir"
|
||||
}
|
||||
|
||||
print_inspect() {
|
||||
local wiki_root="$1"
|
||||
local schema_version language optional_dirs legacy_mode purpose_file cache_file
|
||||
|
||||
validate_layout "$wiki_root"
|
||||
|
||||
schema_version="$(resolved_schema_version "$wiki_root")"
|
||||
language="$(resolved_language "$wiki_root")"
|
||||
optional_dirs="$(missing_optional_raw_dirs "$wiki_root")"
|
||||
purpose_file="$(file_presence "$wiki_root" "purpose.md")"
|
||||
cache_file="$(file_presence "$wiki_root" ".wiki-cache.json")"
|
||||
|
||||
if [ "$schema_version" = "1.0" ] || [ "$optional_dirs" != "-" ] || [ "$purpose_file" = "missing" ] || [ "$cache_file" = "missing" ]; then
|
||||
legacy_mode="yes"
|
||||
else
|
||||
legacy_mode="no"
|
||||
fi
|
||||
|
||||
printf 'wiki_root=%s\n' "$wiki_root"
|
||||
printf 'schema_version=%s\n' "$schema_version"
|
||||
printf 'language=%s\n' "$language"
|
||||
printf 'legacy_mode=%s\n' "$legacy_mode"
|
||||
printf 'migration_required=no\n'
|
||||
printf 'missing_optional_raw_dirs=%s\n' "$optional_dirs"
|
||||
printf 'purpose_file=%s\n' "$purpose_file"
|
||||
printf 'cache_file=%s\n' "$cache_file"
|
||||
}
|
||||
|
||||
ensure_source_dir() {
|
||||
local wiki_root="$1"
|
||||
local source_id="$2"
|
||||
local raw_dir
|
||||
|
||||
validate_layout "$wiki_root"
|
||||
raw_dir="$(source_raw_dir "$source_id")"
|
||||
|
||||
mkdir -p "$wiki_root/$raw_dir"
|
||||
printf '%s\n' "$wiki_root/$raw_dir"
|
||||
}
|
||||
|
||||
command_name="${1:-}"
|
||||
|
||||
case "$command_name" in
|
||||
inspect)
|
||||
[ "$#" -eq 2 ] || { usage; exit 1; }
|
||||
print_inspect "$2"
|
||||
;;
|
||||
validate)
|
||||
[ "$#" -eq 2 ] || { usage; exit 1; }
|
||||
print_inspect "$2" > /dev/null
|
||||
;;
|
||||
ensure-source-dir)
|
||||
[ "$#" -eq 3 ] || { usage; exit 1; }
|
||||
ensure_source_dir "$2" "$3"
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
371
llm-wiki/scripts/wiki-link-cli.js
Normal file
371
llm-wiki/scripts/wiki-link-cli.js
Normal file
@@ -0,0 +1,371 @@
|
||||
#!/usr/bin/env node
|
||||
"use strict";
|
||||
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
const {
|
||||
assembleGraphArtifactPair,
|
||||
commitGraphArtifactPair,
|
||||
prepareOfflineWarningPayload,
|
||||
serializeJsonForHtmlScript,
|
||||
verifyGraphArtifactPair
|
||||
} = require("./lib/graph-warning-bundle");
|
||||
const { normalizeRelativePosixPath } = require("./lib/wiki-file-discovery");
|
||||
const { scanKnowledgeBaseLinks, validatePortableMarkdownFilename } = require("./lib/wiki-link-index");
|
||||
const { renderWikilinkReplacement } = require("./lib/wikilink-parser");
|
||||
|
||||
function usage() {
|
||||
return [
|
||||
"Usage:",
|
||||
" node scripts/wiki-link-cli.js graph <kb-root> <output-dir> [--test-mode]",
|
||||
" node scripts/wiki-link-cli.js commit-pair <kb-root> <graph-input> <warning-groups> <candidate-sets> <graph-path>",
|
||||
" node scripts/wiki-link-cli.js warning-embed <kb-root> <graph-path> <warning-path> <output-path>",
|
||||
" node scripts/wiki-link-cli.js check <kb-root> [--strict] [--json]",
|
||||
" node scripts/wiki-link-cli.js rename-scan <kb-root> <source-path> <new-name>"
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function sanitizeEntries(entries) {
|
||||
return entries.map((item) => ({
|
||||
path: item.path,
|
||||
kind: item.kind,
|
||||
editable: item.editable,
|
||||
graphType: item.graphType
|
||||
}));
|
||||
}
|
||||
|
||||
function sanitizeInventory(inventory) {
|
||||
return {
|
||||
graphSources: sanitizeEntries(inventory.graphSources),
|
||||
lintSources: sanitizeEntries(inventory.lintSources),
|
||||
renameEditableSources: sanitizeEntries(inventory.renameEditableSources),
|
||||
renameReadOnlySources: sanitizeEntries(inventory.renameReadOnlySources),
|
||||
targets: sanitizeEntries(inventory.targets),
|
||||
fileSetSha256: inventory.fileSetSha256
|
||||
};
|
||||
}
|
||||
|
||||
function hasStrictErrors(report) {
|
||||
return report.groups.some((group) => group.severity === "error");
|
||||
}
|
||||
|
||||
function buildCheckReport(kbRoot, policy) {
|
||||
const scan = scanKnowledgeBaseLinks(kbRoot, policy);
|
||||
return {
|
||||
policy,
|
||||
inventory: sanitizeInventory(scan.inventory),
|
||||
edges: scan.edges,
|
||||
candidate_sets: scan.candidate_sets,
|
||||
groups: scan.groups,
|
||||
occurrences: scan.occurrences,
|
||||
source_metadata: scan.source_documents.map(({ _content, ...document }) => document),
|
||||
stale_pending_wrappers: scan.stale_pending_wrappers,
|
||||
metrics: scan.metrics
|
||||
};
|
||||
}
|
||||
|
||||
function writeJson(filePath, value) {
|
||||
fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
|
||||
}
|
||||
|
||||
function writeGraphScanFiles(kbRoot, outputDir) {
|
||||
const scan = scanKnowledgeBaseLinks(kbRoot, "graph");
|
||||
fs.mkdirSync(outputDir, { recursive: true });
|
||||
const nodes = scan.source_documents
|
||||
.filter((document) => document.graph_type)
|
||||
.map((document) => ({
|
||||
id: document.source_path,
|
||||
label: document.label,
|
||||
type: document.graph_type,
|
||||
source_path: document.source_path,
|
||||
_content: document._content,
|
||||
_signals: document._signals
|
||||
}))
|
||||
.sort((left, right) => left.id.localeCompare(right.id, "en"));
|
||||
const edges = scan.edges.map((edge, index) => ({
|
||||
id: `e${index + 1}`,
|
||||
from: edge.from,
|
||||
to: edge.to,
|
||||
type: edge.confidence,
|
||||
confidence: edge.confidence,
|
||||
relation_type: edge.relation_type
|
||||
}));
|
||||
writeJson(path.join(outputDir, "nodes.json"), nodes);
|
||||
writeJson(path.join(outputDir, "edges.json"), edges);
|
||||
writeJson(path.join(outputDir, "warning-groups.json"), scan.groups);
|
||||
writeJson(path.join(outputDir, "candidate-sets.json"), scan.candidate_sets);
|
||||
writeJson(path.join(outputDir, "scan-metrics.json"), scan.metrics);
|
||||
return { nodes, edges, scan };
|
||||
}
|
||||
|
||||
async function commitPair(args) {
|
||||
const graphData = JSON.parse(fs.readFileSync(args.graphInput, "utf8"));
|
||||
const groups = JSON.parse(fs.readFileSync(args.warningGroups, "utf8"));
|
||||
const candidateSets = JSON.parse(fs.readFileSync(args.candidateSets, "utf8"));
|
||||
const kbRootReal = fs.realpathSync.native(args.kbRoot);
|
||||
const requestedGraphPath = path.resolve(args.graphPath);
|
||||
const graphPath = path.join(
|
||||
fs.realpathSync.native(path.dirname(requestedGraphPath)),
|
||||
path.basename(requestedGraphPath)
|
||||
);
|
||||
const warningPath = path.join(path.dirname(graphPath), "graph-warnings.json");
|
||||
const detailsRef = path.relative(kbRootReal, warningPath).split(path.sep).join("/");
|
||||
const pair = assembleGraphArtifactPair({ graphData, groups, candidateSets, detailsRef });
|
||||
await commitGraphArtifactPair({
|
||||
kbRoot: kbRootReal,
|
||||
graphPath,
|
||||
warningPath,
|
||||
pair
|
||||
});
|
||||
}
|
||||
|
||||
async function writeWarningEmbed(args) {
|
||||
const verified = await verifyGraphArtifactPair({
|
||||
kbRoot: args.kbRoot,
|
||||
graphPath: args.graphPath,
|
||||
warningPath: args.warningPath
|
||||
});
|
||||
let payload;
|
||||
let scriptBytes;
|
||||
if (verified.status === "available") {
|
||||
const prepared = prepareOfflineWarningPayload({
|
||||
summary: verified.graphData.meta.warning_summary,
|
||||
bundle: verified.warningBundle
|
||||
});
|
||||
payload = prepared.payload;
|
||||
scriptBytes = prepared.scriptBytes;
|
||||
} else {
|
||||
payload = {
|
||||
summary: sanitizeUnavailableSummary(verified.summary),
|
||||
details_status: "unavailable",
|
||||
details_unavailable_reason: verified.reason,
|
||||
warning_details_truncated: false,
|
||||
omitted_group_count: 0,
|
||||
omitted_candidate_set_count: 0
|
||||
};
|
||||
scriptBytes = serializeJsonForHtmlScript(payload);
|
||||
}
|
||||
fs.writeFileSync(args.outputPath, scriptBytes);
|
||||
}
|
||||
|
||||
function sanitizeUnavailableSummary(summary) {
|
||||
if (!summary || typeof summary !== "object") return {};
|
||||
const safe = {};
|
||||
for (const key of ["build_id", "details_sha256"]) {
|
||||
if (typeof summary[key] === "string" && /^[a-f0-9]{64}$/.test(summary[key])) {
|
||||
safe[key] = summary[key];
|
||||
}
|
||||
}
|
||||
for (const key of ["total_groups", "total_occurrences", "error_occurrences", "warning_occurrences"]) {
|
||||
if (Number.isSafeInteger(summary[key]) && summary[key] >= 0) safe[key] = summary[key];
|
||||
}
|
||||
if (summary.by_code && typeof summary.by_code === "object" && !Array.isArray(summary.by_code)) {
|
||||
safe.by_code = Object.fromEntries(
|
||||
Object.entries(summary.by_code)
|
||||
.filter(([, count]) => Number.isSafeInteger(count) && count >= 0),
|
||||
);
|
||||
}
|
||||
if (typeof summary.details_ref === "string") {
|
||||
try {
|
||||
const normalized = normalizeRelativePosixPath(summary.details_ref);
|
||||
if (normalized === summary.details_ref && path.posix.basename(normalized) === "graph-warnings.json") {
|
||||
safe.details_ref = normalized;
|
||||
}
|
||||
} catch (_) {
|
||||
// Never copy an invalid or machine-local path into an offline artifact.
|
||||
}
|
||||
}
|
||||
return safe;
|
||||
}
|
||||
|
||||
function replacementTargetForOccurrence(occurrence, targetPath) {
|
||||
return targetPath;
|
||||
}
|
||||
|
||||
function buildRenameScanReport(kbRoot, sourcePath, newName) {
|
||||
const scan = scanKnowledgeBaseLinks(kbRoot, "rename");
|
||||
const candidateSets = new Map(scan.candidate_sets.map((item) => [item.candidate_set_id, item.candidates]));
|
||||
const sourceEntry = scan.inventory.graphSources.find((item) => item.path === sourcePath);
|
||||
if (!sourceEntry) {
|
||||
throw new Error(`Source is not a formal graph page: ${sourcePath}`);
|
||||
}
|
||||
|
||||
const validation = validatePortableMarkdownFilename(sourcePath, newName, scan.inventory.targets);
|
||||
if (!validation.ok) {
|
||||
throw new Error(`Invalid target name (${validation.reason}): ${newName}`);
|
||||
}
|
||||
|
||||
const renameTargetPath = validation.target_path;
|
||||
const editable = [];
|
||||
const readOnly = [];
|
||||
const ambiguous = [];
|
||||
|
||||
for (const occurrence of scan.occurrences) {
|
||||
const destination = occurrence.read_only ? readOnly : editable;
|
||||
const resolution = occurrence.resolution;
|
||||
|
||||
if (resolution.status === "resolved" && resolution.target_path === sourcePath) {
|
||||
destination.push({
|
||||
source_path: occurrence.source_path,
|
||||
file_sha256: occurrence.file_sha256,
|
||||
start_byte: occurrence.start_byte,
|
||||
end_byte: occurrence.end_byte,
|
||||
raw_link: occurrence.raw_link,
|
||||
replacement: renderWikilinkReplacement(
|
||||
occurrence,
|
||||
replacementTargetForOccurrence(occurrence, renameTargetPath)
|
||||
)
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const candidatePaths = candidateSets.get(resolution.candidate_set_id) || [];
|
||||
if (resolution.status === "ambiguous" && candidatePaths.includes(sourcePath)) {
|
||||
const rendered_candidates = candidatePaths.map((candidatePath) => ({
|
||||
candidate_path: candidatePath,
|
||||
replacement: renderWikilinkReplacement(
|
||||
occurrence,
|
||||
candidatePath === sourcePath
|
||||
? replacementTargetForOccurrence(occurrence, renameTargetPath)
|
||||
: candidatePath
|
||||
)
|
||||
}));
|
||||
ambiguous.push({
|
||||
source_path: occurrence.source_path,
|
||||
classification: occurrence.read_only ? "read_only" : "editable",
|
||||
read_only: occurrence.read_only,
|
||||
file_sha256: occurrence.file_sha256,
|
||||
start_byte: occurrence.start_byte,
|
||||
end_byte: occurrence.end_byte,
|
||||
raw_link: occurrence.raw_link,
|
||||
candidate_paths: candidatePaths,
|
||||
rendered_candidates
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
file_set_sha256: scan.inventory.fileSetSha256,
|
||||
source_path: sourcePath,
|
||||
target_path: renameTargetPath,
|
||||
validation,
|
||||
editable_occurrences: editable,
|
||||
read_only_occurrences: readOnly,
|
||||
ambiguous_occurrences: ambiguous,
|
||||
metrics: scan.metrics
|
||||
};
|
||||
}
|
||||
|
||||
function assertExactFlags(flags, allowedFlags) {
|
||||
const seen = new Set();
|
||||
for (const flag of flags) {
|
||||
if (!allowedFlags.has(flag)) {
|
||||
throw new Error(`Unknown argument: ${flag}`);
|
||||
}
|
||||
if (seen.has(flag)) {
|
||||
throw new Error(`Duplicate argument: ${flag}`);
|
||||
}
|
||||
seen.add(flag);
|
||||
}
|
||||
return seen;
|
||||
}
|
||||
|
||||
function parseArguments(command, rest) {
|
||||
if (command === "graph") {
|
||||
if (rest.length < 2 || rest.length > 3 || rest[0].startsWith("--") || rest[1].startsWith("--")) {
|
||||
throw new Error("graph requires <kb-root> <output-dir> [--test-mode]");
|
||||
}
|
||||
const flags = assertExactFlags(rest.slice(2), new Set(["--test-mode"]));
|
||||
return { kbRoot: rest[0], outputDir: rest[1], testMode: flags.has("--test-mode") };
|
||||
}
|
||||
|
||||
if (command === "check") {
|
||||
if (rest.length < 1 || rest[0].startsWith("--")) {
|
||||
throw new Error("check requires <kb-root> [--strict] [--json]");
|
||||
}
|
||||
const flags = assertExactFlags(rest.slice(1), new Set(["--strict", "--json"]));
|
||||
return { kbRoot: rest[0], strict: flags.has("--strict"), json: flags.has("--json") };
|
||||
}
|
||||
|
||||
if (command === "commit-pair") {
|
||||
if (rest.length !== 5 || rest.some((value) => value.startsWith("--"))) {
|
||||
throw new Error("commit-pair requires <kb-root> <graph-input> <warning-groups> <candidate-sets> <graph-path>");
|
||||
}
|
||||
return {
|
||||
kbRoot: rest[0],
|
||||
graphInput: rest[1],
|
||||
warningGroups: rest[2],
|
||||
candidateSets: rest[3],
|
||||
graphPath: rest[4]
|
||||
};
|
||||
}
|
||||
|
||||
if (command === "warning-embed") {
|
||||
if (rest.length !== 4 || rest.some((value) => value.startsWith("--"))) {
|
||||
throw new Error("warning-embed requires <kb-root> <graph-path> <warning-path> <output-path>");
|
||||
}
|
||||
return { kbRoot: rest[0], graphPath: rest[1], warningPath: rest[2], outputPath: rest[3] };
|
||||
}
|
||||
|
||||
if (command === "rename-scan") {
|
||||
if (rest.length !== 3 || rest.some((value) => value.startsWith("--"))) {
|
||||
throw new Error("rename-scan requires <kb-root> <source-path> <new-name>");
|
||||
}
|
||||
return { kbRoot: rest[0], sourcePath: rest[1], newName: rest[2] };
|
||||
}
|
||||
|
||||
throw new Error(`Unknown command: ${command}`);
|
||||
}
|
||||
|
||||
async function main(argv) {
|
||||
const [command, ...rest] = argv;
|
||||
|
||||
if (!command) {
|
||||
console.error(usage());
|
||||
return 1;
|
||||
}
|
||||
|
||||
try {
|
||||
const args = parseArguments(command, rest);
|
||||
if (command === "graph") {
|
||||
const result = writeGraphScanFiles(args.kbRoot, args.outputDir);
|
||||
process.stdout.write(`Graph scan wrote ${result.nodes.length} nodes and ${result.edges.length} edges to ${args.outputDir}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (command === "commit-pair") {
|
||||
await commitPair(args);
|
||||
process.stdout.write(`Graph artifact pair committed: ${args.graphPath}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (command === "warning-embed") {
|
||||
await writeWarningEmbed(args);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (command === "check") {
|
||||
const report = buildCheckReport(args.kbRoot, "lint");
|
||||
|
||||
process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
|
||||
return args.strict && hasStrictErrors(report) ? 2 : 0;
|
||||
}
|
||||
|
||||
if (command === "rename-scan") {
|
||||
const report = buildRenameScanReport(args.kbRoot, args.sourcePath, args.newName);
|
||||
process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
|
||||
return 0;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(error.message);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
main(process.argv.slice(2)).then(
|
||||
(code) => { process.exitCode = code; },
|
||||
(error) => {
|
||||
console.error(error && error.message ? error.message : String(error));
|
||||
process.exitCode = 1;
|
||||
}
|
||||
);
|
||||
Reference in New Issue
Block a user