350 lines
9.3 KiB
Bash
Executable File
350 lines
9.3 KiB
Bash
Executable File
#!/bin/bash
|
||
# 统一来源总表读取与验证脚本
|
||
# 权威数据文件：source-registry.tsv（来源定义）、source-record-contract.tsv（字段契约）
|
||
|
||
set -euo pipefail
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||
CONTRACT_FILE="$SCRIPT_DIR/source-record-contract.tsv"
|
||
REGISTRY_FILE="$SCRIPT_DIR/source-registry.tsv"
|
||
|
||
usage() {
|
||
cat <<'EOF'
|
||
用法：
|
||
bash scripts/source-registry.sh fields
|
||
bash scripts/source-registry.sh list
|
||
bash scripts/source-registry.sh get <source_id>
|
||
bash scripts/source-registry.sh match-url <url>
|
||
bash scripts/source-registry.sh match-file <path>
|
||
bash scripts/source-registry.sh list-by-category <core_builtin|optional_adapter|manual_only>
|
||
bash scripts/source-registry.sh unique-dependencies <bundled|install_time|none>
|
||
bash scripts/source-registry.sh validate
|
||
EOF
|
||
}
|
||
|
||
require_file() {
|
||
local file="$1"
|
||
|
||
[ -f "$file" ] || {
|
||
echo "缺少文件：$file" >&2
|
||
exit 1
|
||
}
|
||
}
|
||
|
||
expect_header() {
|
||
local file="$1"
|
||
local expected="$2"
|
||
local actual
|
||
|
||
actual="$(head -n 1 "$file")"
|
||
[ "$actual" = "$expected" ] || {
|
||
echo "表头不匹配：$file" >&2
|
||
echo "期望：$expected" >&2
|
||
echo "实际：$actual" >&2
|
||
exit 1
|
||
}
|
||
}
|
||
|
||
validate_contract() {
|
||
require_file "$CONTRACT_FILE"
|
||
expect_header "$CONTRACT_FILE" $'field_name\trequiredness\tfilled_by\tvalue_rule'
|
||
|
||
awk -F '\t' '
|
||
BEGIN {
|
||
required["source_id"] = 1
|
||
required["source_label"] = 1
|
||
required["source_category"] = 1
|
||
required["input_mode"] = 1
|
||
required["raw_dir"] = 1
|
||
required["original_ref"] = 1
|
||
required["ingest_text"] = 1
|
||
required["adapter_name"] = 1
|
||
required["fallback_hint"] = 1
|
||
}
|
||
NR == 1 { next }
|
||
{
|
||
if ($1 == "" || $2 == "" || $3 == "" || $4 == "") {
|
||
printf("source-record-contract.tsv 第 %d 行存在空字段\n", NR) > "/dev/stderr"
|
||
failed = 1
|
||
}
|
||
|
||
seen[$1] += 1
|
||
}
|
||
END {
|
||
for (field in required) {
|
||
if (seen[field] != 1) {
|
||
printf("source-record-contract.tsv 缺少或重复字段：%s\n", field) > "/dev/stderr"
|
||
failed = 1
|
||
}
|
||
}
|
||
|
||
exit failed ? 1 : 0
|
||
}
|
||
' "$CONTRACT_FILE"
|
||
}
|
||
|
||
validate_registry() {
|
||
require_file "$REGISTRY_FILE"
|
||
expect_header "$REGISTRY_FILE" $'source_id\tsource_label\tsource_category\tinput_mode\tmatch_rule\traw_dir\tadapter_name\tdependency_name\tdependency_type\tfallback_hint'
|
||
|
||
awk -F '\t' '
|
||
NR == 1 { next }
|
||
{
|
||
if ($1 == "" || $2 == "" || $3 == "" || $4 == "" || $5 == "" || $6 == "" || $10 == "") {
|
||
printf("source-registry.tsv 第 %d 行存在空字段\n", NR) > "/dev/stderr"
|
||
failed = 1
|
||
}
|
||
|
||
if ($3 != "core_builtin" && $3 != "optional_adapter" && $3 != "manual_only") {
|
||
printf("source-registry.tsv 第 %d 行存在未知分类：%s\n", NR, $3) > "/dev/stderr"
|
||
failed = 1
|
||
}
|
||
|
||
if ($4 != "url" && $4 != "file" && $4 != "text" && $4 != "asset") {
|
||
printf("source-registry.tsv 第 %d 行存在未知输入模式：%s\n", NR, $4) > "/dev/stderr"
|
||
failed = 1
|
||
}
|
||
|
||
if ($4 == "url" && $5 !~ /^url_host:/) {
|
||
printf("source-registry.tsv 第 %d 行 URL 来源必须声明 url_host 规则：%s\n", NR, $5) > "/dev/stderr"
|
||
failed = 1
|
||
}
|
||
|
||
if ($4 == "file" && $5 !~ /^file_ext:/) {
|
||
printf("source-registry.tsv 第 %d 行文件来源必须声明 file_ext 规则：%s\n", NR, $5) > "/dev/stderr"
|
||
failed = 1
|
||
}
|
||
|
||
if ($4 == "text" && $5 !~ /^text:/) {
|
||
printf("source-registry.tsv 第 %d 行文本来源必须声明 text 规则：%s\n", NR, $5) > "/dev/stderr"
|
||
failed = 1
|
||
}
|
||
|
||
if ($4 == "asset" && $5 !~ /^asset:/) {
|
||
printf("source-registry.tsv 第 %d 行附件来源必须声明 asset 规则：%s\n", NR, $5) > "/dev/stderr"
|
||
failed = 1
|
||
}
|
||
|
||
if ($6 !~ /^raw\//) {
|
||
printf("source-registry.tsv 第 %d 行 raw_dir 必须位于 raw/ 下：%s\n", NR, $6) > "/dev/stderr"
|
||
failed = 1
|
||
}
|
||
|
||
if (seen[$1]++) {
|
||
printf("source-registry.tsv source_id 重复：%s\n", $1) > "/dev/stderr"
|
||
failed = 1
|
||
}
|
||
|
||
category_seen[$3] = 1
|
||
|
||
if ($3 == "optional_adapter") {
|
||
if ($7 == "-" || $8 == "-" || $9 == "none") {
|
||
printf("source-registry.tsv 第 %d 行 optional_adapter 缺少依赖信息\n", NR) > "/dev/stderr"
|
||
failed = 1
|
||
}
|
||
} else if ($7 != "-" || $8 != "-" || $9 != "none") {
|
||
printf("source-registry.tsv 第 %d 行非外挂来源不应声明依赖\n", NR) > "/dev/stderr"
|
||
failed = 1
|
||
}
|
||
}
|
||
END {
|
||
if (!category_seen["core_builtin"]) {
|
||
print "source-registry.tsv 缺少 core_builtin 来源" > "/dev/stderr"
|
||
failed = 1
|
||
}
|
||
|
||
if (!category_seen["optional_adapter"]) {
|
||
print "source-registry.tsv 缺少 optional_adapter 来源" > "/dev/stderr"
|
||
failed = 1
|
||
}
|
||
|
||
if (!category_seen["manual_only"]) {
|
||
print "source-registry.tsv 缺少 manual_only 来源" > "/dev/stderr"
|
||
failed = 1
|
||
}
|
||
|
||
exit failed ? 1 : 0
|
||
}
|
||
' "$REGISTRY_FILE"
|
||
}
|
||
|
||
print_contract() {
|
||
validate_contract
|
||
cat "$CONTRACT_FILE"
|
||
}
|
||
|
||
print_registry() {
|
||
validate_registry
|
||
cat "$REGISTRY_FILE"
|
||
}
|
||
|
||
get_source() {
|
||
local source_id="$1"
|
||
|
||
validate_registry
|
||
|
||
awk -F '\t' -v source_id="$source_id" '
|
||
NR == 1 { next }
|
||
$1 == source_id {
|
||
print
|
||
found = 1
|
||
}
|
||
END {
|
||
exit found ? 0 : 1
|
||
}
|
||
' "$REGISTRY_FILE"
|
||
}
|
||
|
||
extract_url_host() {
|
||
local url="$1"
|
||
local rest host
|
||
|
||
rest="${url#*://}"
|
||
if [ "$rest" = "$url" ]; then
|
||
rest="$url"
|
||
fi
|
||
|
||
rest="${rest#*@}"
|
||
host="${rest%%/*}"
|
||
host="${host%%\?*}"
|
||
host="${host%%#*}"
|
||
host="${host%%:*}"
|
||
|
||
printf '%s\n' "$host" | tr '[:upper:]' '[:lower:]'
|
||
}
|
||
|
||
host_matches_pattern() {
|
||
local host="$1"
|
||
local pattern="$2"
|
||
|
||
case "$host" in
|
||
"$pattern"|*."$pattern")
|
||
return 0
|
||
;;
|
||
*)
|
||
return 1
|
||
;;
|
||
esac
|
||
}
|
||
|
||
match_url() {
|
||
local url="$1"
|
||
local host row source_id source_label source_category input_mode match_rule raw_dir adapter_name dependency_name dependency_type fallback_hint
|
||
local fallback_row=""
|
||
local pattern pattern_list
|
||
|
||
validate_registry
|
||
host="$(extract_url_host "$url")"
|
||
|
||
while IFS=$'\t' read -r source_id source_label source_category input_mode match_rule raw_dir adapter_name dependency_name dependency_type fallback_hint; do
|
||
[ "$source_id" = "source_id" ] && continue
|
||
[ "$input_mode" = "url" ] || continue
|
||
|
||
pattern_list="${match_rule#url_host:}"
|
||
if [ "$pattern_list" = "*" ]; then
|
||
fallback_row="$(printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$source_id" "$source_label" "$source_category" "$input_mode" "$match_rule" "$raw_dir" "$adapter_name" "$dependency_name" "$dependency_type" "$fallback_hint")"
|
||
continue
|
||
fi
|
||
|
||
for pattern in ${pattern_list//,/ }; do
|
||
if host_matches_pattern "$host" "$pattern"; then
|
||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$source_id" "$source_label" "$source_category" "$input_mode" "$match_rule" "$raw_dir" "$adapter_name" "$dependency_name" "$dependency_type" "$fallback_hint"
|
||
return 0
|
||
fi
|
||
done
|
||
done < "$REGISTRY_FILE"
|
||
|
||
[ -n "$fallback_row" ] || return 1
|
||
printf '%s\n' "$fallback_row"
|
||
}
|
||
|
||
match_file() {
|
||
local path="$1"
|
||
local lowered_path source_id source_label source_category input_mode match_rule raw_dir adapter_name dependency_name dependency_type fallback_hint
|
||
local extension_list extension
|
||
|
||
validate_registry
|
||
lowered_path="$(printf '%s\n' "$path" | tr '[:upper:]' '[:lower:]')"
|
||
|
||
while IFS=$'\t' read -r source_id source_label source_category input_mode match_rule raw_dir adapter_name dependency_name dependency_type fallback_hint; do
|
||
[ "$source_id" = "source_id" ] && continue
|
||
[ "$input_mode" = "file" ] || continue
|
||
|
||
extension_list="${match_rule#file_ext:}"
|
||
for extension in ${extension_list//,/ }; do
|
||
case "$lowered_path" in
|
||
*"$extension")
|
||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$source_id" "$source_label" "$source_category" "$input_mode" "$match_rule" "$raw_dir" "$adapter_name" "$dependency_name" "$dependency_type" "$fallback_hint"
|
||
return 0
|
||
;;
|
||
esac
|
||
done
|
||
done < "$REGISTRY_FILE"
|
||
|
||
return 1
|
||
}
|
||
|
||
list_by_category() {
|
||
local category="$1"
|
||
|
||
validate_registry
|
||
|
||
awk -F '\t' -v category="$category" '
|
||
NR == 1 { next }
|
||
$3 == category { print }
|
||
' "$REGISTRY_FILE"
|
||
}
|
||
|
||
list_unique_dependencies() {
|
||
local dependency_type="$1"
|
||
|
||
validate_registry
|
||
|
||
awk -F '\t' -v dependency_type="$dependency_type" '
|
||
NR == 1 { next }
|
||
$9 == dependency_type && $8 != "-" { print $8 }
|
||
' "$REGISTRY_FILE" | sort -u
|
||
}
|
||
|
||
command_name="${1:-}"
|
||
|
||
case "$command_name" in
|
||
fields)
|
||
[ "$#" -eq 1 ] || { usage; exit 1; }
|
||
print_contract
|
||
;;
|
||
list)
|
||
[ "$#" -eq 1 ] || { usage; exit 1; }
|
||
print_registry
|
||
;;
|
||
get)
|
||
[ "$#" -eq 2 ] || { usage; exit 1; }
|
||
get_source "$2"
|
||
;;
|
||
match-url)
|
||
[ "$#" -eq 2 ] || { usage; exit 1; }
|
||
match_url "$2"
|
||
;;
|
||
match-file)
|
||
[ "$#" -eq 2 ] || { usage; exit 1; }
|
||
match_file "$2"
|
||
;;
|
||
list-by-category)
|
||
[ "$#" -eq 2 ] || { usage; exit 1; }
|
||
list_by_category "$2"
|
||
;;
|
||
unique-dependencies)
|
||
[ "$#" -eq 2 ] || { usage; exit 1; }
|
||
list_unique_dependencies "$2"
|
||
;;
|
||
validate)
|
||
[ "$#" -eq 1 ] || { usage; exit 1; }
|
||
validate_contract
|
||
validate_registry
|
||
;;
|
||
*)
|
||
usage
|
||
exit 1
|
||
;;
|
||
esac
|