Add baoyu-skills package

baoyu-skills/skills/baoyu-translate/scripts/bun.lock

@@ -0,0 +1,26 @@
+{
+  "lockfileVersion": 1,
+  "workspaces": {
+    "": {
+      "name": "baoyu-translate-chunk",
+      "dependencies": {
+        "markdown-it": "14.1.1",
+      },
+    },
+  },
+  "packages": {
+    "argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="],
+
+    "entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="],
+
+    "linkify-it": ["linkify-it@5.0.0", "", { "dependencies": { "uc.micro": "^2.0.0" } }, "sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ=="],
+
+    "markdown-it": ["markdown-it@14.1.1", "", { "dependencies": { "argparse": "^2.0.1", "entities": "^4.4.0", "linkify-it": "^5.0.0", "mdurl": "^2.0.0", "punycode.js": "^2.3.1", "uc.micro": "^2.1.0" }, "bin": { "markdown-it": "bin/markdown-it.mjs" } }, "sha512-BuU2qnTti9YKgK5N+IeMubp14ZUKUUw7yeJbkjtosvHiP0AZ5c8IAgEMk79D0eC8F23r4Ac/q8cAIFdm2FtyoA=="],
+
+    "mdurl": ["mdurl@2.0.0", "", {}, "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w=="],
+
+    "punycode.js": ["punycode.js@2.3.1", "", {}, "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA=="],
+
+    "uc.micro": ["uc.micro@2.1.0", "", {}, "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A=="],
+  }
+}

baoyu-skills/skills/baoyu-translate/scripts/chunk.ts

@@ -0,0 +1,335 @@
+import { mkdirSync, readFileSync, writeFileSync } from "fs"
+import { dirname, join } from "path"
+import MarkdownIt from "markdown-it"
+
+type BlockKind =
+  | "heading"
+  | "thematicBreak"
+  | "html"
+  | "code"
+  | "flow"
+
+interface Block {
+  kind: BlockKind
+  md: string
+  words: number
+}
+
+interface Chunk {
+  blocks: Block[]
+  words: number
+}
+
+export interface ChunkCliOptions {
+  file: string
+  maxWords: number
+  outputDir: string
+}
+
+export interface ChunkResult {
+  source: string
+  chunks: number
+  output_dir: string
+  frontmatter: boolean
+  words_per_chunk: number[]
+}
+
+const parser = new MarkdownIt({ html: true })
+
+export function formatChunkUsage(command: string): string {
+  return `Usage: ${command} <file> [--max-words 5000] [--output-dir <dir>]`
+}
+
+export function runChunkCli(args: string[], command = "chunk.ts"): number {
+  const parsed = parseChunkCliArgs(args)
+
+  if ("help" in parsed) {
+    console.log(formatChunkUsage(command))
+    return 0
+  }
+
+  if ("error" in parsed) {
+    console.error(parsed.error)
+    console.error(formatChunkUsage(command))
+    return 1
+  }
+
+  const result = chunkMarkdownFile(parsed.file, {
+    maxWords: parsed.maxWords,
+    outputDir: parsed.outputDir,
+  })
+
+  console.log(JSON.stringify(result))
+  return 0
+}
+
+export function chunkMarkdownFile(
+  file: string,
+  options: { maxWords?: number; outputDir?: string } = {}
+): ChunkResult {
+  const maxWords = options.maxWords ?? 5000
+  const outputDir = options.outputDir ?? ""
+
+  const rawContent = normalizeNewlines(readFileSync(file, "utf-8"))
+  const { frontmatter, body } = extractFrontmatter(rawContent)
+  const chunks = buildChunks(parseMarkdown(body), maxWords)
+
+  const dir = outputDir ? join(outputDir, "chunks") : join(dirname(file), "chunks")
+  mkdirSync(dir, { recursive: true })
+
+  if (frontmatter) {
+    writeFileSync(join(dir, "frontmatter.md"), frontmatter)
+  }
+
+  chunks.forEach((chunk, index) => {
+    const num = String(index + 1).padStart(2, "0")
+    writeFileSync(join(dir, `chunk-${num}.md`), chunk.blocks.map(block => block.md).join("\n\n"))
+  })
+
+  return {
+    source: file,
+    chunks: chunks.length,
+    output_dir: dir,
+    frontmatter: Boolean(frontmatter),
+    words_per_chunk: chunks.map(chunk => chunk.words),
+  }
+}
+
+function parseChunkCliArgs(args: string[]):
+  | ChunkCliOptions
+  | { help: true }
+  | { error: string } {
+  let file = ""
+  let maxWords = 5000
+  let outputDir = ""
+
+  for (let index = 0; index < args.length; index += 1) {
+    const arg = args[index]
+
+    if (arg === "-h" || arg === "--help") {
+      return { help: true }
+    }
+
+    if (arg === "--max-words") {
+      const value = args[index + 1]
+      if (!value) return { error: "Missing value for --max-words" }
+      maxWords = parsePositiveInt(value, 0)
+      if (maxWords <= 0) return { error: `Invalid --max-words value: ${value}` }
+      index += 1
+      continue
+    }
+
+    if (arg === "--output-dir") {
+      const value = args[index + 1]
+      if (!value) return { error: "Missing value for --output-dir" }
+      outputDir = value
+      index += 1
+      continue
+    }
+
+    if (arg.startsWith("-")) {
+      return { error: `Unknown option: ${arg}` }
+    }
+
+    if (!file) {
+      file = arg
+      continue
+    }
+
+    return { error: `Unexpected positional argument: ${arg}` }
+  }
+
+  if (!file) {
+    return { error: "Missing input file" }
+  }
+
+  return { file, maxWords, outputDir }
+}
+
+function parsePositiveInt(value: string | undefined, fallback: number): number {
+  if (!value) return fallback
+  const parsed = Number.parseInt(value, 10)
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback
+}
+
+function normalizeNewlines(text: string): string {
+  return text.replace(/^\uFEFF/, "").replace(/\r\n?/g, "\n")
+}
+
+function trimBoundaryBlankLines(text: string): string {
+  return text.replace(/^\n+/, "").replace(/\n+$/, "")
+}
+
+function extractFrontmatter(content: string): { frontmatter: string; body: string } {
+  const lines = content.split("\n")
+  if (lines[0] !== "---") {
+    return { frontmatter: "", body: content }
+  }
+
+  for (let index = 1; index < lines.length; index += 1) {
+    if (lines[index] === "---" || lines[index] === "...") {
+      return {
+        frontmatter: lines.slice(0, index + 1).join("\n"),
+        body: lines.slice(index + 1).join("\n").replace(/^\n+/, ""),
+      }
+    }
+  }
+
+  return { frontmatter: "", body: content }
+}
+
+function parseMarkdown(content: string): Block[] {
+  if (!content.trim()) return []
+
+  const lines = content.split("\n")
+  const tokens = parser.parse(content, {})
+  const blocks: Block[] = []
+
+  for (const token of tokens) {
+    if (!token.map || token.level !== 0) continue
+    if (token.nesting !== 1 && token.nesting !== 0) continue
+
+    const [startLine, endLine] = token.map
+    const md = trimBoundaryBlankLines(lines.slice(startLine, endLine).join("\n"))
+    if (!md) continue
+
+    blocks.push(makeBlock(tokenTypeToBlockKind(token.type), md))
+  }
+
+  if (blocks.length === 0) {
+    const body = trimBoundaryBlankLines(content)
+    if (body) {
+      blocks.push(makeBlock("flow", body))
+    }
+  }
+
+  return blocks
+}
+
+function tokenTypeToBlockKind(tokenType: string): BlockKind {
+  if (tokenType === "heading_open") return "heading"
+  if (tokenType === "hr") return "thematicBreak"
+  if (tokenType === "html_block") return "html"
+  if (tokenType === "fence" || tokenType === "code_block") return "code"
+  return "flow"
+}
+
+function makeBlock(kind: BlockKind, md: string): Block {
+  return {
+    kind,
+    md: trimBoundaryBlankLines(md),
+    words: countWords(md),
+  }
+}
+
+function buildChunks(blocks: Block[], maxWordsPerChunk: number): Chunk[] {
+  const sections = splitIntoSections(blocks)
+  const normalizedBlocks: Block[] = []
+
+  for (const section of sections) {
+    const sectionWords = section.reduce((sum, block) => sum + block.words, 0)
+    if (sectionWords <= maxWordsPerChunk) {
+      normalizedBlocks.push(makeBlock("flow", section.map(block => block.md).join("\n\n")))
+      continue
+    }
+
+    for (const block of section) {
+      normalizedBlocks.push(...splitOversizedBlock(block, maxWordsPerChunk))
+    }
+  }
+
+  const chunks: Chunk[] = []
+  let currentBlocks: Block[] = []
+  let currentWords = 0
+
+  for (const block of normalizedBlocks) {
+    if (currentWords + block.words > maxWordsPerChunk && currentBlocks.length > 0) {
+      chunks.push({ blocks: currentBlocks, words: currentWords })
+      currentBlocks = [block]
+      currentWords = block.words
+      continue
+    }
+
+    currentBlocks.push(block)
+    currentWords += block.words
+  }
+
+  if (currentBlocks.length > 0) {
+    chunks.push({ blocks: currentBlocks, words: currentWords })
+  }
+
+  return chunks
+}
+
+function splitIntoSections(blocks: Block[]): Block[][] {
+  const sections: Block[][] = []
+  let current: Block[] = []
+
+  for (const block of blocks) {
+    if (block.kind === "heading" && current.length > 0) {
+      sections.push(current)
+      current = [block]
+      continue
+    }
+
+    current.push(block)
+  }
+
+  if (current.length > 0) {
+    sections.push(current)
+  }
+
+  return sections
+}
+
+function splitOversizedBlock(block: Block, maxWordsPerChunk: number): Block[] {
+  if (block.words <= maxWordsPerChunk) return [block]
+
+  if (
+    block.kind === "heading"
+    || block.kind === "thematicBreak"
+    || block.kind === "html"
+    || block.kind === "code"
+  ) {
+    return [block]
+  }
+
+  const lines = block.md.split("\n")
+  if (lines.length <= 1) {
+    return [block]
+  }
+
+  const splitBlocks: Block[] = []
+  let buffer: string[] = []
+  let bufferWords = 0
+
+  for (const line of lines) {
+    const lineWords = countWords(line)
+    if (bufferWords + lineWords > maxWordsPerChunk && buffer.length > 0) {
+      splitBlocks.push(makeBlock(block.kind, buffer.join("\n")))
+      buffer = [line]
+      bufferWords = lineWords
+      continue
+    }
+
+    buffer.push(line)
+    bufferWords += lineWords
+  }
+
+  if (buffer.length > 0) {
+    splitBlocks.push(makeBlock(block.kind, buffer.join("\n")))
+  }
+
+  return splitBlocks
+}
+
+function countWords(text: string): number {
+  const cleaned = text.replace(/[#*`\[\]()>|_~-]/g, " ")
+  const cjk = cleaned.match(/[\u4e00-\u9fff\u3400-\u4dbf\uf900-\ufaff]/g)
+  const latin = cleaned.match(/[a-zA-Z0-9]+/g)
+  return (cjk?.length ?? 0) + (latin?.length ?? 0)
+}
+
+if (import.meta.main) {
+  process.exit(runChunkCli(process.argv.slice(2), process.argv[1] ?? "chunk.ts"))
+}

baoyu-skills/skills/baoyu-translate/scripts/main.ts

@@ -0,0 +1,55 @@
+#!/usr/bin/env bun
+import path from "node:path"
+import process from "node:process"
+import { runChunkCli } from "./chunk.js"
+
+function formatScriptCommand(fallback: string): string {
+  const raw = process.argv[1]
+  const displayPath = raw
+    ? (() => {
+        const relative = path.relative(process.cwd(), raw)
+        return relative && !relative.startsWith("..") ? relative : raw
+      })()
+    : fallback
+
+  const quotedPath = displayPath.includes(" ")
+    ? `"${displayPath.replace(/"/g, '\\"')}"`
+    : displayPath
+
+  return `npx -y bun ${quotedPath}`
+}
+
+function printUsage(exitCode: number): never {
+  const cmd = formatScriptCommand("scripts/main.ts")
+  console.log(`Baoyu Translate CLI
+
+Usage:
+  ${cmd} <file> [--max-words 5000] [--output-dir <dir>]
+  ${cmd} chunk <file> [--max-words 5000] [--output-dir <dir>]
+
+Commands:
+  chunk              Split markdown into chunks
+
+Options:
+  --max-words <n>    Maximum words per chunk (default: 5000)
+  --output-dir <dir> Write chunks into <dir>/chunks/
+  -h, --help         Show help
+`)
+  process.exit(exitCode)
+}
+
+const args = process.argv.slice(2)
+
+if (args.length === 0) {
+  printUsage(1)
+}
+
+if (args[0] === "-h" || args[0] === "--help") {
+  printUsage(0)
+}
+
+if (args[0] === "chunk") {
+  process.exit(runChunkCli(args.slice(1), `${formatScriptCommand("scripts/main.ts")} chunk`))
+}
+
+process.exit(runChunkCli(args, formatScriptCommand("scripts/main.ts")))

baoyu-skills/skills/baoyu-translate/scripts/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "baoyu-translate-chunk",
+  "private": true,
+  "dependencies": {
+    "markdown-it": "14.1.1"
+  }
+}