Skip to content

finos/morphir-scala / .claude/skills/kb

Manages the Morphir knowledge base under kb/ — OKF bundles and concept documents. Use when adding content to a bundle, creating a new bundle, checking the knowledge base for conformance or provenance drift, building or querying its SQLite index, managing intent through its lifecycle, recording or reading architectural decisions as Decision Records, or navigating, searching and listing its bundles, concepts and links.

FileKindSize
.gitignoreother180 B
.mill-versionother19 B
KbCheck.scalaother19.7 KB
KbDecision.scalaother12.6 KB
KbIndex.scalaother21.9 KB
KbIntent.scalaother14.0 KB
KbIntentEdit.scalaother10.9 KB
KbModel.scalaother11.7 KB
KbRefresh.scalaother8.9 KB
KbRender.scalaother9.0 KB
KbScaffold.scalaother9.5 KB
KbStore.scalaother12.0 KB
KbSync.scalaother46.9 KB
KbTests.scalaother71.5 KB
SKILL.mdskill md10.8 KB
kbexecutablescript459 B
kb.batscript485 B
kb.scalaother43.3 KB
millexecutablescript6.5 KB
mill.batscript9.7 KB
references/authoring.mdmarkdown4.7 KB
references/checks.mdmarkdown12.0 KB
references/commands.mdmarkdown12.8 KB
references/decisions.mdmarkdown7.4 KB
references/divergence.mdmarkdown3.4 KB
references/index-db.mdmarkdown5.2 KB
references/sync.mdmarkdown16.7 KB
test-resources/META-INF/services/kyo.test.Testother503 B

KbStore.scala

View on GitHub
//| scalaVersion: 3.9.0
//| moduleDeps: [KbModel.scala]
//| mvnDeps:
//| - io.getkyo::kyo-core:1.0.0-RC7
//| - org.yaml:snakeyaml:2.6
//| - org.commonmark:commonmark:0.29.0

/** Loading and parsing: filesystem to [[Kb]].
  *
  * Frontmatter is parsed with SnakeYAML (full YAML, so nested `sources` entries survive), and the body with
  * commonmark-java (so links come from a real parser rather than a regex that would trip over code fences). File
  * access goes through `kyo.Path`, so everything below the pure-parsing layer is a `Sync` effect.
  */

import kyo.*
import scala.jdk.CollectionConverters.*
import org.commonmark.node.{AbstractVisitor, Link, Node, Text}
import org.commonmark.parser.{IncludeSourceSpans, Parser}
import org.yaml.snakeyaml.{LoaderOptions, Yaml}
import org.yaml.snakeyaml.constructor.SafeConstructor

object KbStore:

  private val parser: Parser =
    Parser.builder().includeSourceSpans(IncludeSourceSpans.BLOCKS_AND_INLINES).build()

  private def newYaml(): Yaml =
    val opts = LoaderOptions()
    opts.setAllowDuplicateKeys(false)
    Yaml(SafeConstructor(opts))

  // ------------------------------------------------------------ pure parsing

  /** Splits a leading `---` fenced YAML block off the front of a document.
    *
    * Returns `(rawFrontmatterOrNone, body)`. A document with no leading fence yields `(None, wholeText)`.
    */
  def splitFrontmatter(text: String): (Option[String], String) =
    val normalized = text.replace("\r\n", "\n")
    if !normalized.startsWith("---\n") then (None, normalized)
    else
      val rest = normalized.drop(4)
      findClosingFence(rest) match
        case Some(i) => (Some(rest.take(i)), rest.drop(i).dropWhile(_ != '\n').drop(1))
        case None => (None, normalized)

  private def findClosingFence(s: String): Option[Int] =
    var idx = 0
    var result: Option[Int] = None
    var done = false
    while !done && !result.isDefined do
      val lineEnd = s.indexOf('\n', idx) match
        case -1 => s.length
        case n => n
      if s.substring(idx, lineEnd).trim == "---" then result = Some(idx)
      else if lineEnd >= s.length then done = true
      else idx = lineEnd + 1
    result

  /** Parses a YAML frontmatter block. Returns Left with the parser message when the YAML is malformed. */
  def parseFrontmatter(raw: String): Either[String, Frontmatter] =
    try
      newYaml().load[Object](raw) match
        case null => Right(Frontmatter.Empty)
        case m: java.util.Map[?, ?] =>
          Right(Frontmatter(m.asScala.toMap.map((k, v) => k.toString -> v.asInstanceOf[Any])))
        case other => Left(s"frontmatter is ${other.getClass.getSimpleName}, expected a mapping")
    catch case e: Exception => Left(Option(e.getMessage).getOrElse(e.toString).linesIterator.next())

  private def lineOf(node: Node): Int =
    var cur: Node | Null = node
    var line = 0
    while cur != null && line == 0 do
      val spans = cur.getSourceSpans
      if spans != null && !spans.isEmpty then line = spans.get(0).getLineIndex + 1
      cur = cur.getParent
    line

  private def linkText(link: Link): String =
    val sb = StringBuilder()
    link.accept(new AbstractVisitor:
      override def visit(t: Text): Unit = sb.append(t.getLiteral)
    )
    sb.toString

  /** Extracts every inline link from a markdown body, with 1-based line numbers.
    *
    * `frontmatterLines` shifts line numbers so they refer to the original file rather than the stripped body.
    */
  def extractLinks(body: String, frontmatterLines: Int): Seq[LinkRef] =
    val doc = parser.parse(body)
    val out = collection.mutable.ArrayBuffer.empty[LinkRef]
    doc.accept(new AbstractVisitor:
      override def visit(l: Link): Unit =
        out += LinkRef(linkText(l), l.getDestination, lineOf(l) + frontmatterLines)
        visitChildren(l)
    )
    out.toSeq

  def kindOf(rel: Seq[String]): DocKind =
    rel.lastOption match
      case Some("log.md") => DocKind.Log
      case Some("index.md") => if rel.length == 1 then DocKind.RootIndex else DocKind.SubIndex
      case _ => DocKind.Concept

  /** Builds a Doc from already-read text — the pure core of `loadDoc`. */
  def parseDoc(file: Path, bundleRoot: Path, text: String): Doc =
    val (rawFm, body) = splitFrontmatter(text)
    val parsed = rawFm.map(parseFrontmatter)
    val fmLines = rawFm.map(_.count(_ == '\n') + 2).getOrElse(0)
    val rel = KbPath.segmentsUnder(file, bundleRoot).getOrElse(file.parts.lastOption.toSeq)
    Doc(
      file = file,
      bundleRoot = bundleRoot,
      rel = rel,
      kind = kindOf(rel),
      frontmatter = parsed.flatMap(_.toOption),
      hasFrontmatterBlock = rawFm.isDefined,
      frontmatterError = parsed.collect { case Left(e) => e },
      body = body,
      links = extractLinks(body, fmLines),
      frontmatterLines = fmLines
    )

  // ------------------------------------------------------------------- I/O

  def isMarkdown(p: Path): Boolean = p.parts.lastOption.exists(_.endsWith(".md"))

  def loadDoc(file: Path, bundleRoot: Path): Doc < (Sync & Abort[Throwable]) =
    Path.runReadOnly {
      file.read.map(parseDoc(file, bundleRoot, _))
    }

  private def isBundleRoot(dir: Path): Boolean < (Sync & Abort[Throwable]) =
    Path.runReadOnly {
      val idx = dir / "index.md"
      idx.exists.map {
        case false => false
        case true =>
          idx.read.map { text =>
            splitFrontmatter(text)._1.flatMap(parseFrontmatter(_).toOption).exists(_.has("okf_version"))
          }
      }
    }

  /** Finds bundle roots under `start`, not descending into a bundle once one is found. */
  def findBundleRoots(start: Path): Chunk[Path] < (Sync & Abort[Throwable]) =
    Path.runReadOnly {
      start.exists.map {
        case false => Chunk.empty[Path]
        case true =>
          isBundleRoot(start).map {
            case true => Chunk(start)
            case false =>
              start.list.map { entries =>
                Kyo.foreach(entries) { e =>
                  e.isDirectory.map {
                    case true => findBundleRoots(e)
                    case false => Chunk.empty[Path]
                  }
                }.map(_.flattenChunk)
              }
          }
      }
    }

  /** Every markdown file at or below `dir`.
    *
    * Recurses with `list` rather than `walk`: `walk` is a stream and drags `Async` and `Scope` into every caller's
    * effect row, which buys nothing for directory trees this small.
    */
  def markdownUnder(dir: Path): Chunk[Path] < (Sync & Abort[Throwable]) =
    Path.runReadOnly {
      dir.list.map { entries =>
        Kyo.foreach(entries) { e =>
          e.isDirectory.map {
            case true => markdownUnder(e)
            case false => if isMarkdown(e) then Chunk(e) else Chunk.empty[Path]
          }
        }.map(_.flattenChunk)
      }
    }

  /** Every file at or below `dir` that is *not* markdown — the mirrored assets of a sync bundle. */
  def nonMarkdownUnder(dir: Path): Chunk[Path] < (Sync & Abort[Throwable]) =
    Path.runReadOnly {
      dir.exists.map {
        case false => Chunk.empty[Path]
        case true =>
          dir.list.map { entries =>
            Kyo.foreach(entries) { e =>
              e.isDirectory.map {
                case true => nonMarkdownUnder(e)
                case false => if isMarkdown(e) then Chunk.empty[Path] else Chunk(e)
              }
            }.map(_.flattenChunk)
          }
      }
    }

  /** The `root:` of a bundle's `sync.yaml`, as path segments, when the bundle mirrors an upstream repository.
    *
    * Read here rather than in KbSync because loading has to know which files are vendored before anything else can,
    * and KbSync sits downstream of this file.
    */
  def mirrorSegments(bundleRoot: Path): Option[Seq[String]] < (Sync & Abort[Throwable]) =
    Path.runReadOnly {
      val manifest = bundleRoot / "sync.yaml"
      manifest.exists.map {
        case false => (None: Option[Seq[String]])
        case true =>
          manifest.read.map { raw =>
            val root =
              try
                newYaml().load[Object](raw) match
                  case m: java.util.Map[?, ?] =>
                    m.asScala.toMap.map((k, v) => k.toString -> v).get("root").collect { case s: String => s }
                  case _ => None
              catch case _: Exception => None
            Some(root.getOrElse("sources").split("/").filter(_.nonEmpty).toSeq)
          }
      }
    }

  def loadBundle(root: Path, kbRoot: Path): Bundle < (Sync & Abort[Throwable]) =
    for
      mirror <- mirrorSegments(root)
      files <- markdownUnder(root)
      loaded <- Kyo.foreach(files)(loadDoc(_, root))
      assetFiles <- mirror match
        case None => (Chunk.empty[Path]: Chunk[Path] < (Sync & Abort[Throwable]))
        case Some(segs) => nonMarkdownUnder(segs.foldLeft(root)(_ / _))
    yield
      val segs = KbPath.segmentsUnder(root, kbRoot / "bundles").getOrElse(root.parts.lastOption.toSeq)
      val isVendored: Doc => Boolean =
        d => mirror.exists(m => d.rel.length > m.length && d.rel.startsWith(m))
      // Inside the mirror, `index.md` and `log.md` are upstream's own files and carry upstream's own frontmatter.
      // OKF reserves those names for the *bundle*, so the reservation stops at the mirror boundary.
      val docs = loaded.map(d => if isVendored(d) then d.copy(vendored = true, kind = DocKind.Concept) else d)
      val sorted = docs.sortBy(_.rel.mkString("/"))
      val assets = assetFiles.map { f =>
        Asset(f, root, KbPath.segmentsUnder(f, root).getOrElse(f.parts.lastOption.toSeq))
      }.sortBy(_.rel.mkString("/")).toSeq
      Bundle(
        root = root,
        name = root.parts.lastOption.getOrElse("?"),
        group = Option.when(segs.length > 1)(segs.dropRight(1).mkString("/")),
        index = sorted.find(_.kind == DocKind.RootIndex).getOrElse(
          sys.error(s"${KbPath.render(root)} has no root index.md")
        ),
        log = sorted.find(d => d.kind == DocKind.Log && d.rel.length == 1),
        subIndexes = sorted.filter(_.kind == DocKind.SubIndex),
        concepts = sorted.filter(_.isConcept),
        assets = assets,
        mirror = mirror
      )

  /** Loads the whole knowledge base rooted at `kbRoot` — the directory holding `bundles/`. */
  def load(kbRoot: Path): Kb < (Sync & Abort[Throwable]) =
    Path.runReadOnly {
      val bundlesDir = kbRoot / "bundles"
      for
        roots <- findBundleRoots(bundlesDir)
        bundles <- Kyo.foreach(roots.sortBy(_.parts.mkString("/")))(loadBundle(_, kbRoot))
        present <- bundlesDir.exists
        allMd <- if present then markdownUnder(bundlesDir) else (Chunk.empty[Path]: Chunk[Path] < (Sync & Abort[Throwable]))
      yield
        val claimed = bundles.map(_.root)
        // A README.md in a grouping directory is expected and correct — it is how a group announces itself without
        // being mistaken for a bundle root.
        val strays = allMd
          .filterNot(p => claimed.exists(r => KbPath.isUnder(p, r)))
          .filterNot(_.parts.lastOption.contains("README.md"))
        Kb(kbRoot, bundles.toSeq, strays.toSeq)
    }

  /** Resolves a link destination to a path, or None when it is external or an anchor. */
  def resolveLink(doc: Doc, link: LinkRef): Option[Path] =
    if link.isExternal || link.isAnchorOnly || link.dest.isEmpty then None
    else
      val dest = link.dest.takeWhile(_ != '#')
      if dest.isEmpty then None
      else if dest.startsWith("/") then Some(doc.bundleRoot / dest.stripPrefix("/"))
      else
        // Relative to the containing directory, resolving any `..` segments.
        //
        // Built up from the directory Path rather than from a rebuilt segment list: an absolute path's leading empty
        // segment was being dropped along with `.` and `""`, quietly turning every resolved relative link into a
        // relative path that could never exist. Nothing caught it because authored concepts link bundle-relative.
        val dir = Path(doc.file.parts.dropRight(1)*)
        val resolved = dest.split('/').foldLeft(dir) { (acc, seg) =>
          seg match
            case "." | "" => acc
            case ".." => Path(acc.parts.dropRight(1)*)
            case s => acc / s
        }
        Some(resolved)
deprecation-and-migrationaddyosmani/agent-skillsManages deprecation and migration. Use when removing old systems, APIs, or features. Use when migrating users from one implementation to another. Use when migrating a database schema in production, such as renaming or dropping a column without downtime (expand/contract). Use when deciding whether to maintain or sunset existing code.Stars: 119kUpdated Claude CodeCodexVerifiedInstall
performance-optimizationaddyosmani/agent-skillsOptimizes application performance across frontend, backend, queries, and databases. Use when performance requirements exist, when you suspect performance regressions, when Core Web Vitals or load times need improvement, when N+1 query patterns need fixing, or when profiling reveals bottlenecks.Stars: 119kUpdated Claude CodeCodexVerifiedInstall
agent-cost-reportthedotmack/claude-memBelievable agent cost report for any period, default the last 7 full days PT, not counting today. Measured tokens from Claude Code transcripts priced at OpenRouter list prices (ESTIMATED), measured provider spend when a sanctioned source exists, note-taker cost separate, Timing-style HTML/PDF plus report.json, line-items.csv, evidence.json.Stars: 118kUpdated Claude CodeCodexVerifiedInstall
agent-cost-reportthedotmack/claude-memBelievable agent cost report for any period, default the last 7 full days PT, not counting today. Measured tokens from Claude Code transcripts priced at OpenRouter list prices (ESTIMATED), measured provider spend when a sanctioned source exists, note-taker cost separate, Timing-style HTML/PDF plus report.json, line-items.csv, evidence.json.Stars: 118kUpdated Claude CodeCodexVerifiedInstall