From 027dcb4e4ea9e44c68aad9f86212f24b274903b0 Mon Sep 17 00:00:00 2001
From: meels <meelstorm@gmail.com>
Date: Tue, 28 Jul 2026 13:06:28 +0200
Subject: [PATCH] feat: derive source pipeline from raw path claims

---
 .obsidian/plugins/webinar-dash/main.js        | 32 ++++++-
 .../webinar-dash/test/pipeline.test.js        | 84 +++++++++++++++++++
 2 files changed, 115 insertions(+), 1 deletion(-)
 create mode 100644 .obsidian/plugins/webinar-dash/test/pipeline.test.js

diff --git a/.obsidian/plugins/webinar-dash/main.js b/.obsidian/plugins/webinar-dash/main.js
index 6ef0af9..8689bde 100644
--- a/.obsidian/plugins/webinar-dash/main.js
+++ b/.obsidian/plugins/webinar-dash/main.js
@@ -21,6 +21,36 @@ const DEFAULTS = {
   conceptDir: "wiki/concepts",
 };
 
+const RAW_PATH_RE = /^\s*-\s*\*\*Raw path:\*\*\s*`([^`]+)`/m;
+
+function extractRawPath(text) {
+  const m = String(text).match(RAW_PATH_RE);
+  return m ? m[1].trim() : null;
+}
+
+function derivePipeline({ rawFiles, sourcePages }) {
+  const claimed = new Map();
+  for (const page of sourcePages) {
+    if (page.rawPath) claimed.set(page.rawPath, page);
+  }
+
+  const processed = [];
+  const unprocessed = [];
+  for (const file of rawFiles) {
+    const page = claimed.get(file.path);
+    if (page) processed.push(Object.assign({}, file, { page }));
+    else unprocessed.push(file);
+  }
+
+  const rawPaths = new Set(rawFiles.map((f) => f.path));
+  const orphaned = sourcePages.filter((p) => !p.rawPath || !rawPaths.has(p.rawPath));
+
+  unprocessed.sort((a, b) => a.name.localeCompare(b.name));
+  processed.sort((a, b) => b.page.name.localeCompare(a.page.name));
+
+  return { processed, unprocessed, orphaned };
+}
+
 function parseConfig(source) {
   const cfg = Object.assign({}, DEFAULTS);
   for (const line of String(source).split(/\r?\n/)) {
@@ -45,4 +75,4 @@ class WebinarDashPlugin extends PluginBase {
 
 module.exports = WebinarDashPlugin;
 module.exports.default = WebinarDashPlugin;
-module.exports.__test__ = { parseConfig, STATIONS, DEFAULTS };
+module.exports.__test__ = { parseConfig, extractRawPath, derivePipeline, STATIONS, DEFAULTS };
diff --git a/.obsidian/plugins/webinar-dash/test/pipeline.test.js b/.obsidian/plugins/webinar-dash/test/pipeline.test.js
new file mode 100644
index 0000000..ebd7fe9
--- /dev/null
+++ b/.obsidian/plugins/webinar-dash/test/pipeline.test.js
@@ -0,0 +1,84 @@
+"use strict";
+const test = require("node:test");
+const assert = require("node:assert/strict");
+const { extractRawPath, derivePipeline } = require("../main.js").__test__;
+
+test("extractRawPath pulls the backticked path", () => {
+  const page = [
+    "# You're reading way too much code",
+    "",
+    "#source",
+    "",
+    "## Source Metadata",
+    "",
+    "- **Date:** YouTube video, 24:11",
+    "- **Raw path:** `raw/sources/You're reading way too much code.md`",
+    "- **Source type:** video essay",
+  ].join("\n");
+  assert.equal(extractRawPath(page), "raw/sources/You're reading way too much code.md");
+});
+
+test("extractRawPath handles cyrillic and em dashes", () => {
+  const page = "- **Raw path:** `raw/sources/Скиллы на базе git — новая память AI-агентов.md`";
+  assert.equal(extractRawPath(page), "raw/sources/Скиллы на базе git — новая память AI-агентов.md");
+});
+
+test("extractRawPath returns null when the line is absent", () => {
+  assert.equal(extractRawPath("# A page\n\n#source\n\nNo metadata here."), null);
+});
+
+test("derivePipeline splits claimed from unclaimed raw files", () => {
+  const rawFiles = [
+    { path: "raw/sources/Nina interview.md", name: "Nina interview.md", size: 6614 },
+    { path: "raw/sources/Webinar script.md", name: "Webinar script.md", size: 15841 },
+  ];
+  const sourcePages = [
+    {
+      path: "wiki/sources/2026-07-14-nina-interview.md",
+      name: "2026-07-14-nina-interview.md",
+      rawPath: "raw/sources/Nina interview.md",
+    },
+  ];
+  const out = derivePipeline({ rawFiles, sourcePages });
+  assert.equal(out.processed.length, 1);
+  assert.equal(out.processed[0].name, "Nina interview.md");
+  assert.equal(out.processed[0].page.name, "2026-07-14-nina-interview.md");
+  assert.equal(out.unprocessed.length, 1);
+  assert.equal(out.unprocessed[0].name, "Webinar script.md");
+  assert.equal(out.orphaned.length, 0);
+});
+
+test("derivePipeline reports source pages whose raw file is gone", () => {
+  const out = derivePipeline({
+    rawFiles: [],
+    sourcePages: [
+      { path: "wiki/sources/x.md", name: "x.md", rawPath: "raw/sources/deleted.md" },
+    ],
+  });
+  assert.equal(out.orphaned.length, 1);
+  assert.equal(out.orphaned[0].name, "x.md");
+});
+
+test("derivePipeline treats a page with no raw path as orphaned", () => {
+  const out = derivePipeline({
+    rawFiles: [{ path: "raw/sources/a.md", name: "a.md", size: 10 }],
+    sourcePages: [{ path: "wiki/sources/y.md", name: "y.md", rawPath: null }],
+  });
+  assert.equal(out.orphaned.length, 1);
+  assert.equal(out.unprocessed.length, 1);
+});
+
+test("derivePipeline sorts unprocessed by name and processed newest first", () => {
+  const out = derivePipeline({
+    rawFiles: [
+      { path: "raw/sources/b.md", name: "b.md", size: 1 },
+      { path: "raw/sources/a.md", name: "a.md", size: 1 },
+      { path: "raw/sources/c.md", name: "c.md", size: 1 },
+    ],
+    sourcePages: [
+      { path: "wiki/sources/2026-07-14-x.md", name: "2026-07-14-x.md", rawPath: "raw/sources/c.md" },
+    ],
+  });
+  assert.deepEqual(out.unprocessed.map((f) => f.name), ["a.md", "b.md"]);
+  assert.equal(out.processed.length, 1);
+});