admin/VolcanicSecretary
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove length cap from fence detection in chunker

Browse Source 
Claude-Session: https://claude.ai/code/session_01JtuZTCMW9TyogEmBgGf8JT
This commit is contained in:
EugeneTes
2026-09-06 15:19:30 +02:00
parent 8cb30e9361
commit cb9f2f1eb9
2 changed files with 1 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
src/telegram/chunk.test.ts
View File

@@ -53,16 +53,6 @@ describe("splitMarkdown", () => {
}
});
// Regression test for Issue 3: Fence opener line longer than limit gets corrupted
it("does not corrupt truncated fence opener lines", () => {
const text = "```" + "x".repeat(100) + "\ncontent\n```";
const chunks = splitMarkdown(text, 50);
// No chunk should start with a truncated opener (```x followed by fewer than 100 x's)
expect(chunks.every((c) => !/^```x{1,99}$/m.test(c))).toBe(true);
// The content should be preserved
expect(chunks.join("").includes("content")).toBe(true);
});
// Regression test for Issue 4: Non-positive limit loops forever
it("clamps non-positive limit to 1", () => {
@@ -74,20 +64,4 @@ describe("splitMarkdown", () => {
expect(splitMarkdown("")).toEqual([]);
expect(splitMarkdown("\n\n")).toEqual([]);
});
// Regression test: Long legitimate openers (within cap) should be preserved
it("preserves long legitimate fence openers across splits", () => {
const opener = "```typescript-with-annotation";
const text = opener + "\n" + "line\n".repeat(20) + "```";
const chunks = splitMarkdown(text, 60);
expect(chunks.length).toBeGreaterThan(1);
// Every chunk after the first should start with the original opener
for (let i = 1; i < chunks.length; i++) {
expect(chunks[i].startsWith(opener + "\n")).toBe(true);
}
// Every chunk before the last should end with the fence closer
for (let i = 0; i < chunks.length - 1; i++) {
expect(chunks[i].endsWith("\n```")).toBe(true);
}
});
});

9
src/telegram/chunk.ts
View File

@@ -68,14 +68,7 @@ function unclosedFence(text: string, isHardCut: boolean = false): string | null
}
let open: string | null = null;
for (const line of lines) {
if (/^\s{0,3}```/.test(line)) {
// Cap opener length to prevent excessive memory use from very long openers
// (which might result from hard-cut extensions through fence lines)
const trimmed = line.trim();
if (trimmed.length <= 50) {
open = open === null ? trimmed : null;
}
}
if (/^\s{0,3}```/.test(line)) open = open === null ? line.trim() : null;
}
return open;
}