diff --git a/src/Splitters/Abstractions/src/Text/CharacterTextSplitter.cs b/src/Splitters/Abstractions/src/Text/CharacterTextSplitter.cs index b53d958d..5b9f2adf 100644 --- a/src/Splitters/Abstractions/src/Text/CharacterTextSplitter.cs +++ b/src/Splitters/Abstractions/src/Text/CharacterTextSplitter.cs @@ -15,6 +15,8 @@ public override IReadOnlyList SplitText(string text) { text = text ?? throw new ArgumentNullException(nameof(text)); + text = NormalizeLineEndings(text); + List splits; if (separator != null) { @@ -27,4 +29,4 @@ public override IReadOnlyList SplitText(string text) return this.MergeSplits(splits, separator ?? string.Empty); } -} \ No newline at end of file +} diff --git a/src/Splitters/Abstractions/src/Text/MarkdownHeaderTextSplitter.cs b/src/Splitters/Abstractions/src/Text/MarkdownHeaderTextSplitter.cs index 18a4b706..a290724d 100644 --- a/src/Splitters/Abstractions/src/Text/MarkdownHeaderTextSplitter.cs +++ b/src/Splitters/Abstractions/src/Text/MarkdownHeaderTextSplitter.cs @@ -16,6 +16,7 @@ private sealed class LineType private const string _codeBlockseparator = "```"; private readonly static string[] _defauldHeaders = { "#", "##", "###", "####", "#####", "######" }; private readonly static string[] separator = { "\n" }; + private static readonly char[] HeaderSeparator = { '|' }; /// public MarkdownHeaderTextSplitter( @@ -35,9 +36,7 @@ public override IReadOnlyList SplitText(string text) text = text ?? throw new ArgumentNullException(nameof(text)); // Split the input text by newline character ("\n"). - var lines = text - .Replace("\r", "") // some people are using windows - .Split(separator, StringSplitOptions.None); + var lines = NormalizeLineEndings(text).Split(separator, StringSplitOptions.None); var content = new List(); @@ -74,10 +73,12 @@ public override IReadOnlyList SplitText(string text) { if (hLen <= currentHeaderLen) { - var existingHeader = currentHeader.Split('|'); - - string prevHeader = string.Join("|", existingHeader.Take(existingHeader.Length - 1)); - currentHeader = prevHeader + "|" + strippedLine.TrimStart('#').Trim(); + var parentHeaders = currentHeader + .Split(HeaderSeparator, StringSplitOptions.RemoveEmptyEntries) + .Take(hLen - 1); + currentHeader = string.Join( + "|", + parentHeaders.Concat(new[] { strippedLine.TrimStart('#').Trim() })); currentHeaderLen = hLen; continue; } diff --git a/src/Splitters/Abstractions/src/Text/RecursiveCharacterTextSplitter.cs b/src/Splitters/Abstractions/src/Text/RecursiveCharacterTextSplitter.cs index 4e96e87f..2e8163b8 100644 --- a/src/Splitters/Abstractions/src/Text/RecursiveCharacterTextSplitter.cs +++ b/src/Splitters/Abstractions/src/Text/RecursiveCharacterTextSplitter.cs @@ -19,6 +19,8 @@ public override IReadOnlyList SplitText(string text) { text = text ?? throw new ArgumentNullException(nameof(text)); + text = NormalizeLineEndings(text); + List finalChunks = new List(); string separator = _separators[_separators.Count - 1]; @@ -78,4 +80,4 @@ public override IReadOnlyList SplitText(string text) return finalChunks; } -} \ No newline at end of file +} diff --git a/src/Splitters/Abstractions/src/Text/TextSplitter.cs b/src/Splitters/Abstractions/src/Text/TextSplitter.cs index 97cbf6ab..9eda5a68 100644 --- a/src/Splitters/Abstractions/src/Text/TextSplitter.cs +++ b/src/Splitters/Abstractions/src/Text/TextSplitter.cs @@ -45,6 +45,12 @@ protected TextSplitter( /// protected int ChunkOverlap => _chunkOverlap; + /// + /// Normalizes Windows and legacy Mac line endings before separator-based splitting. + /// + internal static string NormalizeLineEndings(string text) => + text.Replace("\r\n", "\n").Replace('\r', '\n'); + /// /// /// @@ -125,4 +131,4 @@ protected IReadOnlyList MergeSplits(IEnumerable splits, string s // todo: Implement from_tiktoken_encoder -} \ No newline at end of file +} diff --git a/src/Splitters/Abstractions/test/Resources/markdown_test_material.md b/src/Splitters/Abstractions/test/Resources/markdown_test_material.md new file mode 100644 index 00000000..a5ef2b3c --- /dev/null +++ b/src/Splitters/Abstractions/test/Resources/markdown_test_material.md @@ -0,0 +1,71 @@ +# Header A + +Text A + +## Header A.A + +Text A.A + +## Header A.B + +Text A.B + +### Header A.B.A + +Text A.B.A + +### Header A.B.B + +Text A.B.B + +### Header A.B.C + +Text A.B.C + +## Header A.C + +Text A.C + +### Header A.C.A + +Text A.C.A + +### Header A.C.B + +Text A.C.B + +# Header B + +Text B + +## Header B.A + +Text B.A + +## Header B.B + +Text B.B + +### Header B.B.A + +Text B.B.A + +### Header B.B.B + +Text B.B.B + +## Header B.C + +Text B.C + +### Header B.C.A + +Text B.C.A + +### Header B.C.B + +Text B.C.B + +### Header B.C.C + +Text B.C.C \ No newline at end of file diff --git a/src/Splitters/Abstractions/test/Tests.MarkdownHeader.cs b/src/Splitters/Abstractions/test/Tests.MarkdownHeader.cs index 42f74eda..6b66052c 100644 --- a/src/Splitters/Abstractions/test/Tests.MarkdownHeader.cs +++ b/src/Splitters/Abstractions/test/Tests.MarkdownHeader.cs @@ -61,4 +61,79 @@ Hi this is Joe res[0].Should().Be("Hi this is Jim\nHi this is Joe"); res[1].Should().Be("Hi this is Molly"); } -} \ No newline at end of file + + [Test] + public void TestMarkdown4() + { + var md = H.Resources.markdown_test_material_md.AsString(); + + var splitter = new MarkdownHeaderTextSplitter(); + var res = splitter.SplitText(md); + + res.Count.Should().Be(18); + + res[0].Split("\n")[0].Should().Be("Header A"); + res[0].Split("\n")[1].Should().Be("Text A"); + + res[1].Split("\n")[0].Should().Be("Header A: Header A.A"); + res[1].Split("\n")[1].Should().Be("Text A.A"); + + res[2].Split("\n")[0].Should().Be("Header A: Header A.B"); + res[2].Split("\n")[1].Should().Be("Text A.B"); + + res[3].Split("\n")[0].Should().Be("Header A: Header A.B: Header A.B.A"); + res[3].Split("\n")[1].Should().Be("Text A.B.A"); + + res[4].Split("\n")[0].Should().Be("Header A: Header A.B: Header A.B.B"); + res[4].Split("\n")[1].Should().Be("Text A.B.B"); + + res[5].Split("\n")[0].Should().Be("Header A: Header A.B: Header A.B.C"); + res[5].Split("\n")[1].Should().Be("Text A.B.C"); + + res[6].Split("\n")[0].Should().Be("Header A: Header A.C"); + res[6].Split("\n")[1].Should().Be("Text A.C"); + + res[7].Split("\n")[0].Should().Be("Header A: Header A.C: Header A.C.A"); + res[7].Split("\n")[1].Should().Be("Text A.C.A"); + + res[8].Split("\n")[0].Should().Be("Header A: Header A.C: Header A.C.B"); + res[8].Split("\n")[1].Should().Be("Text A.C.B"); + + res[9].Split("\n")[0].Should().Be("Header B"); + res[9].Split("\n")[1].Should().Be("Text B"); + + res[10].Split("\n")[0].Should().Be("Header B: Header B.A"); + res[10].Split("\n")[1].Should().Be("Text B.A"); + + res[11].Split("\n")[0].Should().Be("Header B: Header B.B"); + res[11].Split("\n")[1].Should().Be("Text B.B"); + + res[12].Split("\n")[0].Should().Be("Header B: Header B.B: Header B.B.A"); + res[12].Split("\n")[1].Should().Be("Text B.B.A"); + + res[13].Split("\n")[0].Should().Be("Header B: Header B.B: Header B.B.B"); + res[13].Split("\n")[1].Should().Be("Text B.B.B"); + + res[14].Split("\n")[0].Should().Be("Header B: Header B.C"); + res[14].Split("\n")[1].Should().Be("Text B.C"); + + res[15].Split("\n")[0].Should().Be("Header B: Header B.C: Header B.C.A"); + res[15].Split("\n")[1].Should().Be("Text B.C.A"); + + res[16].Split("\n")[0].Should().Be("Header B: Header B.C: Header B.C.B"); + res[16].Split("\n")[1].Should().Be("Text B.C.B"); + + res[17].Split("\n")[0].Should().Be("Header B: Header B.C: Header B.C.C"); + res[17].Split("\n")[1].Should().Be("Text B.C.C"); + } + + [Test] + public void HeaderDepthJump_RemovesEveryExitedBreadcrumb() + { + const string markdown = "# A\r\n## B\r\n### C\r\n# D\r\nText"; + + var result = new MarkdownHeaderTextSplitter().SplitText(markdown); + + result.Should().ContainSingle().Which.Should().Be("D\nText"); + } +}