Text splitter bugfixes

src/Splitters/Abstractions/src/Text/CharacterTextSplitter.cs

@@ -15,6 +15,8 @@ public override IReadOnlyList<string> SplitText(string text)
     {
         text = text ?? throw new ArgumentNullException(nameof(text));
 
+        text = NormalizeLineEndings(text);
+
         List<string> splits;
         if (separator != null)
         {
@@ -27,4 +29,4 @@ public override IReadOnlyList<string> SplitText(string text)
 
         return this.MergeSplits(splits, separator ?? string.Empty);
     }
-}
+}

src/Splitters/Abstractions/src/Text/MarkdownHeaderTextSplitter.cs

@@ -16,6 +16,7 @@ private sealed class LineType
     private const string _codeBlockseparator = "```";
     private readonly static string[] _defauldHeaders = { "#", "##", "###", "####", "#####", "######" };
     private readonly static string[] separator = { "\n" };
+    private static readonly char[] HeaderSeparator = { '|' };
 
     /// <inheritdoc/>
     public MarkdownHeaderTextSplitter(
@@ -35,9 +36,7 @@ public override IReadOnlyList<string> SplitText(string text)
         text = text ?? throw new ArgumentNullException(nameof(text));
 
         // Split the input text by newline character ("\n").
-        var lines = text
-            .Replace("\r", "") // some people are using windows
-            .Split(separator, StringSplitOptions.None);
+        var lines = NormalizeLineEndings(text).Split(separator, StringSplitOptions.None);
 
 
         var content = new List<LineType>();
@@ -74,10 +73,12 @@ public override IReadOnlyList<string> SplitText(string text)
             {
                 if (hLen <= currentHeaderLen)
                 {
-                    var existingHeader = currentHeader.Split('|');
-
-                    string prevHeader = string.Join("|", existingHeader.Take(existingHeader.Length - 1));
-                    currentHeader = prevHeader + "|" + strippedLine.TrimStart('#').Trim();
+                    var parentHeaders = currentHeader
+                        .Split(HeaderSeparator, StringSplitOptions.RemoveEmptyEntries)
+                        .Take(hLen - 1);
+                    currentHeader = string.Join(
+                        "|",
+                        parentHeaders.Concat(new[] { strippedLine.TrimStart('#').Trim() }));
                     currentHeaderLen = hLen;
                     continue;
                 }

src/Splitters/Abstractions/src/Text/RecursiveCharacterTextSplitter.cs

@@ -19,6 +19,8 @@ public override IReadOnlyList<string> SplitText(string text)
     {
         text = text ?? throw new ArgumentNullException(nameof(text));
 
+        text = NormalizeLineEndings(text);
+
         List<string> finalChunks = new List<string>();
         string separator = _separators[_separators.Count - 1];
 
@@ -78,4 +80,4 @@ public override IReadOnlyList<string> SplitText(string text)
 
         return finalChunks;
     }
-}
+}

src/Splitters/Abstractions/src/Text/TextSplitter.cs

@@ -45,6 +45,12 @@ protected TextSplitter(
     /// </summary>
     protected int ChunkOverlap => _chunkOverlap;
 
+    /// <summary>
+    /// Normalizes Windows and legacy Mac line endings before separator-based splitting.
+    /// </summary>
+    internal static string NormalizeLineEndings(string text) =>
+        text.Replace("\r\n", "\n").Replace('\r', '\n');
+
     /// <summary>
     /// 
     /// </summary>
@@ -125,4 +131,4 @@ protected IReadOnlyList<string> MergeSplits(IEnumerable<string> splits, string s
     // todo: Implement from_tiktoken_encoder
 
 
-}
+}

src/Splitters/Abstractions/test/Resources/markdown_test_material.md

@@ -0,0 +1,71 @@
+# Header A
+
+Text A
+
+## Header A.A
+
+Text A.A
+
+## Header A.B
+
+Text A.B
+
+### Header A.B.A
+
+Text A.B.A
+
+### Header A.B.B
+
+Text A.B.B
+
+### Header A.B.C
+
+Text A.B.C
+
+## Header A.C
+
+Text A.C
+
+### Header A.C.A
+
+Text A.C.A
+
+### Header A.C.B
+
+Text A.C.B
+
+# Header B
+
+Text B
+
+## Header B.A
+
+Text B.A
+
+## Header B.B
+
+Text B.B
+
+### Header B.B.A
+
+Text B.B.A
+
+### Header B.B.B
+
+Text B.B.B
+
+## Header B.C
+
+Text B.C
+
+### Header B.C.A
+
+Text B.C.A
+
+### Header B.C.B
+
+Text B.C.B
+
+### Header B.C.C
+
+Text B.C.C

src/Splitters/Abstractions/test/Tests.MarkdownHeader.cs

@@ -61,4 +61,79 @@ Hi this is Joe
         res[0].Should().Be("Hi this is Jim\nHi this is Joe");
         res[1].Should().Be("Hi this is Molly");
     }
-}
+
+    [Test]
+    public void TestMarkdown4()
+    {
+        var md = H.Resources.markdown_test_material_md.AsString();
+
+        var splitter = new MarkdownHeaderTextSplitter();
+        var res = splitter.SplitText(md);
+
+        res.Count.Should().Be(18);
+
+        res[0].Split("\n")[0].Should().Be("Header A");
+        res[0].Split("\n")[1].Should().Be("Text A");
+
+        res[1].Split("\n")[0].Should().Be("Header A: Header A.A");
+        res[1].Split("\n")[1].Should().Be("Text A.A");
+
+        res[2].Split("\n")[0].Should().Be("Header A: Header A.B");
+        res[2].Split("\n")[1].Should().Be("Text A.B");
+
+        res[3].Split("\n")[0].Should().Be("Header A: Header A.B: Header A.B.A");
+        res[3].Split("\n")[1].Should().Be("Text A.B.A");
+
+        res[4].Split("\n")[0].Should().Be("Header A: Header A.B: Header A.B.B");
+        res[4].Split("\n")[1].Should().Be("Text A.B.B");
+
+        res[5].Split("\n")[0].Should().Be("Header A: Header A.B: Header A.B.C");
+        res[5].Split("\n")[1].Should().Be("Text A.B.C");
+
+        res[6].Split("\n")[0].Should().Be("Header A: Header A.C");
+        res[6].Split("\n")[1].Should().Be("Text A.C");
+
+        res[7].Split("\n")[0].Should().Be("Header A: Header A.C: Header A.C.A");
+        res[7].Split("\n")[1].Should().Be("Text A.C.A");
+
+        res[8].Split("\n")[0].Should().Be("Header A: Header A.C: Header A.C.B");
+        res[8].Split("\n")[1].Should().Be("Text A.C.B");
+
+        res[9].Split("\n")[0].Should().Be("Header B");
+        res[9].Split("\n")[1].Should().Be("Text B");
+
+        res[10].Split("\n")[0].Should().Be("Header B: Header B.A");
+        res[10].Split("\n")[1].Should().Be("Text B.A");
+
+        res[11].Split("\n")[0].Should().Be("Header B: Header B.B");
+        res[11].Split("\n")[1].Should().Be("Text B.B");
+
+        res[12].Split("\n")[0].Should().Be("Header B: Header B.B: Header B.B.A");
+        res[12].Split("\n")[1].Should().Be("Text B.B.A");
+
+        res[13].Split("\n")[0].Should().Be("Header B: Header B.B: Header B.B.B");
+        res[13].Split("\n")[1].Should().Be("Text B.B.B");
+
+        res[14].Split("\n")[0].Should().Be("Header B: Header B.C");
+        res[14].Split("\n")[1].Should().Be("Text B.C");
+
+        res[15].Split("\n")[0].Should().Be("Header B: Header B.C: Header B.C.A");
+        res[15].Split("\n")[1].Should().Be("Text B.C.A");
+
+        res[16].Split("\n")[0].Should().Be("Header B: Header B.C: Header B.C.B");
+        res[16].Split("\n")[1].Should().Be("Text B.C.B");
+
+        res[17].Split("\n")[0].Should().Be("Header B: Header B.C: Header B.C.C");
+        res[17].Split("\n")[1].Should().Be("Text B.C.C");
+    }
+
+    [Test]
+    public void HeaderDepthJump_RemovesEveryExitedBreadcrumb()
+    {
+        const string markdown = "# A\r\n## B\r\n### C\r\n# D\r\nText";
+
+        var result = new MarkdownHeaderTextSplitter().SplitText(markdown);
+
+        result.Should().ContainSingle().Which.Should().Be("D\nText");
+    }
+}