Merge pull request #22624 from github/tausbn/unified-corpus-source-skeletons

unified: Output source code skeletons in AST dump

Author: tausbn

shared/yeast/doc/yeast.md

@@ -235,6 +235,17 @@ yeast::trees!(ctx,
 (identifier #{name})         // an identifier from a Rust variable
 ```
 
+For reviewing locations, `DumpOptions::show_abridged_source` prints each node's
+source range with every direct child replaced by its field name in Unicode
+angle brackets. This keeps delimiters and other parent-owned syntax visible
+without repeating entire subtrees:
+
+```text
+return_expr source="return ⟨value⟩"
+  value:
+    call_expr source="⟨callee⟩(⟨argument⟩)"
+```
+
 ### Optional fields (`?`)
 
 A `?` on a field's value makes that field fallible. If a `#{expr}` anywhere

shared/yeast/src/dump.rs

@@ -15,13 +15,17 @@ pub struct DumpOptions {
     pub show_locations: bool,
     /// Whether to include source text for leaf nodes.
     pub show_content: bool,
+    /// Whether to include each node's source range with direct-child ranges
+    /// replaced by their field names in `⟨angle brackets⟩`.
+    pub show_abridged_source: bool,
 }
 
 impl Default for DumpOptions {
     fn default() -> Self {
         Self {
             show_locations: false,
             show_content: true,
+            show_abridged_source: false,
         }
     }
 }
@@ -226,6 +230,10 @@ fn dump_node(
         }
     }
 
+    if options.show_abridged_source {
+        write_source_skeleton(ast, node, source, out);
+    }
+
     if let Some(context) = type_check {
         if let Some(err) =
             type_error_for_node(context.schema, node, context.expected, context.parent_field)
@@ -409,6 +417,10 @@ fn dump_node_inline(
         }
     }
 
+    if options.show_abridged_source {
+        write_source_skeleton(ast, node, source, out);
+    }
+
     if let Some(context) = type_check {
         if let Some(err) =
             type_error_for_node(context.schema, node, context.expected, context.parent_field)
@@ -424,6 +436,246 @@ fn is_leaf(node: &Node) -> bool {
     node.fields.is_empty()
 }
 
+enum SourceSkeleton {
+    Missing,
+    Text(String),
+    Invalid(String),
+}
+
+fn write_source_skeleton(ast: &Ast, node: &Node, source: &str, out: &mut String) {
+    match source_skeleton(ast, node, source) {
+        SourceSkeleton::Missing => write!(out, " source=<no location>").unwrap(),
+        SourceSkeleton::Text(text) => write!(out, " source={text:?}").unwrap(),
+        SourceSkeleton::Invalid(error) => write!(out, " source=<invalid: {error}>").unwrap(),
+    }
+}
+
+fn node_source_range(node: &Node) -> Option<crate::Range> {
+    match node.content {
+        NodeContent::Range(range) => Some(range),
+        _ => node.source_range,
+    }
+}
+
+fn source_skeleton(ast: &Ast, node: &Node, source: &str) -> SourceSkeleton {
+    let Some(parent) = node_source_range(node) else {
+        return SourceSkeleton::Missing;
+    };
+    let parent = parent.start_byte..parent.end_byte;
+    if parent.start > parent.end || source.get(parent.clone()).is_none() {
+        return SourceSkeleton::Invalid(format!(
+            "node range {}..{} is outside the source or not on UTF-8 boundaries",
+            parent.start, parent.end
+        ));
+    }
+
+    let mut children = Vec::new();
+    for (field_id, child_ids) in &node.fields {
+        let field_name = if *field_id == CHILD_FIELD {
+            "child"
+        } else {
+            ast.field_name_for_id(*field_id).unwrap_or("?")
+        };
+        for child_id in child_ids {
+            let Some(child) = ast.get_node(*child_id) else {
+                continue;
+            };
+            if !child.is_named() {
+                continue;
+            }
+            let Some(child) = node_source_range(child) else {
+                continue;
+            };
+            let child = child.start_byte..child.end_byte;
+            if child.start > child.end || source.get(child.clone()).is_none() {
+                return SourceSkeleton::Invalid(format!(
+                    "child range {}..{} is outside the source or not on UTF-8 boundaries",
+                    child.start, child.end
+                ));
+            }
+            if child.start < parent.start || child.end > parent.end {
+                return SourceSkeleton::Invalid(format!(
+                    "child range {}..{} is outside node range {}..{}",
+                    child.start, child.end, parent.start, parent.end
+                ));
+            }
+            if child.start == child.end {
+                continue;
+            }
+            children.push((child, field_name));
+        }
+    }
+    children.sort_by_key(|(range, field_name)| (range.start, range.end, *field_name));
+
+    let mut result = String::new();
+    let mut cursor = parent.start;
+    for (range, field_name) in children {
+        if cursor < range.start {
+            result.push_str(source.get(cursor..range.start).unwrap());
+        }
+        result.push('⟨');
+        result.push_str(field_name);
+        result.push('⟩');
+        cursor = cursor.max(range.end);
+    }
+    result.push_str(source.get(cursor..parent.end).unwrap());
+    SourceSkeleton::Text(result)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::{NodeContent, Point, Range};
+    use std::collections::BTreeMap;
+
+    fn range(start: usize, end: usize) -> Range {
+        Range {
+            start_byte: start,
+            end_byte: end,
+            start_point: Point::new(0, start),
+            end_point: Point::new(0, end),
+        }
+    }
+
+    fn dump_with_children(source: &str, parent_range: Range, children: &[(&str, Range)]) -> String {
+        let mut ast = Ast::with_schema(crate::schema::Schema::new());
+        let parent_kind = ast.register_kind("parent");
+        let child_kind = ast.register_kind("child");
+        let mut fields = BTreeMap::new();
+        for (field_name, range) in children {
+            let field = ast.register_field(field_name);
+            let child = ast.create_node_with_range(
+                child_kind,
+                NodeContent::Range(*range),
+                BTreeMap::new(),
+                true,
+                None,
+            );
+            fields.entry(field).or_insert_with(Vec::new).push(child);
+        }
+        let parent = ast.create_node_with_range(
+            parent_kind,
+            NodeContent::Range(parent_range),
+            fields,
+            true,
+            None,
+        );
+        ast.set_root(parent);
+
+        dump_ast_with_options(
+            &ast,
+            parent,
+            source,
+            &DumpOptions {
+                show_locations: false,
+                show_content: false,
+                show_abridged_source: true,
+            },
+        )
+    }
+
+    #[test]
+    fn source_skeleton_elides_direct_children_and_ignores_empty_ranges() {
+        let source = "αbefore(child)afterω";
+        let child_start = source.find("child").unwrap();
+        let child_end = child_start + "child".len();
+        let dump = dump_with_children(
+            source,
+            range(0, source.len()),
+            &[
+                ("value", range(child_start, child_end)),
+                ("marker", range(child_start, child_start)),
+            ],
+        );
+
+        assert!(dump.starts_with("parent source=\"αbefore(⟨value⟩)afterω\"\n"));
+    }
+
+    #[test]
+    fn source_skeleton_preserves_unnamed_tokens() {
+        let source = "x = 1";
+        let runner: crate::Runner = crate::Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]);
+        let ast = runner.run(source).unwrap();
+        let dump = dump_ast_with_options(
+            &ast,
+            ast.get_root(),
+            source,
+            &DumpOptions {
+                show_locations: false,
+                show_content: false,
+                show_abridged_source: true,
+            },
+        );
+
+        assert!(dump.contains("assignment source=\"⟨left⟩ = ⟨right⟩\""));
+    }
+
+    #[test]
+    fn source_skeleton_validates_empty_child_ranges() {
+        let cases = [
+            (
+                "abcdef",
+                range(0, 3),
+                range(4, 4),
+                "child range 4..4 is outside node range 0..3",
+            ),
+            (
+                "abcdef",
+                range(0, 6),
+                range(7, 7),
+                "child range 7..7 is outside the source or not on UTF-8 boundaries",
+            ),
+            (
+                "αbc",
+                range(0, 4),
+                range(1, 1),
+                "child range 1..1 is outside the source or not on UTF-8 boundaries",
+            ),
+        ];
+
+        for (source, parent, child, error) in cases {
+            let dump = dump_with_children(source, parent, &[("marker", child)]);
+            assert!(
+                dump.starts_with(&format!("parent source=<invalid: {error}>\n")),
+                "unexpected dump: {dump}"
+            );
+        }
+    }
+
+    #[test]
+    fn source_skeleton_keeps_adjacent_child_fields_separate() {
+        let source = "abcdef";
+        let dump = dump_with_children(
+            source,
+            range(0, source.len()),
+            &[("left", range(1, 3)), ("right", range(3, 5))],
+        );
+
+        assert!(dump.starts_with("parent source=\"a⟨left⟩⟨right⟩f\"\n"));
+    }
+
+    #[test]
+    fn source_skeleton_keeps_overlapping_child_fields_separate() {
+        let source = "abcdef";
+        let dump = dump_with_children(
+            source,
+            range(0, source.len()),
+            &[("left", range(1, 4)), ("right", range(3, 5))],
+        );
+
+        assert!(dump.starts_with("parent source=\"a⟨left⟩⟨right⟩f\"\n"));
+    }
+
+    #[test]
+    fn source_skeleton_reports_children_outside_the_parent() {
+        let source = "abcdefghi";
+        let dump = dump_with_children(source, range(0, 6), &[("child", range(7, 9))]);
+
+        assert!(dump
+            .starts_with("parent source=<invalid: child range 7..9 is outside node range 0..6>\n"));
+    }
+}
+
 fn node_content(node: &Node, source: &str) -> String {
     match &node.content {
         NodeContent::DynamicString(s) if !s.is_empty() => s.clone(),

unified/extractor/tests/corpus/swift/closures/closure-with-capture-list.output

@@ -60,24 +60,24 @@ sourceFile
 
 ---
 
-top_level
+top_level source="⟨body⟩"
   body:
-    block
+    block source="⟨stmt⟩"
       stmt:
-        variable_declaration
-          modifier: modifier "let"
-          pattern: identifier "f"
+        variable_declaration source="⟨modifier⟩ ⟨pattern⟩ = ⟨value⟩"
+          modifier: modifier "let" source="let"
+          pattern: identifier "f" source="f"
           value:
-            function_expr
+            function_expr source="⟨body⟩⟨capture_declaration⟩"
               capture_declaration:
-                variable_declaration
-                  modifier: modifier "weak"
-                  pattern: identifier "self"
+                variable_declaration source="⟨modifier⟩ ⟨pattern⟩"
+                  modifier: modifier "weak" source="weak"
+                  pattern: identifier "self" source="self"
               body:
-                block
+                block source="{ [weak self] in ⟨stmt⟩ }"
                   stmt:
-                    call_expr
+                    call_expr source="⟨callee⟩()"
                       callee:
-                        member_access_expr
-                          base: identifier "self"
-                          member_name_node: identifier "doThing"
+                        member_access_expr source="⟨base⟩?.⟨member_name_node⟩"
+                          base: identifier "self" source="self"
+                          member_name_node: identifier "doThing" source="doThing"

unified/extractor/tests/corpus/swift/closures/closure-with-explicit-parameters.output

@@ -62,24 +62,24 @@ sourceFile
 
 ---
 
-top_level
+top_level source="⟨body⟩"
   body:
-    block
+    block source="⟨stmt⟩"
       stmt:
-        variable_declaration
-          modifier: modifier "let"
-          pattern: identifier "f"
+        variable_declaration source="⟨modifier⟩ ⟨pattern⟩ = ⟨value⟩"
+          modifier: modifier "let" source="let"
+          pattern: identifier "f" source="f"
           value:
-            function_expr
+            function_expr source="⟨body⟩⟨parameter⟩⟨return_type⟩"
               parameter:
-                parameter
-                  type: identifier "Int"
-                  pattern: identifier "x"
-              return_type: identifier "Int"
+                parameter source="⟨pattern⟩: ⟨type⟩"
+                  type: identifier "Int" source="Int"
+                  pattern: identifier "x" source="x"
+              return_type: identifier "Int" source="Int"
               body:
-                block
+                block source="{ (x: Int) -> Int in ⟨stmt⟩ }"
                   stmt:
-                    binary_expr
-                      left: identifier "x"
-                      operator: infix_operator "*"
-                      right: int_literal "2"
+                    binary_expr source="⟨left⟩ ⟨operator⟩ ⟨right⟩"
+                      left: identifier "x" source="x"
+                      operator: infix_operator "*" source="*"
+                      right: int_literal "2" source="2"