diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d7b51a171d4b..836e39a37b9b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -72,6 +72,9 @@ jobs: - name: Install Rust Problem Matcher run: echo "::add-matcher::.github/rust.json" + - name: Check without building tests + run: cargo check --features in-rust-tree -p proc-macro-srv-cli + - name: Test run: cargo test --features in-rust-tree -p proc-macro-srv -p proc-macro-srv-cli -p proc-macro-api -- --quiet diff --git a/Cargo.lock b/Cargo.lock index cf4ba4404b36..c8ee61483777 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1348,9 +1348,9 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libmimalloc-sys" @@ -1867,7 +1867,8 @@ dependencies = [ "intern", "paths", "postcard", - "proc-macro-srv", + "proc-macro-api", + "ra-ap-rustc_lexer", "rayon", "rustc-hash 2.1.2", "semver", @@ -1888,10 +1889,12 @@ dependencies = [ "intern", "line-index 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "paths", + "proc-macro-api", "proc-macro-test", "rustc-hash 2.1.2", "span", "stdx", + "tt", ] [[package]] @@ -2375,9 +2378,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" +checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" [[package]] name = "rustc-hash" diff --git a/crates/cfg/src/tests.rs b/crates/cfg/src/tests.rs index 45cba042b33b..265534cf697c 100644 --- a/crates/cfg/src/tests.rs +++ b/crates/cfg/src/tests.rs @@ -23,7 +23,7 @@ fn assert_parse_result(input: &str, expected: CfgExpr) { pred_ast.syntax(), DummyTestSpanMap, DUMMY, - DocCommentDesugarMode::ProcMacro, + DocCommentDesugarMode::Keep, ); let cfg = CfgExpr::parse(&tt); assert_eq!(cfg, expected); @@ -39,7 +39,7 @@ fn check_dnf(input: &str, expect: Expect) { pred_ast.syntax(), DummyTestSpanMap, DUMMY, - DocCommentDesugarMode::ProcMacro, + DocCommentDesugarMode::Keep, ); let cfg = CfgExpr::parse(&tt); let actual = format!("#![cfg({})]", DnfExpr::new(&cfg)); @@ -57,7 +57,7 @@ fn check_why_inactive(input: &str, opts: &CfgOptions, expect: Expect) { pred_ast.syntax(), DummyTestSpanMap, DUMMY, - DocCommentDesugarMode::ProcMacro, + DocCommentDesugarMode::Keep, ); let cfg = CfgExpr::parse(&tt); let dnf = DnfExpr::new(&cfg); @@ -77,7 +77,7 @@ fn check_enable_hints(input: &str, opts: &CfgOptions, expected_hints: &[&str]) { pred_ast.syntax(), DummyTestSpanMap, DUMMY, - DocCommentDesugarMode::ProcMacro, + DocCommentDesugarMode::Keep, ); let cfg = CfgExpr::parse(&tt); let dnf = DnfExpr::new(&cfg); diff --git a/crates/hir-def/Cargo.toml b/crates/hir-def/Cargo.toml index eb1e774b8f41..9319bb409143 100644 --- a/crates/hir-def/Cargo.toml +++ b/crates/hir-def/Cargo.toml @@ -56,3 +56,6 @@ in-rust-tree = ["hir-expand/in-rust-tree"] [lints] workspace = true + +[package.metadata.cargo-machete] +ignored = ["ra-ap-rustc_parse_format"] diff --git a/crates/hir-def/src/item_tree/attrs.rs b/crates/hir-def/src/item_tree/attrs.rs index 4ee2f56be19f..f1e604f64b9c 100644 --- a/crates/hir-def/src/item_tree/attrs.rs +++ b/crates/hir-def/src/item_tree/attrs.rs @@ -21,7 +21,7 @@ use hir_expand::{ use intern::{Interned, Symbol, sym}; use syntax::{AstNode, ast}; use syntax_bridge::DocCommentDesugarMode; -use tt::token_to_literal; +use tt::literal_from_str; use crate::item_tree::lower::Ctx; @@ -64,11 +64,13 @@ impl AttrsOrCfg { ast::Meta::KeyValueMeta(meta) => { let span = span_map.span_for(path_range); let input = meta.expr().and_then(|value| { - if let ast::Expr::Literal(value) = value { - Some(Box::new(AttrInput::Literal(token_to_literal( + if let ast::Expr::Literal(value) = value + && let Ok(lit) = literal_from_str( value.token().text(), span_map.span_for(value.syntax().text_range()), - )))) + ) + { + Some(Box::new(AttrInput::Literal(lit))) } else { None } @@ -84,7 +86,8 @@ impl AttrsOrCfg { .unwrap_or_else(|| meta.syntax().clone()), span_map, span, - DocCommentDesugarMode::ProcMacro, + // FIXME: This won't be correct once we support args for macro_rules attributes. + DocCommentDesugarMode::Keep, ); let input = Some(Box::new(AttrInput::TokenTree(tt))); (span, input) diff --git a/crates/hir-def/src/lib.rs b/crates/hir-def/src/lib.rs index 77d00072fd2d..f378129c9673 100644 --- a/crates/hir-def/src/lib.rs +++ b/crates/hir-def/src/lib.rs @@ -9,11 +9,9 @@ #![cfg_attr(feature = "in-rust-tree", feature(rustc_private))] -#[cfg(feature = "in-rust-tree")] -extern crate rustc_parse_format; - -#[cfg(not(feature = "in-rust-tree"))] -extern crate ra_ap_rustc_parse_format as rustc_parse_format; +stdx::rustc_crates! { + extern crate rustc_parse_format or ra_ap_rustc_parse_format; +} pub extern crate ra_ap_rustc_abi as layout; pub extern crate ra_ap_rustc_abi as rustc_abi; diff --git a/crates/hir-def/src/macro_expansion_tests/mbe/matching.rs b/crates/hir-def/src/macro_expansion_tests/mbe/matching.rs index bbadcf8794bf..d74a8d6d12ee 100644 --- a/crates/hir-def/src/macro_expansion_tests/mbe/matching.rs +++ b/crates/hir-def/src/macro_expansion_tests/mbe/matching.rs @@ -257,3 +257,84 @@ macro_rules! m { "#]], ); } + +#[test] +fn doc_comment_is_ignored() { + check( + r#" +macro_rules! m { + ( + /// hello + ) => {}; +} + +m!(); + "#, + expect![[r#" +macro_rules! m { + ( + /// hello + ) => {}; +} + + + "#]], + ); + check( + r#" +macro_rules! m { + () => { + macro_rules! m2 { + (/** hello */) => {} + } + }; +} + +m!(); +m2!(); + "#, + expect![[r#" +macro_rules! m { + () => { + macro_rules! m2 { + (/** hello */) => {} + } + }; +} + +macro_rules !m2 { + (/** hello */ + ) = > {} +} + + "#]], + ); + check( + r#" +macro_rules! m { + ($($t:tt)*) => { + macro_rules! m2 { + ($($t)*) => {} + } + }; +} + +m!(/** hello */); +m2!(); + "#, + expect![[r#" +macro_rules! m { + ($($t:tt)*) => { + macro_rules! m2 { + ($($t)*) => {} + } + }; +} + +macro_rules !m2 { + (#[doc = r" hello "]) = > {} +} +/* error: unexpected token in input */ + "#]], + ); +} diff --git a/crates/hir-def/src/macro_expansion_tests/mod.rs b/crates/hir-def/src/macro_expansion_tests/mod.rs index 188f57f7dda7..34ec1dd94407 100644 --- a/crates/hir-def/src/macro_expansion_tests/mod.rs +++ b/crates/hir-def/src/macro_expansion_tests/mod.rs @@ -325,7 +325,7 @@ fn pretty_print_macro_expansion( (T!['{'], T!['}']) => "", (T![=], _) | (_, T![=]) => " ", (_, T!['{']) => " ", - (T![;] | T!['{'] | T!['}'], _) => "\n", + (T![;] | T!['{'] | T!['}'] | T![inner_doc_comment] | T![outer_doc_comment], _) => "\n", (_, T!['}']) => "\n", _ if (prev_kind.is_any_identifier() || prev_kind == LIFETIME_IDENT diff --git a/crates/hir-def/src/macro_expansion_tests/proc_macros.rs b/crates/hir-def/src/macro_expansion_tests/proc_macros.rs index 8c91cf6793a5..7c83431f2491 100644 --- a/crates/hir-def/src/macro_expansion_tests/proc_macros.rs +++ b/crates/hir-def/src/macro_expansion_tests/proc_macros.rs @@ -50,7 +50,8 @@ mod foo { #[attr1] #[attr2] struct S; -#[doc = " Foo"] mod foo { +/// Foo +mod foo { # ![foo] # ![doc = "123..."] # ![attr2] @@ -293,8 +294,11 @@ struct S; #[doc = "doc attr"] struct S; -#[doc = " doc string \\n with newline"] -#[doc = "\n MultiLines Doc\n MultiLines Doc\n"] +/// doc string \n with newline +/** + MultiLines Doc + MultiLines Doc +*/ #[doc = "doc attr"] struct S;"##]], ); } diff --git a/crates/hir-expand/src/attrs.rs b/crates/hir-expand/src/attrs.rs index 185298d5f937..9676408796b5 100644 --- a/crates/hir-expand/src/attrs.rs +++ b/crates/hir-expand/src/attrs.rs @@ -437,7 +437,7 @@ impl AttrId { tt.syntax(), SpanMap::RealSpanMap(&span_map), span_map.span_for_range(tt.syntax().text_range()), - DocCommentDesugarMode::ProcMacro, + DocCommentDesugarMode::Keep, ); let Some((_, _, derive_tts)) = parse_path_comma_token_tree(db, &tt).nth(derive_index as usize) diff --git a/crates/hir-expand/src/builtin/derive_macro.rs b/crates/hir-expand/src/builtin/derive_macro.rs index f67f09f0dc05..52cfb703e438 100644 --- a/crates/hir-expand/src/builtin/derive_macro.rs +++ b/crates/hir-expand/src/builtin/derive_macro.rs @@ -295,7 +295,7 @@ fn parse_adt_from_syntax( it.syntax(), tm, call_site, - DocCommentDesugarMode::ProcMacro, + DocCommentDesugarMode::Keep, ) } None => { @@ -309,7 +309,7 @@ fn parse_adt_from_syntax( it.syntax(), tm, call_site, - DocCommentDesugarMode::ProcMacro, + DocCommentDesugarMode::Keep, ) }), ast::TypeOrConstParam::Const(_) => None, @@ -322,7 +322,7 @@ fn parse_adt_from_syntax( ty.syntax(), tm, call_site, - DocCommentDesugarMode::ProcMacro, + DocCommentDesugarMode::Keep, ) }) .unwrap_or_else(|| { @@ -343,7 +343,7 @@ fn parse_adt_from_syntax( it.syntax(), tm, call_site, - DocCommentDesugarMode::ProcMacro, + DocCommentDesugarMode::Keep, ) }) .collect() @@ -380,7 +380,7 @@ fn parse_adt_from_syntax( it.syntax(), tm, call_site, - DocCommentDesugarMode::ProcMacro, + DocCommentDesugarMode::Keep, ) }) .collect(); @@ -664,7 +664,7 @@ fn coerce_shared_target( FxHashMap::default(), remove, span, - DocCommentDesugarMode::ProcMacro, + DocCommentDesugarMode::Keep, |_, _| (true, Vec::new()), ); @@ -1464,7 +1464,7 @@ fn coerce_pointee_expand( self_for_traits.syntax(), &span_map, span, - DocCommentDesugarMode::ProcMacro, + DocCommentDesugarMode::Keep, ); let info = match parse_adt_from_syntax(&adt, &span_map, span) { Ok(it) => it, diff --git a/crates/hir-expand/src/builtin/fn_macro.rs b/crates/hir-expand/src/builtin/fn_macro.rs index 44579304d364..0d4ddce03f63 100644 --- a/crates/hir-expand/src/builtin/fn_macro.rs +++ b/crates/hir-expand/src/builtin/fn_macro.rs @@ -842,7 +842,7 @@ fn include_expand( &editioned_file_id.parse(db).syntax_node(), crate::HirFileId::from(editioned_file_id).span_map(db), span, - syntax_bridge::DocCommentDesugarMode::ProcMacro, + syntax_bridge::DocCommentDesugarMode::Keep, )) } diff --git a/crates/hir-expand/src/cfg_process.rs b/crates/hir-expand/src/cfg_process.rs index 45a81d3b3650..382af41a15ea 100644 --- a/crates/hir-expand/src/cfg_process.rs +++ b/crates/hir-expand/src/cfg_process.rs @@ -297,10 +297,16 @@ pub(crate) fn attr_macro_input_to_token_tree( span_map: SpanMap<'_>, span: Span, is_derive: bool, + is_declarative: bool, censor_item_tree_attr_ids: &[AttrId], krate: Crate, ) -> (tt::TopSubtree, SyntaxFixupUndoInfo) { - let fixups = fixup::fixup_syntax(span_map, node, span, DocCommentDesugarMode::ProcMacro); + let doc_comment_mode = if is_declarative { + DocCommentDesugarMode::DesugarMbeInput + } else { + DocCommentDesugarMode::Keep + }; + let fixups = fixup::fixup_syntax(span_map, node, span, doc_comment_mode); ( syntax_bridge::syntax_node_to_token_tree_modified( node, @@ -308,7 +314,7 @@ pub(crate) fn attr_macro_input_to_token_tree( fixups.append, fixups.remove, span, - DocCommentDesugarMode::ProcMacro, + doc_comment_mode, macro_input_callback(db, is_derive, censor_item_tree_attr_ids, krate, span, span_map), ), fixups.undo_info, diff --git a/crates/hir-expand/src/declarative.rs b/crates/hir-expand/src/declarative.rs index a3c9047d764f..3337f94a0cc8 100644 --- a/crates/hir-expand/src/declarative.rs +++ b/crates/hir-expand/src/declarative.rs @@ -136,7 +136,7 @@ impl AstId { map.span_for_range( macro_rules.macro_rules_token().unwrap().text_range(), ), - DocCommentDesugarMode::Mbe, + DocCommentDesugarMode::Keep, ); mbe::DeclarativeMacro::parse_macro_rules(&tt, ctx_edition) @@ -158,14 +158,14 @@ impl AstId { args.syntax(), map, span, - DocCommentDesugarMode::Mbe, + DocCommentDesugarMode::Keep, ) }); let body = syntax_bridge::syntax_node_to_token_tree( body.syntax(), map, span, - DocCommentDesugarMode::Mbe, + DocCommentDesugarMode::Keep, ); mbe::DeclarativeMacro::parse_macro2(args.as_ref(), &body, ctx_edition) diff --git a/crates/hir-expand/src/eager.rs b/crates/hir-expand/src/eager.rs index bddec50c91c0..2c96f1bfef8a 100644 --- a/crates/hir-expand/src/eager.rs +++ b/crates/hir-expand/src/eager.rs @@ -98,7 +98,7 @@ pub fn expand_eager_macro_input( &expanded_eager_input, arg_map, *span, - DocCommentDesugarMode::Mbe, + DocCommentDesugarMode::Keep, ); subtree.set_top_subtree_delimiter_kind(crate::tt::DelimiterKind::Invisible); diff --git a/crates/hir-expand/src/fixup.rs b/crates/hir-expand/src/fixup.rs index 2fda00d6af8d..47d134f5066a 100644 --- a/crates/hir-expand/src/fixup.rs +++ b/crates/hir-expand/src/fixup.rs @@ -446,7 +446,7 @@ mod tests { span_map, &parsed.syntax_node(), span_map.span_for_range(TextRange::empty(0.into())), - DocCommentDesugarMode::Mbe, + DocCommentDesugarMode::Keep, ); let mut tt = syntax_bridge::syntax_node_to_token_tree_modified( &parsed.syntax_node(), @@ -454,7 +454,7 @@ mod tests { fixups.append, fixups.remove, span_map.span_for_range(TextRange::empty(0.into())), - DocCommentDesugarMode::Mbe, + DocCommentDesugarMode::Keep, |_, _| (true, Vec::new()), ); @@ -494,7 +494,7 @@ mod tests { &parsed.syntax_node(), span_map, span_map.span_for_range(TextRange::empty(0.into())), - DocCommentDesugarMode::Mbe, + DocCommentDesugarMode::Keep, ); assert!( check_subtree_eq(&tt, &original_as_tt), diff --git a/crates/hir-expand/src/lib.rs b/crates/hir-expand/src/lib.rs index e646c21ec8c5..0e072a0b7ee5 100644 --- a/crates/hir-expand/src/lib.rs +++ b/crates/hir-expand/src/lib.rs @@ -610,10 +610,10 @@ impl MacroCallId { tt.syntax(), map, span, - if loc.def.is_proc_macro() { - DocCommentDesugarMode::ProcMacro + if loc.def.is_declarative() { + DocCommentDesugarMode::DesugarMbeInput } else { - DocCommentDesugarMode::Mbe + DocCommentDesugarMode::Keep }, ); if loc.def.is_proc_macro() { @@ -647,6 +647,7 @@ impl MacroCallId { map, span, is_derive, + loc.def.is_declarative(), censor_item_tree_attr_ids, loc.krate, ); @@ -804,10 +805,10 @@ impl MacroCallId { speculative_args, span_map, span, - if loc.def.is_proc_macro() { - DocCommentDesugarMode::ProcMacro + if loc.def.is_declarative() { + DocCommentDesugarMode::DesugarMbeInput } else { - DocCommentDesugarMode::Mbe + DocCommentDesugarMode::Keep }, ), SyntaxFixupUndoInfo::NONE, @@ -817,7 +818,11 @@ impl MacroCallId { speculative_args, span_map, span, - DocCommentDesugarMode::ProcMacro, + if loc.def.is_declarative() { + DocCommentDesugarMode::DesugarMbeInput + } else { + DocCommentDesugarMode::Keep + }, ), SyntaxFixupUndoInfo::NONE, ), @@ -833,6 +838,7 @@ impl MacroCallId { span_map, span, true, + loc.def.is_declarative(), attr_ids, loc.krate, ) @@ -844,6 +850,7 @@ impl MacroCallId { span_map, span, false, + loc.def.is_declarative(), attr_ids, loc.krate, ) @@ -867,7 +874,11 @@ impl MacroCallId { token_tree.syntax(), span_map, span, - DocCommentDesugarMode::ProcMacro, + if loc.def.is_declarative() { + DocCommentDesugarMode::DesugarMbeInput + } else { + DocCommentDesugarMode::Keep + }, ); tree.set_top_subtree_delimiter_kind(tt::DelimiterKind::Invisible); tree.set_top_subtree_delimiter_span(tt::DelimSpan::from_single(span)); @@ -887,7 +898,11 @@ impl MacroCallId { tt.syntax(), span_map, span, - DocCommentDesugarMode::ProcMacro, + if loc.def.is_declarative() { + DocCommentDesugarMode::DesugarMbeInput + } else { + DocCommentDesugarMode::Keep + }, ); attr_arg.set_top_subtree_delimiter_kind(tt::DelimiterKind::Invisible); Some(attr_arg) @@ -1062,6 +1077,10 @@ impl MacroDefId { matches!(self.kind, MacroDefKind::ProcMacro(..)) } + pub fn is_declarative(&self) -> bool { + matches!(self.kind, MacroDefKind::Declarative(..)) + } + pub fn is_attribute(&self) -> bool { match self.kind { MacroDefKind::BuiltInAttr(..) diff --git a/crates/load-cargo/src/lib.rs b/crates/load-cargo/src/lib.rs index bf2331a40f9b..db2f6c5443f6 100644 --- a/crates/load-cargo/src/lib.rs +++ b/crates/load-cargo/src/lib.rs @@ -24,8 +24,8 @@ use ide_db::{ }; use itertools::Itertools; use proc_macro_api::{ - MacroDylib, ProcMacroClient, bidirectional_protocol::msg::{ParentSpan, SubRequest, SubResponse}, + client::{MacroDylib, ProcMacroClient}, }; use project_model::{CargoConfig, PackageRoot, ProjectManifest, ProjectWorkspace}; use span::{Span, SpanAnchor, SyntaxContext}; @@ -558,7 +558,7 @@ fn load_crate_graph_into_db( } fn expander_to_proc_macro( - expander: proc_macro_api::ProcMacro, + expander: proc_macro_api::client::ProcMacro, ignored_macros: &[Box], ) -> ProcMacro { let name = expander.name(); @@ -577,7 +577,7 @@ fn expander_to_proc_macro( } #[derive(Debug, PartialEq, Eq)] -struct Expander(proc_macro_api::ProcMacro); +struct Expander(proc_macro_api::client::ProcMacro); impl ProcMacroExpander for Expander { fn expand( diff --git a/crates/mbe/Cargo.toml b/crates/mbe/Cargo.toml index b6e55b9360a3..b2f587bf91fc 100644 --- a/crates/mbe/Cargo.toml +++ b/crates/mbe/Cargo.toml @@ -39,3 +39,6 @@ in-rust-tree = ["parser/in-rust-tree", "tt/in-rust-tree"] [lints] workspace = true + +[package.metadata.cargo-machete] +ignored = ["ra-ap-rustc_lexer"] diff --git a/crates/mbe/src/benchmark.rs b/crates/mbe/src/benchmark.rs index 603fee73064e..9002233cb60f 100644 --- a/crates/mbe/src/benchmark.rs +++ b/crates/mbe/src/benchmark.rs @@ -83,7 +83,7 @@ fn macro_rules_fixtures_tt() -> FxHashMap { rule.token_tree().unwrap().syntax(), DummyTestSpanMap, DUMMY, - DocCommentDesugarMode::Mbe, + DocCommentDesugarMode::Keep, ); (id, def_tt) }) @@ -169,8 +169,7 @@ fn invocation_fixtures( None => (), Some(kind) => panic!("Unhandled kind {kind:?}"), }, - Op::Literal(it) => builder.push(tt::Leaf::from(it.clone())), - Op::Ident(it) => builder.push(tt::Leaf::from(it.clone())), + Op::Leaf(it) => builder.push(it.clone()), Op::Punct(puncts) => { for punct in puncts.as_slice() { builder.push(tt::Leaf::from(*punct)); @@ -191,8 +190,7 @@ fn invocation_fixtures( && let Some(sep) = separator { match &**sep { - Separator::Literal(it) => builder.push(tt::Leaf::Literal(it.clone())), - Separator::Ident(it) => builder.push(tt::Leaf::Ident(it.clone())), + Separator::Leaf(it) => builder.push(it.clone()), Separator::Puncts(puncts) => { for it in puncts { builder.push(tt::Leaf::Punct(*it)) diff --git a/crates/mbe/src/expander/matcher.rs b/crates/mbe/src/expander/matcher.rs index 2f5c4876476d..8850c6d3bc41 100644 --- a/crates/mbe/src/expander/matcher.rs +++ b/crates/mbe/src/expander/matcher.rs @@ -344,6 +344,24 @@ struct MatchState<'t> { is_error: bool, } +fn token_name_eq(t1: &tt::Leaf, t2: &tt::Leaf) -> bool { + match (t1, t2) { + (tt::Leaf::Literal(t1), tt::Leaf::Literal(t2)) => { + t1.kind == t2.kind + && t1.text_and_suffix == t2.text_and_suffix + && t1.suffix_len == t2.suffix_len + } + (tt::Leaf::Ident(t1), tt::Leaf::Ident(t2)) => t1.sym == t2.sym && t1.is_raw == t2.is_raw, + (tt::Leaf::DocComment(_), tt::Leaf::DocComment(_)) => { + unreachable!("this function should not be used for doc comments") + } + (tt::Leaf::Punct(_), tt::Leaf::Punct(_)) => { + unreachable!("this function should not be used for puncts") + } + _ => false, + } +} + /// Process the matcher positions of `cur_items` until it is empty. In the process, this will /// produce more items in `next_items`, `eof_items`, and `bb_items`. /// @@ -516,30 +534,19 @@ fn match_loop_inner<'t>( } } } - OpDelimited::Op(Op::Literal(lhs)) => { - if let Ok(rhs) = src.clone().expect_leaf() { - if matches!(&rhs, tt::Leaf::Literal(it) if it.text_and_suffix == lhs.text_and_suffix) - { - item.dot.next(); - } else { - res.add_err(ExpandError::new( - *rhs.span(), - ExpandErrorKind::UnexpectedToken, - )); - item.is_error = true; - } - } else { - res.add_err(ExpandError::binding_error( - src.clone().next().map_or(delim_span.close, |it| it.first_span()), - format!("expected literal: `{lhs}`"), - )); - item.is_error = true; + OpDelimited::Op(Op::Leaf(lhs)) => { + if matches!(lhs, tt::Leaf::DocComment(_)) { + // From rustc: + // If it's a doc comment, we just ignore it and move on to the next tt in the + // matcher. This is a bug, but #95267 showed that existing programs rely on this + // behaviour, and changing it would require some care and a transition period. + item.dot.next(); + cur_items.push(item); + continue; } - try_push!(next_items, item); - } - OpDelimited::Op(Op::Ident(lhs)) => { + if let Ok(rhs) = src.clone().expect_leaf() { - if matches!(&rhs, tt::Leaf::Ident(it) if it.sym == lhs.sym) { + if token_name_eq(lhs, &rhs) { item.dot.next(); } else { res.add_err(ExpandError::new( @@ -551,7 +558,14 @@ fn match_loop_inner<'t>( } else { res.add_err(ExpandError::binding_error( src.clone().next().map_or(delim_span.close, |it| it.first_span()), - format!("expected ident: `{lhs}`"), + format!( + "expected {}: `{lhs}`", + match lhs { + tt::Leaf::Punct(_) | tt::Leaf::DocComment(_) => unreachable!(), + tt::Leaf::Literal(_) => "literal", + tt::Leaf::Ident(_) => "ident", + }, + ), )); item.is_error = true; } @@ -871,7 +885,7 @@ fn collect_vars(collector_fun: &mut impl FnMut(Symbol), pattern: &MetaTemplate) Op::Var { name, .. } => collector_fun(name.clone()), Op::Subtree { tokens, .. } => collect_vars(collector_fun, tokens), Op::Repeat { tokens, .. } => collect_vars(collector_fun, tokens), - Op::Literal(_) | Op::Ident(_) | Op::Punct(_) => {} + Op::Leaf(_) | Op::Punct(_) => {} Op::Ignore { .. } | Op::Index { .. } | Op::Count { .. } @@ -952,16 +966,8 @@ impl<'a> Iterator for OpDelimitedIter<'a> { fn expect_separator(iter: &mut TtIter<'_>, separator: &Separator) -> bool { let mut fork = iter.clone(); let ok = match separator { - Separator::Ident(lhs) => match fork.expect_ident_or_underscore() { - Ok(rhs) => rhs.sym == lhs.sym, - Err(_) => false, - }, - Separator::Literal(lhs) => match fork.expect_literal() { - Ok(rhs) => match rhs { - tt::Leaf::Literal(rhs) => rhs.text_and_suffix == lhs.text_and_suffix, - tt::Leaf::Ident(rhs) => rhs.sym == lhs.text_and_suffix, - tt::Leaf::Punct(_) => false, - }, + Separator::Leaf(lhs) => match fork.expect_leaf() { + Ok(rhs) => token_name_eq(lhs, &rhs), Err(_) => false, }, Separator::Puncts(lhs) => match fork.expect_glued_punct() { diff --git a/crates/mbe/src/expander/transcriber.rs b/crates/mbe/src/expander/transcriber.rs index 440e75eb7e1b..0e6934e293b6 100644 --- a/crates/mbe/src/expander/transcriber.rs +++ b/crates/mbe/src/expander/transcriber.rs @@ -173,16 +173,11 @@ fn expand_subtree( let mut err = None; 'ops: for op in template.iter() { match op { - Op::Literal(it) => builder.push(tt::Leaf::from({ + Op::Leaf(it) => builder.push({ let mut it = it.clone(); - marker(&mut it.span); + marker(it.span_mut()); it - })), - Op::Ident(it) => builder.push(tt::Leaf::from({ - let mut it = it.clone(); - marker(&mut it.span); - it - })), + }), Op::Punct(puncts) => { builder.extend(puncts.iter().map(|punct| { tt::Leaf::from({ @@ -508,8 +503,7 @@ fn expand_repeat( if let Some(sep) = separator { match sep { - Separator::Ident(ident) => builder.push(tt::Leaf::from(ident.clone())), - Separator::Literal(lit) => builder.push(tt::Leaf::from(lit.clone())), + Separator::Leaf(leaf) => builder.push(leaf.clone()), Separator::Puncts(puncts) => { for &punct in puncts { builder.push(tt::Leaf::from(punct)); diff --git a/crates/mbe/src/lib.rs b/crates/mbe/src/lib.rs index 6de9b4275ce2..13b71b5afdef 100644 --- a/crates/mbe/src/lib.rs +++ b/crates/mbe/src/lib.rs @@ -8,10 +8,9 @@ #![cfg_attr(feature = "in-rust-tree", feature(rustc_private))] -#[cfg(not(feature = "in-rust-tree"))] -extern crate ra_ap_rustc_lexer as rustc_lexer; -#[cfg(feature = "in-rust-tree")] -extern crate rustc_lexer; +stdx::rustc_crates! { + extern crate rustc_lexer or ra_ap_rustc_lexer; +} mod expander; mod macro_call_style; diff --git a/crates/mbe/src/parser.rs b/crates/mbe/src/parser.rs index 7c3d451d0465..4e6d3e7dd884 100644 --- a/crates/mbe/src/parser.rs +++ b/crates/mbe/src/parser.rs @@ -100,9 +100,8 @@ pub(crate) enum Op { Concat { elements: Box<[ConcatMetaVarExprElem]>, span: Span }, Repeat { tokens: MetaTemplate, kind: RepeatKind, separator: Option> }, Subtree { tokens: MetaTemplate, delimiter: tt::Delimiter }, - Literal(tt::Literal), + Leaf(tt::Leaf), Punct(Box>), - Ident(tt::Ident), } #[derive(Clone, Debug, PartialEq, Eq)] @@ -152,33 +151,20 @@ pub(crate) enum MetaVarKind { Literal, } -#[derive(Clone, Debug, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub(crate) enum Separator { - Literal(tt::Literal), - Ident(tt::Ident), + /// Putting a doc comment as the separator is not an error, but there is no way to match it - + /// doc comments in the input are desugared early, while in the macro they're kept as one token, + /// and those aren't the same. This seems like an oversight in rustc (especially given that + /// the fact that doc comments in the macro are ignored during matching is itself an oversight + /// that cannot be fixed because it'll be breaking). + /// + /// It's still possible to have them in the transcriber. + Leaf(tt::Leaf), Puncts(ArrayVec), Lifetime(tt::Punct, tt::Ident), } -// Note that when we compare a Separator, we just care about its textual value. -impl PartialEq for Separator { - fn eq(&self, other: &Separator) -> bool { - use Separator::*; - - match (self, other) { - (Ident(a), Ident(b)) => a.sym == b.sym, - (Literal(a), Literal(b)) => a.text_and_suffix == b.text_and_suffix, - (Puncts(a), Puncts(b)) if a.len() == b.len() => { - let a_iter = a.iter().map(|a| a.char); - let b_iter = b.iter().map(|b| b.char); - a_iter.eq(b_iter) - } - (Lifetime(_, a), Lifetime(_, b)) => a.sym == b.sym, - _ => false, - } - } -} - #[derive(Clone, Copy)] enum Mode { Pattern, @@ -231,11 +217,11 @@ fn next_op( TtElement::Leaf(leaf) => match leaf { tt::Leaf::Ident(ident) if ident.sym == sym::crate_ => { // We simply produce identifier `$crate` here. And it will be resolved when lowering ast to Path. - Op::Ident(tt::Ident { + Op::Leaf(tt::Leaf::Ident(tt::Ident { sym: sym::dollar_crate, span: ident.span, is_raw: tt::IdentIsRaw::No, - }) + })) } tt::Leaf::Ident(ident) => { let kind = eat_fragment_kind(edition, src, mode)?; @@ -261,7 +247,7 @@ fn next_op( Box::new(res) }), }, - tt::Leaf::Punct(_) | tt::Leaf::Literal(_) => { + tt::Leaf::Punct(_) | tt::Leaf::Literal(_) | tt::Leaf::DocComment(_) => { return Err(ParseError::expected("expected ident")); } }, @@ -270,12 +256,17 @@ fn next_op( TtElement::Leaf(tt::Leaf::Literal(it)) => { src.next().expect("first token already peeked"); - Op::Literal(it.clone()) + Op::Leaf(tt::Leaf::Literal(it)) } TtElement::Leaf(tt::Leaf::Ident(it)) => { src.next().expect("first token already peeked"); - Op::Ident(it.clone()) + Op::Leaf(tt::Leaf::Ident(it)) + } + + TtElement::Leaf(tt::Leaf::DocComment(it)) => { + src.next().expect("first token already peeked"); + Op::Leaf(tt::Leaf::DocComment(it)) } TtElement::Leaf(tt::Leaf::Punct(_)) => { @@ -356,18 +347,22 @@ fn parse_repeat(src: &mut TtIter<'_>) -> Result<(Option, RepeatKind), match tt { tt::Leaf::Ident(ident) => match separator { Separator::Puncts(puncts) if puncts.is_empty() => { - separator = Separator::Ident(ident.clone()); + separator = Separator::Leaf(tt::Leaf::Ident(ident)); } Separator::Puncts(puncts) => match puncts.as_slice() { [tt::Punct { char: '\'', .. }] => { - separator = Separator::Lifetime(puncts[0], ident.clone()); + separator = Separator::Lifetime(puncts[0], ident); } _ => return Err(ParseError::InvalidRepeat), }, _ => return Err(ParseError::InvalidRepeat), }, - tt::Leaf::Literal(_) if has_sep => return Err(ParseError::InvalidRepeat), - tt::Leaf::Literal(lit) => separator = Separator::Literal(lit.clone()), + tt::Leaf::Literal(_) | tt::Leaf::DocComment(_) if has_sep => { + return Err(ParseError::InvalidRepeat); + } + leaf @ (tt::Leaf::Literal(_) | tt::Leaf::DocComment(_)) => { + separator = Separator::Leaf(leaf) + } tt::Leaf::Punct(punct) => { let repeat_kind = match punct.char { '*' => RepeatKind::ZeroOrMore, diff --git a/crates/mbe/src/tests.rs b/crates/mbe/src/tests.rs index 9e93d1869d04..fd382141290c 100644 --- a/crates/mbe/src/tests.rs +++ b/crates/mbe/src/tests.rs @@ -138,20 +138,20 @@ struct MyTraitMap2 IDENT MyTraitMap2 1:Root[0000, 0]@8..19#ROOT2024 SUBTREE {} 0:Root[0000, 0]@48..49#ROOT2024 0:Root[0000, 0]@100..101#ROOT2024 IDENT map 0:Root[0000, 0]@58..61#ROOT2024 - PUNCH : [alone] 0:Root[0000, 0]@61..62#ROOT2024 - PUNCH : [joint] 0:Root[0000, 0]@63..64#ROOT2024 - PUNCH : [alone] 0:Root[0000, 0]@64..65#ROOT2024 + PUNCT : [alone] 0:Root[0000, 0]@61..62#ROOT2024 + PUNCT : [joint] 0:Root[0000, 0]@63..64#ROOT2024 + PUNCT : [alone] 0:Root[0000, 0]@64..65#ROOT2024 IDENT std 0:Root[0000, 0]@65..68#ROOT2024 - PUNCH : [joint] 0:Root[0000, 0]@68..69#ROOT2024 - PUNCH : [alone] 0:Root[0000, 0]@69..70#ROOT2024 + PUNCT : [joint] 0:Root[0000, 0]@68..69#ROOT2024 + PUNCT : [alone] 0:Root[0000, 0]@69..70#ROOT2024 IDENT collections 0:Root[0000, 0]@70..81#ROOT2024 - PUNCH : [joint] 0:Root[0000, 0]@81..82#ROOT2024 - PUNCH : [alone] 0:Root[0000, 0]@82..83#ROOT2024 + PUNCT : [joint] 0:Root[0000, 0]@81..82#ROOT2024 + PUNCT : [alone] 0:Root[0000, 0]@82..83#ROOT2024 IDENT HashSet 0:Root[0000, 0]@83..90#ROOT2024 - PUNCH < [alone] 0:Root[0000, 0]@90..91#ROOT2024 + PUNCT < [alone] 0:Root[0000, 0]@90..91#ROOT2024 SUBTREE () 0:Root[0000, 0]@91..92#ROOT2024 0:Root[0000, 0]@92..93#ROOT2024 - PUNCH > [joint] 0:Root[0000, 0]@93..94#ROOT2024 - PUNCH , [alone] 0:Root[0000, 0]@94..95#ROOT2024 + PUNCT > [joint] 0:Root[0000, 0]@93..94#ROOT2024 + PUNCT , [alone] 0:Root[0000, 0]@94..95#ROOT2024 struct MyTraitMap2 { map: ::std::collections::HashSet<()>, @@ -186,22 +186,22 @@ fn main() { SUBTREE () 1:Root[0000, 0]@8..9#ROOT2024 1:Root[0000, 0]@9..10#ROOT2024 SUBTREE {} 1:Root[0000, 0]@11..12#ROOT2024 1:Root[0000, 0]@61..62#ROOT2024 LITERAL Integer 1 1:Root[0000, 0]@17..18#ROOT2024 - PUNCH ; [alone] 1:Root[0000, 0]@18..19#ROOT2024 + PUNCT ; [alone] 1:Root[0000, 0]@18..19#ROOT2024 LITERAL Float 1.0 1:Root[0000, 0]@24..27#ROOT2024 - PUNCH ; [alone] 1:Root[0000, 0]@27..28#ROOT2024 + PUNCT ; [alone] 1:Root[0000, 0]@27..28#ROOT2024 SUBTREE () 1:Root[0000, 0]@33..34#ROOT2024 1:Root[0000, 0]@39..40#ROOT2024 SUBTREE () 1:Root[0000, 0]@34..35#ROOT2024 1:Root[0000, 0]@37..38#ROOT2024 LITERAL Integer 1 1:Root[0000, 0]@35..36#ROOT2024 - PUNCH , [alone] 1:Root[0000, 0]@36..37#ROOT2024 - PUNCH , [alone] 1:Root[0000, 0]@38..39#ROOT2024 - PUNCH . [alone] 1:Root[0000, 0]@40..41#ROOT2024 + PUNCT , [alone] 1:Root[0000, 0]@36..37#ROOT2024 + PUNCT , [alone] 1:Root[0000, 0]@38..39#ROOT2024 + PUNCT . [alone] 1:Root[0000, 0]@40..41#ROOT2024 LITERAL Float 0.0 1:Root[0000, 0]@41..44#ROOT2024 - PUNCH ; [alone] 1:Root[0000, 0]@44..45#ROOT2024 + PUNCT ; [alone] 1:Root[0000, 0]@44..45#ROOT2024 IDENT let 1:Root[0000, 0]@50..53#ROOT2024 IDENT x 1:Root[0000, 0]@54..55#ROOT2024 - PUNCH = [alone] 1:Root[0000, 0]@56..57#ROOT2024 + PUNCT = [alone] 1:Root[0000, 0]@56..57#ROOT2024 LITERAL Integer 1 1:Root[0000, 0]@58..59#ROOT2024 - PUNCH ; [alone] 1:Root[0000, 0]@59..60#ROOT2024 + PUNCT ; [alone] 1:Root[0000, 0]@59..60#ROOT2024 fn main(){ 1; @@ -229,12 +229,12 @@ fn expr_2021() { expect![[r#" SUBTREE $$ 1:Root[0000, 0]@0..25#ROOT2024 1:Root[0000, 0]@0..25#ROOT2024 IDENT _ 1:Root[0000, 0]@5..6#ROOT2024 - PUNCH ; [joint] 0:Root[0000, 0]@36..37#ROOT2024 + PUNCT ; [joint] 0:Root[0000, 0]@36..37#ROOT2024 SUBTREE () 0:Root[0000, 0]@34..35#ROOT2024 0:Root[0000, 0]@34..35#ROOT2024 IDENT const 1:Root[0000, 0]@12..17#ROOT2024 SUBTREE {} 1:Root[0000, 0]@18..19#ROOT2024 1:Root[0000, 0]@22..23#ROOT2024 LITERAL Integer 1 1:Root[0000, 0]@20..21#ROOT2024 - PUNCH ; [alone] 0:Root[0000, 0]@39..40#ROOT2024 + PUNCT ; [alone] 0:Root[0000, 0]@39..40#ROOT2024 _; (const { @@ -261,7 +261,7 @@ fn expr_2021() { } SUBTREE $$ 1:Root[0000, 0]@0..8#ROOT2024 1:Root[0000, 0]@0..8#ROOT2024 - PUNCH ; [alone] 0:Root[0000, 0]@39..40#ROOT2024 + PUNCT ; [alone] 0:Root[0000, 0]@39..40#ROOT2024 ;"#]], ); @@ -285,7 +285,7 @@ fn expr_2021() { } SUBTREE $$ 1:Root[0000, 0]@0..18#ROOT2024 1:Root[0000, 0]@0..18#ROOT2024 - PUNCH ; [alone] 0:Root[0000, 0]@39..40#ROOT2024 + PUNCT ; [alone] 0:Root[0000, 0]@39..40#ROOT2024 ;"#]], ); @@ -307,24 +307,24 @@ fn expr_2021() { expect![[r#" SUBTREE $$ 1:Root[0000, 0]@0..76#ROOT2024 1:Root[0000, 0]@0..76#ROOT2024 LITERAL Integer 4 1:Root[0000, 0]@5..6#ROOT2024 - PUNCH ; [joint] 0:Root[0000, 0]@41..42#ROOT2024 + PUNCT ; [joint] 0:Root[0000, 0]@41..42#ROOT2024 LITERAL Str literal 1:Root[0000, 0]@12..21#ROOT2024 - PUNCH ; [joint] 0:Root[0000, 0]@41..42#ROOT2024 + PUNCT ; [joint] 0:Root[0000, 0]@41..42#ROOT2024 SUBTREE () 0:Root[0000, 0]@39..40#ROOT2024 0:Root[0000, 0]@39..40#ROOT2024 IDENT funcall 1:Root[0000, 0]@27..34#ROOT2024 SUBTREE () 1:Root[0000, 0]@34..35#ROOT2024 1:Root[0000, 0]@35..36#ROOT2024 - PUNCH ; [joint] 0:Root[0000, 0]@41..42#ROOT2024 + PUNCT ; [joint] 0:Root[0000, 0]@41..42#ROOT2024 SUBTREE () 0:Root[0000, 0]@39..40#ROOT2024 0:Root[0000, 0]@39..40#ROOT2024 IDENT future 1:Root[0000, 0]@42..48#ROOT2024 - PUNCH . [alone] 1:Root[0000, 0]@48..49#ROOT2024 + PUNCT . [alone] 1:Root[0000, 0]@48..49#ROOT2024 IDENT await 1:Root[0000, 0]@49..54#ROOT2024 - PUNCH ; [joint] 0:Root[0000, 0]@41..42#ROOT2024 + PUNCT ; [joint] 0:Root[0000, 0]@41..42#ROOT2024 SUBTREE () 0:Root[0000, 0]@39..40#ROOT2024 0:Root[0000, 0]@39..40#ROOT2024 IDENT break 1:Root[0000, 0]@60..65#ROOT2024 - PUNCH ' [joint] 1:Root[0000, 0]@66..67#ROOT2024 + PUNCT ' [joint] 1:Root[0000, 0]@66..67#ROOT2024 IDENT foo 1:Root[0000, 0]@67..70#ROOT2024 IDENT bar 1:Root[0000, 0]@71..74#ROOT2024 - PUNCH ; [alone] 0:Root[0000, 0]@44..45#ROOT2024 + PUNCT ; [alone] 0:Root[0000, 0]@44..45#ROOT2024 4; "literal"; @@ -352,7 +352,7 @@ fn expr_2021() { } SUBTREE $$ 1:Root[0000, 0]@0..8#ROOT2024 1:Root[0000, 0]@0..8#ROOT2024 - PUNCH ; [alone] 0:Root[0000, 0]@44..45#ROOT2024 + PUNCT ; [alone] 0:Root[0000, 0]@44..45#ROOT2024 ;"#]], ); @@ -371,7 +371,7 @@ fn minus_belongs_to_literal() { "-1", expect![[r#" SUBTREE $$ 1:Root[0000, 0]@0..2#ROOT2024 1:Root[0000, 0]@0..2#ROOT2024 - PUNCH - [alone] 0:Root[0000, 0]@10..11#ROOT2024 + PUNCT - [alone] 0:Root[0000, 0]@10..11#ROOT2024 LITERAL Integer 1 0:Root[0000, 0]@11..12#ROOT2024 -1"#]], @@ -380,7 +380,7 @@ fn minus_belongs_to_literal() { "- 1", expect![[r#" SUBTREE $$ 1:Root[0000, 0]@0..3#ROOT2024 1:Root[0000, 0]@0..3#ROOT2024 - PUNCH - [alone] 0:Root[0000, 0]@10..11#ROOT2024 + PUNCT - [alone] 0:Root[0000, 0]@10..11#ROOT2024 LITERAL Integer 1 0:Root[0000, 0]@11..12#ROOT2024 -1"#]], @@ -389,7 +389,7 @@ fn minus_belongs_to_literal() { "-2", expect![[r#" SUBTREE $$ 1:Root[0000, 0]@0..2#ROOT2024 1:Root[0000, 0]@0..2#ROOT2024 - PUNCH - [alone] 0:Root[0000, 0]@25..26#ROOT2024 + PUNCT - [alone] 0:Root[0000, 0]@25..26#ROOT2024 LITERAL Integer 2 0:Root[0000, 0]@27..28#ROOT2024 -2"#]], @@ -398,7 +398,7 @@ fn minus_belongs_to_literal() { "- 2", expect![[r#" SUBTREE $$ 1:Root[0000, 0]@0..3#ROOT2024 1:Root[0000, 0]@0..3#ROOT2024 - PUNCH - [alone] 0:Root[0000, 0]@25..26#ROOT2024 + PUNCT - [alone] 0:Root[0000, 0]@25..26#ROOT2024 LITERAL Integer 2 0:Root[0000, 0]@27..28#ROOT2024 -2"#]], @@ -407,7 +407,7 @@ fn minus_belongs_to_literal() { "-3.0", expect![[r#" SUBTREE $$ 1:Root[0000, 0]@0..4#ROOT2024 1:Root[0000, 0]@0..4#ROOT2024 - PUNCH - [alone] 0:Root[0000, 0]@43..44#ROOT2024 + PUNCT - [alone] 0:Root[0000, 0]@43..44#ROOT2024 LITERAL Float 3.0 0:Root[0000, 0]@45..48#ROOT2024 -3.0"#]], @@ -416,7 +416,7 @@ fn minus_belongs_to_literal() { "- 3.0", expect![[r#" SUBTREE $$ 1:Root[0000, 0]@0..5#ROOT2024 1:Root[0000, 0]@0..5#ROOT2024 - PUNCH - [alone] 0:Root[0000, 0]@43..44#ROOT2024 + PUNCT - [alone] 0:Root[0000, 0]@43..44#ROOT2024 LITERAL Float 3.0 0:Root[0000, 0]@45..48#ROOT2024 -3.0"#]], @@ -433,7 +433,7 @@ fn minus_belongs_to_literal() { "@-1", expect![[r#" SUBTREE $$ 1:Root[0000, 0]@0..3#ROOT2024 1:Root[0000, 0]@0..3#ROOT2024 - PUNCH - [alone] 1:Root[0000, 0]@1..2#ROOT2024 + PUNCT - [alone] 1:Root[0000, 0]@1..2#ROOT2024 LITERAL Integer 1 1:Root[0000, 0]@2..3#ROOT2024 -1"#]], @@ -450,7 +450,7 @@ fn minus_belongs_to_literal() { "@-1.0", expect![[r#" SUBTREE $$ 1:Root[0000, 0]@0..5#ROOT2024 1:Root[0000, 0]@0..5#ROOT2024 - PUNCH - [alone] 1:Root[0000, 0]@1..2#ROOT2024 + PUNCT - [alone] 1:Root[0000, 0]@1..2#ROOT2024 LITERAL Float 1.0 1:Root[0000, 0]@2..5#ROOT2024 -1.0"#]], diff --git a/crates/parser/Cargo.toml b/crates/parser/Cargo.toml index 2bdf8d76fbc6..8710f2c51228 100644 --- a/crates/parser/Cargo.toml +++ b/crates/parser/Cargo.toml @@ -19,6 +19,7 @@ rustc-literal-escaper.workspace = true tracing.workspace = true edition.workspace = true +stdx.workspace = true winnow = { version = "0.7.13", default-features = false } [dev-dependencies] @@ -32,3 +33,6 @@ in-rust-tree = [] [lints] workspace = true + +[package.metadata.cargo-machete] +ignored = ["ra-ap-rustc_lexer"] diff --git a/crates/parser/src/lib.rs b/crates/parser/src/lib.rs index 5900d7cfeda9..ba9b3d39a8f5 100644 --- a/crates/parser/src/lib.rs +++ b/crates/parser/src/lib.rs @@ -21,12 +21,12 @@ #![allow(rustdoc::private_intra_doc_links)] #![cfg_attr(feature = "in-rust-tree", feature(rustc_private))] -#[cfg(not(feature = "in-rust-tree"))] -extern crate ra_ap_rustc_lexer as rustc_lexer; #[cfg(feature = "in-rust-tree")] extern crate rustc_driver as _; -#[cfg(feature = "in-rust-tree")] -extern crate rustc_lexer; + +stdx::rustc_crates! { + extern crate rustc_lexer or ra_ap_rustc_lexer; +} mod event; mod frontmatter; diff --git a/crates/proc-macro-api/Cargo.toml b/crates/proc-macro-api/Cargo.toml index 7342e0ecdcf6..e88f43178868 100644 --- a/crates/proc-macro-api/Cargo.toml +++ b/crates/proc-macro-api/Cargo.toml @@ -22,20 +22,36 @@ indexmap.workspace = true # local deps paths = { workspace = true, features = ["serde1"] } -tt.workspace = true +tt = { path = "../tt", version = "0.0.0", default-features = false } stdx.workspace = true -proc-macro-srv = {workspace = true, optional = true} # span = {workspace = true, default-features = false} does not work -span = { path = "../span", version = "0.0.0", default-features = false} +span = { path = "../span", version = "0.0.0", default-features = false } + +ra-ap-rustc_lexer.workspace = true intern.workspace = true postcard.workspace = true semver.workspace = true rayon.workspace = true +[dev-dependencies] +# Enable both features for test. +proc-macro-api = { path = "../proc-macro-api", features = [ + "in-ra", + "in-proc-macro-srv", +] } + [features] -in-rust-tree = ["proc-macro-srv", "proc-macro-srv/in-rust-tree"] -default = [] +default = ["in-ra"] +in-rust-tree = ["tt/in-rust-tree"] +in-ra = ["tt/in-ra"] +in-proc-macro-srv = [] [lints] workspace = true + +[package.metadata.rust-analyzer] +rustc_private = true + +[package.metadata.cargo-machete] +ignored = ["ra-ap-rustc_lexer"] diff --git a/crates/proc-macro-api/src/bidirectional_protocol.rs b/crates/proc-macro-api/src/bidirectional_protocol.rs index f070b1c9a334..c475ebfbbe43 100644 --- a/crates/proc-macro-api/src/bidirectional_protocol.rs +++ b/crates/proc-macro-api/src/bidirectional_protocol.rs @@ -1,214 +1,8 @@ //! Bidirectional protocol methods -use std::{ - io::{self, BufRead, Write}, - panic::{AssertUnwindSafe, catch_unwind}, - sync::Arc, -}; - -use paths::AbsPath; -use span::Span; - -use crate::{ - ProcMacro, ProcMacroKind, ServerError, - bidirectional_protocol::msg::{ - ApiVersionCheck, BidirectionalMessage, ExpandMacro, ExpandMacroData, ExpnGlobals, - ListMacros, Request, Response, SubRequest, SubResponse, - }, - legacy_protocol::{ - SpanMode, - msg::{ - FlatTree, ServerConfig, SpanDataIndexMap, deserialize_span_data_index_map, - serialize_span_data_index_map, - }, - }, - process::ProcMacroServerProcess, - transport::postcard, -}; - pub mod msg; +#[cfg(feature = "in-ra")] +mod sender; -pub type SubCallback<'a> = &'a dyn Fn(SubRequest) -> Result; - -pub fn run_conversation( - writer: &mut dyn Write, - reader: &mut dyn BufRead, - buf: &mut Vec, - msg: BidirectionalMessage, - callback: SubCallback<'_>, -) -> Result { - let encoded = postcard::encode(&msg).map_err(wrap_encode)?; - postcard::write(writer, &encoded).map_err(wrap_io("failed to write initial request"))?; - - loop { - let maybe_buf = postcard::read(reader, buf).map_err(wrap_io("failed to read message"))?; - let Some(b) = maybe_buf else { - return Err(ServerError { - message: "proc-macro server closed the stream".into(), - io: Some(Arc::new(io::Error::new(io::ErrorKind::UnexpectedEof, "closed"))), - }); - }; - - let msg: BidirectionalMessage = postcard::decode(b).map_err(wrap_decode)?; - - match msg { - BidirectionalMessage::Response(response) => { - return Ok(BidirectionalMessage::Response(response)); - } - BidirectionalMessage::SubRequest(sr) => { - // TODO: Avoid `AssertUnwindSafe` by making the callback `UnwindSafe` once `SourceDatabase` - // becomes unwind-safe (currently blocked by `parking_lot::RwLock` in the VFS). - let resp = match catch_unwind(AssertUnwindSafe(|| callback(sr))) { - Ok(Ok(resp)) => BidirectionalMessage::SubResponse(resp), - Ok(Err(err)) => BidirectionalMessage::SubResponse(SubResponse::Cancel { - reason: err.to_string(), - }), - Err(_) => BidirectionalMessage::SubResponse(SubResponse::Cancel { - reason: "callback panicked or was cancelled".into(), - }), - }; - - let encoded = postcard::encode(&resp).map_err(wrap_encode)?; - postcard::write(writer, &encoded) - .map_err(wrap_io("failed to write sub-response"))?; - } - _ => { - return Err(ServerError { - message: format!("unexpected message {:?}", msg), - io: None, - }); - } - } - } -} - -fn wrap_io(msg: &'static str) -> impl Fn(io::Error) -> ServerError { - move |err| ServerError { message: msg.into(), io: Some(Arc::new(err)) } -} - -fn wrap_encode(err: io::Error) -> ServerError { - ServerError { message: "failed to encode message".into(), io: Some(Arc::new(err)) } -} - -fn wrap_decode(err: io::Error) -> ServerError { - ServerError { message: "failed to decode message".into(), io: Some(Arc::new(err)) } -} - -pub(crate) fn version_check( - srv: &ProcMacroServerProcess, - callback: SubCallback<'_>, -) -> Result { - let request = BidirectionalMessage::Request(Request::ApiVersionCheck(ApiVersionCheck {})); - - let response_payload = run_request(srv, request, callback)?; - - match response_payload { - BidirectionalMessage::Response(Response::ApiVersionCheck(version)) => Ok(version), - other => { - Err(ServerError { message: format!("unexpected response: {:?}", other), io: None }) - } - } -} - -/// Enable support for rust-analyzer span mode if the server supports it. -pub(crate) fn enable_rust_analyzer_spans( - srv: &ProcMacroServerProcess, - callback: SubCallback<'_>, -) -> Result { - let request = BidirectionalMessage::Request(Request::SetConfig(ServerConfig { - span_mode: SpanMode::RustAnalyzer, - })); - - let response_payload = run_request(srv, request, callback)?; - - match response_payload { - BidirectionalMessage::Response(Response::SetConfig(ServerConfig { span_mode })) => { - Ok(span_mode) - } - _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), - } -} - -/// Finds proc-macros in a given dynamic library. -pub(crate) fn find_proc_macros( - srv: &ProcMacroServerProcess, - dylib_path: &AbsPath, - callback: SubCallback<'_>, -) -> Result, String>, ServerError> { - let request = BidirectionalMessage::Request(Request::ListMacros(ListMacros { - dylib_path: dylib_path.to_path_buf().into(), - })); - - let response_payload = run_request(srv, request, callback)?; - - match response_payload { - BidirectionalMessage::Response(Response::ListMacros(it)) => Ok(it), - _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), - } -} - -pub(crate) fn expand( - proc_macro: &ProcMacro, - process: &ProcMacroServerProcess, - subtree: tt::SubtreeView<'_>, - attr: Option>, - env: Vec<(String, String)>, - def_site: Span, - call_site: Span, - mixed_site: Span, - current_dir: String, - callback: SubCallback<'_>, -) -> Result, crate::ServerError> { - let version = process.version(); - let mut span_data_table = SpanDataIndexMap::default(); - let def_site = span_data_table.insert_full(def_site).0; - let call_site = span_data_table.insert_full(call_site).0; - let mixed_site = span_data_table.insert_full(mixed_site).0; - let task = BidirectionalMessage::Request(Request::ExpandMacro(Box::new(ExpandMacro { - data: ExpandMacroData { - macro_body: FlatTree::from_subtree(subtree, version, &mut span_data_table), - macro_name: proc_macro.name.to_string(), - attributes: attr - .map(|subtree| FlatTree::from_subtree(subtree, version, &mut span_data_table)), - has_global_spans: ExpnGlobals { def_site, call_site, mixed_site }, - span_data_table: if process.rust_analyzer_spans() { - serialize_span_data_index_map(&span_data_table) - } else { - Vec::new() - }, - }, - lib: proc_macro.dylib_path.to_path_buf().into(), - env, - current_dir: Some(current_dir), - }))); - - let response_payload = run_request(process, task, callback)?; - - match response_payload { - BidirectionalMessage::Response(Response::ExpandMacro(it)) => Ok(it - .map(|resp| { - FlatTree::to_subtree_resolved( - resp.tree, - version, - &deserialize_span_data_index_map(&resp.span_data_table), - ) - }) - .map_err(|msg| msg.0)), - _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), - } -} - -fn run_request( - srv: &ProcMacroServerProcess, - msg: BidirectionalMessage, - callback: SubCallback<'_>, -) -> Result { - if let Some(err) = srv.exited() { - return Err(err.clone()); - } - srv.run_bidirectional(msg, callback) -} - -pub fn reject_subrequests(req: SubRequest) -> Result { - Err(ServerError { message: format!("{req:?} sub-request not supported here"), io: None }) -} +#[cfg(feature = "in-ra")] +pub use self::sender::*; diff --git a/crates/proc-macro-api/src/bidirectional_protocol/msg.rs b/crates/proc-macro-api/src/bidirectional_protocol/msg.rs index 22aac04c28e0..05c576e63e0c 100644 --- a/crates/proc-macro-api/src/bidirectional_protocol/msg.rs +++ b/crates/proc-macro-api/src/bidirectional_protocol/msg.rs @@ -1,4 +1,7 @@ //! Bidirectional protocol messages + +#![expect(clippy::large_enum_variant, reason = "this is just for serialization")] + use std::{ io::{self, BufRead, Write}, ops::Range, @@ -10,7 +13,8 @@ use serde::{Deserialize, Serialize}; use crate::{ ProcMacroKind, - legacy_protocol::msg::{FlatTree, Message, PanicMessage, ServerConfig}, + flat::FlatTree, + legacy_protocol::msg::{Message, PanicMessage, ServerConfig}, transport::postcard, }; diff --git a/crates/proc-macro-api/src/bidirectional_protocol/sender.rs b/crates/proc-macro-api/src/bidirectional_protocol/sender.rs new file mode 100644 index 000000000000..0884b6496626 --- /dev/null +++ b/crates/proc-macro-api/src/bidirectional_protocol/sender.rs @@ -0,0 +1,211 @@ +//! Functions for the sender side, i.e. the rust-analyzer side. + +use std::{ + io::{self, BufRead, Write}, + panic::{AssertUnwindSafe, catch_unwind}, + sync::Arc, +}; + +use paths::AbsPath; +use span::Span; + +use crate::{ + ProcMacroKind, + bidirectional_protocol::msg::{ + ApiVersionCheck, BidirectionalMessage, ExpandMacro, ExpandMacroData, ExpnGlobals, + ListMacros, Request, Response, SubRequest, SubResponse, + }, + client::{ProcMacro, ServerError}, + flat::{ + FlatTree, SpanDataIndexMap, deserialize_span_data_index_map, serialize_span_data_index_map, + }, + legacy_protocol::msg::{ServerConfig, SpanMode}, + process::ProcMacroServerProcess, + transport::postcard, +}; + +pub type SubCallback<'a> = &'a dyn Fn(SubRequest) -> Result; + +pub fn run_conversation( + writer: &mut dyn Write, + reader: &mut dyn BufRead, + buf: &mut Vec, + msg: BidirectionalMessage, + callback: SubCallback<'_>, +) -> Result { + let encoded = postcard::encode(&msg).map_err(wrap_encode)?; + postcard::write(writer, &encoded).map_err(wrap_io("failed to write initial request"))?; + + loop { + let maybe_buf = postcard::read(reader, buf).map_err(wrap_io("failed to read message"))?; + let Some(b) = maybe_buf else { + return Err(ServerError { + message: "proc-macro server closed the stream".into(), + io: Some(Arc::new(io::Error::new(io::ErrorKind::UnexpectedEof, "closed"))), + }); + }; + + let msg: BidirectionalMessage = postcard::decode(b).map_err(wrap_decode)?; + + match msg { + BidirectionalMessage::Response(response) => { + return Ok(BidirectionalMessage::Response(response)); + } + BidirectionalMessage::SubRequest(sr) => { + // TODO: Avoid `AssertUnwindSafe` by making the callback `UnwindSafe` once `SourceDatabase` + // becomes unwind-safe (currently blocked by `parking_lot::RwLock` in the VFS). + let resp = match catch_unwind(AssertUnwindSafe(|| callback(sr))) { + Ok(Ok(resp)) => BidirectionalMessage::SubResponse(resp), + Ok(Err(err)) => BidirectionalMessage::SubResponse(SubResponse::Cancel { + reason: err.to_string(), + }), + Err(_) => BidirectionalMessage::SubResponse(SubResponse::Cancel { + reason: "callback panicked or was cancelled".into(), + }), + }; + + let encoded = postcard::encode(&resp).map_err(wrap_encode)?; + postcard::write(writer, &encoded) + .map_err(wrap_io("failed to write sub-response"))?; + } + _ => { + return Err(ServerError { + message: format!("unexpected message {:?}", msg), + io: None, + }); + } + } + } +} + +fn wrap_io(msg: &'static str) -> impl Fn(io::Error) -> ServerError { + move |err| ServerError { message: msg.into(), io: Some(Arc::new(err)) } +} + +fn wrap_encode(err: io::Error) -> ServerError { + ServerError { message: "failed to encode message".into(), io: Some(Arc::new(err)) } +} + +fn wrap_decode(err: io::Error) -> ServerError { + ServerError { message: "failed to decode message".into(), io: Some(Arc::new(err)) } +} + +pub(crate) fn version_check( + srv: &ProcMacroServerProcess, + callback: SubCallback<'_>, +) -> Result { + let request = BidirectionalMessage::Request(Request::ApiVersionCheck(ApiVersionCheck {})); + + let response_payload = run_request(srv, request, callback)?; + + match response_payload { + BidirectionalMessage::Response(Response::ApiVersionCheck(version)) => Ok(version), + other => { + Err(ServerError { message: format!("unexpected response: {:?}", other), io: None }) + } + } +} + +/// Enable support for rust-analyzer span mode if the server supports it. +pub(crate) fn enable_rust_analyzer_spans( + srv: &ProcMacroServerProcess, + callback: SubCallback<'_>, +) -> Result { + let request = BidirectionalMessage::Request(Request::SetConfig(ServerConfig { + span_mode: SpanMode::RustAnalyzer, + })); + + let response_payload = run_request(srv, request, callback)?; + + match response_payload { + BidirectionalMessage::Response(Response::SetConfig(ServerConfig { span_mode })) => { + Ok(span_mode) + } + _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), + } +} + +/// Finds proc-macros in a given dynamic library. +pub(crate) fn find_proc_macros( + srv: &ProcMacroServerProcess, + dylib_path: &AbsPath, + callback: SubCallback<'_>, +) -> Result, String>, ServerError> { + let request = BidirectionalMessage::Request(Request::ListMacros(ListMacros { + dylib_path: dylib_path.to_path_buf().into(), + })); + + let response_payload = run_request(srv, request, callback)?; + + match response_payload { + BidirectionalMessage::Response(Response::ListMacros(it)) => Ok(it), + _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), + } +} + +pub(crate) fn expand( + proc_macro: &ProcMacro, + process: &ProcMacroServerProcess, + subtree: tt::SubtreeView<'_>, + attr: Option>, + env: Vec<(String, String)>, + def_site: Span, + call_site: Span, + mixed_site: Span, + current_dir: String, + callback: SubCallback<'_>, +) -> Result, ServerError> { + let version = process.version(); + let mut span_data_table = SpanDataIndexMap::default(); + let def_site = span_data_table.insert_full(def_site).0; + let call_site = span_data_table.insert_full(call_site).0; + let mixed_site = span_data_table.insert_full(mixed_site).0; + let task = BidirectionalMessage::Request(Request::ExpandMacro(Box::new(ExpandMacro { + data: ExpandMacroData { + macro_body: FlatTree::from_subtree(subtree, version, &mut span_data_table), + macro_name: proc_macro.name.to_string(), + attributes: attr + .map(|subtree| FlatTree::from_subtree(subtree, version, &mut span_data_table)), + has_global_spans: ExpnGlobals { def_site, call_site, mixed_site }, + span_data_table: if process.rust_analyzer_spans() { + serialize_span_data_index_map(&span_data_table) + } else { + Vec::new() + }, + }, + lib: proc_macro.dylib_path.to_path_buf().into(), + env, + current_dir: Some(current_dir), + }))); + + let response_payload = run_request(process, task, callback)?; + + match response_payload { + BidirectionalMessage::Response(Response::ExpandMacro(it)) => Ok(it + .map(|resp| { + FlatTree::to_subtree( + resp.tree, + version, + &deserialize_span_data_index_map(&resp.span_data_table), + ) + }) + .map_err(|msg| msg.0)), + _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), + } +} + +fn run_request( + srv: &ProcMacroServerProcess, + msg: BidirectionalMessage, + callback: SubCallback<'_>, +) -> Result { + if let Some(err) = srv.exited() { + return Err(err.clone()); + } + + crate::flat::with_serialization_version(srv.version(), || srv.run_bidirectional(msg, callback)) +} + +pub fn reject_subrequests(req: SubRequest) -> Result { + Err(ServerError { message: format!("{req:?} sub-request not supported here"), io: None }) +} diff --git a/crates/proc-macro-api/src/client.rs b/crates/proc-macro-api/src/client.rs new file mode 100644 index 000000000000..23638850331f --- /dev/null +++ b/crates/proc-macro-api/src/client.rs @@ -0,0 +1,214 @@ +//! Definitions and operations for the proc macro client operated in rust-analyzer. + +use paths::{AbsPath, AbsPathBuf}; +use semver::Version; +use span::{ErasedFileAstId, FIXUP_ERASED_FILE_AST_ID_MARKER, Span}; +use std::{fmt, io, sync::Arc, time::SystemTime}; + +use crate::{ + ProcMacroKind, ProtocolFormat, + bidirectional_protocol::SubCallback, + pool::ProcMacroServerPool, + process::{self, ProcMacroServerProcess}, + version, +}; + +/// A handle to proc-macro server process pool which load dylibs with macros (.so or .dll) +/// and runs actual macro expansion functions. +#[derive(Debug, Clone)] +pub struct ProcMacroClient { + /// Currently, the proc macro process expands all procedural macros sequentially. + /// + /// That means that concurrent salsa requests may block each other when expanding proc macros, + /// which is unfortunate, but simple and good enough for the time being. + pool: Arc, + /// The path to the proc-macro server binary. + path: AbsPathBuf, +} + +/// Represents a dynamically loaded library containing procedural macros. +pub struct MacroDylib { + pub(crate) path: AbsPathBuf, +} + +impl MacroDylib { + /// Creates a new MacroDylib instance with the given path. + pub fn new(path: AbsPathBuf) -> MacroDylib { + MacroDylib { path } + } +} + +/// A handle to a specific proc-macro (a `#[proc_macro]` annotated function). +/// +/// It exists within the context of a specific proc-macro server -- currently +/// we share a single expander process for all macros within a workspace. +#[derive(Debug, Clone)] +pub struct ProcMacro { + pub(crate) pool: ProcMacroServerPool, + pub(crate) dylib_path: Arc, + pub(crate) name: Box, + pub(crate) kind: ProcMacroKind, + pub(crate) dylib_last_modified: Option, +} + +impl Eq for ProcMacro {} +impl PartialEq for ProcMacro { + fn eq(&self, other: &Self) -> bool { + self.name == other.name + && self.kind == other.kind + && self.dylib_path == other.dylib_path + && self.dylib_last_modified == other.dylib_last_modified + } +} + +/// Represents errors encountered when communicating with the proc-macro server. +#[derive(Clone, Debug)] +pub struct ServerError { + pub message: String, + pub io: Option>, +} + +impl fmt::Display for ServerError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.message.fmt(f)?; + if let Some(io) = &self.io { + f.write_str(": ")?; + io.fmt(f)?; + } + Ok(()) + } +} + +impl ProcMacroClient { + /// Spawns an external process as the proc macro server and returns a client connected to it. + pub fn spawn<'a>( + process_path: &AbsPath, + env: impl IntoIterator< + Item = (impl AsRef, &'a Option>), + > + Clone, + version: Option<&Version>, + num_process: usize, + ) -> io::Result { + let pool_size = num_process; + let mut workers = Vec::with_capacity(pool_size); + for _ in 0..pool_size { + let worker = ProcMacroServerProcess::spawn(process_path, env.clone(), version)?; + workers.push(worker); + } + + let pool = ProcMacroServerPool::new(workers); + Ok(ProcMacroClient { pool: Arc::new(pool), path: process_path.to_owned() }) + } + + /// Invokes `spawn` and returns a client connected to the resulting read and write handles. + /// + /// The `process_path` is used for `Self::server_path`. This function is mainly used for testing. + pub fn with_io_channels( + process_path: &AbsPath, + spawn: impl Fn( + Option, + ) -> io::Result<( + Box, + Box, + Box, + )> + Clone, + version: Option<&Version>, + num_process: usize, + ) -> io::Result { + let pool_size = num_process; + let mut workers = Vec::with_capacity(pool_size); + for _ in 0..pool_size { + let worker = + ProcMacroServerProcess::run(spawn.clone(), version, || "".to_owned())?; + workers.push(worker); + } + + let pool = ProcMacroServerPool::new(workers); + Ok(ProcMacroClient { pool: Arc::new(pool), path: process_path.to_owned() }) + } + + /// Returns the absolute path to the proc-macro server. + pub fn server_path(&self) -> &AbsPath { + &self.path + } + + /// Loads a proc-macro dylib into the server process returning a list of `ProcMacro`s loaded. + pub fn load_dylib(&self, dylib: MacroDylib) -> Result, ServerError> { + self.pool.load_dylib(&dylib) + } + + /// Checks if the proc-macro server has exited. + pub fn exited(&self) -> Option<&ServerError> { + self.pool.exited() + } +} + +impl ProcMacro { + /// Returns the name of the procedural macro. + pub fn name(&self) -> &str { + &self.name + } + + /// Returns the type of procedural macro. + pub fn kind(&self) -> ProcMacroKind { + self.kind + } + + pub(crate) fn needs_fixup_change(&self) -> bool { + let version = self.pool.version(); + (version::RUST_ANALYZER_SPAN_SUPPORT..version::HASHED_AST_ID).contains(&version) + } + + /// On some server versions, the fixup ast id is different than ours. So change it to match. + pub(crate) fn change_fixup_to_match_old_server(&self, tt: &mut tt::TopSubtree) { + const OLD_FIXUP_AST_ID: ErasedFileAstId = ErasedFileAstId::from_raw(!0 - 1); + tt.change_every_ast_id(|ast_id| { + if *ast_id == FIXUP_ERASED_FILE_AST_ID_MARKER { + *ast_id = OLD_FIXUP_AST_ID; + } else if *ast_id == OLD_FIXUP_AST_ID { + // Swap between them, that means no collision plus the change can be reversed by doing itself. + *ast_id = FIXUP_ERASED_FILE_AST_ID_MARKER; + } + }); + } + + /// Expands the procedural macro by sending an expansion request to the server. + /// This includes span information and environmental context. + pub fn expand( + &self, + subtree: tt::SubtreeView<'_>, + attr: Option>, + env: Vec<(String, String)>, + def_site: Span, + call_site: Span, + mixed_site: Span, + current_dir: String, + callback: Option>, + ) -> Result, ServerError> { + let (mut subtree, mut attr) = (subtree, attr); + let (mut subtree_changed, mut attr_changed); + if self.needs_fixup_change() { + subtree_changed = tt::TopSubtree::from_subtree(subtree); + self.change_fixup_to_match_old_server(&mut subtree_changed); + subtree = subtree_changed.view(); + + if let Some(attr) = &mut attr { + attr_changed = tt::TopSubtree::from_subtree(*attr); + self.change_fixup_to_match_old_server(&mut attr_changed); + *attr = attr_changed.view(); + } + } + + self.pool.pick_process()?.expand( + self, + subtree, + attr, + env, + def_site, + call_site, + mixed_site, + current_dir, + callback, + ) + } +} diff --git a/crates/proc-macro-api/src/flat.rs b/crates/proc-macro-api/src/flat.rs new file mode 100644 index 000000000000..775c857aa5c4 --- /dev/null +++ b/crates/proc-macro-api/src/flat.rs @@ -0,0 +1,856 @@ +//! Serialization-friendly representation of `tt::TopSubtree`. +//! +//! It is possible to serialize `TopSubtree` recursively, as a tree, but using +//! arbitrary-nested trees in JSON is problematic, as they can cause the JSON +//! parser to overflow the stack. +//! +//! Additionally, such implementation would be pretty verbose, and we do care +//! about performance here a bit. +//! +//! So what this module does is dumping a `tt::TopSubtree` into a bunch of flat +//! array of numbers. +//! +//! ```json +//! { +//! // Array of subtrees, each subtree is represented by 4 numbers: +//! // id of delimiter, delimiter kind, index of first child in `token_tree`, +//! // index of last child in `token_tree` +//! "subtree":[4294967295,0,0,5,2,2,5,5], +//! // 2 ints per literal: [token id, index into `text`] +//! "literal":[4294967295,1], +//! // 3 ints per punct: [token id, char, spacing] +//! "punct":[4294967295,64,1], +//! // 2 ints per ident: [token id, index into `text`] +//! "ident": [0,0,1,1], +//! // children of all subtrees, concatenated. Each child is represented as `index << shift_indices_by | tag` +//! // where tag denotes one of subtree, literal, punct or ident. +//! "token_tree":[3,7,1,4], +//! // Strings shared by idents and literals +//! "text": ["struct","Foo"] +//! } +//! ``` +//! +//! We probably should replace most of the code here with bincode someday, but, +//! as we don't have bincode in Cargo.toml yet, let's stick with serde_json for +//! the time being. + +#[cfg(feature = "in-proc-macro-srv")] +mod proc_macro_srv_side; +#[cfg(feature = "in-ra")] +mod ra_side; + +use std::{borrow::Borrow, cell::Cell, collections::VecDeque, marker::PhantomData}; + +use intern::{Symbol, sym}; +use rustc_hash::FxHashMap; +use serde::{Deserialize, Serialize}; +use span::{EditionedFileId, ErasedFileAstId, Span, SpanAnchor, SyntaxContext, TextRange}; +use stdx::always; + +use crate::{ + legacy_protocol::SpanId, + version::{DOC_COMMENT_LEAF, ENCODE_CLOSE_SPAN_VERSION, EXTENDED_LEAF_DATA}, +}; + +pub type SpanDataIndexMap = + indexmap::IndexSet>; + +pub fn serialize_span_data_index_map(map: &SpanDataIndexMap) -> Vec { + map.iter() + .map(|span| { + [ + span.anchor.file_id.as_u32(), + span.anchor.ast_id.into_raw(), + span.range.start().into(), + span.range.end().into(), + span.ctx.into_u32(), + ] + }) + .collect::>() + .into_flattened() +} + +pub fn deserialize_span_data_index_map(map: &[u32]) -> SpanDataIndexMap { + let (chunks, remainder) = map.as_chunks(); + assert!(remainder.is_empty()); + chunks + .iter() + .map(|&[file_id, ast_id, start, end, e]| { + Span { + anchor: SpanAnchor { + file_id: EditionedFileId::from_raw(file_id), + ast_id: ErasedFileAstId::from_raw(ast_id), + }, + range: TextRange::new(start.into(), end.into()), + // SAFETY: We only receive spans from the server. If someone mess up the communication UB can happen, + // but that will be their problem. + ctx: unsafe { SyntaxContext::from_u32(e) }, + } + }) + .collect() +} + +fn tag_bit_width(version: u32) -> u32 { + if version >= DOC_COMMENT_LEAF { 3 } else { 2 } +} + +/// [`FlatTree`] when `version < DOC_COMMENT_LEAF`, because `postcard` is non-self-describing and does not support `skip_serializing_if`. +#[derive(Serialize, Deserialize, Debug, Clone)] +struct FlatTreePreDocCommentLeafParts { + subtree: Vec, + literal: Vec, + punct: Vec, + ident: Vec, + token_tree: Vec, + text: Vec, +} + +/// [`FlatTree`] when `version >= DOC_COMMENT_LEAF`. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct FlatTreePostDocCommentLeafParts { + pre_doc_comment_leaf: FlatTreePreDocCommentLeafParts, + doc_comments: Vec, +} + +#[derive(Debug, Clone)] +pub struct FlatTree(pub FlatTreePostDocCommentLeafParts); + +impl FlatTree { + fn from_pre_doc_comments(value: FlatTreePreDocCommentLeafParts) -> Self { + Self(FlatTreePostDocCommentLeafParts { + pre_doc_comment_leaf: value, + doc_comments: Vec::new(), + }) + } + + fn from_post_doc_comments(value: FlatTreePostDocCommentLeafParts) -> Self { + Self(value) + } + + fn as_pre_doc_comments(&self) -> &FlatTreePreDocCommentLeafParts { + let FlatTreePostDocCommentLeafParts { pre_doc_comment_leaf: _, doc_comments } = &self.0; + always!(doc_comments.is_empty()); + &self.0.pre_doc_comment_leaf + } + + fn as_post_doc_comments(&self) -> &FlatTreePostDocCommentLeafParts { + &self.0 + } +} + +thread_local! { + static IN_FLIGHT_SERIALIZATION_VERSION: Cell> = const { Cell::new(None) }; +} + +/// We need to see the version during serialization, because it impacts the shape of the `FlatTree` +/// and postcard is non-self-describing. +/// +/// `serde` provides `DeserializeSeed` to pass data to deserializers, but not to serializers and there is no derive for +/// it (external crates have but we don't want to import them just for this). So we smuggle it in a thread local instead. +/// +/// Note: while the proc macro server side always has the same version, the r-a side might handle multiple servers with +/// different versions. +pub fn with_serialization_version(version: u32, f: impl FnOnce() -> T) -> T { + struct Guard; + impl Drop for Guard { + fn drop(&mut self) { + IN_FLIGHT_SERIALIZATION_VERSION.set(None); + } + } + + let _guard = Guard; + + std::assert_matches!( + IN_FLIGHT_SERIALIZATION_VERSION.replace(Some(version)), + None, + "cannot set serialization version mid-[de]serialization", + ); + + f() +} + +fn serialization_version() -> u32 { + IN_FLIGHT_SERIALIZATION_VERSION + .get() + .expect("`FlatTree` serialization version must be set during [de]serialization") +} + +impl Serialize for FlatTree { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + match serialization_version() { + (..DOC_COMMENT_LEAF) => self.as_pre_doc_comments().serialize(serializer), + (DOC_COMMENT_LEAF..) => self.as_post_doc_comments().serialize(serializer), + } + } +} + +impl<'de> Deserialize<'de> for FlatTree { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + match serialization_version() { + (..DOC_COMMENT_LEAF) => FlatTreePreDocCommentLeafParts::deserialize(deserializer) + .map(Self::from_pre_doc_comments), + (DOC_COMMENT_LEAF..) => FlatTreePostDocCommentLeafParts::deserialize(deserializer) + .map(Self::from_post_doc_comments), + } + } +} + +impl FlatTree { + fn deserialize<'a, ST: SpanTransformer, W: WriterTrait<'a, ST::Span>>( + top_subtree: W::Subtree, + version: u32, + span_data_table: &mut ST::Table, + ) -> FlatTree { + let mut w = Writer:: { + string_table: FxHashMap::default(), + work: VecDeque::new(), + span_data_table, + tag_bit_width: tag_bit_width(version), + + subtree: Vec::new(), + literal: Vec::new(), + punct: Vec::new(), + ident: Vec::new(), + doc_comment: Vec::new(), + token_tree: Vec::new(), + text: Vec::new(), + version, + }; + w.write_subtree(top_subtree); + + FlatTree(FlatTreePostDocCommentLeafParts { + pre_doc_comment_leaf: FlatTreePreDocCommentLeafParts { + subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { + write_vec(w.subtree, SubtreeRepr::write_with_close_span) + } else { + write_vec(w.subtree, SubtreeRepr::write) + }, + literal: if version >= EXTENDED_LEAF_DATA { + write_vec(w.literal, LiteralRepr::write_with_kind) + } else { + write_vec(w.literal, LiteralRepr::write) + }, + punct: write_vec(w.punct, PunctRepr::write), + ident: if version >= EXTENDED_LEAF_DATA { + write_vec(w.ident, IdentRepr::write_with_rawness) + } else { + write_vec(w.ident, IdentRepr::write) + }, + token_tree: w.token_tree, + text: w.text, + }, + doc_comments: write_vec(w.doc_comment, DocCommentRepr::write), + }) + } + + fn serialize>( + self, + version: u32, + span_data_table: &ST::Table, + ) -> (tt::Delimiter, Vec) { + let tag_bit_width = tag_bit_width(version); + Reader:: { + subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { + read_vec(self.0.pre_doc_comment_leaf.subtree, SubtreeRepr::read_with_close_span) + } else { + read_vec(self.0.pre_doc_comment_leaf.subtree, SubtreeRepr::read) + }, + literal: if version >= EXTENDED_LEAF_DATA { + read_vec(self.0.pre_doc_comment_leaf.literal, LiteralRepr::read_with_kind) + } else { + read_vec(self.0.pre_doc_comment_leaf.literal, LiteralRepr::read) + }, + punct: read_vec(self.0.pre_doc_comment_leaf.punct, PunctRepr::read), + ident: if version >= EXTENDED_LEAF_DATA { + read_vec(self.0.pre_doc_comment_leaf.ident, IdentRepr::read_with_rawness) + } else { + read_vec(self.0.pre_doc_comment_leaf.ident, IdentRepr::read) + }, + doc_comment: read_vec(self.0.doc_comments, DocCommentRepr::read), + token_tree: self.0.pre_doc_comment_leaf.token_tree, + text: self.0.pre_doc_comment_leaf.text, + span_data_table, + version, + tag_bit_width, + tag_mask: (1 << tag_bit_width) - 1, + _marker: PhantomData, + } + .read() + } +} + +#[derive(Debug)] +struct SubtreeRepr { + open: SpanId, + close: SpanId, + kind: tt::DelimiterKind, + tt: [u32; 2], +} + +#[derive(Debug)] +struct LiteralRepr { + id: SpanId, + text: u32, + suffix: u32, + kind: u16, +} + +#[derive(Debug)] +struct PunctRepr { + id: SpanId, + char: char, + spacing: tt::Spacing, +} + +#[derive(Debug)] +struct IdentRepr { + id: SpanId, + text: u32, + is_raw: bool, +} + +#[derive(Debug)] +struct DocCommentRepr { + id: SpanId, + text_with_comment_signs: u32, + is_inner: bool, + is_block: bool, +} + +fn read_vec T, const N: usize>(xs: Vec, f: F) -> Vec { + let (chunks, remainder) = xs.as_chunks(); + assert!(remainder.is_empty()); + chunks.iter().map(|chunk| f(*chunk)).collect() +} + +fn write_vec [u32; N], const N: usize>(xs: Vec, f: F) -> Vec { + xs.into_iter().map(f).collect::>().into_flattened() +} + +impl SubtreeRepr { + fn write(self) -> [u32; 4] { + let kind = match self.kind { + tt::DelimiterKind::Invisible => 0, + tt::DelimiterKind::Parenthesis => 1, + tt::DelimiterKind::Brace => 2, + tt::DelimiterKind::Bracket => 3, + }; + [self.open.0, kind, self.tt[0], self.tt[1]] + } + fn read([open, kind, lo, len]: [u32; 4]) -> SubtreeRepr { + let kind = match kind { + 0 => tt::DelimiterKind::Invisible, + 1 => tt::DelimiterKind::Parenthesis, + 2 => tt::DelimiterKind::Brace, + 3 => tt::DelimiterKind::Bracket, + other => panic!("bad kind {other}"), + }; + SubtreeRepr { open: SpanId(open), close: SpanId(!0), kind, tt: [lo, len] } + } + fn write_with_close_span(self) -> [u32; 5] { + let kind = match self.kind { + tt::DelimiterKind::Invisible => 0, + tt::DelimiterKind::Parenthesis => 1, + tt::DelimiterKind::Brace => 2, + tt::DelimiterKind::Bracket => 3, + }; + [self.open.0, self.close.0, kind, self.tt[0], self.tt[1]] + } + fn read_with_close_span([open, close, kind, lo, len]: [u32; 5]) -> SubtreeRepr { + let kind = match kind { + 0 => tt::DelimiterKind::Invisible, + 1 => tt::DelimiterKind::Parenthesis, + 2 => tt::DelimiterKind::Brace, + 3 => tt::DelimiterKind::Bracket, + other => panic!("bad kind {other}"), + }; + SubtreeRepr { open: SpanId(open), close: SpanId(close), kind, tt: [lo, len] } + } +} + +impl LiteralRepr { + fn write(self) -> [u32; 2] { + [self.id.0, self.text] + } + fn read([id, text]: [u32; 2]) -> LiteralRepr { + LiteralRepr { id: SpanId(id), text, kind: 0, suffix: !0 } + } + fn write_with_kind(self) -> [u32; 4] { + [self.id.0, self.text, self.kind as u32, self.suffix] + } + fn read_with_kind([id, text, kind, suffix]: [u32; 4]) -> LiteralRepr { + LiteralRepr { id: SpanId(id), text, kind: kind as u16, suffix } + } +} + +impl PunctRepr { + fn write(self) -> [u32; 3] { + let spacing = match self.spacing { + tt::Spacing::Alone | tt::Spacing::JointHidden => 0, + tt::Spacing::Joint => 1, + }; + [self.id.0, self.char as u32, spacing] + } + fn read([id, char, spacing]: [u32; 3]) -> PunctRepr { + let spacing = match spacing { + 0 => tt::Spacing::Alone, + 1 => tt::Spacing::Joint, + other => panic!("bad spacing {other}"), + }; + PunctRepr { id: SpanId(id), char: char.try_into().unwrap(), spacing } + } +} + +impl IdentRepr { + fn write(self) -> [u32; 2] { + [self.id.0, self.text] + } + fn read(data: [u32; 2]) -> IdentRepr { + IdentRepr { id: SpanId(data[0]), text: data[1], is_raw: false } + } + fn write_with_rawness(self) -> [u32; 3] { + [self.id.0, self.text, self.is_raw as u32] + } + fn read_with_rawness([id, text, is_raw]: [u32; 3]) -> IdentRepr { + IdentRepr { id: SpanId(id), text, is_raw: is_raw == 1 } + } +} + +impl DocCommentRepr { + fn write(self) -> [u32; 3] { + [ + self.id.0, + self.text_with_comment_signs, + u16::from_le_bytes([self.is_inner.into(), self.is_block.into()]).into(), + ] + } + + fn read([id, text_with_comment_signs, is_inner_and_is_block]: [u32; 3]) -> DocCommentRepr { + let [is_inner, is_block] = (is_inner_and_is_block as u16).to_le_bytes(); + DocCommentRepr { + id: SpanId(id), + text_with_comment_signs, + is_inner: is_inner != 0, + is_block: is_block != 0, + } + } +} + +pub trait SpanTransformer { + type Table; + type Span: Copy + 'static; + fn token_id_of(table: &mut Self::Table, s: Self::Span) -> SpanId; + fn span_for_token_id(table: &Self::Table, id: SpanId) -> Self::Span; +} +impl SpanTransformer for SpanId { + type Table = (); + type Span = Self; + fn token_id_of((): &mut Self::Table, token_id: Self::Span) -> SpanId { + token_id + } + + fn span_for_token_id((): &Self::Table, id: SpanId) -> Self::Span { + id + } +} +impl SpanTransformer for Span { + type Table = SpanDataIndexMap; + type Span = Self; + fn token_id_of(table: &mut Self::Table, span: Self::Span) -> SpanId { + SpanId(table.insert_full(span).0 as u32) + } + fn span_for_token_id(table: &Self::Table, id: SpanId) -> Self::Span { + *table.get_index(id.0 as usize).unwrap_or_else(|| &table[0]) + } +} + +enum SubtreeOrLeafRef<'a, Span, W: WriterTrait<'a, Span>> { + Subtree(W::Subtree), + Leaf(W::Leaf), +} + +enum WorkItem<'a, Span, W: WriterTrait<'a, Span>> { + Subtree(W::SubtreeIter), + DesugaredDocCommentSubtree(tt::DocComment), +} + +trait WriterTrait<'a, Span>: Sized { + type Subtree; + type Leaf: Borrow>; + + type SubtreeIter: Clone; + + fn subtree_data(subtree: &Self::Subtree) -> (tt::Delimiter, Self::SubtreeIter); + fn subtree_len(subtree: &Self::Subtree) -> usize; + + fn subtree_iter_next(iter: &mut Self::SubtreeIter) -> Option>; +} + +struct Writer<'a, 'span, ST: SpanTransformer, W: WriterTrait<'a, ST::Span>> { + work: VecDeque<(usize, usize, WorkItem<'a, ST::Span, W>)>, + string_table: FxHashMap, u32>, + span_data_table: &'span mut ST::Table, + version: u32, + tag_bit_width: u32, + + subtree: Vec, + literal: Vec, + punct: Vec, + ident: Vec, + doc_comment: Vec, + token_tree: Vec, + text: Vec, +} + +impl<'a, ST: SpanTransformer, W: WriterTrait<'a, ST::Span>> Writer<'a, '_, ST, W> { + fn write_subtree(&mut self, root: W::Subtree) { + self.enqueue(root); + while let Some((idx, len, subtree)) = self.work.pop_front() { + self.subtree(idx, len, subtree); + } + } + + fn subtree(&mut self, idx: usize, n_tt: usize, subtree: WorkItem<'a, ST::Span, W>) { + let mut first_tt = self.token_tree.len(); + self.token_tree.resize(first_tt + n_tt, !0); + + self.subtree[idx].tt = [first_tt as u32, (first_tt + n_tt) as u32]; + + let mut push_tt = |this: &mut Self, idx_tag| { + this.token_tree[first_tt] = idx_tag; + first_tt += 1; + }; + + let mut subtree = match subtree { + WorkItem::Subtree(it) => it, + WorkItem::DesugaredDocCommentSubtree(doc_comment) => { + let doc_ident = self.ident(&tt::Ident { + sym: sym::doc, + span: doc_comment.span, + is_raw: tt::IdentIsRaw::No, + }); + push_tt(self, doc_ident); + let eq_punct = self.punct(&tt::Punct { + char: '=', + spacing: tt::Spacing::Alone, + span: doc_comment.span, + }); + push_tt(self, eq_punct); + let doc_literal = self.literal(&doc_comment.literal_for_proc_macros()); + push_tt(self, doc_literal); + return; + } + }; + + while let Some(child) = W::subtree_iter_next(&mut subtree) { + let idx_tag = match child { + SubtreeOrLeafRef::Subtree(subtree) => { + let idx = self.enqueue(subtree); + idx << self.tag_bit_width + } + SubtreeOrLeafRef::Leaf(leaf) => match leaf.borrow() { + tt::Leaf::Literal(lit) => self.literal(lit), + tt::Leaf::Punct(punct) => self.punct(punct), + tt::Leaf::Ident(ident) => self.ident(ident), + tt::Leaf::DocComment(doc_comment) => { + if self.version >= DOC_COMMENT_LEAF { + let idx = self.doc_comment.len() as u32; + let id = self.token_id_of(doc_comment.span); + let text = self.intern_owned( + doc_comment.text_with_comment_signs.as_str().to_owned(), + ); + let is_inner = doc_comment.doc_style == tt::DocCommentStyle::Inner; + let is_block = doc_comment.comment_style == tt::CommentStyle::Block; + self.doc_comment.push(DocCommentRepr { + id, + text_with_comment_signs: text, + is_inner, + is_block, + }); + (idx << self.tag_bit_width) | 0b100 + } else { + let hash_punct = self.punct(&tt::Punct { + char: '#', + spacing: tt::Spacing::Alone, + span: doc_comment.span, + }); + push_tt(self, hash_punct); + if doc_comment.doc_style == tt::DocCommentStyle::Inner { + let bang_punct = self.punct(&tt::Punct { + char: '!', + spacing: tt::Spacing::Alone, + span: doc_comment.span, + }); + push_tt(self, bang_punct); + } + + /// `doc`, `=`, and the literal. + const DESUGARED_DOC_COMMENT_SUBTREE_LEN: usize = 3; + let idx = self.subtree.len(); + let kind = tt::DelimiterKind::Bracket; + let span = self.token_id_of(doc_comment.span); + self.subtree.push(SubtreeRepr { + open: span, + close: span, + kind, + tt: [!0, !0], + }); + self.work.push_back(( + idx, + DESUGARED_DOC_COMMENT_SUBTREE_LEN, + WorkItem::DesugaredDocCommentSubtree(doc_comment.clone()), + )); + push_tt(self, idx as u32); + + return; + } + } + }, + }; + push_tt(self, idx_tag); + } + } + + fn ident(&mut self, ident: &tt::Ident) -> u32 { + let idx = self.ident.len() as u32; + let id = self.token_id_of(ident.span); + let text = if self.version >= EXTENDED_LEAF_DATA { + self.intern_owned(ident.sym.as_str().to_owned()) + } else if ident.is_raw.yes() { + self.intern_owned(format!("r#{}", ident.sym.as_str(),)) + } else { + self.intern_owned(ident.sym.as_str().to_owned()) + }; + self.ident.push(IdentRepr { id, text, is_raw: ident.is_raw.yes() }); + (idx << self.tag_bit_width) | 0b011 + } + + fn punct(&mut self, punct: &tt::Punct) -> u32 { + let idx = self.punct.len() as u32; + let id = self.token_id_of(punct.span); + self.punct.push(PunctRepr { char: punct.char, spacing: punct.spacing, id }); + (idx << self.tag_bit_width) | 0b010 + } + + fn literal(&mut self, lit: &tt::Literal) -> u32 { + let idx = self.literal.len() as u32; + let id = self.token_id_of(lit.span); + let (text, suffix) = if self.version >= EXTENDED_LEAF_DATA { + let (text, suffix) = lit.text_and_suffix(); + ( + self.intern_owned(text.to_owned()), + if suffix.is_empty() { !0 } else { self.intern_owned(suffix.to_owned()) }, + ) + } else { + (self.intern_owned(format!("{lit}")), !0) + }; + self.literal.push(LiteralRepr { + id, + text, + kind: u16::from_le_bytes(match lit.kind { + tt::LitKind::Err(_) => [0, 0], + tt::LitKind::Byte => [1, 0], + tt::LitKind::Char => [2, 0], + tt::LitKind::Integer => [3, 0], + tt::LitKind::Float => [4, 0], + tt::LitKind::Str => [5, 0], + tt::LitKind::StrRaw(r) => [6, r], + tt::LitKind::ByteStr => [7, 0], + tt::LitKind::ByteStrRaw(r) => [8, r], + tt::LitKind::CStr => [9, 0], + tt::LitKind::CStrRaw(r) => [10, r], + }), + suffix, + }); + (idx << self.tag_bit_width) | 0b001 + } + + fn enqueue(&mut self, subtree: W::Subtree) -> u32 { + let idx = self.subtree.len(); + let (delimiter, contents) = W::subtree_data(&subtree); + let len = if self.version >= DOC_COMMENT_LEAF { + W::subtree_len(&subtree) + } else { + // We need to count doc comments as multiple items. + let mut contents = contents.clone(); + let contents = std::iter::from_fn(move || W::subtree_iter_next(&mut contents)); + contents + .map(|item| { + if let SubtreeOrLeafRef::Leaf(leaf) = item + && let tt::Leaf::DocComment(doc_comment) = leaf.borrow() + { + // `#`, `!` if inner, and `[...]`. + 2 + usize::from(doc_comment.doc_style == tt::DocCommentStyle::Inner) + } else { + 1 + } + }) + .sum() + }; + let open = self.token_id_of(delimiter.open); + let close = self.token_id_of(delimiter.close); + let delimiter_kind = delimiter.kind; + self.subtree.push(SubtreeRepr { open, close, kind: delimiter_kind, tt: [!0, !0] }); + self.work.push_back((idx, len, WorkItem::Subtree(contents))); + idx as u32 + } + + fn token_id_of(&mut self, span: ST::Span) -> SpanId { + ST::token_id_of(self.span_data_table, span) + } + + fn intern_owned(&mut self, text: String) -> u32 { + let table = &mut self.text; + *self.string_table.entry(text.clone().into()).or_insert_with(|| { + let idx = table.len(); + table.push(text); + idx as u32 + }) + } +} + +trait ReaderTrait { + type TokenTree; + + fn leaf(leaf: tt::Leaf) -> Self::TokenTree; + + fn append_subtree( + delimiter: tt::Delimiter, + children: Vec, + insert_into: &mut Vec, + ); +} + +struct Reader<'span, ST: SpanTransformer, R: ReaderTrait> { + version: u32, + tag_bit_width: u32, + tag_mask: u32, + subtree: Vec, + literal: Vec, + punct: Vec, + ident: Vec, + doc_comment: Vec, + token_tree: Vec, + text: Vec, + span_data_table: &'span ST::Table, + _marker: PhantomData, +} + +impl> Reader<'_, ST, R> { + pub(crate) fn read(self) -> (tt::Delimiter, Vec) { + let mut res: Vec, Vec)>> = + (0..self.subtree.len()).map(|_| None).collect(); + let read_span = |id| ST::span_for_token_id(self.span_data_table, id); + for i in (0..self.subtree.len()).rev() { + let repr = &self.subtree[i]; + let token_trees = &self.token_tree[repr.tt[0] as usize..repr.tt[1] as usize]; + let delimiter = tt::Delimiter { + open: read_span(repr.open), + close: read_span(repr.close), + kind: repr.kind, + }; + let mut s = Vec::new(); + for &idx_tag in token_trees { + let tag = idx_tag & self.tag_mask; + let idx = (idx_tag >> self.tag_bit_width) as usize; + match tag { + // XXX: we iterate subtrees in reverse to guarantee + // that this unwrap doesn't fire. + 0b000 => { + let (delimiter, subtree) = res[idx].take().unwrap(); + R::append_subtree(delimiter, subtree, &mut s); + } + 0b001 => { + use tt::LitKind::*; + let repr = &self.literal[idx]; + let text = self.text[repr.text as usize].as_str(); + let span = read_span(repr.id); + s.push(R::leaf(tt::Leaf::Literal(if self.version >= EXTENDED_LEAF_DATA { + tt::Literal::new( + text, + span, + match u16::to_le_bytes(repr.kind) { + [0, _] => Err(()), + [1, _] => Byte, + [2, _] => Char, + [3, _] => Integer, + [4, _] => Float, + [5, _] => Str, + [6, r] => StrRaw(r), + [7, _] => ByteStr, + [8, r] => ByteStrRaw(r), + [9, _] => CStr, + [10, r] => CStrRaw(r), + _ => unreachable!(), + }, + if repr.suffix != !0 { + self.text[repr.suffix as usize].as_str() + } else { + "" + }, + ) + } else { + tt::literal_from_str_or_err(text, span) + }))) + } + 0b010 => { + let repr = &self.punct[idx]; + s.push(R::leaf(tt::Leaf::Punct(tt::Punct { + char: repr.char, + spacing: repr.spacing, + span: read_span(repr.id), + }))) + } + 0b011 => { + let repr = &self.ident[idx]; + let text = self.text[repr.text as usize].as_str(); + let (is_raw, text) = if self.version >= EXTENDED_LEAF_DATA { + ( + if repr.is_raw { tt::IdentIsRaw::Yes } else { tt::IdentIsRaw::No }, + text, + ) + } else { + tt::IdentIsRaw::split_from_symbol(text) + }; + s.push(R::leaf(tt::Leaf::Ident(tt::Ident { + sym: Symbol::intern(text), + span: read_span(repr.id), + is_raw, + }))) + } + 0b100 => { + let repr = &self.doc_comment[idx]; + let text_with_comment_signs = + self.text[repr.text_with_comment_signs as usize].as_str(); + let doc_style = if repr.is_inner { + tt::DocCommentStyle::Inner + } else { + tt::DocCommentStyle::Outer + }; + let comment_style = if repr.is_block { + tt::CommentStyle::Block + } else { + tt::CommentStyle::Line + }; + s.push(R::leaf(tt::Leaf::DocComment(tt::DocComment { + text_with_comment_signs: Symbol::intern(text_with_comment_signs), + span: read_span(repr.id), + doc_style, + comment_style, + }))) + } + other => panic!("bad tag: {other}"), + } + } + res[i] = Some((delimiter, s)); + } + + res[0].take().unwrap() + } +} diff --git a/crates/proc-macro-api/src/flat/proc_macro_srv_side.rs b/crates/proc-macro-api/src/flat/proc_macro_srv_side.rs new file mode 100644 index 000000000000..44abe30faa1d --- /dev/null +++ b/crates/proc-macro-api/src/flat/proc_macro_srv_side.rs @@ -0,0 +1,91 @@ +//! Conversion from/to the flat tree for the proc macro server. + +use crate::{ + flat::{FlatTree, ReaderTrait, SpanTransformer, SubtreeOrLeafRef, WriterTrait}, + token_stream::{Group, SpanLike, TokenStream, TokenTree}, +}; + +struct Writer; + +impl<'a, Span: Copy + 'a> WriterTrait<'a, Span> for Writer { + type Subtree = &'a Group; + type Leaf = &'a tt::Leaf; + + type SubtreeIter = Option>>; + + fn subtree_data(subtree: &Self::Subtree) -> (tt::Delimiter, Self::SubtreeIter) { + (subtree.delimiter, subtree.stream.as_ref().map(|it| it.iter())) + } + + fn subtree_len(subtree: &Self::Subtree) -> usize { + subtree.stream_len() + } + + fn subtree_iter_next(iter: &mut Self::SubtreeIter) -> Option> { + iter.as_mut()?.next().map(|item| match item { + TokenTree::Leaf(leaf) => SubtreeOrLeafRef::Leaf(leaf), + TokenTree::Group(group) => SubtreeOrLeafRef::Subtree(group), + }) + } +} + +struct Reader; + +impl ReaderTrait for Reader { + type TokenTree = TokenTree; + + fn leaf(leaf: tt::Leaf) -> Self::TokenTree { + TokenTree::Leaf(leaf) + } + + fn append_subtree( + delimiter: tt::Delimiter, + children: Vec, + insert_into: &mut Vec, + ) { + insert_into.push(TokenTree::Group(Group { + delimiter, + stream: TokenStream::new_or_empty(children), + })); + } +} + +impl FlatTree { + pub fn from_tokenstream( + tokenstream: TokenStream, + call_site: ST::Span, + version: u32, + span_data_table: &mut ST::Table, + ) -> FlatTree { + let root = if let Some(group) = tokenstream.as_single_group() { + group.clone() + } else { + Group { + delimiter: tt::Delimiter { + open: call_site, + close: call_site, + kind: tt::DelimiterKind::Invisible, + }, + stream: Some(tokenstream), + } + }; + FlatTree::deserialize::(&root, version, span_data_table) + } + + pub fn to_tokenstream>( + self, + version: u32, + span_data_table: &ST::Table, + ) -> TokenStream { + let (top_delimiter, top_children) = self.serialize::(version, span_data_table); + let result = if top_delimiter.kind == tt::DelimiterKind::Invisible { + top_children + } else { + vec![TokenTree::Group(Group { + delimiter: top_delimiter, + stream: TokenStream::new_or_empty(top_children), + })] + }; + TokenStream::new(result) + } +} diff --git a/crates/proc-macro-api/src/flat/ra_side.rs b/crates/proc-macro-api/src/flat/ra_side.rs new file mode 100644 index 000000000000..387881fc5f9f --- /dev/null +++ b/crates/proc-macro-api/src/flat/ra_side.rs @@ -0,0 +1,77 @@ +//! Conversion from/to the flat tree for rust-analyzer. + +use tt::Span; + +use crate::flat::{FlatTree, ReaderTrait, SpanDataIndexMap, SubtreeOrLeafRef, WriterTrait}; + +struct Writer; + +impl<'a> WriterTrait<'a, Span> for Writer { + type Subtree = (tt::Subtree, tt::TtIter<'a>); + type Leaf = tt::Leaf; + + type SubtreeIter = tt::TtIter<'a>; + + fn subtree_data((subtree, iter): &Self::Subtree) -> (tt::Delimiter, Self::SubtreeIter) { + (subtree.delimiter, iter.clone()) + } + + fn subtree_len((_subtree, iter): &Self::Subtree) -> usize { + // FIXME: `count()` walks over the iterator. + iter.clone().count() + } + + fn subtree_iter_next(iter: &mut Self::SubtreeIter) -> Option> { + iter.next().map(|item| match item { + tt::TtElement::Leaf(leaf) => SubtreeOrLeafRef::Leaf(leaf), + tt::TtElement::Subtree(subtree, iter) => SubtreeOrLeafRef::Subtree((subtree, iter)), + }) + } +} + +struct Reader; + +impl ReaderTrait for Reader { + type TokenTree = tt::TokenTree; + + fn leaf(leaf: tt::Leaf) -> Self::TokenTree { + tt::TokenTree::Leaf(leaf) + } + + fn append_subtree( + delimiter: tt::Delimiter, + children: Vec, + insert_into: &mut Vec, + ) { + insert_into + .push(tt::TokenTree::Subtree(tt::Subtree { delimiter, len: children.len() as u32 })); + insert_into.extend(children); + } +} + +impl FlatTree { + pub fn from_subtree( + subtree: tt::SubtreeView<'_>, + version: u32, + span_data_table: &mut SpanDataIndexMap, + ) -> FlatTree { + FlatTree::deserialize::( + (subtree.top_subtree(), subtree.iter()), + version, + span_data_table, + ) + } + + pub fn to_subtree(self, version: u32, span_data_table: &SpanDataIndexMap) -> tt::TopSubtree { + let (top_delimiter, mut top_children) = + self.serialize::(version, span_data_table); + top_children.insert( + 0, + tt::TokenTree::Subtree(tt::Subtree { + delimiter: top_delimiter, + len: top_children.len() as u32, + }), + ); + tt::TopSubtree::from_serialized(top_children) + } +} diff --git a/crates/proc-macro-api/src/legacy_protocol.rs b/crates/proc-macro-api/src/legacy_protocol.rs index ee1795d39c2e..929729e0492f 100644 --- a/crates/proc-macro-api/src/legacy_protocol.rs +++ b/crates/proc-macro-api/src/legacy_protocol.rs @@ -1,27 +1,11 @@ //! The initial proc-macro-srv protocol, soon to be deprecated. pub mod msg; +#[cfg(feature = "in-ra")] +mod sender; -use std::{ - io::{BufRead, Write}, - sync::Arc, -}; - -use paths::AbsPath; -use span::Span; - -use crate::{ - ProcMacro, ProcMacroKind, ServerError, - legacy_protocol::msg::{ - ExpandMacro, ExpandMacroData, ExpnGlobals, FlatTree, Message, Request, Response, - ServerConfig, SpanDataIndexMap, deserialize_span_data_index_map, - flat::serialize_span_data_index_map, - }, - process::ProcMacroServerProcess, - version, -}; - -pub(crate) use crate::legacy_protocol::msg::SpanMode; +#[cfg(feature = "in-ra")] +pub(crate) use self::sender::*; /// Legacy span type, only defined here as it is still used by the proc-macro server. /// While rust-analyzer doesn't use this anymore at all, RustRover relies on the legacy type for @@ -34,136 +18,3 @@ impl std::fmt::Debug for SpanId { self.0.fmt(f) } } - -pub(crate) fn version_check(srv: &ProcMacroServerProcess) -> Result { - let request = Request::ApiVersionCheck {}; - let response = send_task(srv, request)?; - - match response { - Response::ApiVersionCheck(version) => Ok(version), - _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), - } -} - -/// Enable support for rust-analyzer span mode if the server supports it. -pub(crate) fn enable_rust_analyzer_spans( - srv: &ProcMacroServerProcess, -) -> Result { - let request = Request::SetConfig(ServerConfig { span_mode: SpanMode::RustAnalyzer }); - let response = send_task(srv, request)?; - - match response { - Response::SetConfig(ServerConfig { span_mode }) => Ok(span_mode), - _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), - } -} - -/// Finds proc-macros in a given dynamic library. -pub(crate) fn find_proc_macros( - srv: &ProcMacroServerProcess, - dylib_path: &AbsPath, -) -> Result, String>, ServerError> { - let request = Request::ListMacros { dylib_path: dylib_path.to_path_buf().into() }; - - let response = send_task(srv, request)?; - - match response { - Response::ListMacros(it) => Ok(it), - _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), - } -} - -pub(crate) fn expand( - proc_macro: &ProcMacro, - process: &ProcMacroServerProcess, - subtree: tt::SubtreeView<'_>, - attr: Option>, - env: Vec<(String, String)>, - def_site: Span, - call_site: Span, - mixed_site: Span, - current_dir: String, -) -> Result, crate::ServerError> { - let version = process.version(); - let mut span_data_table = SpanDataIndexMap::default(); - let def_site = span_data_table.insert_full(def_site).0; - let call_site = span_data_table.insert_full(call_site).0; - let mixed_site = span_data_table.insert_full(mixed_site).0; - let task = ExpandMacro { - data: ExpandMacroData { - macro_body: FlatTree::from_subtree(subtree, version, &mut span_data_table), - macro_name: proc_macro.name.to_string(), - attributes: attr - .map(|subtree| FlatTree::from_subtree(subtree, version, &mut span_data_table)), - has_global_spans: ExpnGlobals { - serialize: version >= version::HAS_GLOBAL_SPANS, - def_site, - call_site, - mixed_site, - }, - span_data_table: if process.rust_analyzer_spans() { - serialize_span_data_index_map(&span_data_table) - } else { - Vec::new() - }, - }, - lib: proc_macro.dylib_path.to_path_buf().into(), - env, - current_dir: Some(current_dir), - }; - - let response = send_task(process, Request::ExpandMacro(Box::new(task)))?; - - match response { - Response::ExpandMacro(it) => Ok(it - .map(|tree| { - let mut expanded = FlatTree::to_subtree_resolved(tree, version, &span_data_table); - if proc_macro.needs_fixup_change() { - proc_macro.change_fixup_to_match_old_server(&mut expanded); - } - expanded - }) - .map_err(|msg| msg.0)), - Response::ExpandMacroExtended(it) => Ok(it - .map(|resp| { - let mut expanded = FlatTree::to_subtree_resolved( - resp.tree, - version, - &deserialize_span_data_index_map(&resp.span_data_table), - ); - if proc_macro.needs_fixup_change() { - proc_macro.change_fixup_to_match_old_server(&mut expanded); - } - expanded - }) - .map_err(|msg| msg.0)), - _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), - } -} - -/// Sends a request to the proc-macro server and waits for a response. -fn send_task(srv: &ProcMacroServerProcess, req: Request) -> Result { - if let Some(server_error) = srv.exited() { - return Err(server_error.clone()); - } - - srv.send_task_legacy::<_, _>(send_request, req) -} - -/// Sends a request to the server and reads the response. -fn send_request( - mut writer: &mut dyn Write, - mut reader: &mut dyn BufRead, - req: Request, - buf: &mut String, -) -> Result, ServerError> { - req.write(&mut writer).map_err(|err| ServerError { - message: "failed to write request".into(), - io: Some(Arc::new(err)), - })?; - let res = Response::read(&mut reader, buf).map_err(|err| ServerError { - message: "failed to read response".into(), - io: Some(Arc::new(err)), - })?; - Ok(res) -} diff --git a/crates/proc-macro-api/src/legacy_protocol/msg.rs b/crates/proc-macro-api/src/legacy_protocol/msg.rs index 9b71a8b70c8f..539e9bf6f5a3 100644 --- a/crates/proc-macro-api/src/legacy_protocol/msg.rs +++ b/crates/proc-macro-api/src/legacy_protocol/msg.rs @@ -1,6 +1,4 @@ //! Defines messages for cross-process message passing based on `ndjson` wire protocol -pub(crate) mod flat; -pub use self::flat::*; use std::io::{self, BufRead, Write}; @@ -8,7 +6,7 @@ use paths::Utf8PathBuf; use serde::de::DeserializeOwned; use serde_derive::{Deserialize, Serialize}; -use crate::{ProcMacroKind, transport::json}; +use crate::{ProcMacroKind, flat::FlatTree, transport::json}; /// Represents requests sent from the client to the proc-macro-srv. #[derive(Debug, Serialize, Deserialize)] @@ -370,13 +368,15 @@ mod tests { current_dir: Default::default(), }; - let json = serde_json::to_string(&task).unwrap(); - // println!("{}", json); - let back: ExpandMacro = serde_json::from_str(&json).unwrap(); + let back: ExpandMacro = crate::flat::with_serialization_version(v, || { + let json = serde_json::to_string(&task).unwrap(); + // println!("{}", json); + serde_json::from_str(&json).unwrap() + }); assert_eq!( tt, - back.data.macro_body.to_subtree_resolved(v, &span_data_table), + back.data.macro_body.to_subtree(v, &span_data_table), "version: {v}" ); } @@ -384,7 +384,6 @@ mod tests { } #[test] - #[cfg(feature = "in-rust-tree")] fn test_proc_macro_rpc_works_ts() { for tt in [ fixture_token_tree_top_many_none, @@ -395,18 +394,19 @@ mod tests { for v in version::RUST_ANALYZER_SPAN_SUPPORT..=version::CURRENT_API_VERSION { let mut span_data_table = Default::default(); let flat_tree = FlatTree::from_subtree(tt.view(), v, &mut span_data_table); - assert_eq!( - tt, - flat_tree.clone().to_subtree_resolved(v, &span_data_table), - "version: {v}" - ); - let ts = flat_tree.to_tokenstream_resolved(v, &span_data_table, |a, b| a.cover(b)); + assert_eq!(tt, flat_tree.clone().to_subtree(v, &span_data_table), "version: {v}"); + let ts = flat_tree.to_tokenstream::(v, &span_data_table); let call_site = *span_data_table.first().unwrap(); let mut span_data_table = Default::default(); assert_eq!( tt, - FlatTree::from_tokenstream(ts.clone(), v, call_site, &mut span_data_table) - .to_subtree_resolved(v, &span_data_table), + FlatTree::from_tokenstream::( + ts.clone(), + call_site, + v, + &mut span_data_table + ) + .to_subtree(v, &span_data_table), "version: {v}, ts:\n{ts:#?}" ); } diff --git a/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs b/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs deleted file mode 100644 index b9b6247b54fa..000000000000 --- a/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs +++ /dev/null @@ -1,980 +0,0 @@ -//! Serialization-friendly representation of `tt::TopSubtree`. -//! -//! It is possible to serialize `TopSubtree` recursively, as a tree, but using -//! arbitrary-nested trees in JSON is problematic, as they can cause the JSON -//! parser to overflow the stack. -//! -//! Additionally, such implementation would be pretty verbose, and we do care -//! about performance here a bit. -//! -//! So what this module does is dumping a `tt::TopSubtree` into a bunch of flat -//! array of numbers. -//! -//! ```json -//! { -//! // Array of subtrees, each subtree is represented by 4 numbers: -//! // id of delimiter, delimiter kind, index of first child in `token_tree`, -//! // index of last child in `token_tree` -//! "subtree":[4294967295,0,0,5,2,2,5,5], -//! // 2 ints per literal: [token id, index into `text`] -//! "literal":[4294967295,1], -//! // 3 ints per punct: [token id, char, spacing] -//! "punct":[4294967295,64,1], -//! // 2 ints per ident: [token id, index into `text`] -//! "ident": [0,0,1,1], -//! // children of all subtrees, concatenated. Each child is represented as `index << 2 | tag` -//! // where tag denotes one of subtree, literal, punct or ident. -//! "token_tree":[3,7,1,4], -//! // Strings shared by idents and literals -//! "text": ["struct","Foo"] -//! } -//! ``` -//! -//! We probably should replace most of the code here with bincode someday, but, -//! as we don't have bincode in Cargo.toml yet, let's stick with serde_json for -//! the time being. - -#[cfg(feature = "in-rust-tree")] -use proc_macro_srv::TokenStream; - -use std::collections::VecDeque; - -use intern::Symbol; -use rustc_hash::FxHashMap; -use serde_derive::{Deserialize, Serialize}; -use span::{EditionedFileId, ErasedFileAstId, Span, SpanAnchor, SyntaxContext, TextRange}; - -use crate::{ - legacy_protocol::SpanId, - version::{ENCODE_CLOSE_SPAN_VERSION, EXTENDED_LEAF_DATA}, -}; - -pub type SpanDataIndexMap = - indexmap::IndexSet>; - -pub fn serialize_span_data_index_map(map: &SpanDataIndexMap) -> Vec { - map.iter() - .map(|span| { - [ - span.anchor.file_id.as_u32(), - span.anchor.ast_id.into_raw(), - span.range.start().into(), - span.range.end().into(), - span.ctx.into_u32(), - ] - }) - .collect::>() - .into_flattened() -} - -pub fn deserialize_span_data_index_map(map: &[u32]) -> SpanDataIndexMap { - let (chunks, remainder) = map.as_chunks(); - assert!(remainder.is_empty()); - chunks - .iter() - .map(|&[file_id, ast_id, start, end, e]| { - Span { - anchor: SpanAnchor { - file_id: EditionedFileId::from_raw(file_id), - ast_id: ErasedFileAstId::from_raw(ast_id), - }, - range: TextRange::new(start.into(), end.into()), - // SAFETY: We only receive spans from the server. If someone mess up the communication UB can happen, - // but that will be their problem. - ctx: unsafe { SyntaxContext::from_u32(e) }, - } - }) - .collect() -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct FlatTree { - subtree: Vec, - literal: Vec, - punct: Vec, - ident: Vec, - token_tree: Vec, - text: Vec, -} - -struct SubtreeRepr { - open: SpanId, - close: SpanId, - kind: tt::DelimiterKind, - tt: [u32; 2], -} - -struct LiteralRepr { - id: SpanId, - text: u32, - suffix: u32, - kind: u16, -} - -struct PunctRepr { - id: SpanId, - char: char, - spacing: tt::Spacing, -} - -struct IdentRepr { - id: SpanId, - text: u32, - is_raw: bool, -} - -impl FlatTree { - pub fn from_subtree( - subtree: tt::SubtreeView<'_>, - version: u32, - span_data_table: &mut SpanDataIndexMap, - ) -> FlatTree { - let mut w = Writer:: { - string_table: FxHashMap::default(), - work: VecDeque::new(), - span_data_table, - - subtree: Vec::new(), - literal: Vec::new(), - punct: Vec::new(), - ident: Vec::new(), - token_tree: Vec::new(), - text: Vec::new(), - version, - }; - w.write_subtree(subtree); - - FlatTree { - subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { - write_vec(w.subtree, SubtreeRepr::write_with_close_span) - } else { - write_vec(w.subtree, SubtreeRepr::write) - }, - literal: if version >= EXTENDED_LEAF_DATA { - write_vec(w.literal, LiteralRepr::write_with_kind) - } else { - write_vec(w.literal, LiteralRepr::write) - }, - punct: write_vec(w.punct, PunctRepr::write), - ident: if version >= EXTENDED_LEAF_DATA { - write_vec(w.ident, IdentRepr::write_with_rawness) - } else { - write_vec(w.ident, IdentRepr::write) - }, - token_tree: w.token_tree, - text: w.text, - } - } - - pub fn to_subtree_resolved( - self, - version: u32, - span_data_table: &SpanDataIndexMap, - ) -> tt::TopSubtree { - Reader:: { - subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { - read_vec(self.subtree, SubtreeRepr::read_with_close_span) - } else { - read_vec(self.subtree, SubtreeRepr::read) - }, - literal: if version >= EXTENDED_LEAF_DATA { - read_vec(self.literal, LiteralRepr::read_with_kind) - } else { - read_vec(self.literal, LiteralRepr::read) - }, - punct: read_vec(self.punct, PunctRepr::read), - ident: if version >= EXTENDED_LEAF_DATA { - read_vec(self.ident, IdentRepr::read_with_rawness) - } else { - read_vec(self.ident, IdentRepr::read) - }, - token_tree: self.token_tree, - text: self.text, - span_data_table, - version, - } - .read_subtree() - } -} - -#[cfg(feature = "in-rust-tree")] -impl FlatTree { - pub fn from_tokenstream( - tokenstream: proc_macro_srv::TokenStream, - version: u32, - call_site: Span, - span_data_table: &mut SpanDataIndexMap, - ) -> FlatTree { - let mut w = Writer:: { - string_table: FxHashMap::default(), - work: VecDeque::new(), - span_data_table, - - subtree: Vec::new(), - literal: Vec::new(), - punct: Vec::new(), - ident: Vec::new(), - token_tree: Vec::new(), - text: Vec::new(), - version, - }; - w.write_tokenstream(call_site, &tokenstream); - - FlatTree { - subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { - write_vec(w.subtree, SubtreeRepr::write_with_close_span) - } else { - write_vec(w.subtree, SubtreeRepr::write) - }, - literal: if version >= EXTENDED_LEAF_DATA { - write_vec(w.literal, LiteralRepr::write_with_kind) - } else { - write_vec(w.literal, LiteralRepr::write) - }, - punct: write_vec(w.punct, PunctRepr::write), - ident: if version >= EXTENDED_LEAF_DATA { - write_vec(w.ident, IdentRepr::write_with_rawness) - } else { - write_vec(w.ident, IdentRepr::write) - }, - token_tree: w.token_tree, - text: w.text, - } - } - - pub fn from_tokenstream_raw>( - tokenstream: proc_macro_srv::TokenStream, - call_site: T::Span, - version: u32, - ) -> FlatTree { - let mut w = Writer:: { - string_table: FxHashMap::default(), - work: VecDeque::new(), - span_data_table: &mut (), - - subtree: Vec::new(), - literal: Vec::new(), - punct: Vec::new(), - ident: Vec::new(), - token_tree: Vec::new(), - text: Vec::new(), - version, - }; - w.write_tokenstream(call_site, &tokenstream); - - FlatTree { - subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { - write_vec(w.subtree, SubtreeRepr::write_with_close_span) - } else { - write_vec(w.subtree, SubtreeRepr::write) - }, - literal: if version >= EXTENDED_LEAF_DATA { - write_vec(w.literal, LiteralRepr::write_with_kind) - } else { - write_vec(w.literal, LiteralRepr::write) - }, - punct: write_vec(w.punct, PunctRepr::write), - ident: if version >= EXTENDED_LEAF_DATA { - write_vec(w.ident, IdentRepr::write_with_rawness) - } else { - write_vec(w.ident, IdentRepr::write) - }, - token_tree: w.token_tree, - text: w.text, - } - } - - pub fn to_tokenstream_unresolved>( - self, - version: u32, - span_join: impl Fn(T::Span, T::Span) -> T::Span, - ) -> proc_macro_srv::TokenStream { - Reader:: { - subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { - read_vec(self.subtree, SubtreeRepr::read_with_close_span) - } else { - read_vec(self.subtree, SubtreeRepr::read) - }, - literal: if version >= EXTENDED_LEAF_DATA { - read_vec(self.literal, LiteralRepr::read_with_kind) - } else { - read_vec(self.literal, LiteralRepr::read) - }, - punct: read_vec(self.punct, PunctRepr::read), - ident: if version >= EXTENDED_LEAF_DATA { - read_vec(self.ident, IdentRepr::read_with_rawness) - } else { - read_vec(self.ident, IdentRepr::read) - }, - token_tree: self.token_tree, - text: self.text, - span_data_table: &(), - version, - } - .read_tokenstream(span_join) - } - - pub fn to_tokenstream_resolved( - self, - version: u32, - span_data_table: &SpanDataIndexMap, - span_join: impl Fn(Span, Span) -> Span, - ) -> proc_macro_srv::TokenStream { - Reader:: { - subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { - read_vec(self.subtree, SubtreeRepr::read_with_close_span) - } else { - read_vec(self.subtree, SubtreeRepr::read) - }, - literal: if version >= EXTENDED_LEAF_DATA { - read_vec(self.literal, LiteralRepr::read_with_kind) - } else { - read_vec(self.literal, LiteralRepr::read) - }, - punct: read_vec(self.punct, PunctRepr::read), - ident: if version >= EXTENDED_LEAF_DATA { - read_vec(self.ident, IdentRepr::read_with_rawness) - } else { - read_vec(self.ident, IdentRepr::read) - }, - token_tree: self.token_tree, - text: self.text, - span_data_table, - version, - } - .read_tokenstream(span_join) - } -} - -fn read_vec T, const N: usize>(xs: Vec, f: F) -> Vec { - let (chunks, remainder) = xs.as_chunks(); - assert!(remainder.is_empty()); - chunks.iter().map(|chunk| f(*chunk)).collect() -} - -fn write_vec [u32; N], const N: usize>(xs: Vec, f: F) -> Vec { - xs.into_iter().map(f).collect::>().into_flattened() -} - -impl SubtreeRepr { - fn write(self) -> [u32; 4] { - let kind = match self.kind { - tt::DelimiterKind::Invisible => 0, - tt::DelimiterKind::Parenthesis => 1, - tt::DelimiterKind::Brace => 2, - tt::DelimiterKind::Bracket => 3, - }; - [self.open.0, kind, self.tt[0], self.tt[1]] - } - fn read([open, kind, lo, len]: [u32; 4]) -> SubtreeRepr { - let kind = match kind { - 0 => tt::DelimiterKind::Invisible, - 1 => tt::DelimiterKind::Parenthesis, - 2 => tt::DelimiterKind::Brace, - 3 => tt::DelimiterKind::Bracket, - other => panic!("bad kind {other}"), - }; - SubtreeRepr { open: SpanId(open), close: SpanId(!0), kind, tt: [lo, len] } - } - fn write_with_close_span(self) -> [u32; 5] { - let kind = match self.kind { - tt::DelimiterKind::Invisible => 0, - tt::DelimiterKind::Parenthesis => 1, - tt::DelimiterKind::Brace => 2, - tt::DelimiterKind::Bracket => 3, - }; - [self.open.0, self.close.0, kind, self.tt[0], self.tt[1]] - } - fn read_with_close_span([open, close, kind, lo, len]: [u32; 5]) -> SubtreeRepr { - let kind = match kind { - 0 => tt::DelimiterKind::Invisible, - 1 => tt::DelimiterKind::Parenthesis, - 2 => tt::DelimiterKind::Brace, - 3 => tt::DelimiterKind::Bracket, - other => panic!("bad kind {other}"), - }; - SubtreeRepr { open: SpanId(open), close: SpanId(close), kind, tt: [lo, len] } - } -} - -impl LiteralRepr { - fn write(self) -> [u32; 2] { - [self.id.0, self.text] - } - fn read([id, text]: [u32; 2]) -> LiteralRepr { - LiteralRepr { id: SpanId(id), text, kind: 0, suffix: !0 } - } - fn write_with_kind(self) -> [u32; 4] { - [self.id.0, self.text, self.kind as u32, self.suffix] - } - fn read_with_kind([id, text, kind, suffix]: [u32; 4]) -> LiteralRepr { - LiteralRepr { id: SpanId(id), text, kind: kind as u16, suffix } - } -} - -impl PunctRepr { - fn write(self) -> [u32; 3] { - let spacing = match self.spacing { - tt::Spacing::Alone | tt::Spacing::JointHidden => 0, - tt::Spacing::Joint => 1, - }; - [self.id.0, self.char as u32, spacing] - } - fn read([id, char, spacing]: [u32; 3]) -> PunctRepr { - let spacing = match spacing { - 0 => tt::Spacing::Alone, - 1 => tt::Spacing::Joint, - other => panic!("bad spacing {other}"), - }; - PunctRepr { id: SpanId(id), char: char.try_into().unwrap(), spacing } - } -} - -impl IdentRepr { - fn write(self) -> [u32; 2] { - [self.id.0, self.text] - } - fn read(data: [u32; 2]) -> IdentRepr { - IdentRepr { id: SpanId(data[0]), text: data[1], is_raw: false } - } - fn write_with_rawness(self) -> [u32; 3] { - [self.id.0, self.text, self.is_raw as u32] - } - fn read_with_rawness([id, text, is_raw]: [u32; 3]) -> IdentRepr { - IdentRepr { id: SpanId(id), text, is_raw: is_raw == 1 } - } -} - -pub trait SpanTransformer { - type Table; - type Span: Copy; - fn token_id_of(table: &mut Self::Table, s: Self::Span) -> SpanId; - fn span_for_token_id(table: &Self::Table, id: SpanId) -> Self::Span; -} -impl SpanTransformer for SpanId { - type Table = (); - type Span = Self; - fn token_id_of((): &mut Self::Table, token_id: Self::Span) -> SpanId { - token_id - } - - fn span_for_token_id((): &Self::Table, id: SpanId) -> Self::Span { - id - } -} -impl SpanTransformer for Span { - type Table = SpanDataIndexMap; - type Span = Self; - fn token_id_of(table: &mut Self::Table, span: Self::Span) -> SpanId { - SpanId(table.insert_full(span).0 as u32) - } - fn span_for_token_id(table: &Self::Table, id: SpanId) -> Self::Span { - *table.get_index(id.0 as usize).unwrap_or_else(|| &table[0]) - } -} - -struct Writer<'a, 'span, S: SpanTransformer, W> { - work: VecDeque<(usize, usize, W)>, - string_table: FxHashMap, u32>, - span_data_table: &'span mut S::Table, - version: u32, - - subtree: Vec, - literal: Vec, - punct: Vec, - ident: Vec, - token_tree: Vec, - text: Vec, -} - -impl<'a, T: SpanTransformer> Writer<'a, '_, T, tt::iter::TtIter<'a>> { - fn write_subtree(&mut self, root: tt::SubtreeView<'a>) { - let subtree = root.top_subtree(); - self.enqueue(&subtree, root.iter()); - while let Some((idx, len, subtree)) = self.work.pop_front() { - self.subtree(idx, len, subtree); - } - } - - #[expect( - clippy::explicit_counter_loop, - reason = "it looks better the current way since we use `first_tt` before the loop" - )] - fn subtree(&mut self, idx: usize, n_tt: usize, subtree: tt::iter::TtIter<'a>) { - let mut first_tt = self.token_tree.len(); - self.token_tree.resize(first_tt + n_tt, !0); - - self.subtree[idx].tt = [first_tt as u32, (first_tt + n_tt) as u32]; - - for child in subtree { - let idx_tag = match child { - tt::iter::TtElement::Subtree(subtree, subtree_iter) => { - let idx = self.enqueue(&subtree, subtree_iter); - idx << 2 - } - tt::iter::TtElement::Leaf(leaf) => match leaf { - tt::Leaf::Literal(lit) => { - let idx = self.literal.len() as u32; - let id = self.token_id_of(lit.span); - let (text, suffix) = if self.version >= EXTENDED_LEAF_DATA { - let (text, suffix) = lit.text_and_suffix(); - ( - self.intern_owned(text.to_owned()), - if suffix.is_empty() { - !0 - } else { - self.intern_owned(suffix.to_owned()) - }, - ) - } else { - (self.intern_owned(format!("{lit}")), !0) - }; - self.literal.push(LiteralRepr { - id, - text, - kind: u16::from_le_bytes(match lit.kind { - tt::LitKind::Err(_) => [0, 0], - tt::LitKind::Byte => [1, 0], - tt::LitKind::Char => [2, 0], - tt::LitKind::Integer => [3, 0], - tt::LitKind::Float => [4, 0], - tt::LitKind::Str => [5, 0], - tt::LitKind::StrRaw(r) => [6, r], - tt::LitKind::ByteStr => [7, 0], - tt::LitKind::ByteStrRaw(r) => [8, r], - tt::LitKind::CStr => [9, 0], - tt::LitKind::CStrRaw(r) => [10, r], - }), - suffix, - }); - (idx << 2) | 0b01 - } - tt::Leaf::Punct(punct) => { - let idx = self.punct.len() as u32; - let id = self.token_id_of(punct.span); - self.punct.push(PunctRepr { char: punct.char, spacing: punct.spacing, id }); - (idx << 2) | 0b10 - } - tt::Leaf::Ident(ident) => { - let idx = self.ident.len() as u32; - let id = self.token_id_of(ident.span); - let text = if self.version >= EXTENDED_LEAF_DATA { - self.intern_owned(ident.sym.as_str().to_owned()) - } else if ident.is_raw.yes() { - self.intern_owned(format!("r#{}", ident.sym.as_str(),)) - } else { - self.intern_owned(ident.sym.as_str().to_owned()) - }; - self.ident.push(IdentRepr { id, text, is_raw: ident.is_raw.yes() }); - (idx << 2) | 0b11 - } - }, - }; - self.token_tree[first_tt] = idx_tag; - first_tt += 1; - } - } - - fn enqueue(&mut self, subtree: &tt::Subtree, contents: tt::iter::TtIter<'a>) -> u32 { - let idx = self.subtree.len(); - let open = self.token_id_of(subtree.delimiter.open); - let close = self.token_id_of(subtree.delimiter.close); - let delimiter_kind = subtree.delimiter.kind; - self.subtree.push(SubtreeRepr { open, close, kind: delimiter_kind, tt: [!0, !0] }); - // FIXME: `count()` walks over the entire iterator. - self.work.push_back((idx, contents.clone().count(), contents)); - idx as u32 - } -} - -impl<'a, T: SpanTransformer, U> Writer<'a, '_, T, U> { - fn token_id_of(&mut self, span: T::Span) -> SpanId { - T::token_id_of(self.span_data_table, span) - } - - #[cfg(feature = "in-rust-tree")] - pub(crate) fn intern(&mut self, text: &'a str) -> u32 { - let table = &mut self.text; - *self.string_table.entry(text.into()).or_insert_with(|| { - let idx = table.len(); - table.push(text.to_owned()); - idx as u32 - }) - } - - pub(crate) fn intern_owned(&mut self, text: String) -> u32 { - let table = &mut self.text; - *self.string_table.entry(text.clone().into()).or_insert_with(|| { - let idx = table.len(); - table.push(text); - idx as u32 - }) - } -} - -#[cfg(feature = "in-rust-tree")] -impl<'a, T: SpanTransformer> - Writer<'a, '_, T, Option>> -{ - fn write_tokenstream( - &mut self, - call_site: T::Span, - root: &'a proc_macro_srv::TokenStream, - ) { - let call_site = self.token_id_of(call_site); - if let Some(group) = root.as_single_group() { - self.enqueue(group); - } else { - self.subtree.push(SubtreeRepr { - open: call_site, - close: call_site, - kind: tt::DelimiterKind::Invisible, - tt: [!0, !0], - }); - self.work.push_back((0, root.len(), Some(root.iter()))); - } - while let Some((idx, len, group)) = self.work.pop_front() { - self.group(idx, len, group); - } - } - - fn group( - &mut self, - idx: usize, - n_tt: usize, - group: Option>, - ) { - let mut first_tt = self.token_tree.len(); - self.token_tree.resize(first_tt + n_tt, !0); - - self.subtree[idx].tt = [first_tt as u32, (first_tt + n_tt) as u32]; - - for tt in group.into_iter().flatten() { - let idx_tag = match tt { - proc_macro_srv::TokenTree::Group(group) => { - let idx = self.enqueue(group); - idx << 2 - } - proc_macro_srv::TokenTree::Literal(lit) => { - let idx = self.literal.len() as u32; - let id = self.token_id_of(lit.span); - let (text, suffix) = if self.version >= EXTENDED_LEAF_DATA { - ( - self.intern(lit.symbol.as_str()), - lit.suffix.as_ref().map(|s| self.intern(s.as_str())).unwrap_or(!0), - ) - } else { - (self.intern_owned(proc_macro_srv::literal_to_string(lit)), !0) - }; - self.literal.push(LiteralRepr { - id, - text, - kind: u16::from_le_bytes(match lit.kind { - proc_macro_srv::LitKind::ErrWithGuar => [0, 0], - proc_macro_srv::LitKind::Byte => [1, 0], - proc_macro_srv::LitKind::Char => [2, 0], - proc_macro_srv::LitKind::Integer => [3, 0], - proc_macro_srv::LitKind::Float => [4, 0], - proc_macro_srv::LitKind::Str => [5, 0], - proc_macro_srv::LitKind::StrRaw(r) => [6, r], - proc_macro_srv::LitKind::ByteStr => [7, 0], - proc_macro_srv::LitKind::ByteStrRaw(r) => [8, r], - proc_macro_srv::LitKind::CStr => [9, 0], - proc_macro_srv::LitKind::CStrRaw(r) => [10, r], - }), - suffix, - }); - (idx << 2) | 0b01 - } - proc_macro_srv::TokenTree::Punct(punct) => { - let idx = self.punct.len() as u32; - let id = self.token_id_of(punct.span); - self.punct.push(PunctRepr { - char: punct.ch as char, - spacing: if punct.joint { tt::Spacing::Joint } else { tt::Spacing::Alone }, - id, - }); - (idx << 2) | 0b10 - } - proc_macro_srv::TokenTree::Ident(ident) => { - let idx = self.ident.len() as u32; - let id = self.token_id_of(ident.span); - let text = if self.version >= EXTENDED_LEAF_DATA { - self.intern(ident.sym.as_str()) - } else if ident.is_raw { - self.intern_owned(format!("r#{}", ident.sym.as_str(),)) - } else { - self.intern(ident.sym.as_str()) - }; - self.ident.push(IdentRepr { id, text, is_raw: ident.is_raw }); - (idx << 2) | 0b11 - } - }; - self.token_tree[first_tt] = idx_tag; - first_tt += 1; - } - } - - fn enqueue(&mut self, group: &'a proc_macro_srv::Group) -> u32 { - let idx = self.subtree.len(); - let open = self.token_id_of(group.span.open); - let close = self.token_id_of(group.span.close); - let delimiter_kind = match group.delimiter { - proc_macro_srv::Delimiter::Parenthesis => tt::DelimiterKind::Parenthesis, - proc_macro_srv::Delimiter::Brace => tt::DelimiterKind::Brace, - proc_macro_srv::Delimiter::Bracket => tt::DelimiterKind::Bracket, - proc_macro_srv::Delimiter::None => tt::DelimiterKind::Invisible, - }; - self.subtree.push(SubtreeRepr { open, close, kind: delimiter_kind, tt: [!0, !0] }); - self.work.push_back(( - idx, - group.stream.as_ref().map_or(0, |stream| stream.len()), - group.stream.as_ref().map(|ts| ts.iter()), - )); - idx as u32 - } -} - -struct Reader<'span, S: SpanTransformer> { - version: u32, - subtree: Vec, - literal: Vec, - punct: Vec, - ident: Vec, - token_tree: Vec, - text: Vec, - span_data_table: &'span S::Table, -} - -impl> Reader<'_, T> { - pub(crate) fn read_subtree(self) -> tt::TopSubtree { - let mut res: Vec)>> = - vec![None; self.subtree.len()]; - let read_span = |id| T::span_for_token_id(self.span_data_table, id); - for i in (0..self.subtree.len()).rev() { - let repr = &self.subtree[i]; - let token_trees = &self.token_tree[repr.tt[0] as usize..repr.tt[1] as usize]; - let delimiter = tt::Delimiter { - open: read_span(repr.open), - close: read_span(repr.close), - kind: repr.kind, - }; - let mut s = Vec::new(); - for &idx_tag in token_trees { - let tag = idx_tag & 0b11; - let idx = (idx_tag >> 2) as usize; - match tag { - // XXX: we iterate subtrees in reverse to guarantee - // that this unwrap doesn't fire. - 0b00 => { - let (delimiter, subtree) = res[idx].take().unwrap(); - s.push(tt::TokenTree::Subtree(tt::Subtree { - delimiter, - len: subtree.len() as u32, - })); - s.extend(subtree) - } - 0b01 => { - use tt::LitKind::*; - let repr = &self.literal[idx]; - let text = self.text[repr.text as usize].as_str(); - let span = read_span(repr.id); - s.push( - tt::Leaf::Literal(if self.version >= EXTENDED_LEAF_DATA { - tt::Literal::new( - text, - span, - match u16::to_le_bytes(repr.kind) { - [0, _] => Err(()), - [1, _] => Byte, - [2, _] => Char, - [3, _] => Integer, - [4, _] => Float, - [5, _] => Str, - [6, r] => StrRaw(r), - [7, _] => ByteStr, - [8, r] => ByteStrRaw(r), - [9, _] => CStr, - [10, r] => CStrRaw(r), - _ => unreachable!(), - }, - if repr.suffix != !0 { - self.text[repr.suffix as usize].as_str() - } else { - "" - }, - ) - } else { - tt::token_to_literal(text, span) - }) - .into(), - ) - } - 0b10 => { - let repr = &self.punct[idx]; - s.push( - tt::Leaf::Punct(tt::Punct { - char: repr.char, - spacing: repr.spacing, - span: read_span(repr.id), - }) - .into(), - ) - } - 0b11 => { - let repr = &self.ident[idx]; - let text = self.text[repr.text as usize].as_str(); - let (is_raw, text) = if self.version >= EXTENDED_LEAF_DATA { - ( - if repr.is_raw { tt::IdentIsRaw::Yes } else { tt::IdentIsRaw::No }, - text, - ) - } else { - tt::IdentIsRaw::split_from_symbol(text) - }; - s.push( - tt::Leaf::Ident(tt::Ident { - sym: Symbol::intern(text), - span: read_span(repr.id), - is_raw, - }) - .into(), - ) - } - other => panic!("bad tag: {other}"), - } - } - res[i] = Some((delimiter, s)); - } - - let (delimiter, mut res) = res[0].take().unwrap(); - res.insert(0, tt::TokenTree::Subtree(tt::Subtree { delimiter, len: res.len() as u32 })); - tt::TopSubtree::from_serialized(res) - } -} - -#[cfg(feature = "in-rust-tree")] -impl Reader<'_, T> { - pub(crate) fn read_tokenstream( - self, - span_join: impl Fn(T::Span, T::Span) -> T::Span, - ) -> proc_macro_srv::TokenStream { - let mut res: Vec>> = vec![None; self.subtree.len()]; - let read_span = |id| T::span_for_token_id(self.span_data_table, id); - for i in (0..self.subtree.len()).rev() { - let repr = &self.subtree[i]; - let token_trees = &self.token_tree[repr.tt[0] as usize..repr.tt[1] as usize]; - - let stream = token_trees - .iter() - .copied() - .map(|idx_tag| { - let tag = idx_tag & 0b11; - let idx = (idx_tag >> 2) as usize; - match tag { - // XXX: we iterate subtrees in reverse to guarantee - // that this unwrap doesn't fire. - 0b00 => proc_macro_srv::TokenTree::Group(res[idx].take().unwrap()), - 0b01 => { - let repr = &self.literal[idx]; - let text = self.text[repr.text as usize].as_str(); - let span = read_span(repr.id); - proc_macro_srv::TokenTree::Literal( - if self.version >= EXTENDED_LEAF_DATA { - proc_macro_srv::Literal { - symbol: Symbol::intern(text), - span, - kind: match u16::to_le_bytes(repr.kind) { - [0, _] => proc_macro_srv::LitKind::ErrWithGuar, - [1, _] => proc_macro_srv::LitKind::Byte, - [2, _] => proc_macro_srv::LitKind::Char, - [3, _] => proc_macro_srv::LitKind::Integer, - [4, _] => proc_macro_srv::LitKind::Float, - [5, _] => proc_macro_srv::LitKind::Str, - [6, r] => proc_macro_srv::LitKind::StrRaw(r), - [7, _] => proc_macro_srv::LitKind::ByteStr, - [8, r] => proc_macro_srv::LitKind::ByteStrRaw(r), - [9, _] => proc_macro_srv::LitKind::CStr, - [10, r] => proc_macro_srv::LitKind::CStrRaw(r), - _ => unreachable!(), - }, - suffix: if repr.suffix != !0 { - Some(Symbol::intern( - self.text[repr.suffix as usize].as_str(), - )) - } else { - None - }, - } - } else { - proc_macro_srv::literal_from_str(text, span).unwrap_or_else( - |_| proc_macro_srv::Literal { - symbol: Symbol::intern("internal error"), - span, - kind: proc_macro_srv::LitKind::ErrWithGuar, - suffix: None, - }, - ) - }, - ) - } - 0b10 => { - let repr = &self.punct[idx]; - proc_macro_srv::TokenTree::Punct(proc_macro_srv::Punct { - ch: repr.char as u8, - joint: repr.spacing == tt::Spacing::Joint, - span: read_span(repr.id), - }) - } - 0b11 => { - let repr = &self.ident[idx]; - let text = self.text[repr.text as usize].as_str(); - let (is_raw, text) = if self.version >= EXTENDED_LEAF_DATA { - ( - if repr.is_raw { - tt::IdentIsRaw::Yes - } else { - tt::IdentIsRaw::No - }, - text, - ) - } else { - tt::IdentIsRaw::split_from_symbol(text) - }; - proc_macro_srv::TokenTree::Ident(proc_macro_srv::Ident { - sym: Symbol::intern(text), - span: read_span(repr.id), - is_raw: is_raw.yes(), - }) - } - other => panic!("bad tag: {other}"), - } - }) - .collect::>(); - let open = read_span(repr.open); - let close = read_span(repr.close); - let g = proc_macro_srv::Group { - delimiter: match repr.kind { - tt::DelimiterKind::Parenthesis => proc_macro_srv::Delimiter::Parenthesis, - tt::DelimiterKind::Brace => proc_macro_srv::Delimiter::Brace, - tt::DelimiterKind::Bracket => proc_macro_srv::Delimiter::Bracket, - tt::DelimiterKind::Invisible => proc_macro_srv::Delimiter::None, - }, - stream: if stream.is_empty() { None } else { Some(TokenStream::new(stream)) }, - span: proc_macro_srv::DelimSpan { - open, - close, - // FIXME: The protocol does not yet encode entire spans ... - entire: span_join(open, close), - }, - }; - res[i] = Some(g); - } - let group = res[0].take().unwrap(); - if group.delimiter == proc_macro_srv::Delimiter::None { - group.stream.unwrap_or_default() - } else { - TokenStream::new(vec![proc_macro_srv::TokenTree::Group(group)]) - } - } -} diff --git a/crates/proc-macro-api/src/legacy_protocol/sender.rs b/crates/proc-macro-api/src/legacy_protocol/sender.rs new file mode 100644 index 000000000000..e0521422c0f4 --- /dev/null +++ b/crates/proc-macro-api/src/legacy_protocol/sender.rs @@ -0,0 +1,158 @@ +//! Functions for the sender side, i.e. the rust-analyzer side. + +use std::{ + io::{BufRead, Write}, + sync::Arc, +}; + +use paths::AbsPath; +use span::Span; + +use crate::{ + ProcMacroKind, + client::{ProcMacro, ServerError}, + flat::{ + FlatTree, SpanDataIndexMap, deserialize_span_data_index_map, serialize_span_data_index_map, + }, + legacy_protocol::msg::{ + ExpandMacro, ExpandMacroData, ExpnGlobals, Message, Request, Response, ServerConfig, + SpanMode, + }, + process::ProcMacroServerProcess, + version, +}; + +pub(crate) fn version_check(srv: &ProcMacroServerProcess) -> Result { + let request = Request::ApiVersionCheck {}; + let response = send_task(srv, request)?; + + match response { + Response::ApiVersionCheck(version) => Ok(version), + _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), + } +} + +/// Enable support for rust-analyzer span mode if the server supports it. +pub(crate) fn enable_rust_analyzer_spans( + srv: &ProcMacroServerProcess, +) -> Result { + let request = Request::SetConfig(ServerConfig { span_mode: SpanMode::RustAnalyzer }); + let response = send_task(srv, request)?; + + match response { + Response::SetConfig(ServerConfig { span_mode }) => Ok(span_mode), + _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), + } +} + +/// Finds proc-macros in a given dynamic library. +pub(crate) fn find_proc_macros( + srv: &ProcMacroServerProcess, + dylib_path: &AbsPath, +) -> Result, String>, ServerError> { + let request = Request::ListMacros { dylib_path: dylib_path.to_path_buf().into() }; + + let response = send_task(srv, request)?; + + match response { + Response::ListMacros(it) => Ok(it), + _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), + } +} + +pub(crate) fn expand( + proc_macro: &ProcMacro, + process: &ProcMacroServerProcess, + subtree: tt::SubtreeView<'_>, + attr: Option>, + env: Vec<(String, String)>, + def_site: Span, + call_site: Span, + mixed_site: Span, + current_dir: String, +) -> Result, ServerError> { + let version = process.version(); + let mut span_data_table = SpanDataIndexMap::default(); + let def_site = span_data_table.insert_full(def_site).0; + let call_site = span_data_table.insert_full(call_site).0; + let mixed_site = span_data_table.insert_full(mixed_site).0; + let task = ExpandMacro { + data: ExpandMacroData { + macro_body: FlatTree::from_subtree(subtree, version, &mut span_data_table), + macro_name: proc_macro.name.to_string(), + attributes: attr + .map(|subtree| FlatTree::from_subtree(subtree, version, &mut span_data_table)), + has_global_spans: ExpnGlobals { + serialize: version >= version::HAS_GLOBAL_SPANS, + def_site, + call_site, + mixed_site, + }, + span_data_table: if process.rust_analyzer_spans() { + serialize_span_data_index_map(&span_data_table) + } else { + Vec::new() + }, + }, + lib: proc_macro.dylib_path.to_path_buf().into(), + env, + current_dir: Some(current_dir), + }; + + let response = send_task(process, Request::ExpandMacro(Box::new(task)))?; + + match response { + Response::ExpandMacro(it) => Ok(it + .map(|tree| { + let mut expanded = FlatTree::to_subtree(tree, version, &span_data_table); + if proc_macro.needs_fixup_change() { + proc_macro.change_fixup_to_match_old_server(&mut expanded); + } + expanded + }) + .map_err(|msg| msg.0)), + Response::ExpandMacroExtended(it) => Ok(it + .map(|resp| { + let mut expanded = FlatTree::to_subtree( + resp.tree, + version, + &deserialize_span_data_index_map(&resp.span_data_table), + ); + if proc_macro.needs_fixup_change() { + proc_macro.change_fixup_to_match_old_server(&mut expanded); + } + expanded + }) + .map_err(|msg| msg.0)), + _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), + } +} + +/// Sends a request to the proc-macro server and waits for a response. +fn send_task(srv: &ProcMacroServerProcess, req: Request) -> Result { + if let Some(server_error) = srv.exited() { + return Err(server_error.clone()); + } + + crate::flat::with_serialization_version(srv.version(), || { + srv.send_task_legacy::<_, _>(send_request, req) + }) +} + +/// Sends a request to the server and reads the response. +fn send_request( + mut writer: &mut dyn Write, + mut reader: &mut dyn BufRead, + req: Request, + buf: &mut String, +) -> Result, ServerError> { + req.write(&mut writer).map_err(|err| ServerError { + message: "failed to write request".into(), + io: Some(Arc::new(err)), + })?; + let res = Response::read(&mut reader, buf).map_err(|err| ServerError { + message: "failed to read response".into(), + io: Some(Arc::new(err)), + })?; + Ok(res) +} diff --git a/crates/proc-macro-api/src/lib.rs b/crates/proc-macro-api/src/lib.rs index 4b5e25e48801..66afbcd55a28 100644 --- a/crates/proc-macro-api/src/lib.rs +++ b/crates/proc-macro-api/src/lib.rs @@ -5,30 +5,30 @@ //! is used to provide basic infrastructure for communication between two //! processes: Client (RA itself), Server (the external program) -#![cfg_attr(not(feature = "in-rust-tree"), allow(unused_crate_dependencies))] -#![cfg_attr( - feature = "in-rust-tree", - feature(proc_macro_internals, proc_macro_diagnostic, proc_macro_span, rustc_private) -)] +#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))] #![allow(internal_features, unused_features)] #[cfg(feature = "in-rust-tree")] extern crate rustc_driver as _; +stdx::rustc_crates! { + extern crate rustc_lexer or ra_ap_rustc_lexer; +} + pub mod bidirectional_protocol; +#[cfg(feature = "in-ra")] +pub mod client; +pub mod flat; pub mod legacy_protocol; +#[cfg(feature = "in-ra")] pub mod pool; +#[cfg(feature = "in-ra")] pub mod process; +#[cfg(feature = "in-proc-macro-srv")] +pub mod token_stream; pub mod transport; -use paths::{AbsPath, AbsPathBuf}; -use semver::Version; -use span::{ErasedFileAstId, FIXUP_ERASED_FILE_AST_ID_MARKER, Span}; -use std::{fmt, io, sync::Arc, time::SystemTime}; - -use crate::{ - bidirectional_protocol::SubCallback, pool::ProcMacroServerPool, process::ProcMacroServerProcess, -}; +use std::fmt; /// The versions of the server protocol pub mod version { @@ -40,9 +40,10 @@ pub mod version { /// Whether literals encode their kind as an additional u32 field and idents their rawness as a u32 field. pub const EXTENDED_LEAF_DATA: u32 = 5; pub const HASHED_AST_ID: u32 = 6; + pub const DOC_COMMENT_LEAF: u32 = 7; /// Current API version of the proc-macro protocol. - pub const CURRENT_API_VERSION: u32 = HASHED_AST_ID; + pub const CURRENT_API_VERSION: u32 = DOC_COMMENT_LEAF; } /// Protocol format for communication between client and server. @@ -77,203 +78,3 @@ pub enum ProcMacroKind { #[serde(rename(serialize = "FuncLike", deserialize = "FuncLike"))] Bang, } - -/// A handle to proc-macro server process pool which load dylibs with macros (.so or .dll) -/// and runs actual macro expansion functions. -#[derive(Debug, Clone)] -pub struct ProcMacroClient { - /// Currently, the proc macro process expands all procedural macros sequentially. - /// - /// That means that concurrent salsa requests may block each other when expanding proc macros, - /// which is unfortunate, but simple and good enough for the time being. - pool: Arc, - /// The path to the proc-macro server binary. - path: AbsPathBuf, -} - -/// Represents a dynamically loaded library containing procedural macros. -pub struct MacroDylib { - path: AbsPathBuf, -} - -impl MacroDylib { - /// Creates a new MacroDylib instance with the given path. - pub fn new(path: AbsPathBuf) -> MacroDylib { - MacroDylib { path } - } -} - -/// A handle to a specific proc-macro (a `#[proc_macro]` annotated function). -/// -/// It exists within the context of a specific proc-macro server -- currently -/// we share a single expander process for all macros within a workspace. -#[derive(Debug, Clone)] -pub struct ProcMacro { - pool: ProcMacroServerPool, - dylib_path: Arc, - name: Box, - kind: ProcMacroKind, - dylib_last_modified: Option, -} - -impl Eq for ProcMacro {} -impl PartialEq for ProcMacro { - fn eq(&self, other: &Self) -> bool { - self.name == other.name - && self.kind == other.kind - && self.dylib_path == other.dylib_path - && self.dylib_last_modified == other.dylib_last_modified - } -} - -/// Represents errors encountered when communicating with the proc-macro server. -#[derive(Clone, Debug)] -pub struct ServerError { - pub message: String, - pub io: Option>, -} - -impl fmt::Display for ServerError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.message.fmt(f)?; - if let Some(io) = &self.io { - f.write_str(": ")?; - io.fmt(f)?; - } - Ok(()) - } -} - -impl ProcMacroClient { - /// Spawns an external process as the proc macro server and returns a client connected to it. - pub fn spawn<'a>( - process_path: &AbsPath, - env: impl IntoIterator< - Item = (impl AsRef, &'a Option>), - > + Clone, - version: Option<&Version>, - num_process: usize, - ) -> io::Result { - let pool_size = num_process; - let mut workers = Vec::with_capacity(pool_size); - for _ in 0..pool_size { - let worker = ProcMacroServerProcess::spawn(process_path, env.clone(), version)?; - workers.push(worker); - } - - let pool = ProcMacroServerPool::new(workers); - Ok(ProcMacroClient { pool: Arc::new(pool), path: process_path.to_owned() }) - } - - /// Invokes `spawn` and returns a client connected to the resulting read and write handles. - /// - /// The `process_path` is used for `Self::server_path`. This function is mainly used for testing. - pub fn with_io_channels( - process_path: &AbsPath, - spawn: impl Fn( - Option, - ) -> io::Result<( - Box, - Box, - Box, - )> + Clone, - version: Option<&Version>, - num_process: usize, - ) -> io::Result { - let pool_size = num_process; - let mut workers = Vec::with_capacity(pool_size); - for _ in 0..pool_size { - let worker = - ProcMacroServerProcess::run(spawn.clone(), version, || "".to_owned())?; - workers.push(worker); - } - - let pool = ProcMacroServerPool::new(workers); - Ok(ProcMacroClient { pool: Arc::new(pool), path: process_path.to_owned() }) - } - - /// Returns the absolute path to the proc-macro server. - pub fn server_path(&self) -> &AbsPath { - &self.path - } - - /// Loads a proc-macro dylib into the server process returning a list of `ProcMacro`s loaded. - pub fn load_dylib(&self, dylib: MacroDylib) -> Result, ServerError> { - self.pool.load_dylib(&dylib) - } - - /// Checks if the proc-macro server has exited. - pub fn exited(&self) -> Option<&ServerError> { - self.pool.exited() - } -} - -impl ProcMacro { - /// Returns the name of the procedural macro. - pub fn name(&self) -> &str { - &self.name - } - - /// Returns the type of procedural macro. - pub fn kind(&self) -> ProcMacroKind { - self.kind - } - - fn needs_fixup_change(&self) -> bool { - let version = self.pool.version(); - (version::RUST_ANALYZER_SPAN_SUPPORT..version::HASHED_AST_ID).contains(&version) - } - - /// On some server versions, the fixup ast id is different than ours. So change it to match. - fn change_fixup_to_match_old_server(&self, tt: &mut tt::TopSubtree) { - const OLD_FIXUP_AST_ID: ErasedFileAstId = ErasedFileAstId::from_raw(!0 - 1); - tt.change_every_ast_id(|ast_id| { - if *ast_id == FIXUP_ERASED_FILE_AST_ID_MARKER { - *ast_id = OLD_FIXUP_AST_ID; - } else if *ast_id == OLD_FIXUP_AST_ID { - // Swap between them, that means no collision plus the change can be reversed by doing itself. - *ast_id = FIXUP_ERASED_FILE_AST_ID_MARKER; - } - }); - } - - /// Expands the procedural macro by sending an expansion request to the server. - /// This includes span information and environmental context. - pub fn expand( - &self, - subtree: tt::SubtreeView<'_>, - attr: Option>, - env: Vec<(String, String)>, - def_site: Span, - call_site: Span, - mixed_site: Span, - current_dir: String, - callback: Option>, - ) -> Result, ServerError> { - let (mut subtree, mut attr) = (subtree, attr); - let (mut subtree_changed, mut attr_changed); - if self.needs_fixup_change() { - subtree_changed = tt::TopSubtree::from_subtree(subtree); - self.change_fixup_to_match_old_server(&mut subtree_changed); - subtree = subtree_changed.view(); - - if let Some(attr) = &mut attr { - attr_changed = tt::TopSubtree::from_subtree(*attr); - self.change_fixup_to_match_old_server(&mut attr_changed); - *attr = attr_changed.view(); - } - } - - self.pool.pick_process()?.expand( - self, - subtree, - attr, - env, - def_site, - call_site, - mixed_site, - current_dir, - callback, - ) - } -} diff --git a/crates/proc-macro-api/src/pool.rs b/crates/proc-macro-api/src/pool.rs index e6541823da58..1f449a12399e 100644 --- a/crates/proc-macro-api/src/pool.rs +++ b/crates/proc-macro-api/src/pool.rs @@ -3,7 +3,10 @@ use std::sync::Arc; use rayon::iter::{IntoParallelIterator, ParallelIterator}; -use crate::{MacroDylib, ProcMacro, ServerError, process::ProcMacroServerProcess}; +use crate::{ + client::{MacroDylib, ProcMacro, ServerError}, + process::ProcMacroServerProcess, +}; #[derive(Debug, Clone)] pub(crate) struct ProcMacroServerPool { diff --git a/crates/proc-macro-api/src/process.rs b/crates/proc-macro-api/src/process.rs index 035c12669c8f..2781d2b4db0b 100644 --- a/crates/proc-macro-api/src/process.rs +++ b/crates/proc-macro-api/src/process.rs @@ -17,13 +17,14 @@ use span::Span; use stdx::JodChild; use crate::{ - ProcMacro, ProcMacroKind, ProtocolFormat, ServerError, + ProcMacroKind, ProtocolFormat, bidirectional_protocol::{ self, SubCallback, msg::{BidirectionalMessage, SubResponse}, reject_subrequests, }, - legacy_protocol::{self, SpanMode}, + client::{ProcMacro, ServerError}, + legacy_protocol::{self, msg::SpanMode}, version, }; diff --git a/crates/proc-macro-api/src/token_stream.rs b/crates/proc-macro-api/src/token_stream.rs new file mode 100644 index 000000000000..ad0f163b8cb6 --- /dev/null +++ b/crates/proc-macro-api/src/token_stream.rs @@ -0,0 +1,634 @@ +//! The proc-macro server token stream implementation. + +use core::fmt; +use std::{mem, rc::Rc}; + +use intern::Symbol; +use tt::{ + Delimiter, DelimiterKind, DocComment, Ident, IdentIsRaw, Leaf, LitKind, Punct, Spacing, + literal_from_lexer, +}; + +/// Trait for allowing tests to parse tokenstreams with dynamic span ranges +pub trait SpanLike: Copy { + fn derive_ranged(&self, range: std::ops::Range) -> Self; + + fn cover(self, other: Self) -> Self; +} + +#[derive(Debug, Clone)] +pub struct Group { + pub delimiter: Delimiter, + pub stream: Option>, +} + +impl Group { + pub fn stream_len(&self) -> usize { + self.stream.as_ref().map_or(0, |it| it.len()) + } +} + +#[derive(Clone)] +pub enum TokenTree { + Leaf(Leaf), + Group(Group), +} + +#[derive(Clone)] +#[expect(clippy::rc_buffer, reason = "we commonly mutate this via `Rc::make_mut()`")] +pub struct TokenStream(Rc>>); + +impl Default for TokenStream { + fn default() -> Self { + Self(Default::default()) + } +} + +impl TokenStream { + #[inline] + pub fn new(tts: Vec>) -> TokenStream { + TokenStream(Rc::new(tts)) + } + + #[inline] + pub fn new_or_empty(tts: Vec>) -> Option> { + if tts.is_empty() { None } else { Some(TokenStream(Rc::new(tts))) } + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + #[inline] + pub fn len(&self) -> usize { + self.0.len() + } + + #[inline] + pub fn iter(&self) -> std::slice::Iter<'_, TokenTree> { + self.0.iter() + } + + #[inline] + pub fn as_slice(&self) -> &[TokenTree] { + &self.0 + } + + #[inline] + pub fn as_single_group(&self) -> Option<&Group> { + match &**self.0 { + [TokenTree::Group(group)] => Some(group), + _ => None, + } + } + + pub fn from_str(s: &str, span: S) -> Result + where + S: SpanLike + Copy, + { + let mut groups = Vec::new(); + groups.push((DelimiterKind::Invisible, 0..0, vec![])); + let mut offset = 0; + let mut tokens = rustc_lexer::tokenize(s, rustc_lexer::FrontmatterAllowed::No).peekable(); + while let Some(token) = tokens.next() { + let range = offset..offset + token.len as usize; + offset += token.len as usize; + + let mut spacing = || { + let is_joint = tokens.peek().is_some_and(|token| { + matches!( + token.kind, + rustc_lexer::TokenKind::RawLifetime + | rustc_lexer::TokenKind::GuardedStrPrefix + | rustc_lexer::TokenKind::Lifetime { .. } + | rustc_lexer::TokenKind::Semi + | rustc_lexer::TokenKind::Comma + | rustc_lexer::TokenKind::Dot + | rustc_lexer::TokenKind::OpenParen + | rustc_lexer::TokenKind::CloseParen + | rustc_lexer::TokenKind::OpenBrace + | rustc_lexer::TokenKind::CloseBrace + | rustc_lexer::TokenKind::OpenBracket + | rustc_lexer::TokenKind::CloseBracket + | rustc_lexer::TokenKind::At + | rustc_lexer::TokenKind::Pound + | rustc_lexer::TokenKind::Tilde + | rustc_lexer::TokenKind::Question + | rustc_lexer::TokenKind::Colon + | rustc_lexer::TokenKind::Dollar + | rustc_lexer::TokenKind::Eq + | rustc_lexer::TokenKind::Bang + | rustc_lexer::TokenKind::Lt + | rustc_lexer::TokenKind::Gt + | rustc_lexer::TokenKind::Minus + | rustc_lexer::TokenKind::And + | rustc_lexer::TokenKind::Or + | rustc_lexer::TokenKind::Plus + | rustc_lexer::TokenKind::Star + | rustc_lexer::TokenKind::Slash + | rustc_lexer::TokenKind::Percent + | rustc_lexer::TokenKind::Caret + ) + }); + if is_joint { Spacing::Joint } else { Spacing::Alone } + }; + + let Some((open_delim, _, tokenstream)) = groups.last_mut() else { + return Err("Unbalanced delimiters".to_owned()); + }; + match token.kind { + rustc_lexer::TokenKind::OpenParen => { + groups.push((DelimiterKind::Parenthesis, range, vec![])) + } + rustc_lexer::TokenKind::CloseParen if *open_delim != DelimiterKind::Parenthesis => { + return if *open_delim == DelimiterKind::Invisible { + Err("Unexpected ')'".to_owned()) + } else { + Err("Expected ')'".to_owned()) + }; + } + rustc_lexer::TokenKind::CloseParen => { + let (delimiter, open_range, stream) = groups.pop().unwrap(); + groups.last_mut().ok_or_else(|| "Unbalanced delimiters".to_owned())?.2.push( + TokenTree::Group(Group { + delimiter: Delimiter { + open: span.derive_ranged(open_range), + close: span.derive_ranged(range), + kind: delimiter, + }, + stream: TokenStream::new_or_empty(stream), + }), + ); + } + rustc_lexer::TokenKind::OpenBrace => { + groups.push((DelimiterKind::Brace, range, vec![])) + } + rustc_lexer::TokenKind::CloseBrace if *open_delim != DelimiterKind::Brace => { + return if *open_delim == DelimiterKind::Invisible { + Err("Unexpected '}'".to_owned()) + } else { + Err("Expected '}'".to_owned()) + }; + } + rustc_lexer::TokenKind::CloseBrace => { + let (delimiter, open_range, stream) = groups.pop().unwrap(); + groups.last_mut().ok_or_else(|| "Unbalanced delimiters".to_owned())?.2.push( + TokenTree::Group(Group { + delimiter: Delimiter { + open: span.derive_ranged(open_range), + close: span.derive_ranged(range), + kind: delimiter, + }, + stream: TokenStream::new_or_empty(stream), + }), + ); + } + rustc_lexer::TokenKind::OpenBracket => { + groups.push((DelimiterKind::Bracket, range, vec![])) + } + rustc_lexer::TokenKind::CloseBracket if *open_delim != DelimiterKind::Bracket => { + return if *open_delim == DelimiterKind::Invisible { + Err("Unexpected ']'".to_owned()) + } else { + Err("Expected ']'".to_owned()) + }; + } + rustc_lexer::TokenKind::CloseBracket => { + let (delimiter, open_range, stream) = groups.pop().unwrap(); + groups.last_mut().ok_or_else(|| "Unbalanced delimiters".to_owned())?.2.push( + TokenTree::Group(Group { + delimiter: Delimiter { + open: span.derive_ranged(open_range), + close: span.derive_ranged(range), + kind: delimiter, + }, + stream: TokenStream::new_or_empty(stream), + }), + ); + } + rustc_lexer::TokenKind::LineComment { doc_style: None } + | rustc_lexer::TokenKind::BlockComment { doc_style: None, terminated: _ } => { + continue; + } + rustc_lexer::TokenKind::LineComment { doc_style: Some(doc_style) } => { + let text = &s[range.clone()]; + let span = span.derive_ranged(range); + tokenstream.push(TokenTree::Leaf(Leaf::DocComment(DocComment { + text_with_comment_signs: Symbol::intern(text), + span, + doc_style: tt::DocCommentStyle::from_lexer(doc_style), + comment_style: tt::CommentStyle::Line, + }))); + } + rustc_lexer::TokenKind::BlockComment { doc_style: Some(doc_style), terminated } => { + if !terminated { + return Err("unterminated block comment".to_owned()); + } + let text = &s[range.clone()]; + let span = span.derive_ranged(range); + tokenstream.push(TokenTree::Leaf(Leaf::DocComment(DocComment { + text_with_comment_signs: Symbol::intern(text), + span, + doc_style: tt::DocCommentStyle::from_lexer(doc_style), + comment_style: tt::CommentStyle::Block, + }))); + } + rustc_lexer::TokenKind::Whitespace => continue, + rustc_lexer::TokenKind::Frontmatter { .. } => unreachable!(), + rustc_lexer::TokenKind::Unknown => { + return Err(format!("Unknown token: `{}`", &s[range])); + } + rustc_lexer::TokenKind::UnknownPrefix => { + return Err(format!("Unknown prefix: `{}`", &s[range])); + } + rustc_lexer::TokenKind::UnknownPrefixLifetime => { + return Err(format!("Unknown lifetime prefix: `{}`", &s[range])); + } + // FIXME: Error on edition >= 2024 ... I dont think the proc-macro server can fetch editions currently + // and whose edition is this? + rustc_lexer::TokenKind::GuardedStrPrefix => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: s.as_bytes()[range.start].into(), + spacing: Spacing::Joint, + span: span.derive_ranged(range.start..range.start + 1), + }))); + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: s.as_bytes()[range.start + 1].into(), + spacing: spacing(), + span: span.derive_ranged(range.start + 1..range.end), + }))) + } + rustc_lexer::TokenKind::Ident => { + tokenstream.push(TokenTree::Leaf(Leaf::Ident(Ident { + sym: Symbol::intern(&s[range.clone()]), + is_raw: IdentIsRaw::No, + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::InvalidIdent => { + return Err(format!("Invalid identifier: `{}`", &s[range])); + } + rustc_lexer::TokenKind::RawIdent => { + let range = range.start + 2..range.end; + tokenstream.push(TokenTree::Leaf(Leaf::Ident(Ident { + sym: Symbol::intern(&s[range.clone()]), + is_raw: IdentIsRaw::Yes, + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Literal { kind, suffix_start } => { + tokenstream.push(TokenTree::Leaf(Leaf::Literal(literal_from_lexer( + &s[range.clone()], + span.derive_ranged(range), + kind, + suffix_start, + )))) + } + rustc_lexer::TokenKind::RawLifetime => { + let range = range.start + 1 + 2..range.end; + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '\'', + spacing: Spacing::Joint, + span: span.derive_ranged(range.start..range.start + 1), + }))); + tokenstream.push(TokenTree::Leaf(Leaf::Ident(Ident { + sym: Symbol::intern(&s[range.clone()]), + is_raw: IdentIsRaw::Yes, + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Lifetime { starts_with_number } => { + if starts_with_number { + return Err("Lifetime cannot start with a number".to_owned()); + } + let range = range.start + 1..range.end; + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '\'', + spacing: Spacing::Joint, + span: span.derive_ranged(range.start..range.start + 1), + }))); + tokenstream.push(TokenTree::Leaf(Leaf::Ident(Ident { + sym: Symbol::intern(&s[range.clone()]), + is_raw: IdentIsRaw::No, + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Semi => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: ';', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Comma => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: ',', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Dot => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '.', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::At => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '@', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Pound => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '#', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Tilde => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '~', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Question => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '?', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Colon => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: ':', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Dollar => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '$', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Eq => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '=', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Bang => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '!', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Lt => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '<', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Gt => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '>', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Minus => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '-', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::And => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '&', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Or => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '|', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Plus => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '+', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Star => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '*', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Slash => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '/', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Caret => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '^', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Percent => { + tokenstream.push(TokenTree::Leaf(Leaf::Punct(Punct { + char: '%', + spacing: spacing(), + span: span.derive_ranged(range), + }))) + } + rustc_lexer::TokenKind::Eof => break, + } + } + if let Some((DelimiterKind::Invisible, _, tokentrees)) = groups.pop() + && groups.is_empty() + { + Ok(TokenStream::new(tokentrees)) + } else { + Err("Mismatched token groups".to_owned()) + } + } +} + +impl fmt::Display for TokenStream { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut emit_whitespace = false; + for tt in self.0.iter() { + display_token_tree(tt, &mut emit_whitespace, f)?; + } + Ok(()) + } +} + +fn display_token_tree( + tt: &TokenTree, + emit_whitespace: &mut bool, + f: &mut std::fmt::Formatter<'_>, +) -> std::fmt::Result { + if mem::take(emit_whitespace) { + write!(f, " ")?; + } + match tt { + TokenTree::Group(Group { delimiter, stream }) => { + let (open, close) = delimiter.kind.display_open_close(); + write!(f, "{open}")?; + if let Some(stream) = stream { + write!(f, "{stream}")?; + } + write!(f, "{close}")?; + } + TokenTree::Leaf(leaf) => { + fmt::Display::fmt(leaf, f)?; + *emit_whitespace = match leaf { + Leaf::Literal(literal) => !matches!( + literal.kind, + LitKind::Str + | LitKind::StrRaw(_) + | LitKind::ByteStr + | LitKind::ByteStrRaw(_) + | LitKind::CStr + | LitKind::CStrRaw(_) + ), + Leaf::Punct(punct) => punct.spacing == Spacing::Alone, + Leaf::Ident(_) => true, + Leaf::DocComment(_) => false, + }; + } + } + Ok(()) +} + +impl fmt::Debug for TokenStream { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + debug_token_stream(self, 0, f) + } +} + +fn debug_token_stream( + ts: &TokenStream, + depth: usize, + f: &mut std::fmt::Formatter<'_>, +) -> std::fmt::Result { + for tt in ts.0.iter() { + debug_token_tree(tt, depth, f)?; + } + Ok(()) +} + +fn debug_token_tree( + tt: &TokenTree, + depth: usize, + f: &mut std::fmt::Formatter<'_>, +) -> std::fmt::Result { + write!(f, "{:indent$}", "", indent = depth * 2)?; + + match tt { + TokenTree::Group(Group { delimiter, stream }) => { + writeln!( + f, + "GROUP {} {:#?} {:#?}", + delimiter.kind.debug_view(), + delimiter.open, + delimiter.close, + )?; + if let Some(stream) = stream { + debug_token_stream(stream, depth + 1, f)?; + } + return Ok(()); + } + TokenTree::Leaf(leaf) => leaf.print_debug(f)?, + } + writeln!(f) +} + +impl TokenStream { + pub fn extend_with_streams(&mut self, streams: std::vec::IntoIter>) { + let vec_mut = Rc::make_mut(&mut self.0); + + vec_mut.reserve(streams.as_slice().iter().map(|item| item.len()).sum()); + streams.into_iter().for_each(|item| vec_mut.extend(item.iter().cloned())); + } +} + +impl FromIterator> for TokenStream { + fn from_iter>>(iter: I) -> Self { + TokenStream::new(Vec::from_iter(iter)) + } +} + +impl Extend> for TokenStream { + fn extend>>(&mut self, iter: T) { + let vec_mut = Rc::make_mut(&mut self.0); + vec_mut.extend(iter); + } +} + +impl SpanLike for () { + fn derive_ranged(&self, _: std::ops::Range) -> Self { + *self + } + + fn cover(self, _other: Self) -> Self { + self + } +} + +impl SpanLike for span::Span { + fn derive_ranged(&self, range: std::ops::Range) -> Self { + span::Span { + range: span::TextRange::new( + span::TextSize::new(range.start as u32), + span::TextSize::new(range.end as u32), + ), + anchor: self.anchor, + ctx: self.ctx, + } + } + + fn cover(self, other: Self) -> Self { + self.cover(other) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ts_to_string() { + let token_stream = + TokenStream::from_str("{} () [] <> ;/., \"gfhdgfuiofghd\" 0f32 r#\"dff\"# 'r#lt", ()) + .unwrap(); + assert_eq!(token_stream.to_string(), "{}()[]<> ;/., \"gfhdgfuiofghd\"0f32 r#\"dff\"#'r#lt"); + } + + #[test] + fn doc_comment_from_str() { + let token_stream = TokenStream::from_str("/// foo", ()).unwrap(); + assert_eq!(token_stream.to_string(), "/// foo\n"); + } +} diff --git a/crates/proc-macro-srv-cli/Cargo.toml b/crates/proc-macro-srv-cli/Cargo.toml index 44e19f2d3c3b..cfc157596cf7 100644 --- a/crates/proc-macro-srv-cli/Cargo.toml +++ b/crates/proc-macro-srv-cli/Cargo.toml @@ -15,26 +15,33 @@ doctest = false [dependencies] proc-macro-srv.workspace = true -proc-macro-api.workspace = true -clap = {version = "4.5.42", default-features = false, features = ["std"]} +proc-macro-api = { path = "../proc-macro-api", version = "0.0.0", default-features = false, features = [ + "in-proc-macro-srv", +] } +clap = { version = "4.5.42", default-features = false, features = ["std"] } +# span = { workspace = true, default-features = false } does not work +span = { path = "../span", version = "0.0.0", default-features = false } [dev-dependencies] expect-test.workspace = true paths.workspace = true -# span = {workspace = true, default-features = false} does not work -span = { path = "../span", default-features = false} -tt.workspace = true intern.workspace = true # used as proc macro test target proc-macro-test.path = "../proc-macro-srv/proc-macro-test" +# Enable the in-ra feature for tests. +proc-macro-api = { path = "../proc-macro-api", default-features = false, features = [ + "in-proc-macro-srv", + "in-ra", +] } +tt.workspace = true + [features] default = [] # default = ["in-rust-tree"] in-rust-tree = ["proc-macro-srv/in-rust-tree", "proc-macro-api/in-rust-tree"] - [[bin]] name = "rust-analyzer-proc-macro-srv" path = "src/main.rs" diff --git a/crates/proc-macro-srv-cli/src/lib.rs b/crates/proc-macro-srv-cli/src/lib.rs index c330928fbc9c..e51708a4a025 100644 --- a/crates/proc-macro-srv-cli/src/lib.rs +++ b/crates/proc-macro-srv-cli/src/lib.rs @@ -4,7 +4,88 @@ #![cfg(feature = "in-rust-tree")] #![feature(rustc_private)] +#![expect(clippy::print_stdout, clippy::print_stderr)] extern crate rustc_driver as _; pub mod main_loop; +mod version; + +use clap::{Command, ValueEnum}; +use proc_macro_api::ProtocolFormat; + +pub fn main() -> std::io::Result<()> { + let v = std::env::var("RUST_ANALYZER_INTERNALS_DO_NOT_USE"); + if v.is_err() { + eprintln!( + "This is an IDE implementation detail, you can use this tool by exporting RUST_ANALYZER_INTERNALS_DO_NOT_USE." + ); + eprintln!( + "Note that this tool's API is highly unstable and may break without prior notice" + ); + std::process::exit(122); + } + let matches = Command::new("proc-macro-srv") + .args(&[ + clap::Arg::new("format") + .long("format") + .action(clap::ArgAction::Set) + .default_value("json-legacy") + .value_parser(clap::builder::EnumValueParser::::new()), + clap::Arg::new("version") + .long("version") + .action(clap::ArgAction::SetTrue) + .help("Prints the version of the proc-macro-srv"), + ]) + .get_matches(); + if matches.get_flag("version") { + println!("rust-analyzer-proc-macro-srv {}", version::version()); + return Ok(()); + } + let &format = matches + .get_one::("format") + .expect("format value should always be present"); + + let mut stdin = std::io::BufReader::new(std::io::stdin()); + let mut stdout = std::io::stdout(); + + main_loop::run(&mut stdin, &mut stdout, format.into()) +} + +/// Wrapper for CLI argument parsing that implements `ValueEnum`. +#[derive(Copy, Clone)] +struct ProtocolFormatArg(ProtocolFormat); + +impl From for ProtocolFormat { + fn from(arg: ProtocolFormatArg) -> Self { + arg.0 + } +} + +impl ValueEnum for ProtocolFormatArg { + fn value_variants<'a>() -> &'a [Self] { + &[ + ProtocolFormatArg(ProtocolFormat::JsonLegacy), + ProtocolFormatArg(ProtocolFormat::BidirectionalPostcardPrototype), + ] + } + + fn to_possible_value(&self) -> Option { + match self.0 { + ProtocolFormat::JsonLegacy => Some(clap::builder::PossibleValue::new("json-legacy")), + ProtocolFormat::BidirectionalPostcardPrototype => { + Some(clap::builder::PossibleValue::new("bidirectional-postcard-prototype")) + } + } + } + + fn from_str(input: &str, _ignore_case: bool) -> Result { + match input { + "json-legacy" => Ok(ProtocolFormatArg(ProtocolFormat::JsonLegacy)), + "bidirectional-postcard-prototype" => { + Ok(ProtocolFormatArg(ProtocolFormat::BidirectionalPostcardPrototype)) + } + _ => Err(format!("unknown protocol format: {input}")), + } + } +} diff --git a/crates/proc-macro-srv-cli/src/main.rs b/crates/proc-macro-srv-cli/src/main.rs index 926633df628d..c6669577054e 100644 --- a/crates/proc-macro-srv-cli/src/main.rs +++ b/crates/proc-macro-srv-cli/src/main.rs @@ -2,104 +2,17 @@ //! Driver for proc macro server #![cfg_attr(feature = "in-rust-tree", feature(rustc_private))] #![cfg_attr(not(feature = "in-rust-tree"), allow(unused_crate_dependencies))] -#![allow(clippy::print_stdout, clippy::print_stderr)] #[cfg(feature = "in-rust-tree")] extern crate rustc_driver as _; -mod version; - -use clap::{Command, ValueEnum}; -use proc_macro_api::ProtocolFormat; - -#[cfg(feature = "in-rust-tree")] -use proc_macro_srv_cli::main_loop::run; - fn main() -> std::io::Result<()> { - let v = std::env::var("RUST_ANALYZER_INTERNALS_DO_NOT_USE"); - if v.is_err() { - eprintln!( - "This is an IDE implementation detail, you can use this tool by exporting RUST_ANALYZER_INTERNALS_DO_NOT_USE." - ); - eprintln!( - "Note that this tool's API is highly unstable and may break without prior notice" - ); - std::process::exit(122); - } - let matches = Command::new("proc-macro-srv") - .args(&[ - clap::Arg::new("format") - .long("format") - .action(clap::ArgAction::Set) - .default_value("json-legacy") - .value_parser(clap::builder::EnumValueParser::::new()), - clap::Arg::new("version") - .long("version") - .action(clap::ArgAction::SetTrue) - .help("Prints the version of the proc-macro-srv"), - ]) - .get_matches(); - if matches.get_flag("version") { - println!("rust-analyzer-proc-macro-srv {}", version::version()); - return Ok(()); - } - let &format = matches - .get_one::("format") - .expect("format value should always be present"); - - let mut stdin = std::io::BufReader::new(std::io::stdin()); - let mut stdout = std::io::stdout(); - - run(&mut stdin, &mut stdout, format.into()) -} - -/// Wrapper for CLI argument parsing that implements `ValueEnum`. -#[derive(Copy, Clone)] -struct ProtocolFormatArg(ProtocolFormat); - -impl From for ProtocolFormat { - fn from(arg: ProtocolFormatArg) -> Self { - arg.0 - } -} - -impl ValueEnum for ProtocolFormatArg { - fn value_variants<'a>() -> &'a [Self] { - &[ - ProtocolFormatArg(ProtocolFormat::JsonLegacy), - ProtocolFormatArg(ProtocolFormat::BidirectionalPostcardPrototype), - ] - } - - fn to_possible_value(&self) -> Option { - match self.0 { - ProtocolFormat::JsonLegacy => Some(clap::builder::PossibleValue::new("json-legacy")), - ProtocolFormat::BidirectionalPostcardPrototype => { - Some(clap::builder::PossibleValue::new("bidirectional-postcard-prototype")) - } - } - } - - fn from_str(input: &str, _ignore_case: bool) -> Result { - match input { - "json-legacy" => Ok(ProtocolFormatArg(ProtocolFormat::JsonLegacy)), - "bidirectional-postcard-prototype" => { - Ok(ProtocolFormatArg(ProtocolFormat::BidirectionalPostcardPrototype)) - } - _ => Err(format!("unknown protocol format: {input}")), - } + cfg_select! { + feature = "in-rust-tree" => proc_macro_srv_cli::main(), + _ => Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "proc-macro-srv-cli needs to be compiled with the `in-rust-tree` feature to function" + .to_owned(), + )), } } - -#[cfg(not(feature = "in-rust-tree"))] -fn run( - _: &mut std::io::BufReader, - _: &mut std::io::Stdout, - _: ProtocolFormat, -) -> std::io::Result<()> { - Err(std::io::Error::new( - std::io::ErrorKind::Unsupported, - "proc-macro-srv-cli needs to be compiled with the `in-rust-tree` feature to function" - .to_owned(), - )) -} diff --git a/crates/proc-macro-srv-cli/src/main_loop.rs b/crates/proc-macro-srv-cli/src/main_loop.rs index 6697b6380dd2..f36955a1ce4a 100644 --- a/crates/proc-macro-srv-cli/src/main_loop.rs +++ b/crates/proc-macro-srv-cli/src/main_loop.rs @@ -1,22 +1,24 @@ //! The main loop of the proc-macro server. -use proc_macro_api::bidirectional_protocol::msg::{ApiVersionCheck, ListMacros}; -use proc_macro_api::{ - ProtocolFormat, bidirectional_protocol::msg as bidirectional, legacy_protocol::msg as legacy, - version::CURRENT_API_VERSION, -}; + use std::panic::{panic_any, resume_unwind}; use std::{ io::{self, BufRead, Write}, ops::Range, }; -use legacy::Message; - +use proc_macro_api::{ + ProtocolFormat, + bidirectional_protocol::msg::{self as bidirectional, ApiVersionCheck, ListMacros}, + flat::{self, SpanTransformer}, + legacy_protocol::msg::{self as legacy, Message}, + version::CURRENT_API_VERSION, +}; use proc_macro_srv::{EnvSnapshot, ProcMacroClientError, ProcMacroPanicMarker, SpanId}; +use span::Span; struct SpanTrans; -impl legacy::SpanTransformer for SpanTrans { +impl SpanTransformer for SpanTrans { type Table = (); type Span = SpanId; fn token_id_of( @@ -38,10 +40,10 @@ pub fn run( stdout: &mut (dyn Write + Send + Sync), format: ProtocolFormat, ) -> io::Result<()> { - match format { + proc_macro_api::flat::with_serialization_version(CURRENT_API_VERSION, || match format { ProtocolFormat::JsonLegacy => run_old(stdin, stdout), ProtocolFormat::BidirectionalPostcardPrototype => run_new(stdin, stdout), - } + }) } fn run_new( @@ -138,14 +140,9 @@ fn handle_expand_id( let call_site = SpanId(call_site as u32); let mixed_site = SpanId(mixed_site as u32); - let macro_body = - macro_body.to_tokenstream_unresolved::(CURRENT_API_VERSION, |_, b| b); - let attributes = attributes - .map(|it| it.to_tokenstream_unresolved::(CURRENT_API_VERSION, |_, b| b)); - let mut tracked_env = Default::default(); let res = srv - .expand( + .expand::( lib, &env, current_dir, @@ -156,11 +153,9 @@ fn handle_expand_id( call_site, mixed_site, &mut tracked_env, + &mut (), None, ) - .map(|it| { - legacy::FlatTree::from_tokenstream_raw::(it, call_site, CURRENT_API_VERSION) - }) .map(|tree| bidirectional::ExpandMacroResponse { tree, span_data_table: vec![], @@ -415,27 +410,16 @@ fn handle_expand_ra( }, } = task; - let mut span_data_table = legacy::deserialize_span_data_index_map(&span_data_table); + let mut span_data_table = flat::deserialize_span_data_index_map(&span_data_table); let def_site = span_data_table[def_site]; let call_site = span_data_table[call_site]; let mixed_site = span_data_table[mixed_site]; - let macro_body = - macro_body.to_tokenstream_resolved(CURRENT_API_VERSION, &span_data_table, |a, b| { - srv.join_spans(a, b).unwrap_or(b) - }); - - let attributes = attributes.map(|it| { - it.to_tokenstream_resolved(CURRENT_API_VERSION, &span_data_table, |a, b| { - srv.join_spans(a, b).unwrap_or(b) - }) - }); - let mut tracked_env = Default::default(); let res = srv - .expand( + .expand::( lib, &env, current_dir, @@ -446,19 +430,10 @@ fn handle_expand_ra( call_site, mixed_site, &mut tracked_env, + &mut span_data_table, Some(&mut ProcMacroClientHandle { stdin, stdout, buf }), ) - .map(|it| { - ( - legacy::FlatTree::from_tokenstream( - it, - CURRENT_API_VERSION, - call_site, - &mut span_data_table, - ), - legacy::serialize_span_data_index_map(&span_data_table), - ) - }) + .map(|it| (it, flat::serialize_span_data_index_map(&span_data_table))) .map(|(tree, span_data_table)| bidirectional::ExpandMacroResponse { tree, span_data_table, @@ -521,13 +496,7 @@ fn run_old( let call_site = SpanId(call_site as u32); let mixed_site = SpanId(mixed_site as u32); - let macro_body = macro_body - .to_tokenstream_unresolved::(CURRENT_API_VERSION, |_, b| b); - let attributes = attributes.map(|it| { - it.to_tokenstream_unresolved::(CURRENT_API_VERSION, |_, b| b) - }); - - srv.expand( + srv.expand::( lib, &env, current_dir, @@ -538,39 +507,21 @@ fn run_old( call_site, mixed_site, &mut Default::default(), + &mut (), None, ) - .map(|it| { - legacy::FlatTree::from_tokenstream_raw::( - it, - call_site, - CURRENT_API_VERSION, - ) - }) .map_err(|e| e.into_string().unwrap_or_default()) .map_err(legacy::PanicMessage) }), legacy::SpanMode::RustAnalyzer => legacy::Response::ExpandMacroExtended({ let mut span_data_table = - legacy::deserialize_span_data_index_map(&span_data_table); + flat::deserialize_span_data_index_map(&span_data_table); let def_site = span_data_table[def_site]; let call_site = span_data_table[call_site]; let mixed_site = span_data_table[mixed_site]; - let macro_body = macro_body.to_tokenstream_resolved( - CURRENT_API_VERSION, - &span_data_table, - |a, b| srv.join_spans(a, b).unwrap_or(b), - ); - let attributes = attributes.map(|it| { - it.to_tokenstream_resolved( - CURRENT_API_VERSION, - &span_data_table, - |a, b| srv.join_spans(a, b).unwrap_or(b), - ) - }); - srv.expand( + srv.expand::( lib, &env, current_dir, @@ -581,19 +532,10 @@ fn run_old( call_site, mixed_site, &mut Default::default(), + &mut span_data_table, None, ) - .map(|it| { - ( - legacy::FlatTree::from_tokenstream( - it, - CURRENT_API_VERSION, - call_site, - &mut span_data_table, - ), - legacy::serialize_span_data_index_map(&span_data_table), - ) - }) + .map(|it| (it, flat::serialize_span_data_index_map(&span_data_table))) .map(|(tree, span_data_table)| legacy::ExpandMacroExtended { tree, span_data_table, diff --git a/crates/proc-macro-srv-cli/tests/bidirectional_postcard.rs b/crates/proc-macro-srv-cli/tests/bidirectional_postcard.rs index 9c55eed08e52..9f4b62daa6ac 100644 --- a/crates/proc-macro-srv-cli/tests/bidirectional_postcard.rs +++ b/crates/proc-macro-srv-cli/tests/bidirectional_postcard.rs @@ -20,7 +20,8 @@ use proc_macro_api::{ }, reject_subrequests, }, - legacy_protocol::msg::{PanicMessage, ServerConfig, SpanDataIndexMap, SpanMode}, + flat::SpanDataIndexMap, + legacy_protocol::msg::{PanicMessage, ServerConfig, SpanMode}, version::CURRENT_API_VERSION, }; diff --git a/crates/proc-macro-srv-cli/tests/common/utils.rs b/crates/proc-macro-srv-cli/tests/common/utils.rs index b78e10745274..88599ab573c8 100644 --- a/crates/proc-macro-srv-cli/tests/common/utils.rs +++ b/crates/proc-macro-srv-cli/tests/common/utils.rs @@ -7,11 +7,13 @@ use std::{ use paths::Utf8PathBuf; use proc_macro_api::{ - ServerError, bidirectional_protocol::msg::{ BidirectionalMessage, Request as BiRequest, Response as BiResponse, SubRequest, SubResponse, }, - legacy_protocol::msg::{FlatTree, Message, Request, Response, SpanDataIndexMap}, + client::ServerError, + flat::{FlatTree, SpanDataIndexMap}, + legacy_protocol::msg::{Message, Request, Response}, + version::CURRENT_API_VERSION, }; use span::{Edition, EditionedFileId, FileId, Span, SpanAnchor, SyntaxContext, TextRange}; use tt::{Delimiter, DelimiterKind, TopSubtreeBuilder}; @@ -172,7 +174,9 @@ where proc_macro_srv_cli::main_loop::run(&mut server_reader, &mut server_writer, format) }); - let result = test_fn(&mut client_writer, &mut client_reader); + let result = proc_macro_api::flat::with_serialization_version(CURRENT_API_VERSION, || { + test_fn(&mut client_writer, &mut client_reader) + }); drop(client_writer); diff --git a/crates/proc-macro-srv-cli/tests/legacy_json.rs b/crates/proc-macro-srv-cli/tests/legacy_json.rs index f5cbaa7421eb..3dbe67c595c3 100644 --- a/crates/proc-macro-srv-cli/tests/legacy_json.rs +++ b/crates/proc-macro-srv-cli/tests/legacy_json.rs @@ -18,9 +18,10 @@ use common::utils::{ use expect_test::expect; use proc_macro_api::{ ProtocolFormat::JsonLegacy, + flat::SpanDataIndexMap, legacy_protocol::msg::{ ExpandMacro, ExpandMacroData, ExpnGlobals, PanicMessage, Request, Response, ServerConfig, - SpanDataIndexMap, SpanMode, + SpanMode, }, version::CURRENT_API_VERSION, }; diff --git a/crates/proc-macro-srv/Cargo.toml b/crates/proc-macro-srv/Cargo.toml index 0427d0ee74a6..bb8a61d0af07 100644 --- a/crates/proc-macro-srv/Cargo.toml +++ b/crates/proc-macro-srv/Cargo.toml @@ -16,8 +16,12 @@ doctest = false paths.workspace = true rustc-hash.workspace = true # span = {workspace = true, default-features = false} does not work -span = { path = "../span", version = "0.0.0", default-features = false} +span = { path = "../span", version = "0.0.0", default-features = false } intern.workspace = true +tt = { path = "../tt", version = "0.0.0", default-features = false } +proc-macro-api = { path = "../proc-macro-api", version = "0.0.0", default-features = false, features = [ + "in-proc-macro-srv", +] } stdx.workspace = true [dev-dependencies] @@ -29,10 +33,10 @@ proc-macro-test.path = "./proc-macro-test" [features] default = [] -in-rust-tree = [] +in-rust-tree = ["tt/in-rust-tree", "proc-macro-api/in-rust-tree"] [lints] workspace = true [package.metadata.rust-analyzer] -rustc_private=true +rustc_private = true diff --git a/crates/proc-macro-srv/src/bridge.rs b/crates/proc-macro-srv/src/bridge.rs deleted file mode 100644 index fc62f9413a34..000000000000 --- a/crates/proc-macro-srv/src/bridge.rs +++ /dev/null @@ -1,12 +0,0 @@ -//! `proc_macro::bridge` newtypes. - -use rustc_proc_macro::bridge as pm_bridge; - -pub use pm_bridge::{DelimSpan, Diagnostic, ExpnGlobals, LitKind}; - -pub type TokenTree = - pm_bridge::TokenTree, S, intern::Symbol>; -pub type Literal = pm_bridge::Literal; -pub type Group = pm_bridge::Group, S>; -pub type Punct = pm_bridge::Punct; -pub type Ident = pm_bridge::Ident; diff --git a/crates/proc-macro-srv/src/dylib.rs b/crates/proc-macro-srv/src/dylib.rs index 718eb47228eb..2af91ef4d737 100644 --- a/crates/proc-macro-srv/src/dylib.rs +++ b/crates/proc-macro-srv/src/dylib.rs @@ -3,6 +3,7 @@ mod proc_macros; use paths::{Utf8Path, Utf8PathBuf}; +use proc_macro_api::token_stream::TokenStream; use rustc_codegen_ssa::back::metadata::DefaultMetadataLoader; use rustc_interface::util::rustc_version_str; use rustc_proc_macro::bridge; @@ -11,7 +12,7 @@ use stdx::tempfile::NamedTempFile; use crate::{ PanicMessage, ProcMacroClientHandle, ProcMacroKind, ProcMacroSrvSpan, TrackedEnv, - dylib::proc_macros::ProcMacros, token_stream::TokenStream, + dylib::proc_macros::ProcMacros, }; pub(crate) struct Expander { diff --git a/crates/proc-macro-srv/src/dylib/proc_macros.rs b/crates/proc-macro-srv/src/dylib/proc_macros.rs index 4976298d5fcd..5324a02a54ac 100644 --- a/crates/proc-macro-srv/src/dylib/proc_macros.rs +++ b/crates/proc-macro-srv/src/dylib/proc_macros.rs @@ -1,9 +1,9 @@ //! Proc macro ABI -use crate::{ - ProcMacroClientHandle, ProcMacroKind, ProcMacroSrvSpan, TrackedEnv, token_stream::TokenStream, -}; +use proc_macro_api::token_stream::TokenStream; use rustc_proc_macro::bridge; +use crate::{ProcMacroClientHandle, ProcMacroKind, ProcMacroSrvSpan, TrackedEnv}; + impl From for crate::PanicMessage { fn from(p: bridge::PanicMessage) -> Self { Self { message: p.into_string() } diff --git a/crates/proc-macro-srv/src/lib.rs b/crates/proc-macro-srv/src/lib.rs index e38d2ac11bce..1ea4b9010dff 100644 --- a/crates/proc-macro-srv/src/lib.rs +++ b/crates/proc-macro-srv/src/lib.rs @@ -18,15 +18,12 @@ extern crate rustc_codegen_ssa; extern crate rustc_driver as _; extern crate rustc_interface; -extern crate rustc_lexer; extern crate rustc_metadata; extern crate rustc_proc_macro; extern crate rustc_span; -mod bridge; mod dylib; mod server_impl; -mod token_stream; use std::{ collections::hash_map::Entry, @@ -40,17 +37,20 @@ use std::{ }; use paths::{Utf8Path, Utf8PathBuf}; +use proc_macro_api::{ + flat::{FlatTree, SpanTransformer}, + token_stream::SpanLike, + version::CURRENT_API_VERSION, +}; use rustc_hash::{FxHashMap, FxHashSet}; use span::{FIXUP_ERASED_FILE_AST_ID_MARKER, Span}; -pub use crate::server_impl::token_id::SpanId; - pub use rustc_proc_macro::Delimiter; pub use span; -pub use crate::bridge::*; -pub use crate::server_impl::literal_from_str; -pub use crate::token_stream::{TokenStream, TokenStreamIter, literal_to_string}; +pub use tt::literal_from_str; + +pub use crate::server_impl::token_id::SpanId; #[derive(Copy, Clone, Eq, PartialEq, Debug)] pub enum ProcMacroKind { @@ -128,20 +128,26 @@ impl ExpandError { } impl ProcMacroSrv<'_> { - pub fn expand<'a, S: ProcMacroSrvSpan + 'a>( + pub fn expand<'a, ST>( &self, lib: impl AsRef, env: &[(String, String)], current_dir: Option>, macro_name: &str, - macro_body: token_stream::TokenStream, - attribute: Option>, - def_site: S, - call_site: S, - mixed_site: S, + macro_body: FlatTree, + attribute: Option, + def_site: ST::Span, + call_site: ST::Span, + mixed_site: ST::Span, tracked_env: &'a mut TrackedEnv, + span_data_table: &mut ST::Table, callback: Option>, - ) -> Result, ExpandError> { + ) -> Result + where + ST: SpanTransformer, + ST::Span: ProcMacroSrvSpan + SpanLike + 'a, + ST::Table: Send, + { let snapped_env = self.env; let expander = self.expander(lib.as_ref()).map_err(|err| ExpandError::Internal { reason: Some(format!("failed to load macro: {err}")), @@ -156,16 +162,29 @@ impl ProcMacroSrv<'_> { .stack_size(EXPANDER_STACK_SIZE) .name(macro_name.to_owned()) .spawn_scoped(s, move || { - expander.expand( - macro_name, - macro_body, - attribute, - def_site, - call_site, - mixed_site, - tracked_env, - callback, - ) + let macro_body = + macro_body.to_tokenstream::(CURRENT_API_VERSION, span_data_table); + let attribute = attribute + .map(|it| it.to_tokenstream::(CURRENT_API_VERSION, span_data_table)); + expander + .expand( + macro_name, + macro_body, + attribute, + def_site, + call_site, + mixed_site, + tracked_env, + callback, + ) + .map(|result| { + FlatTree::from_tokenstream::( + result, + call_site, + CURRENT_API_VERSION, + span_data_table, + ) + }) }); match thread.unwrap().join() { Ok(res) => res.map_err(ExpandError::Panic), @@ -233,7 +252,7 @@ pub struct TrackedEnv { pub trait ProcMacroSrvSpan: Copy + Send + Sync { type Server<'a>: rustc_proc_macro::bridge::server::Server< - TokenStream = crate::token_stream::TokenStream, + TokenStream = proc_macro_api::token_stream::TokenStream, >; fn make_server<'a>( call_site: Self, diff --git a/crates/proc-macro-srv/src/server_impl.rs b/crates/proc-macro-srv/src/server_impl.rs index bacead1a88da..aaf8f540d9f4 100644 --- a/crates/proc-macro-srv/src/server_impl.rs +++ b/crates/proc-macro-srv/src/server_impl.rs @@ -6,34 +6,6 @@ //! The original idea from fedochet is using proc-macro2 as backend, //! we use tt instead for better integration with RA. +mod bridge; pub(crate) mod rust_analyzer_span; pub(crate) mod token_id; - -pub fn literal_from_str( - s: &str, - span: Span, -) -> Result, ()> { - use rustc_lexer::{LiteralKind, Token, TokenKind}; - let mut tokens = rustc_lexer::tokenize(s, rustc_lexer::FrontmatterAllowed::No); - let minus_or_lit = tokens.next().unwrap_or(Token { kind: TokenKind::Eof, len: 0 }); - - let lit = if minus_or_lit.kind == TokenKind::Minus { - let lit = tokens.next().ok_or(())?; - if !matches!( - lit.kind, - TokenKind::Literal { kind: LiteralKind::Int { .. } | LiteralKind::Float { .. }, .. } - ) { - return Err(()); - } - lit - } else { - minus_or_lit - }; - - if tokens.next().is_some() { - return Err(()); - } - - let TokenKind::Literal { kind, suffix_start } = lit.kind else { return Err(()) }; - Ok(crate::token_stream::literal_from_lexer(s, span, kind, suffix_start)) -} diff --git a/crates/proc-macro-srv/src/server_impl/bridge.rs b/crates/proc-macro-srv/src/server_impl/bridge.rs new file mode 100644 index 000000000000..c8ca5da134dc --- /dev/null +++ b/crates/proc-macro-srv/src/server_impl/bridge.rs @@ -0,0 +1,215 @@ +//! Conversions between proc_macro bridge types and tt types. + +use intern::sym; +use proc_macro_api::token_stream::SpanLike; + +pub(super) mod ours { + pub(crate) type Literal = tt::Literal; + pub(crate) type Punct = tt::Punct; + pub(crate) type Ident = tt::Ident; + pub(crate) type DocComment = tt::DocComment; + pub(crate) type Leaf = tt::Leaf; + pub(crate) type Group = proc_macro_api::token_stream::Group; + pub(crate) type TokenTree = proc_macro_api::token_stream::TokenTree; + pub(crate) type TokenStream = proc_macro_api::token_stream::TokenStream; +} + +#[expect(clippy::module_inception, reason = "this is not a mistake")] +pub(super) mod bridge { + use rustc_proc_macro::bridge as pm_bridge; + + use super::ours; + + pub(crate) use pm_bridge::*; + + pub(crate) type TokenTree = + pm_bridge::TokenTree, Span, intern::Symbol>; + pub(crate) type Literal = pm_bridge::Literal; + pub(crate) type Group = pm_bridge::Group, Span>; + pub(crate) type Punct = pm_bridge::Punct; + pub(crate) type Ident = pm_bridge::Ident; +} + +pub(super) fn literal_into_bridge(literal: ours::Literal) -> bridge::Literal { + let kind = match literal.kind { + tt::LitKind::Byte => bridge::LitKind::Byte, + tt::LitKind::Char => bridge::LitKind::Char, + tt::LitKind::Integer => bridge::LitKind::Integer, + tt::LitKind::Float => bridge::LitKind::Float, + tt::LitKind::Str => bridge::LitKind::Str, + tt::LitKind::StrRaw(count) => bridge::LitKind::StrRaw(count), + tt::LitKind::ByteStr => bridge::LitKind::ByteStr, + tt::LitKind::ByteStrRaw(count) => bridge::LitKind::ByteStrRaw(count), + tt::LitKind::CStr => bridge::LitKind::CStr, + tt::LitKind::CStrRaw(count) => bridge::LitKind::CStrRaw(count), + tt::LitKind::Err(()) => bridge::LitKind::ErrWithGuar, + }; + let (symbol, suffix) = literal.text_and_suffix_symbols(); + bridge::Literal { kind, symbol, suffix, span: literal.span } +} + +fn punct_into_bridge(punct: ours::Punct) -> bridge::Punct { + bridge::Punct { + // FIXME: Is `as u8` correct here? + ch: punct.char as u8, + joint: punct.spacing != tt::Spacing::Alone, + span: punct.span, + } +} + +fn ident_into_bridge(ident: ours::Ident) -> bridge::Ident { + bridge::Ident { sym: ident.sym, is_raw: ident.is_raw.yes(), span: ident.span } +} + +fn group_into_bridge(group: ours::Group) -> bridge::Group { + let delimiter = match group.delimiter.kind { + tt::DelimiterKind::Parenthesis => rustc_proc_macro::Delimiter::Parenthesis, + tt::DelimiterKind::Brace => rustc_proc_macro::Delimiter::Brace, + tt::DelimiterKind::Bracket => rustc_proc_macro::Delimiter::Bracket, + tt::DelimiterKind::Invisible => rustc_proc_macro::Delimiter::None, + }; + let span = bridge::DelimSpan { + open: group.delimiter.open, + close: group.delimiter.close, + entire: group.delimiter.open.cover(group.delimiter.close), + }; + bridge::Group { delimiter, stream: group.stream, span } +} + +fn doc_comment_into_bridge( + doc_comment: ours::DocComment, + output: &mut Vec>, +) { + let is_inner = doc_comment.doc_style == tt::DocCommentStyle::Inner; + let to_reserve = output.spare_capacity_mut().len() + 2 + usize::from(is_inner); + output.reserve(to_reserve); + output.push(bridge::TokenTree::Punct(bridge::Punct { + ch: b'#', + joint: false, + span: doc_comment.span, + })); + if is_inner { + output.push(bridge::TokenTree::Punct(bridge::Punct { + ch: b'!', + joint: false, + span: doc_comment.span, + })); + } + output.push(bridge::TokenTree::Group(bridge::Group { + delimiter: rustc_proc_macro::Delimiter::Bracket, + span: bridge::DelimSpan::from_single(doc_comment.span), + stream: Some(ours::TokenStream::new(vec![ + ours::TokenTree::Leaf(ours::Leaf::Ident(ours::Ident { + sym: sym::doc, + is_raw: tt::IdentIsRaw::No, + span: doc_comment.span, + })), + ours::TokenTree::Leaf(ours::Leaf::Punct(ours::Punct { + char: '=', + spacing: tt::Spacing::Alone, + span: doc_comment.span, + })), + ours::TokenTree::Leaf(ours::Leaf::Literal(doc_comment.literal_for_proc_macros())), + ])), + })); +} + +fn token_tree_into_bridge( + token_tree: ours::TokenTree, + output: &mut Vec>, +) { + let tree = match token_tree { + ours::TokenTree::Leaf(ours::Leaf::Literal(literal)) => { + bridge::TokenTree::Literal(literal_into_bridge(literal)) + } + ours::TokenTree::Leaf(ours::Leaf::Ident(ident)) => { + bridge::TokenTree::Ident(ident_into_bridge(ident)) + } + ours::TokenTree::Leaf(ours::Leaf::Punct(punct)) => { + bridge::TokenTree::Punct(punct_into_bridge(punct)) + } + ours::TokenTree::Leaf(ours::Leaf::DocComment(doc_comment)) => { + return doc_comment_into_bridge(doc_comment, output); + } + ours::TokenTree::Group(group) => bridge::TokenTree::Group(group_into_bridge(group)), + }; + output.push(tree); +} + +pub(super) fn token_stream_into_bridge( + token_stream: ours::TokenStream, +) -> Vec> { + let mut result = Vec::with_capacity(token_stream.len()); + for tree in token_stream.iter() { + token_tree_into_bridge(tree.clone(), &mut result); + } + result +} + +fn literal_from_bridge(literal: bridge::Literal) -> ours::Literal { + let kind = match literal.kind { + bridge::LitKind::Byte => tt::LitKind::Byte, + bridge::LitKind::Char => tt::LitKind::Char, + bridge::LitKind::Integer => tt::LitKind::Integer, + bridge::LitKind::Float => tt::LitKind::Float, + bridge::LitKind::Str => tt::LitKind::Str, + bridge::LitKind::StrRaw(count) => tt::LitKind::StrRaw(count), + bridge::LitKind::ByteStr => tt::LitKind::ByteStr, + bridge::LitKind::ByteStrRaw(count) => tt::LitKind::ByteStrRaw(count), + bridge::LitKind::CStr => tt::LitKind::CStr, + bridge::LitKind::CStrRaw(count) => tt::LitKind::CStrRaw(count), + bridge::LitKind::ErrWithGuar => tt::LitKind::Err(()), + }; + match literal.suffix { + Some(suffix) => { + tt::Literal::new(literal.symbol.as_str(), literal.span, kind, suffix.as_str()) + } + None => { + tt::Literal { text_and_suffix: literal.symbol, span: literal.span, kind, suffix_len: 0 } + } + } +} + +fn punct_from_bridge(punct: bridge::Punct) -> ours::Punct { + ours::Punct { + char: char::from(punct.ch), + spacing: if punct.joint { tt::Spacing::Joint } else { tt::Spacing::Alone }, + span: punct.span, + } +} + +fn ident_from_bridge(ident: bridge::Ident) -> ours::Ident { + ours::Ident { + sym: ident.sym, + is_raw: if ident.is_raw { tt::IdentIsRaw::Yes } else { tt::IdentIsRaw::No }, + span: ident.span, + } +} + +fn group_from_bridge(group: bridge::Group) -> ours::Group { + let kind = match group.delimiter { + rustc_proc_macro::Delimiter::Parenthesis => tt::DelimiterKind::Parenthesis, + rustc_proc_macro::Delimiter::Brace => tt::DelimiterKind::Brace, + rustc_proc_macro::Delimiter::Bracket => tt::DelimiterKind::Bracket, + rustc_proc_macro::Delimiter::None => tt::DelimiterKind::Invisible, + }; + let delimiter = tt::Delimiter { open: group.span.open, close: group.span.close, kind }; + ours::Group { delimiter, stream: group.stream } +} + +pub(super) fn token_tree_from_bridge( + token_tree: bridge::TokenTree, +) -> ours::TokenTree { + match token_tree { + bridge::TokenTree::Literal(literal) => { + ours::TokenTree::Leaf(ours::Leaf::Literal(literal_from_bridge(literal))) + } + bridge::TokenTree::Ident(ident) => { + ours::TokenTree::Leaf(ours::Leaf::Ident(ident_from_bridge(ident))) + } + bridge::TokenTree::Punct(punct) => { + ours::TokenTree::Leaf(ours::Leaf::Punct(punct_from_bridge(punct))) + } + bridge::TokenTree::Group(group) => ours::TokenTree::Group(group_from_bridge(group)), + } +} diff --git a/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs b/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs index 188d24abcc67..95b1092e81d0 100644 --- a/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs +++ b/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs @@ -9,13 +9,28 @@ use std::ops::{Bound, Range}; use intern::Symbol; use rustc_proc_macro::bridge::server; use span::{ErasedFileAstId, Span, TextRange, TextSize}; +use tt::literal_from_str; use crate::{ ProcMacroClientHandle, TrackedEnv, - bridge::{Diagnostic, ExpnGlobals, Literal, TokenTree}, - server_impl::literal_from_str, + server_impl::bridge::{literal_into_bridge, token_stream_into_bridge, token_tree_from_bridge}, }; +mod ours { + use span::Span; + + pub(super) type TokenStream = crate::server_impl::bridge::ours::TokenStream; +} + +mod bridge { + use span::Span; + + pub(super) use crate::server_impl::bridge::bridge::*; + + pub(super) type TokenTree = crate::server_impl::bridge::bridge::TokenTree; + pub(super) type Literal = crate::server_impl::bridge::bridge::Literal; +} + pub struct RaSpanServer<'a> { pub tracked_env: &'a mut TrackedEnv, pub call_site: Span, @@ -26,12 +41,12 @@ pub struct RaSpanServer<'a> { } impl server::Server for RaSpanServer<'_> { - type TokenStream = crate::token_stream::TokenStream; + type TokenStream = ours::TokenStream; type Span = Span; type Symbol = Symbol; - fn globals(&mut self) -> ExpnGlobals { - ExpnGlobals { + fn globals(&mut self) -> bridge::ExpnGlobals { + bridge::ExpnGlobals { def_site: self.def_site, call_site: self.call_site, mixed_site: self.mixed_site, @@ -53,12 +68,13 @@ impl server::Server for RaSpanServer<'_> { self.tracked_env.paths.insert(path.into()); } - fn literal_from_str(&mut self, s: &str) -> Result, String> { + fn literal_from_str(&mut self, s: &str) -> Result { literal_from_str(s, self.call_site) - .map_err(|()| "cannot parse string into literal".to_string()) + .map(literal_into_bridge) + .map_err(|()| "cannot parse string into literal".to_owned()) } - fn emit_diagnostic(&mut self, _: Diagnostic) { + fn emit_diagnostic(&mut self, _: bridge::Diagnostic) { // FIXME handle diagnostic } @@ -81,8 +97,8 @@ impl server::Server for RaSpanServer<'_> { stream.to_string() } - fn ts_from_token_tree(&mut self, tree: TokenTree) -> Self::TokenStream { - Self::TokenStream::new(vec![tree]) + fn ts_from_token_tree(&mut self, tree: bridge::TokenTree) -> Self::TokenStream { + ours::TokenStream::new(vec![token_tree_from_bridge(tree)]) } fn ts_expand_expr(&mut self, self_: &Self::TokenStream) -> Result { @@ -96,17 +112,16 @@ impl server::Server for RaSpanServer<'_> { fn ts_concat_trees( &mut self, - base: Option, - trees: Vec>, + base: Option, + trees: Vec, ) -> Self::TokenStream { + let trees = trees.into_iter().map(token_tree_from_bridge); match base { Some(mut base) => { - for tt in trees { - base.push_tree(tt); - } + base.extend(trees); base } - None => Self::TokenStream::new(trees), + None => trees.collect(), } } @@ -115,15 +130,14 @@ impl server::Server for RaSpanServer<'_> { base: Option, streams: Vec, ) -> Self::TokenStream { - let mut stream = base.unwrap_or_default(); - for s in streams { - stream.push_stream(s); - } + let mut streams = streams.into_iter(); + let mut stream = base.or_else(|| streams.next()).unwrap_or_default(); + stream.extend_with_streams(streams); stream } - fn ts_into_trees(&mut self, stream: Self::TokenStream) -> Vec> { - (*stream.0).clone() + fn ts_into_trees(&mut self, stream: ours::TokenStream) -> Vec { + token_stream_into_bridge(stream) } fn span_debug(&mut self, span: Self::Span) -> String { diff --git a/crates/proc-macro-srv/src/server_impl/token_id.rs b/crates/proc-macro-srv/src/server_impl/token_id.rs index a42476f85672..4e5b15d52446 100644 --- a/crates/proc-macro-srv/src/server_impl/token_id.rs +++ b/crates/proc-macro-srv/src/server_impl/token_id.rs @@ -4,23 +4,48 @@ use std::ops::{Bound, Range}; use intern::Symbol; use rustc_proc_macro::bridge::server; +use tt::literal_from_str; use crate::{ ProcMacroClientHandle, - bridge::{Diagnostic, ExpnGlobals, Literal, TokenTree}, - server_impl::literal_from_str, + server_impl::bridge::{literal_into_bridge, token_stream_into_bridge, token_tree_from_bridge}, }; #[derive(Clone, Copy, PartialEq, Eq, Hash)] pub struct SpanId(pub u32); +impl proc_macro_api::token_stream::SpanLike for crate::SpanId { + fn derive_ranged(&self, _: std::ops::Range) -> Self { + *self + } + + fn cover(self, _other: Self) -> Self { + self + } +} + impl std::fmt::Debug for SpanId { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { self.0.fmt(f) } } -type Span = SpanId; +use SpanId as Span; + +mod ours { + use super::Span; + + pub(super) type TokenStream = crate::server_impl::bridge::ours::TokenStream; +} + +mod bridge { + use super::Span; + + pub(super) use crate::server_impl::bridge::bridge::*; + + pub(super) type TokenTree = crate::server_impl::bridge::bridge::TokenTree; + pub(super) type Literal = crate::server_impl::bridge::bridge::Literal; +} pub struct SpanIdServer<'a> { pub call_site: Span, @@ -30,12 +55,12 @@ pub struct SpanIdServer<'a> { } impl server::Server for SpanIdServer<'_> { - type TokenStream = crate::token_stream::TokenStream; + type TokenStream = ours::TokenStream; type Span = Span; type Symbol = Symbol; - fn globals(&mut self) -> ExpnGlobals { - ExpnGlobals { + fn globals(&mut self) -> bridge::ExpnGlobals { + bridge::ExpnGlobals { def_site: self.def_site, call_site: self.call_site, mixed_site: self.mixed_site, @@ -54,12 +79,13 @@ impl server::Server for SpanIdServer<'_> { fn track_path(&mut self, _: &str) {} - fn literal_from_str(&mut self, s: &str) -> Result, String> { + fn literal_from_str(&mut self, s: &str) -> Result { literal_from_str(s, self.call_site) - .map_err(|()| "cannot parse string into literal".to_string()) + .map(literal_into_bridge) + .map_err(|()| "cannot parse string into literal".to_owned()) } - fn emit_diagnostic(&mut self, _: Diagnostic) {} + fn emit_diagnostic(&mut self, _: bridge::Diagnostic) {} fn ts_drop(&mut self, stream: Self::TokenStream) { drop(stream); @@ -79,8 +105,8 @@ impl server::Server for SpanIdServer<'_> { fn ts_to_string(&mut self, stream: &Self::TokenStream) -> String { stream.to_string() } - fn ts_from_token_tree(&mut self, tree: TokenTree) -> Self::TokenStream { - Self::TokenStream::new(vec![tree]) + fn ts_from_token_tree(&mut self, tree: bridge::TokenTree) -> Self::TokenStream { + Self::TokenStream::new(vec![token_tree_from_bridge(tree)]) } fn ts_expand_expr(&mut self, self_: &Self::TokenStream) -> Result { @@ -90,16 +116,15 @@ impl server::Server for SpanIdServer<'_> { fn ts_concat_trees( &mut self, base: Option, - trees: Vec>, + trees: Vec, ) -> Self::TokenStream { + let trees = trees.into_iter().map(token_tree_from_bridge); match base { Some(mut base) => { - for tt in trees { - base.push_tree(tt); - } + base.extend(trees); base } - None => Self::TokenStream::new(trees), + None => trees.collect(), } } @@ -108,15 +133,14 @@ impl server::Server for SpanIdServer<'_> { base: Option, streams: Vec, ) -> Self::TokenStream { - let mut stream = base.unwrap_or_default(); - for s in streams { - stream.push_stream(s); - } + let mut streams = streams.into_iter(); + let mut stream = base.or_else(|| streams.next()).unwrap_or_default(); + stream.extend_with_streams(streams); stream } - fn ts_into_trees(&mut self, stream: Self::TokenStream) -> Vec> { - (*stream.0).clone() + fn ts_into_trees(&mut self, stream: ours::TokenStream) -> Vec { + token_stream_into_bridge(stream) } fn span_debug(&mut self, span: Self::Span) -> String { diff --git a/crates/proc-macro-srv/src/tests/mod.rs b/crates/proc-macro-srv/src/tests/mod.rs index 7b3752f2a1a4..8efb6e6ef40e 100644 --- a/crates/proc-macro-srv/src/tests/mod.rs +++ b/crates/proc-macro-srv/src/tests/mod.rs @@ -12,42 +12,42 @@ fn test_derive_empty() { "DeriveEmpty", r#"struct S { field: &'r#lt fn(u32) -> &'a r#u32 }"#, expect![[r#" - IDENT 1 struct - IDENT 1 S - GROUP {} 1 1 1 - IDENT 1 field - PUNCT 1 : [alone] - PUNCT 1 & [joint] - PUNCT 1 ' [joint] - IDENT 1 r#lt - IDENT 1 fn - GROUP () 1 1 1 - IDENT 1 u32 - PUNCT 1 - [joint] - PUNCT 1 > [alone] - PUNCT 1 & [joint] - PUNCT 1 ' [joint] - IDENT 1 a - IDENT 1 r#u32 + IDENT struct 1 + IDENT S 1 + GROUP {} 1 1 + IDENT field 1 + PUNCT : [alone] 1 + PUNCT & [joint] 1 + PUNCT ' [joint] 1 + IDENT r#lt 1 + IDENT fn 1 + GROUP () 1 1 + IDENT u32 1 + PUNCT - [joint] 1 + PUNCT > [alone] 1 + PUNCT & [joint] 1 + PUNCT ' [joint] 1 + IDENT a 1 + IDENT r#u32 1 "#]], expect![[r#" - IDENT 42:Root[0000, 0]@0..6#1 struct - IDENT 42:Root[0000, 0]@7..8#1 S - GROUP {} 42:Root[0000, 0]@9..10#1 42:Root[0000, 0]@46..47#1 42:Root[0000, 0]@9..47#1 - IDENT 42:Root[0000, 0]@11..16#1 field - PUNCT 42:Root[0000, 0]@16..17#1 : [alone] - PUNCT 42:Root[0000, 0]@18..19#1 & [joint] - PUNCT 42:Root[0000, 0]@22..23#1 ' [joint] - IDENT 42:Root[0000, 0]@22..24#1 r#lt - IDENT 42:Root[0000, 0]@25..27#1 fn - GROUP () 42:Root[0000, 0]@27..28#1 42:Root[0000, 0]@31..32#1 42:Root[0000, 0]@27..32#1 - IDENT 42:Root[0000, 0]@28..31#1 u32 - PUNCT 42:Root[0000, 0]@33..34#1 - [joint] - PUNCT 42:Root[0000, 0]@34..35#1 > [alone] - PUNCT 42:Root[0000, 0]@36..37#1 & [joint] - PUNCT 42:Root[0000, 0]@38..39#1 ' [joint] - IDENT 42:Root[0000, 0]@38..39#1 a - IDENT 42:Root[0000, 0]@42..45#1 r#u32 + IDENT struct 42:Root[0000, 0]@0..6#1 + IDENT S 42:Root[0000, 0]@7..8#1 + GROUP {} 42:Root[0000, 0]@9..10#1 42:Root[0000, 0]@46..47#1 + IDENT field 42:Root[0000, 0]@11..16#1 + PUNCT : [alone] 42:Root[0000, 0]@16..17#1 + PUNCT & [joint] 42:Root[0000, 0]@18..19#1 + PUNCT ' [joint] 42:Root[0000, 0]@22..23#1 + IDENT r#lt 42:Root[0000, 0]@22..24#1 + IDENT fn 42:Root[0000, 0]@25..27#1 + GROUP () 42:Root[0000, 0]@27..28#1 42:Root[0000, 0]@31..32#1 + IDENT u32 42:Root[0000, 0]@28..31#1 + PUNCT - [joint] 42:Root[0000, 0]@33..34#1 + PUNCT > [alone] 42:Root[0000, 0]@34..35#1 + PUNCT & [joint] 42:Root[0000, 0]@36..37#1 + PUNCT ' [joint] 42:Root[0000, 0]@38..39#1 + IDENT a 42:Root[0000, 0]@38..39#1 + IDENT r#u32 42:Root[0000, 0]@42..45#1 "#]], ); } @@ -65,148 +65,132 @@ pub struct Foo { } "#, expect![[r#" - PUNCT 1 # [joint] - GROUP [] 1 1 1 - IDENT 1 helper - GROUP () 1 1 1 - IDENT 1 build_fn - GROUP () 1 1 1 - IDENT 1 private - PUNCT 1 , [alone] - IDENT 1 name - PUNCT 1 = [alone] - LITER 1 Str partial_build - IDENT 1 pub - IDENT 1 struct - IDENT 1 Foo - GROUP {} 1 1 1 - PUNCT 1 # [alone] - GROUP [] 1 1 1 - IDENT 1 doc - PUNCT 1 = [alone] - LITER 1 Str The domain where this federated instance is running - PUNCT 1 # [joint] - GROUP [] 1 1 1 - IDENT 1 helper - GROUP () 1 1 1 - IDENT 1 setter - GROUP () 1 1 1 - IDENT 1 into - IDENT 1 pub - GROUP () 1 1 1 - IDENT 1 crate - IDENT 1 domain - PUNCT 1 : [alone] - IDENT 1 String - PUNCT 1 , [alone] - - - PUNCT 1 # [joint] - GROUP [] 1 1 1 - IDENT 1 helper - GROUP () 1 1 1 - IDENT 1 build_fn - GROUP () 1 1 1 - IDENT 1 private - PUNCT 1 , [alone] - IDENT 1 name - PUNCT 1 = [alone] - LITER 1 Str partial_build - IDENT 1 pub - IDENT 1 struct - IDENT 1 Foo - GROUP {} 1 1 1 - PUNCT 1 # [alone] - GROUP [] 1 1 1 - IDENT 1 doc - PUNCT 1 = [alone] - LITER 1 Str The domain where this federated instance is running - PUNCT 1 # [joint] - GROUP [] 1 1 1 - IDENT 1 helper - GROUP () 1 1 1 - IDENT 1 setter - GROUP () 1 1 1 - IDENT 1 into - IDENT 1 pub - GROUP () 1 1 1 - IDENT 1 crate - IDENT 1 domain - PUNCT 1 : [alone] - IDENT 1 String - PUNCT 1 , [alone] + PUNCT # [joint] 1 + GROUP [] 1 1 + IDENT helper 1 + GROUP () 1 1 + IDENT build_fn 1 + GROUP () 1 1 + IDENT private 1 + PUNCT , [alone] 1 + IDENT name 1 + PUNCT = [alone] 1 + LITERAL Str partial_build 1 + IDENT pub 1 + IDENT struct 1 + IDENT Foo 1 + GROUP {} 1 1 + DOC_COMMENT Outer Line /// The domain where this federated instance is running 1 + PUNCT # [joint] 1 + GROUP [] 1 1 + IDENT helper 1 + GROUP () 1 1 + IDENT setter 1 + GROUP () 1 1 + IDENT into 1 + IDENT pub 1 + GROUP () 1 1 + IDENT crate 1 + IDENT domain 1 + PUNCT : [alone] 1 + IDENT String 1 + PUNCT , [alone] 1 + + + PUNCT # [joint] 1 + GROUP [] 1 1 + IDENT helper 1 + GROUP () 1 1 + IDENT build_fn 1 + GROUP () 1 1 + IDENT private 1 + PUNCT , [alone] 1 + IDENT name 1 + PUNCT = [alone] 1 + LITERAL Str partial_build 1 + IDENT pub 1 + IDENT struct 1 + IDENT Foo 1 + GROUP {} 1 1 + DOC_COMMENT Outer Line /// The domain where this federated instance is running 1 + PUNCT # [joint] 1 + GROUP [] 1 1 + IDENT helper 1 + GROUP () 1 1 + IDENT setter 1 + GROUP () 1 1 + IDENT into 1 + IDENT pub 1 + GROUP () 1 1 + IDENT crate 1 + IDENT domain 1 + PUNCT : [alone] 1 + IDENT String 1 + PUNCT , [alone] 1 "#]], expect![[r#" - PUNCT 42:Root[0000, 0]@1..2#1 # [joint] - GROUP [] 42:Root[0000, 0]@2..3#1 42:Root[0000, 0]@52..53#1 42:Root[0000, 0]@2..53#1 - IDENT 42:Root[0000, 0]@3..9#1 helper - GROUP () 42:Root[0000, 0]@9..10#1 42:Root[0000, 0]@51..52#1 42:Root[0000, 0]@9..52#1 - IDENT 42:Root[0000, 0]@10..18#1 build_fn - GROUP () 42:Root[0000, 0]@18..19#1 42:Root[0000, 0]@50..51#1 42:Root[0000, 0]@18..51#1 - IDENT 42:Root[0000, 0]@19..26#1 private - PUNCT 42:Root[0000, 0]@26..27#1 , [alone] - IDENT 42:Root[0000, 0]@28..32#1 name - PUNCT 42:Root[0000, 0]@33..34#1 = [alone] - LITER 42:Root[0000, 0]@35..50#1 Str partial_build - IDENT 42:Root[0000, 0]@54..57#1 pub - IDENT 42:Root[0000, 0]@58..64#1 struct - IDENT 42:Root[0000, 0]@65..68#1 Foo - GROUP {} 42:Root[0000, 0]@69..70#1 42:Root[0000, 0]@190..191#1 42:Root[0000, 0]@69..191#1 - PUNCT 42:Root[0000, 0]@0..0#1 # [alone] - GROUP [] 42:Root[0000, 0]@0..0#1 42:Root[0000, 0]@0..0#1 42:Root[0000, 0]@0..0#1 - IDENT 42:Root[0000, 0]@0..0#1 doc - PUNCT 42:Root[0000, 0]@0..0#1 = [alone] - LITER 42:Root[0000, 0]@75..130#1 Str The domain where this federated instance is running - PUNCT 42:Root[0000, 0]@135..136#1 # [joint] - GROUP [] 42:Root[0000, 0]@136..137#1 42:Root[0000, 0]@157..158#1 42:Root[0000, 0]@136..158#1 - IDENT 42:Root[0000, 0]@137..143#1 helper - GROUP () 42:Root[0000, 0]@143..144#1 42:Root[0000, 0]@156..157#1 42:Root[0000, 0]@143..157#1 - IDENT 42:Root[0000, 0]@144..150#1 setter - GROUP () 42:Root[0000, 0]@150..151#1 42:Root[0000, 0]@155..156#1 42:Root[0000, 0]@150..156#1 - IDENT 42:Root[0000, 0]@151..155#1 into - IDENT 42:Root[0000, 0]@163..166#1 pub - GROUP () 42:Root[0000, 0]@166..167#1 42:Root[0000, 0]@172..173#1 42:Root[0000, 0]@166..173#1 - IDENT 42:Root[0000, 0]@167..172#1 crate - IDENT 42:Root[0000, 0]@174..180#1 domain - PUNCT 42:Root[0000, 0]@180..181#1 : [alone] - IDENT 42:Root[0000, 0]@182..188#1 String - PUNCT 42:Root[0000, 0]@188..189#1 , [alone] - - - PUNCT 42:Root[0000, 0]@1..2#1 # [joint] - GROUP [] 42:Root[0000, 0]@2..3#1 42:Root[0000, 0]@52..53#1 42:Root[0000, 0]@2..53#1 - IDENT 42:Root[0000, 0]@3..9#1 helper - GROUP () 42:Root[0000, 0]@9..10#1 42:Root[0000, 0]@51..52#1 42:Root[0000, 0]@9..52#1 - IDENT 42:Root[0000, 0]@10..18#1 build_fn - GROUP () 42:Root[0000, 0]@18..19#1 42:Root[0000, 0]@50..51#1 42:Root[0000, 0]@18..51#1 - IDENT 42:Root[0000, 0]@19..26#1 private - PUNCT 42:Root[0000, 0]@26..27#1 , [alone] - IDENT 42:Root[0000, 0]@28..32#1 name - PUNCT 42:Root[0000, 0]@33..34#1 = [alone] - LITER 42:Root[0000, 0]@35..50#1 Str partial_build - IDENT 42:Root[0000, 0]@54..57#1 pub - IDENT 42:Root[0000, 0]@58..64#1 struct - IDENT 42:Root[0000, 0]@65..68#1 Foo - GROUP {} 42:Root[0000, 0]@69..70#1 42:Root[0000, 0]@190..191#1 42:Root[0000, 0]@69..191#1 - PUNCT 42:Root[0000, 0]@0..0#1 # [alone] - GROUP [] 42:Root[0000, 0]@0..0#1 42:Root[0000, 0]@0..0#1 42:Root[0000, 0]@0..0#1 - IDENT 42:Root[0000, 0]@0..0#1 doc - PUNCT 42:Root[0000, 0]@0..0#1 = [alone] - LITER 42:Root[0000, 0]@75..130#1 Str The domain where this federated instance is running - PUNCT 42:Root[0000, 0]@135..136#1 # [joint] - GROUP [] 42:Root[0000, 0]@136..137#1 42:Root[0000, 0]@157..158#1 42:Root[0000, 0]@136..158#1 - IDENT 42:Root[0000, 0]@137..143#1 helper - GROUP () 42:Root[0000, 0]@143..144#1 42:Root[0000, 0]@156..157#1 42:Root[0000, 0]@143..157#1 - IDENT 42:Root[0000, 0]@144..150#1 setter - GROUP () 42:Root[0000, 0]@150..151#1 42:Root[0000, 0]@155..156#1 42:Root[0000, 0]@150..156#1 - IDENT 42:Root[0000, 0]@151..155#1 into - IDENT 42:Root[0000, 0]@163..166#1 pub - GROUP () 42:Root[0000, 0]@166..167#1 42:Root[0000, 0]@172..173#1 42:Root[0000, 0]@166..173#1 - IDENT 42:Root[0000, 0]@167..172#1 crate - IDENT 42:Root[0000, 0]@174..180#1 domain - PUNCT 42:Root[0000, 0]@180..181#1 : [alone] - IDENT 42:Root[0000, 0]@182..188#1 String - PUNCT 42:Root[0000, 0]@188..189#1 , [alone] + PUNCT # [joint] 42:Root[0000, 0]@1..2#1 + GROUP [] 42:Root[0000, 0]@2..3#1 42:Root[0000, 0]@52..53#1 + IDENT helper 42:Root[0000, 0]@3..9#1 + GROUP () 42:Root[0000, 0]@9..10#1 42:Root[0000, 0]@51..52#1 + IDENT build_fn 42:Root[0000, 0]@10..18#1 + GROUP () 42:Root[0000, 0]@18..19#1 42:Root[0000, 0]@50..51#1 + IDENT private 42:Root[0000, 0]@19..26#1 + PUNCT , [alone] 42:Root[0000, 0]@26..27#1 + IDENT name 42:Root[0000, 0]@28..32#1 + PUNCT = [alone] 42:Root[0000, 0]@33..34#1 + LITERAL Str partial_build 42:Root[0000, 0]@35..50#1 + IDENT pub 42:Root[0000, 0]@54..57#1 + IDENT struct 42:Root[0000, 0]@58..64#1 + IDENT Foo 42:Root[0000, 0]@65..68#1 + GROUP {} 42:Root[0000, 0]@69..70#1 42:Root[0000, 0]@190..191#1 + DOC_COMMENT Outer Line /// The domain where this federated instance is running 42:Root[0000, 0]@75..130#1 + PUNCT # [joint] 42:Root[0000, 0]@135..136#1 + GROUP [] 42:Root[0000, 0]@136..137#1 42:Root[0000, 0]@157..158#1 + IDENT helper 42:Root[0000, 0]@137..143#1 + GROUP () 42:Root[0000, 0]@143..144#1 42:Root[0000, 0]@156..157#1 + IDENT setter 42:Root[0000, 0]@144..150#1 + GROUP () 42:Root[0000, 0]@150..151#1 42:Root[0000, 0]@155..156#1 + IDENT into 42:Root[0000, 0]@151..155#1 + IDENT pub 42:Root[0000, 0]@163..166#1 + GROUP () 42:Root[0000, 0]@166..167#1 42:Root[0000, 0]@172..173#1 + IDENT crate 42:Root[0000, 0]@167..172#1 + IDENT domain 42:Root[0000, 0]@174..180#1 + PUNCT : [alone] 42:Root[0000, 0]@180..181#1 + IDENT String 42:Root[0000, 0]@182..188#1 + PUNCT , [alone] 42:Root[0000, 0]@188..189#1 + + + PUNCT # [joint] 42:Root[0000, 0]@1..2#1 + GROUP [] 42:Root[0000, 0]@2..3#1 42:Root[0000, 0]@52..53#1 + IDENT helper 42:Root[0000, 0]@3..9#1 + GROUP () 42:Root[0000, 0]@9..10#1 42:Root[0000, 0]@51..52#1 + IDENT build_fn 42:Root[0000, 0]@10..18#1 + GROUP () 42:Root[0000, 0]@18..19#1 42:Root[0000, 0]@50..51#1 + IDENT private 42:Root[0000, 0]@19..26#1 + PUNCT , [alone] 42:Root[0000, 0]@26..27#1 + IDENT name 42:Root[0000, 0]@28..32#1 + PUNCT = [alone] 42:Root[0000, 0]@33..34#1 + LITERAL Str partial_build 42:Root[0000, 0]@35..50#1 + IDENT pub 42:Root[0000, 0]@54..57#1 + IDENT struct 42:Root[0000, 0]@58..64#1 + IDENT Foo 42:Root[0000, 0]@65..68#1 + GROUP {} 42:Root[0000, 0]@69..70#1 42:Root[0000, 0]@190..191#1 + DOC_COMMENT Outer Line /// The domain where this federated instance is running 42:Root[0000, 0]@75..130#1 + PUNCT # [joint] 42:Root[0000, 0]@135..136#1 + GROUP [] 42:Root[0000, 0]@136..137#1 42:Root[0000, 0]@157..158#1 + IDENT helper 42:Root[0000, 0]@137..143#1 + GROUP () 42:Root[0000, 0]@143..144#1 42:Root[0000, 0]@156..157#1 + IDENT setter 42:Root[0000, 0]@144..150#1 + GROUP () 42:Root[0000, 0]@150..151#1 42:Root[0000, 0]@155..156#1 + IDENT into 42:Root[0000, 0]@151..155#1 + IDENT pub 42:Root[0000, 0]@163..166#1 + GROUP () 42:Root[0000, 0]@166..167#1 42:Root[0000, 0]@172..173#1 + IDENT crate 42:Root[0000, 0]@167..172#1 + IDENT domain 42:Root[0000, 0]@174..180#1 + PUNCT : [alone] 42:Root[0000, 0]@180..181#1 + IDENT String 42:Root[0000, 0]@182..188#1 + PUNCT , [alone] 42:Root[0000, 0]@188..189#1 "#]], ); } @@ -217,34 +201,34 @@ fn test_derive_error() { "DeriveError", r#"struct S { field: u32 }"#, expect![[r#" - IDENT 1 struct - IDENT 1 S - GROUP {} 1 1 1 - IDENT 1 field - PUNCT 1 : [alone] - IDENT 1 u32 - - - IDENT 1 compile_error - PUNCT 1 ! [joint] - GROUP () 1 1 1 - LITER 1 Str #[derive(DeriveError)] struct S {field : u32} - PUNCT 1 ; [alone] + IDENT struct 1 + IDENT S 1 + GROUP {} 1 1 + IDENT field 1 + PUNCT : [alone] 1 + IDENT u32 1 + + + IDENT compile_error 1 + PUNCT ! [joint] 1 + GROUP () 1 1 + LITERAL Str #[derive(DeriveError)] struct S {field : u32} 1 + PUNCT ; [alone] 1 "#]], expect![[r#" - IDENT 42:Root[0000, 0]@0..6#1 struct - IDENT 42:Root[0000, 0]@7..8#1 S - GROUP {} 42:Root[0000, 0]@9..10#1 42:Root[0000, 0]@22..23#1 42:Root[0000, 0]@9..23#1 - IDENT 42:Root[0000, 0]@11..16#1 field - PUNCT 42:Root[0000, 0]@16..17#1 : [alone] - IDENT 42:Root[0000, 0]@18..21#1 u32 - - - IDENT 42:Root[0000, 0]@0..13#1 compile_error - PUNCT 42:Root[0000, 0]@13..14#1 ! [joint] - GROUP () 42:Root[0000, 0]@14..15#1 42:Root[0000, 0]@62..63#1 42:Root[0000, 0]@14..63#1 - LITER 42:Root[0000, 0]@15..62#1 Str #[derive(DeriveError)] struct S {field : u32} - PUNCT 42:Root[0000, 0]@63..64#1 ; [alone] + IDENT struct 42:Root[0000, 0]@0..6#1 + IDENT S 42:Root[0000, 0]@7..8#1 + GROUP {} 42:Root[0000, 0]@9..10#1 42:Root[0000, 0]@22..23#1 + IDENT field 42:Root[0000, 0]@11..16#1 + PUNCT : [alone] 42:Root[0000, 0]@16..17#1 + IDENT u32 42:Root[0000, 0]@18..21#1 + + + IDENT compile_error 42:Root[0000, 0]@0..13#1 + PUNCT ! [joint] 42:Root[0000, 0]@13..14#1 + GROUP () 42:Root[0000, 0]@14..15#1 42:Root[0000, 0]@62..63#1 + LITERAL Str #[derive(DeriveError)] struct S {field : u32} 42:Root[0000, 0]@15..62#1 + PUNCT ; [alone] 42:Root[0000, 0]@63..64#1 "#]], ); } @@ -255,40 +239,40 @@ fn test_fn_like_macro_noop() { "fn_like_noop", r#"ident, 0, 1, []"#, expect![[r#" - IDENT 1 ident - PUNCT 1 , [alone] - LITER 1 Integer 0 - PUNCT 1 , [alone] - LITER 1 Integer 1 - PUNCT 1 , [alone] - GROUP [] 1 1 1 - - - IDENT 1 ident - PUNCT 1 , [alone] - LITER 1 Integer 0 - PUNCT 1 , [alone] - LITER 1 Integer 1 - PUNCT 1 , [alone] - GROUP [] 1 1 1 + IDENT ident 1 + PUNCT , [alone] 1 + LITERAL Integer 0 1 + PUNCT , [alone] 1 + LITERAL Integer 1 1 + PUNCT , [alone] 1 + GROUP [] 1 1 + + + IDENT ident 1 + PUNCT , [alone] 1 + LITERAL Integer 0 1 + PUNCT , [alone] 1 + LITERAL Integer 1 1 + PUNCT , [alone] 1 + GROUP [] 1 1 "#]], expect![[r#" - IDENT 42:Root[0000, 0]@0..5#1 ident - PUNCT 42:Root[0000, 0]@5..6#1 , [alone] - LITER 42:Root[0000, 0]@7..8#1 Integer 0 - PUNCT 42:Root[0000, 0]@8..9#1 , [alone] - LITER 42:Root[0000, 0]@10..11#1 Integer 1 - PUNCT 42:Root[0000, 0]@11..12#1 , [alone] - GROUP [] 42:Root[0000, 0]@13..14#1 42:Root[0000, 0]@14..15#1 42:Root[0000, 0]@13..15#1 - - - IDENT 42:Root[0000, 0]@0..5#1 ident - PUNCT 42:Root[0000, 0]@5..6#1 , [alone] - LITER 42:Root[0000, 0]@7..8#1 Integer 0 - PUNCT 42:Root[0000, 0]@8..9#1 , [alone] - LITER 42:Root[0000, 0]@10..11#1 Integer 1 - PUNCT 42:Root[0000, 0]@11..12#1 , [alone] - GROUP [] 42:Root[0000, 0]@13..14#1 42:Root[0000, 0]@14..15#1 42:Root[0000, 0]@13..15#1 + IDENT ident 42:Root[0000, 0]@0..5#1 + PUNCT , [alone] 42:Root[0000, 0]@5..6#1 + LITERAL Integer 0 42:Root[0000, 0]@7..8#1 + PUNCT , [alone] 42:Root[0000, 0]@8..9#1 + LITERAL Integer 1 42:Root[0000, 0]@10..11#1 + PUNCT , [alone] 42:Root[0000, 0]@11..12#1 + GROUP [] 42:Root[0000, 0]@13..14#1 42:Root[0000, 0]@14..15#1 + + + IDENT ident 42:Root[0000, 0]@0..5#1 + PUNCT , [alone] 42:Root[0000, 0]@5..6#1 + LITERAL Integer 0 42:Root[0000, 0]@7..8#1 + PUNCT , [alone] 42:Root[0000, 0]@8..9#1 + LITERAL Integer 1 42:Root[0000, 0]@10..11#1 + PUNCT , [alone] 42:Root[0000, 0]@11..12#1 + GROUP [] 42:Root[0000, 0]@13..14#1 42:Root[0000, 0]@14..15#1 "#]], ); } @@ -299,36 +283,36 @@ fn test_fn_like_macro_clone_ident_subtree() { "fn_like_clone_tokens", r#"ident, [ident2, ident3]"#, expect![[r#" - IDENT 1 ident - PUNCT 1 , [alone] - GROUP [] 1 1 1 - IDENT 1 ident2 - PUNCT 1 , [alone] - IDENT 1 ident3 - - - IDENT 1 ident - PUNCT 1 , [alone] - GROUP [] 1 1 1 - IDENT 1 ident2 - PUNCT 1 , [alone] - IDENT 1 ident3 + IDENT ident 1 + PUNCT , [alone] 1 + GROUP [] 1 1 + IDENT ident2 1 + PUNCT , [alone] 1 + IDENT ident3 1 + + + IDENT ident 1 + PUNCT , [alone] 1 + GROUP [] 1 1 + IDENT ident2 1 + PUNCT , [alone] 1 + IDENT ident3 1 "#]], expect![[r#" - IDENT 42:Root[0000, 0]@0..5#1 ident - PUNCT 42:Root[0000, 0]@5..6#1 , [alone] - GROUP [] 42:Root[0000, 0]@7..8#1 42:Root[0000, 0]@22..23#1 42:Root[0000, 0]@7..23#1 - IDENT 42:Root[0000, 0]@8..14#1 ident2 - PUNCT 42:Root[0000, 0]@14..15#1 , [alone] - IDENT 42:Root[0000, 0]@16..22#1 ident3 - - - IDENT 42:Root[0000, 0]@0..5#1 ident - PUNCT 42:Root[0000, 0]@5..6#1 , [alone] - GROUP [] 42:Root[0000, 0]@7..23#1 42:Root[0000, 0]@7..23#1 42:Root[0000, 0]@7..23#1 - IDENT 42:Root[0000, 0]@8..14#1 ident2 - PUNCT 42:Root[0000, 0]@14..15#1 , [alone] - IDENT 42:Root[0000, 0]@16..22#1 ident3 + IDENT ident 42:Root[0000, 0]@0..5#1 + PUNCT , [alone] 42:Root[0000, 0]@5..6#1 + GROUP [] 42:Root[0000, 0]@7..8#1 42:Root[0000, 0]@22..23#1 + IDENT ident2 42:Root[0000, 0]@8..14#1 + PUNCT , [alone] 42:Root[0000, 0]@14..15#1 + IDENT ident3 42:Root[0000, 0]@16..22#1 + + + IDENT ident 42:Root[0000, 0]@0..5#1 + PUNCT , [alone] 42:Root[0000, 0]@5..6#1 + GROUP [] 42:Root[0000, 0]@7..23#1 42:Root[0000, 0]@7..23#1 + IDENT ident2 42:Root[0000, 0]@8..14#1 + PUNCT , [alone] 42:Root[0000, 0]@14..15#1 + IDENT ident3 42:Root[0000, 0]@16..22#1 "#]], ); } @@ -339,16 +323,16 @@ fn test_fn_like_macro_clone_raw_ident() { "fn_like_clone_tokens", "r#async", expect![[r#" - IDENT 1 r#async + IDENT r#async 1 - IDENT 1 r#async + IDENT r#async 1 "#]], expect![[r#" - IDENT 42:Root[0000, 0]@2..7#1 r#async + IDENT r#async 42:Root[0000, 0]@2..7#1 - IDENT 42:Root[0000, 0]@2..7#1 r#async + IDENT r#async 42:Root[0000, 0]@2..7#1 "#]], ); } @@ -359,18 +343,18 @@ fn test_fn_like_fn_like_span_join() { "fn_like_span_join", "foo bar", expect![[r#" - IDENT 1 foo - IDENT 1 bar + IDENT foo 1 + IDENT bar 1 - IDENT 1 r#joined + IDENT r#joined 1 "#]], expect![[r#" - IDENT 42:Root[0000, 0]@0..3#1 foo - IDENT 42:Root[0000, 0]@8..11#1 bar + IDENT foo 42:Root[0000, 0]@0..3#1 + IDENT bar 42:Root[0000, 0]@8..11#1 - IDENT 42:Root[0000, 0]@0..11#1 r#joined + IDENT r#joined 42:Root[0000, 0]@0..11#1 "#]], ); } @@ -381,24 +365,24 @@ fn test_fn_like_fn_like_span_ops() { "fn_like_span_ops", "set_def_site resolved_at_def_site start_span", expect![[r#" - IDENT 1 set_def_site - IDENT 1 resolved_at_def_site - IDENT 1 start_span + IDENT set_def_site 1 + IDENT resolved_at_def_site 1 + IDENT start_span 1 - IDENT 0 set_def_site - IDENT 1 resolved_at_def_site - IDENT 1 start_span + IDENT set_def_site 0 + IDENT resolved_at_def_site 1 + IDENT start_span 1 "#]], expect![[r#" - IDENT 42:Root[0000, 0]@0..12#1 set_def_site - IDENT 42:Root[0000, 0]@13..33#1 resolved_at_def_site - IDENT 42:Root[0000, 0]@34..44#1 start_span + IDENT set_def_site 42:Root[0000, 0]@0..12#1 + IDENT resolved_at_def_site 42:Root[0000, 0]@13..33#1 + IDENT start_span 42:Root[0000, 0]@34..44#1 - IDENT 41:Root[0000, 0]@0..150#1 set_def_site - IDENT 42:Root[0000, 0]@13..33#1 resolved_at_def_site - IDENT 42:Root[0000, 0]@34..34#1 start_span + IDENT set_def_site 41:Root[0000, 0]@0..150#1 + IDENT resolved_at_def_site 42:Root[0000, 0]@13..33#1 + IDENT start_span 42:Root[0000, 0]@34..34#1 "#]], ); } @@ -411,36 +395,36 @@ fn test_fn_like_mk_literals() { expect![[r#" - LITER 1 ByteStr byte_string - LITER 1 Char c - LITER 1 Str string - LITER 1 Str -string - LITER 1 CStr cstring - LITER 1 Float 3.14f64 - LITER 1 Float -3.14f64 - LITER 1 Float 3.14 - LITER 1 Float -3.14 - LITER 1 Integer 123i64 - LITER 1 Integer -123i64 - LITER 1 Integer 123 - LITER 1 Integer -123 + LITERAL ByteStr byte_string 1 + LITERAL Char c 1 + LITERAL Str string 1 + LITERAL Str -string 1 + LITERAL CStr cstring 1 + LITERAL Float 3.14f64 1 + LITERAL Float -3.14f64 1 + LITERAL Float 3.14 1 + LITERAL Float -3.14 1 + LITERAL Integer 123i64 1 + LITERAL Integer -123i64 1 + LITERAL Integer 123 1 + LITERAL Integer -123 1 "#]], expect![[r#" - LITER 42:Root[0000, 0]@0..100#1 ByteStr byte_string - LITER 42:Root[0000, 0]@0..100#1 Char c - LITER 42:Root[0000, 0]@0..100#1 Str string - LITER 42:Root[0000, 0]@0..100#1 Str -string - LITER 42:Root[0000, 0]@0..100#1 CStr cstring - LITER 42:Root[0000, 0]@0..100#1 Float 3.14f64 - LITER 42:Root[0000, 0]@0..100#1 Float -3.14f64 - LITER 42:Root[0000, 0]@0..100#1 Float 3.14 - LITER 42:Root[0000, 0]@0..100#1 Float -3.14 - LITER 42:Root[0000, 0]@0..100#1 Integer 123i64 - LITER 42:Root[0000, 0]@0..100#1 Integer -123i64 - LITER 42:Root[0000, 0]@0..100#1 Integer 123 - LITER 42:Root[0000, 0]@0..100#1 Integer -123 + LITERAL ByteStr byte_string 42:Root[0000, 0]@0..100#1 + LITERAL Char c 42:Root[0000, 0]@0..100#1 + LITERAL Str string 42:Root[0000, 0]@0..100#1 + LITERAL Str -string 42:Root[0000, 0]@0..100#1 + LITERAL CStr cstring 42:Root[0000, 0]@0..100#1 + LITERAL Float 3.14f64 42:Root[0000, 0]@0..100#1 + LITERAL Float -3.14f64 42:Root[0000, 0]@0..100#1 + LITERAL Float 3.14 42:Root[0000, 0]@0..100#1 + LITERAL Float -3.14 42:Root[0000, 0]@0..100#1 + LITERAL Integer 123i64 42:Root[0000, 0]@0..100#1 + LITERAL Integer -123i64 42:Root[0000, 0]@0..100#1 + LITERAL Integer 123 42:Root[0000, 0]@0..100#1 + LITERAL Integer -123 42:Root[0000, 0]@0..100#1 "#]], ); } @@ -453,14 +437,14 @@ fn test_fn_like_mk_idents() { expect![[r#" - IDENT 1 standard - IDENT 1 r#raw + IDENT standard 1 + IDENT r#raw 1 "#]], expect![[r#" - IDENT 42:Root[0000, 0]@0..100#1 standard - IDENT 42:Root[0000, 0]@0..100#1 r#raw + IDENT standard 42:Root[0000, 0]@0..100#1 + IDENT r#raw 42:Root[0000, 0]@0..100#1 "#]], ); } @@ -471,92 +455,92 @@ fn test_fn_like_macro_clone_literals() { "fn_like_clone_tokens", r###"1u16, 2_u32, -4i64, 3.14f32, "hello bridge", "suffixed"suffix, r##"raw"##, 'a', b'b', c"null""###, expect![[r#" - LITER 1 Integer 1u16 - PUNCT 1 , [alone] - LITER 1 Integer 2_u32 - PUNCT 1 , [alone] - PUNCT 1 - [alone] - LITER 1 Integer 4i64 - PUNCT 1 , [alone] - LITER 1 Float 3.14f32 - PUNCT 1 , [alone] - LITER 1 Str hello bridge - PUNCT 1 , [alone] - LITER 1 Str suffixedsuffix - PUNCT 1 , [alone] - LITER 1 StrRaw(2) raw - PUNCT 1 , [alone] - LITER 1 Char a - PUNCT 1 , [alone] - LITER 1 Byte b - PUNCT 1 , [alone] - LITER 1 CStr null - - - LITER 1 Integer 1u16 - PUNCT 1 , [alone] - LITER 1 Integer 2_u32 - PUNCT 1 , [alone] - PUNCT 1 - [alone] - LITER 1 Integer 4i64 - PUNCT 1 , [alone] - LITER 1 Float 3.14f32 - PUNCT 1 , [alone] - LITER 1 Str hello bridge - PUNCT 1 , [alone] - LITER 1 Str suffixedsuffix - PUNCT 1 , [alone] - LITER 1 StrRaw(2) raw - PUNCT 1 , [alone] - LITER 1 Char a - PUNCT 1 , [alone] - LITER 1 Byte b - PUNCT 1 , [alone] - LITER 1 CStr null + LITERAL Integer 1u16 1 + PUNCT , [alone] 1 + LITERAL Integer 2_u32 1 + PUNCT , [alone] 1 + PUNCT - [alone] 1 + LITERAL Integer 4i64 1 + PUNCT , [alone] 1 + LITERAL Float 3.14f32 1 + PUNCT , [alone] 1 + LITERAL Str hello bridge 1 + PUNCT , [alone] 1 + LITERAL Err(()) "suffixed"suffix 1 + PUNCT , [alone] 1 + LITERAL StrRaw(2) raw 1 + PUNCT , [alone] 1 + LITERAL Char a 1 + PUNCT , [alone] 1 + LITERAL Byte b 1 + PUNCT , [alone] 1 + LITERAL CStr null 1 + + + LITERAL Integer 1u16 1 + PUNCT , [alone] 1 + LITERAL Integer 2_u32 1 + PUNCT , [alone] 1 + PUNCT - [alone] 1 + LITERAL Integer 4i64 1 + PUNCT , [alone] 1 + LITERAL Float 3.14f32 1 + PUNCT , [alone] 1 + LITERAL Str hello bridge 1 + PUNCT , [alone] 1 + LITERAL Err(()) "suffixed"suffix 1 + PUNCT , [alone] 1 + LITERAL StrRaw(2) raw 1 + PUNCT , [alone] 1 + LITERAL Char a 1 + PUNCT , [alone] 1 + LITERAL Byte b 1 + PUNCT , [alone] 1 + LITERAL CStr null 1 "#]], expect![[r#" - LITER 42:Root[0000, 0]@0..4#1 Integer 1u16 - PUNCT 42:Root[0000, 0]@4..5#1 , [alone] - LITER 42:Root[0000, 0]@6..11#1 Integer 2_u32 - PUNCT 42:Root[0000, 0]@11..12#1 , [alone] - PUNCT 42:Root[0000, 0]@13..14#1 - [alone] - LITER 42:Root[0000, 0]@14..18#1 Integer 4i64 - PUNCT 42:Root[0000, 0]@18..19#1 , [alone] - LITER 42:Root[0000, 0]@20..27#1 Float 3.14f32 - PUNCT 42:Root[0000, 0]@27..28#1 , [alone] - LITER 42:Root[0000, 0]@29..43#1 Str hello bridge - PUNCT 42:Root[0000, 0]@43..44#1 , [alone] - LITER 42:Root[0000, 0]@45..61#1 Str suffixedsuffix - PUNCT 42:Root[0000, 0]@61..62#1 , [alone] - LITER 42:Root[0000, 0]@63..73#1 StrRaw(2) raw - PUNCT 42:Root[0000, 0]@73..74#1 , [alone] - LITER 42:Root[0000, 0]@75..78#1 Char a - PUNCT 42:Root[0000, 0]@78..79#1 , [alone] - LITER 42:Root[0000, 0]@80..84#1 Byte b - PUNCT 42:Root[0000, 0]@84..85#1 , [alone] - LITER 42:Root[0000, 0]@86..93#1 CStr null - - - LITER 42:Root[0000, 0]@0..4#1 Integer 1u16 - PUNCT 42:Root[0000, 0]@4..5#1 , [alone] - LITER 42:Root[0000, 0]@6..11#1 Integer 2_u32 - PUNCT 42:Root[0000, 0]@11..12#1 , [alone] - PUNCT 42:Root[0000, 0]@13..14#1 - [alone] - LITER 42:Root[0000, 0]@14..18#1 Integer 4i64 - PUNCT 42:Root[0000, 0]@18..19#1 , [alone] - LITER 42:Root[0000, 0]@20..27#1 Float 3.14f32 - PUNCT 42:Root[0000, 0]@27..28#1 , [alone] - LITER 42:Root[0000, 0]@29..43#1 Str hello bridge - PUNCT 42:Root[0000, 0]@43..44#1 , [alone] - LITER 42:Root[0000, 0]@45..61#1 Str suffixedsuffix - PUNCT 42:Root[0000, 0]@61..62#1 , [alone] - LITER 42:Root[0000, 0]@63..73#1 StrRaw(2) raw - PUNCT 42:Root[0000, 0]@73..74#1 , [alone] - LITER 42:Root[0000, 0]@75..78#1 Char a - PUNCT 42:Root[0000, 0]@78..79#1 , [alone] - LITER 42:Root[0000, 0]@80..84#1 Byte b - PUNCT 42:Root[0000, 0]@84..85#1 , [alone] - LITER 42:Root[0000, 0]@86..93#1 CStr null + LITERAL Integer 1u16 42:Root[0000, 0]@0..4#1 + PUNCT , [alone] 42:Root[0000, 0]@4..5#1 + LITERAL Integer 2_u32 42:Root[0000, 0]@6..11#1 + PUNCT , [alone] 42:Root[0000, 0]@11..12#1 + PUNCT - [alone] 42:Root[0000, 0]@13..14#1 + LITERAL Integer 4i64 42:Root[0000, 0]@14..18#1 + PUNCT , [alone] 42:Root[0000, 0]@18..19#1 + LITERAL Float 3.14f32 42:Root[0000, 0]@20..27#1 + PUNCT , [alone] 42:Root[0000, 0]@27..28#1 + LITERAL Str hello bridge 42:Root[0000, 0]@29..43#1 + PUNCT , [alone] 42:Root[0000, 0]@43..44#1 + LITERAL Err(()) "suffixed"suffix 42:Root[0000, 0]@45..61#1 + PUNCT , [alone] 42:Root[0000, 0]@61..62#1 + LITERAL StrRaw(2) raw 42:Root[0000, 0]@63..73#1 + PUNCT , [alone] 42:Root[0000, 0]@73..74#1 + LITERAL Char a 42:Root[0000, 0]@75..78#1 + PUNCT , [alone] 42:Root[0000, 0]@78..79#1 + LITERAL Byte b 42:Root[0000, 0]@80..84#1 + PUNCT , [alone] 42:Root[0000, 0]@84..85#1 + LITERAL CStr null 42:Root[0000, 0]@86..93#1 + + + LITERAL Integer 1u16 42:Root[0000, 0]@0..4#1 + PUNCT , [alone] 42:Root[0000, 0]@4..5#1 + LITERAL Integer 2_u32 42:Root[0000, 0]@6..11#1 + PUNCT , [alone] 42:Root[0000, 0]@11..12#1 + PUNCT - [alone] 42:Root[0000, 0]@13..14#1 + LITERAL Integer 4i64 42:Root[0000, 0]@14..18#1 + PUNCT , [alone] 42:Root[0000, 0]@18..19#1 + LITERAL Float 3.14f32 42:Root[0000, 0]@20..27#1 + PUNCT , [alone] 42:Root[0000, 0]@27..28#1 + LITERAL Str hello bridge 42:Root[0000, 0]@29..43#1 + PUNCT , [alone] 42:Root[0000, 0]@43..44#1 + LITERAL Err(()) "suffixed"suffix 42:Root[0000, 0]@45..61#1 + PUNCT , [alone] 42:Root[0000, 0]@61..62#1 + LITERAL StrRaw(2) raw 42:Root[0000, 0]@63..73#1 + PUNCT , [alone] 42:Root[0000, 0]@73..74#1 + LITERAL Char a 42:Root[0000, 0]@75..78#1 + PUNCT , [alone] 42:Root[0000, 0]@78..79#1 + LITERAL Byte b 42:Root[0000, 0]@80..84#1 + PUNCT , [alone] 42:Root[0000, 0]@84..85#1 + LITERAL CStr null 42:Root[0000, 0]@86..93#1 "#]], ); } @@ -567,56 +551,56 @@ fn test_fn_like_macro_negative_literals() { "fn_like_clone_tokens", r###"-1u16, - 2_u32, -3.14f32, - 2.7"###, expect![[r#" - PUNCT 1 - [alone] - LITER 1 Integer 1u16 - PUNCT 1 , [alone] - PUNCT 1 - [alone] - LITER 1 Integer 2_u32 - PUNCT 1 , [alone] - PUNCT 1 - [alone] - LITER 1 Float 3.14f32 - PUNCT 1 , [alone] - PUNCT 1 - [alone] - LITER 1 Float 2.7 - - - PUNCT 1 - [alone] - LITER 1 Integer 1u16 - PUNCT 1 , [alone] - PUNCT 1 - [alone] - LITER 1 Integer 2_u32 - PUNCT 1 , [alone] - PUNCT 1 - [alone] - LITER 1 Float 3.14f32 - PUNCT 1 , [alone] - PUNCT 1 - [alone] - LITER 1 Float 2.7 + PUNCT - [alone] 1 + LITERAL Integer 1u16 1 + PUNCT , [alone] 1 + PUNCT - [alone] 1 + LITERAL Integer 2_u32 1 + PUNCT , [alone] 1 + PUNCT - [alone] 1 + LITERAL Float 3.14f32 1 + PUNCT , [alone] 1 + PUNCT - [alone] 1 + LITERAL Float 2.7 1 + + + PUNCT - [alone] 1 + LITERAL Integer 1u16 1 + PUNCT , [alone] 1 + PUNCT - [alone] 1 + LITERAL Integer 2_u32 1 + PUNCT , [alone] 1 + PUNCT - [alone] 1 + LITERAL Float 3.14f32 1 + PUNCT , [alone] 1 + PUNCT - [alone] 1 + LITERAL Float 2.7 1 "#]], expect![[r#" - PUNCT 42:Root[0000, 0]@0..1#1 - [alone] - LITER 42:Root[0000, 0]@1..5#1 Integer 1u16 - PUNCT 42:Root[0000, 0]@5..6#1 , [alone] - PUNCT 42:Root[0000, 0]@7..8#1 - [alone] - LITER 42:Root[0000, 0]@9..14#1 Integer 2_u32 - PUNCT 42:Root[0000, 0]@14..15#1 , [alone] - PUNCT 42:Root[0000, 0]@16..17#1 - [alone] - LITER 42:Root[0000, 0]@17..24#1 Float 3.14f32 - PUNCT 42:Root[0000, 0]@24..25#1 , [alone] - PUNCT 42:Root[0000, 0]@26..27#1 - [alone] - LITER 42:Root[0000, 0]@28..31#1 Float 2.7 - - - PUNCT 42:Root[0000, 0]@0..1#1 - [alone] - LITER 42:Root[0000, 0]@1..5#1 Integer 1u16 - PUNCT 42:Root[0000, 0]@5..6#1 , [alone] - PUNCT 42:Root[0000, 0]@7..8#1 - [alone] - LITER 42:Root[0000, 0]@9..14#1 Integer 2_u32 - PUNCT 42:Root[0000, 0]@14..15#1 , [alone] - PUNCT 42:Root[0000, 0]@16..17#1 - [alone] - LITER 42:Root[0000, 0]@17..24#1 Float 3.14f32 - PUNCT 42:Root[0000, 0]@24..25#1 , [alone] - PUNCT 42:Root[0000, 0]@26..27#1 - [alone] - LITER 42:Root[0000, 0]@28..31#1 Float 2.7 + PUNCT - [alone] 42:Root[0000, 0]@0..1#1 + LITERAL Integer 1u16 42:Root[0000, 0]@1..5#1 + PUNCT , [alone] 42:Root[0000, 0]@5..6#1 + PUNCT - [alone] 42:Root[0000, 0]@7..8#1 + LITERAL Integer 2_u32 42:Root[0000, 0]@9..14#1 + PUNCT , [alone] 42:Root[0000, 0]@14..15#1 + PUNCT - [alone] 42:Root[0000, 0]@16..17#1 + LITERAL Float 3.14f32 42:Root[0000, 0]@17..24#1 + PUNCT , [alone] 42:Root[0000, 0]@24..25#1 + PUNCT - [alone] 42:Root[0000, 0]@26..27#1 + LITERAL Float 2.7 42:Root[0000, 0]@28..31#1 + + + PUNCT - [alone] 42:Root[0000, 0]@0..1#1 + LITERAL Integer 1u16 42:Root[0000, 0]@1..5#1 + PUNCT , [alone] 42:Root[0000, 0]@5..6#1 + PUNCT - [alone] 42:Root[0000, 0]@7..8#1 + LITERAL Integer 2_u32 42:Root[0000, 0]@9..14#1 + PUNCT , [alone] 42:Root[0000, 0]@14..15#1 + PUNCT - [alone] 42:Root[0000, 0]@16..17#1 + LITERAL Float 3.14f32 42:Root[0000, 0]@17..24#1 + PUNCT , [alone] 42:Root[0000, 0]@24..25#1 + PUNCT - [alone] 42:Root[0000, 0]@26..27#1 + LITERAL Float 2.7 42:Root[0000, 0]@28..31#1 "#]], ); } @@ -631,36 +615,36 @@ fn test_attr_macro() { r#"mod m {}"#, r#"some arguments"#, expect![[r#" - IDENT 1 mod - IDENT 1 m - GROUP {} 1 1 1 + IDENT mod 1 + IDENT m 1 + GROUP {} 1 1 - IDENT 1 some - IDENT 1 arguments + IDENT some 1 + IDENT arguments 1 - IDENT 1 compile_error - PUNCT 1 ! [joint] - GROUP () 1 1 1 - LITER 1 Str #[attr_error(some arguments)] mod m {} - PUNCT 1 ; [alone] + IDENT compile_error 1 + PUNCT ! [joint] 1 + GROUP () 1 1 + LITERAL Str #[attr_error(some arguments)] mod m {} 1 + PUNCT ; [alone] 1 "#]], expect![[r#" - IDENT 42:Root[0000, 0]@0..3#1 mod - IDENT 42:Root[0000, 0]@4..5#1 m - GROUP {} 42:Root[0000, 0]@6..7#1 42:Root[0000, 0]@7..8#1 42:Root[0000, 0]@6..8#1 + IDENT mod 42:Root[0000, 0]@0..3#1 + IDENT m 42:Root[0000, 0]@4..5#1 + GROUP {} 42:Root[0000, 0]@6..7#1 42:Root[0000, 0]@7..8#1 - IDENT 42:Root[0000, 0]@0..4#1 some - IDENT 42:Root[0000, 0]@5..14#1 arguments + IDENT some 42:Root[0000, 0]@0..4#1 + IDENT arguments 42:Root[0000, 0]@5..14#1 - IDENT 42:Root[0000, 0]@0..13#1 compile_error - PUNCT 42:Root[0000, 0]@13..14#1 ! [joint] - GROUP () 42:Root[0000, 0]@14..15#1 42:Root[0000, 0]@55..56#1 42:Root[0000, 0]@14..56#1 - LITER 42:Root[0000, 0]@15..55#1 Str #[attr_error(some arguments)] mod m {} - PUNCT 42:Root[0000, 0]@56..57#1 ; [alone] + IDENT compile_error 42:Root[0000, 0]@0..13#1 + PUNCT ! [joint] 42:Root[0000, 0]@13..14#1 + GROUP () 42:Root[0000, 0]@14..15#1 42:Root[0000, 0]@55..56#1 + LITERAL Str #[attr_error(some arguments)] mod m {} 42:Root[0000, 0]@15..55#1 + PUNCT ; [alone] 42:Root[0000, 0]@56..57#1 "#]], ); } @@ -722,8 +706,8 @@ fn test_fn_like_span_line_column() { " hello", expect![[r#" - LITER 42:Root[0000, 0]@0..100#1 Integer 2 - LITER 42:Root[0000, 0]@0..100#1 Integer 1 + LITERAL Integer 2 42:Root[0000, 0]@0..100#1 + LITERAL Integer 1 42:Root[0000, 0]@0..100#1 "#]], ); } diff --git a/crates/proc-macro-srv/src/tests/utils.rs b/crates/proc-macro-srv/src/tests/utils.rs index 7f92c66fb69d..b514e8e3d8f9 100644 --- a/crates/proc-macro-srv/src/tests/utils.rs +++ b/crates/proc-macro-srv/src/tests/utils.rs @@ -1,6 +1,7 @@ //! utils used in proc-macro tests use expect_test::Expect; +use proc_macro_api::token_stream::TokenStream; use span::{ EditionedFileId, FileId, ROOT_ERASED_FILE_AST_ID, Span, SpanAnchor, SyntaxContext, TextRange, }; @@ -8,7 +9,6 @@ use std::ops::Range; use crate::{ EnvSnapshot, ProcMacroClientInterface, ProcMacroSrv, SpanId, dylib, proc_macro_test_dylib_path, - token_stream::TokenStream, }; fn make_ctx() -> SyntaxContext { diff --git a/crates/proc-macro-srv/src/token_stream.rs b/crates/proc-macro-srv/src/token_stream.rs deleted file mode 100644 index 5201bb6aeb86..000000000000 --- a/crates/proc-macro-srv/src/token_stream.rs +++ /dev/null @@ -1,767 +0,0 @@ -//! The proc-macro server token stream implementation. - -use core::fmt; -use std::{mem, sync::Arc}; - -use intern::Symbol; -use rustc_lexer::{DocStyle, LiteralKind}; -use rustc_proc_macro::Delimiter; - -use crate::bridge::{DelimSpan, Group, Ident, LitKind, Literal, Punct, TokenTree}; - -/// Trait for allowing tests to parse tokenstreams with dynamic span ranges -pub(crate) trait SpanLike { - fn derive_ranged(&self, range: std::ops::Range) -> Self; -} - -#[derive(Clone)] -pub struct TokenStream(pub(crate) Arc>>); - -impl Default for TokenStream { - fn default() -> Self { - Self(Default::default()) - } -} - -impl TokenStream { - pub fn new(tts: Vec>) -> TokenStream { - TokenStream(Arc::new(tts)) - } - - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } - - pub fn len(&self) -> usize { - self.0.len() - } - - pub fn iter(&self) -> TokenStreamIter<'_, S> { - TokenStreamIter::new(self) - } - - pub fn as_single_group(&self) -> Option<&Group> { - match &**self.0 { - [TokenTree::Group(group)] => Some(group), - _ => None, - } - } - - pub(crate) fn from_str(s: &str, span: S) -> Result - where - S: SpanLike + Copy, - { - let mut groups = Vec::new(); - groups.push((rustc_proc_macro::Delimiter::None, 0..0, vec![])); - let mut offset = 0; - let mut tokens = rustc_lexer::tokenize(s, rustc_lexer::FrontmatterAllowed::No).peekable(); - while let Some(token) = tokens.next() { - let range = offset..offset + token.len as usize; - offset += token.len as usize; - - let mut is_joint = || { - tokens.peek().is_some_and(|token| { - matches!( - token.kind, - rustc_lexer::TokenKind::RawLifetime - | rustc_lexer::TokenKind::GuardedStrPrefix - | rustc_lexer::TokenKind::Lifetime { .. } - | rustc_lexer::TokenKind::Semi - | rustc_lexer::TokenKind::Comma - | rustc_lexer::TokenKind::Dot - | rustc_lexer::TokenKind::OpenParen - | rustc_lexer::TokenKind::CloseParen - | rustc_lexer::TokenKind::OpenBrace - | rustc_lexer::TokenKind::CloseBrace - | rustc_lexer::TokenKind::OpenBracket - | rustc_lexer::TokenKind::CloseBracket - | rustc_lexer::TokenKind::At - | rustc_lexer::TokenKind::Pound - | rustc_lexer::TokenKind::Tilde - | rustc_lexer::TokenKind::Question - | rustc_lexer::TokenKind::Colon - | rustc_lexer::TokenKind::Dollar - | rustc_lexer::TokenKind::Eq - | rustc_lexer::TokenKind::Bang - | rustc_lexer::TokenKind::Lt - | rustc_lexer::TokenKind::Gt - | rustc_lexer::TokenKind::Minus - | rustc_lexer::TokenKind::And - | rustc_lexer::TokenKind::Or - | rustc_lexer::TokenKind::Plus - | rustc_lexer::TokenKind::Star - | rustc_lexer::TokenKind::Slash - | rustc_lexer::TokenKind::Percent - | rustc_lexer::TokenKind::Caret - ) - }) - }; - - let Some((open_delim, _, tokenstream)) = groups.last_mut() else { - return Err("Unbalanced delimiters".to_owned()); - }; - match token.kind { - rustc_lexer::TokenKind::OpenParen => { - groups.push((rustc_proc_macro::Delimiter::Parenthesis, range, vec![])) - } - rustc_lexer::TokenKind::CloseParen if *open_delim != Delimiter::Parenthesis => { - return if *open_delim == Delimiter::None { - Err("Unexpected ')'".to_owned()) - } else { - Err("Expected ')'".to_owned()) - }; - } - rustc_lexer::TokenKind::CloseParen => { - let (delimiter, open_range, stream) = groups.pop().unwrap(); - groups.last_mut().ok_or_else(|| "Unbalanced delimiters".to_owned())?.2.push( - TokenTree::Group(Group { - delimiter, - stream: if stream.is_empty() { - None - } else { - Some(TokenStream::new(stream)) - }, - span: DelimSpan { - entire: span.derive_ranged(open_range.start..range.end), - open: span.derive_ranged(open_range), - close: span.derive_ranged(range), - }, - }), - ); - } - rustc_lexer::TokenKind::OpenBrace => { - groups.push((rustc_proc_macro::Delimiter::Brace, range, vec![])) - } - rustc_lexer::TokenKind::CloseBrace if *open_delim != Delimiter::Brace => { - return if *open_delim == Delimiter::None { - Err("Unexpected '}'".to_owned()) - } else { - Err("Expected '}'".to_owned()) - }; - } - rustc_lexer::TokenKind::CloseBrace => { - let (delimiter, open_range, stream) = groups.pop().unwrap(); - groups.last_mut().ok_or_else(|| "Unbalanced delimiters".to_owned())?.2.push( - TokenTree::Group(Group { - delimiter, - stream: if stream.is_empty() { - None - } else { - Some(TokenStream::new(stream)) - }, - span: DelimSpan { - entire: span.derive_ranged(open_range.start..range.end), - open: span.derive_ranged(open_range), - close: span.derive_ranged(range), - }, - }), - ); - } - rustc_lexer::TokenKind::OpenBracket => { - groups.push((rustc_proc_macro::Delimiter::Bracket, range, vec![])) - } - rustc_lexer::TokenKind::CloseBracket if *open_delim != Delimiter::Bracket => { - return if *open_delim == Delimiter::None { - Err("Unexpected ']'".to_owned()) - } else { - Err("Expected ']'".to_owned()) - }; - } - rustc_lexer::TokenKind::CloseBracket => { - let (delimiter, open_range, stream) = groups.pop().unwrap(); - groups.last_mut().ok_or_else(|| "Unbalanced delimiters".to_owned())?.2.push( - TokenTree::Group(Group { - delimiter, - stream: if stream.is_empty() { - None - } else { - Some(TokenStream::new(stream)) - }, - span: DelimSpan { - entire: span.derive_ranged(open_range.start..range.end), - open: span.derive_ranged(open_range), - close: span.derive_ranged(range), - }, - }), - ); - } - rustc_lexer::TokenKind::LineComment { doc_style: None } - | rustc_lexer::TokenKind::BlockComment { doc_style: None, terminated: _ } => { - continue; - } - rustc_lexer::TokenKind::LineComment { doc_style: Some(doc_style) } => { - let text = &s[range.start + 3..range.end]; - tokenstream.push(TokenTree::Punct(Punct { ch: b'#', joint: false, span })); - if doc_style == DocStyle::Inner { - tokenstream.push(TokenTree::Punct(Punct { ch: b'!', joint: false, span })); - } - tokenstream.push(TokenTree::Group(Group { - delimiter: Delimiter::Bracket, - stream: Some(TokenStream::new(vec![ - TokenTree::Ident(Ident { - sym: Symbol::intern("doc"), - is_raw: false, - span, - }), - TokenTree::Punct(Punct { ch: b'=', joint: false, span }), - TokenTree::Literal(Literal { - kind: LitKind::Str, - symbol: Symbol::intern(&text.escape_debug().to_string()), - suffix: None, - span: span.derive_ranged(range), - }), - ])), - span: DelimSpan { open: span, close: span, entire: span }, - })); - } - rustc_lexer::TokenKind::BlockComment { doc_style: Some(doc_style), terminated } => { - let text = - &s[range.start + 3..if terminated { range.end - 2 } else { range.end }]; - tokenstream.push(TokenTree::Punct(Punct { ch: b'#', joint: false, span })); - if doc_style == DocStyle::Inner { - tokenstream.push(TokenTree::Punct(Punct { ch: b'!', joint: false, span })); - } - tokenstream.push(TokenTree::Group(Group { - delimiter: Delimiter::Bracket, - stream: Some(TokenStream::new(vec![ - TokenTree::Ident(Ident { - sym: Symbol::intern("doc"), - is_raw: false, - span, - }), - TokenTree::Punct(Punct { ch: b'=', joint: false, span }), - TokenTree::Literal(Literal { - kind: LitKind::Str, - symbol: Symbol::intern(&text.escape_debug().to_string()), - suffix: None, - span: span.derive_ranged(range), - }), - ])), - span: DelimSpan { open: span, close: span, entire: span }, - })); - } - rustc_lexer::TokenKind::Whitespace => continue, - rustc_lexer::TokenKind::Frontmatter { .. } => unreachable!(), - rustc_lexer::TokenKind::Unknown => { - return Err(format!("Unknown token: `{}`", &s[range])); - } - rustc_lexer::TokenKind::UnknownPrefix => { - return Err(format!("Unknown prefix: `{}`", &s[range])); - } - rustc_lexer::TokenKind::UnknownPrefixLifetime => { - return Err(format!("Unknown lifetime prefix: `{}`", &s[range])); - } - // FIXME: Error on edition >= 2024 ... I dont think the proc-macro server can fetch editions currently - // and whose edition is this? - rustc_lexer::TokenKind::GuardedStrPrefix => { - tokenstream.push(TokenTree::Punct(Punct { - ch: s.as_bytes()[range.start], - joint: true, - span: span.derive_ranged(range.start..range.start + 1), - })); - tokenstream.push(TokenTree::Punct(Punct { - ch: s.as_bytes()[range.start + 1], - joint: is_joint(), - span: span.derive_ranged(range.start + 1..range.end), - })) - } - rustc_lexer::TokenKind::Ident => tokenstream.push(TokenTree::Ident(Ident { - sym: Symbol::intern(&s[range.clone()]), - is_raw: false, - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::InvalidIdent => { - return Err(format!("Invalid identifier: `{}`", &s[range])); - } - rustc_lexer::TokenKind::RawIdent => { - let range = range.start + 2..range.end; - tokenstream.push(TokenTree::Ident(Ident { - sym: Symbol::intern(&s[range.clone()]), - is_raw: true, - span: span.derive_ranged(range), - })) - } - rustc_lexer::TokenKind::Literal { kind, suffix_start } => { - tokenstream.push(TokenTree::Literal(literal_from_lexer( - &s[range.clone()], - span.derive_ranged(range), - kind, - suffix_start, - ))) - } - rustc_lexer::TokenKind::RawLifetime => { - let range = range.start + 1 + 2..range.end; - tokenstream.push(TokenTree::Punct(Punct { - ch: b'\'', - joint: true, - span: span.derive_ranged(range.start..range.start + 1), - })); - tokenstream.push(TokenTree::Ident(Ident { - sym: Symbol::intern(&s[range.clone()]), - is_raw: true, - span: span.derive_ranged(range), - })) - } - rustc_lexer::TokenKind::Lifetime { starts_with_number } => { - if starts_with_number { - return Err("Lifetime cannot start with a number".to_owned()); - } - let range = range.start + 1..range.end; - tokenstream.push(TokenTree::Punct(Punct { - ch: b'\'', - joint: true, - span: span.derive_ranged(range.start..range.start + 1), - })); - tokenstream.push(TokenTree::Ident(Ident { - sym: Symbol::intern(&s[range.clone()]), - is_raw: false, - span: span.derive_ranged(range), - })) - } - rustc_lexer::TokenKind::Semi => tokenstream.push(TokenTree::Punct(Punct { - ch: b';', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Comma => tokenstream.push(TokenTree::Punct(Punct { - ch: b',', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Dot => tokenstream.push(TokenTree::Punct(Punct { - ch: b'.', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::At => tokenstream.push(TokenTree::Punct(Punct { - ch: b'@', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Pound => tokenstream.push(TokenTree::Punct(Punct { - ch: b'#', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Tilde => tokenstream.push(TokenTree::Punct(Punct { - ch: b'~', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Question => tokenstream.push(TokenTree::Punct(Punct { - ch: b'?', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Colon => tokenstream.push(TokenTree::Punct(Punct { - ch: b':', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Dollar => tokenstream.push(TokenTree::Punct(Punct { - ch: b'$', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Eq => tokenstream.push(TokenTree::Punct(Punct { - ch: b'=', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Bang => tokenstream.push(TokenTree::Punct(Punct { - ch: b'!', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Lt => tokenstream.push(TokenTree::Punct(Punct { - ch: b'<', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Gt => tokenstream.push(TokenTree::Punct(Punct { - ch: b'>', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Minus => tokenstream.push(TokenTree::Punct(Punct { - ch: b'-', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::And => tokenstream.push(TokenTree::Punct(Punct { - ch: b'&', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Or => tokenstream.push(TokenTree::Punct(Punct { - ch: b'|', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Plus => tokenstream.push(TokenTree::Punct(Punct { - ch: b'+', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Star => tokenstream.push(TokenTree::Punct(Punct { - ch: b'*', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Slash => tokenstream.push(TokenTree::Punct(Punct { - ch: b'/', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Caret => tokenstream.push(TokenTree::Punct(Punct { - ch: b'^', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Percent => tokenstream.push(TokenTree::Punct(Punct { - ch: b'%', - joint: is_joint(), - span: span.derive_ranged(range), - })), - rustc_lexer::TokenKind::Eof => break, - } - } - if let Some((Delimiter::None, _, tokentrees)) = groups.pop() - && groups.is_empty() - { - Ok(TokenStream::new(tokentrees)) - } else { - Err("Mismatched token groups".to_owned()) - } - } -} - -impl fmt::Display for TokenStream { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mut emit_whitespace = false; - for tt in self.0.iter() { - display_token_tree(tt, &mut emit_whitespace, f)?; - } - Ok(()) - } -} - -fn display_token_tree( - tt: &TokenTree, - emit_whitespace: &mut bool, - f: &mut std::fmt::Formatter<'_>, -) -> std::fmt::Result { - if mem::take(emit_whitespace) { - write!(f, " ")?; - } - match tt { - TokenTree::Group(Group { delimiter, stream, span: _ }) => { - write!( - f, - "{}", - match delimiter { - rustc_proc_macro::Delimiter::Parenthesis => "(", - rustc_proc_macro::Delimiter::Brace => "{", - rustc_proc_macro::Delimiter::Bracket => "[", - rustc_proc_macro::Delimiter::None => "", - } - )?; - if let Some(stream) = stream { - write!(f, "{stream}")?; - } - write!( - f, - "{}", - match delimiter { - rustc_proc_macro::Delimiter::Parenthesis => ")", - rustc_proc_macro::Delimiter::Brace => "}", - rustc_proc_macro::Delimiter::Bracket => "]", - rustc_proc_macro::Delimiter::None => "", - } - )?; - } - TokenTree::Punct(Punct { ch, joint, span: _ }) => { - *emit_whitespace = !*joint; - write!(f, "{}", *ch as char)?; - } - TokenTree::Ident(Ident { sym, is_raw, span: _ }) => { - if *is_raw { - write!(f, "r#")?; - } - write!(f, "{sym}")?; - *emit_whitespace = true; - } - TokenTree::Literal(lit) => { - display_fmt_literal(lit, f)?; - let joint = match lit.kind { - LitKind::Str - | LitKind::StrRaw(_) - | LitKind::ByteStr - | LitKind::ByteStrRaw(_) - | LitKind::CStr - | LitKind::CStrRaw(_) => true, - _ => false, - }; - *emit_whitespace = !joint; - } - } - Ok(()) -} - -pub fn literal_to_string(literal: &Literal) -> String { - let mut buf = String::new(); - display_fmt_literal(literal, &mut buf).unwrap(); - buf -} - -fn display_fmt_literal(literal: &Literal, f: &mut impl std::fmt::Write) -> fmt::Result { - match literal.kind { - LitKind::Byte => write!(f, "b'{}'", literal.symbol), - LitKind::Char => write!(f, "'{}'", literal.symbol), - LitKind::Integer | LitKind::Float | LitKind::ErrWithGuar => { - write!(f, "{}", literal.symbol) - } - LitKind::Str => write!(f, "\"{}\"", literal.symbol), - LitKind::ByteStr => write!(f, "b\"{}\"", literal.symbol), - LitKind::CStr => write!(f, "c\"{}\"", literal.symbol), - LitKind::StrRaw(num_of_hashes) => { - let num_of_hashes = num_of_hashes as usize; - write!( - f, - r#"r{0:# { - let num_of_hashes = num_of_hashes as usize; - write!( - f, - r#"br{0:# { - let num_of_hashes = num_of_hashes as usize; - write!( - f, - r#"cr{0:# fmt::Debug for TokenStream { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - debug_token_stream(self, 0, f) - } -} - -fn debug_token_stream( - ts: &TokenStream, - depth: usize, - f: &mut std::fmt::Formatter<'_>, -) -> std::fmt::Result { - for tt in ts.0.iter() { - debug_token_tree(tt, depth, f)?; - } - Ok(()) -} - -fn debug_token_tree( - tt: &TokenTree, - depth: usize, - f: &mut std::fmt::Formatter<'_>, -) -> std::fmt::Result { - write!(f, "{:indent$}", "", indent = depth * 2)?; - match tt { - TokenTree::Group(Group { delimiter, stream, span }) => { - writeln!( - f, - "GROUP {}{} {:#?} {:#?} {:#?}", - match delimiter { - rustc_proc_macro::Delimiter::Parenthesis => "(", - rustc_proc_macro::Delimiter::Brace => "{", - rustc_proc_macro::Delimiter::Bracket => "[", - rustc_proc_macro::Delimiter::None => "$", - }, - match delimiter { - rustc_proc_macro::Delimiter::Parenthesis => ")", - rustc_proc_macro::Delimiter::Brace => "}", - rustc_proc_macro::Delimiter::Bracket => "]", - rustc_proc_macro::Delimiter::None => "$", - }, - span.open, - span.close, - span.entire, - )?; - if let Some(stream) = stream { - debug_token_stream(stream, depth + 1, f)?; - } - return Ok(()); - } - TokenTree::Punct(Punct { ch, joint, span }) => write!( - f, - "PUNCT {span:#?} {} {}", - *ch as char, - if *joint { "[joint]" } else { "[alone]" } - )?, - TokenTree::Ident(Ident { sym, is_raw, span }) => { - write!(f, "IDENT {span:#?} ")?; - if *is_raw { - write!(f, "r#")?; - } - write!(f, "{sym}")?; - } - TokenTree::Literal(Literal { kind, symbol, suffix, span }) => write!( - f, - "LITER {span:#?} {kind:?} {symbol}{}", - match suffix { - Some(suffix) => suffix.clone(), - None => Symbol::intern(""), - } - )?, - } - writeln!(f) -} - -impl TokenStream { - /// Push `tt` onto the end of the stream, possibly gluing it to the last - /// token. Uses `make_mut` to maximize efficiency. - pub(crate) fn push_tree(&mut self, tt: TokenTree) { - let vec_mut = Arc::make_mut(&mut self.0); - vec_mut.push(tt); - } - - /// Push `stream` onto the end of the stream, possibly gluing the first - /// token tree to the last token. (No other token trees will be glued.) - /// Uses `make_mut` to maximize efficiency. - pub(crate) fn push_stream(&mut self, stream: TokenStream) { - let vec_mut = Arc::make_mut(&mut self.0); - - let stream_iter = stream.0.iter().cloned(); - - vec_mut.extend(stream_iter); - } -} - -impl FromIterator> for TokenStream { - fn from_iter>>(iter: I) -> Self { - TokenStream::new(iter.into_iter().collect::>>()) - } -} - -#[derive(Clone)] -pub struct TokenStreamIter<'t, S> { - stream: &'t TokenStream, - index: usize, -} - -impl<'t, S> TokenStreamIter<'t, S> { - fn new(stream: &'t TokenStream) -> Self { - TokenStreamIter { stream, index: 0 } - } -} - -impl<'t, S> Iterator for TokenStreamIter<'t, S> { - type Item = &'t TokenTree; - - fn next(&mut self) -> Option<&'t TokenTree> { - self.stream.0.get(self.index).map(|tree| { - self.index += 1; - tree - }) - } -} - -pub(super) fn literal_from_lexer( - s: &str, - span: Span, - kind: rustc_lexer::LiteralKind, - suffix_start: u32, -) -> Literal { - let (kind, start_offset, end_offset) = match kind { - LiteralKind::Int { .. } => (LitKind::Integer, 0, 0), - LiteralKind::Float { .. } => (LitKind::Float, 0, 0), - LiteralKind::Char { terminated } => (LitKind::Char, 1, terminated as usize), - LiteralKind::Byte { terminated } => (LitKind::Byte, 2, terminated as usize), - LiteralKind::Str { terminated } => (LitKind::Str, 1, terminated as usize), - LiteralKind::ByteStr { terminated } => (LitKind::ByteStr, 2, terminated as usize), - LiteralKind::CStr { terminated } => (LitKind::CStr, 2, terminated as usize), - LiteralKind::RawStr { n_hashes } => ( - LitKind::StrRaw(n_hashes.unwrap_or_default()), - 2 + n_hashes.unwrap_or_default() as usize, - 1 + n_hashes.unwrap_or_default() as usize, - ), - LiteralKind::RawByteStr { n_hashes } => ( - LitKind::ByteStrRaw(n_hashes.unwrap_or_default()), - 3 + n_hashes.unwrap_or_default() as usize, - 1 + n_hashes.unwrap_or_default() as usize, - ), - LiteralKind::RawCStr { n_hashes } => ( - LitKind::CStrRaw(n_hashes.unwrap_or_default()), - 3 + n_hashes.unwrap_or_default() as usize, - 1 + n_hashes.unwrap_or_default() as usize, - ), - }; - - let (lit, suffix) = s.split_at(suffix_start as usize); - let lit = &lit[start_offset..lit.len() - end_offset]; - let suffix = match suffix { - "" | "_" => None, - suffix => Some(Symbol::intern(suffix)), - }; - - Literal { kind, symbol: Symbol::intern(lit), suffix, span } -} - -impl SpanLike for crate::SpanId { - fn derive_ranged(&self, _: std::ops::Range) -> Self { - *self - } -} - -impl SpanLike for () { - fn derive_ranged(&self, _: std::ops::Range) -> Self { - *self - } -} - -impl SpanLike for crate::Span { - fn derive_ranged(&self, range: std::ops::Range) -> Self { - crate::Span { - range: span::TextRange::new( - span::TextSize::new(range.start as u32), - span::TextSize::new(range.end as u32), - ), - anchor: self.anchor, - ctx: self.ctx, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn ts_to_string() { - let token_stream = - TokenStream::from_str("{} () [] <> ;/., \"gfhdgfuiofghd\" 0f32 r#\"dff\"# 'r#lt", ()) - .unwrap(); - assert_eq!(token_stream.to_string(), "{}()[]<> ;/., \"gfhdgfuiofghd\"0f32 r#\"dff\"#'r#lt"); - } - - #[test] - fn doc_comment_from_str() { - let token_stream = TokenStream::from_str("/// foo", ()).unwrap(); - assert_eq!(token_stream.to_string(), r#"# [doc = " foo"]"#); - } -} diff --git a/crates/rust-analyzer/src/global_state.rs b/crates/rust-analyzer/src/global_state.rs index c0d87104fbe1..3659c718b177 100644 --- a/crates/rust-analyzer/src/global_state.rs +++ b/crates/rust-analyzer/src/global_state.rs @@ -23,7 +23,7 @@ use parking_lot::{ MappedRwLockReadGuard, Mutex, RwLock, RwLockReadGuard, RwLockUpgradableReadGuard, RwLockWriteGuard, }; -use proc_macro_api::ProcMacroClient; +use proc_macro_api::client::ProcMacroClient; use project_model::{ ManifestPath, ProjectWorkspace, ProjectWorkspaceKind, TargetKind, WorkspaceBuildScripts, }; diff --git a/crates/rust-analyzer/src/reload.rs b/crates/rust-analyzer/src/reload.rs index 039fbeff828e..91014f0f7198 100644 --- a/crates/rust-analyzer/src/reload.rs +++ b/crates/rust-analyzer/src/reload.rs @@ -24,7 +24,7 @@ use itertools::Itertools; use load_cargo::{ProjectFolders, load_proc_macro}; use lsp_types::FileSystemWatcher; use paths::Utf8Path; -use proc_macro_api::ProcMacroClient; +use proc_macro_api::client::ProcMacroClient; use project_model::{ ManifestPath, ProjectWorkspace, ProjectWorkspaceKind, WorkspaceBuildScripts, project_json, }; diff --git a/crates/stdx/src/lib.rs b/crates/stdx/src/lib.rs index dcba06415b5f..7d844c041b0a 100644 --- a/crates/stdx/src/lib.rs +++ b/crates/stdx/src/lib.rs @@ -363,6 +363,32 @@ pub fn slice_tails(this: &[T]) -> impl Iterator { (0..this.len()).map(|i| &this[i..]) } +/// Imports a sysroot crate from the sysroot or from crates.io, depending on whether the `in-rust-tree` +/// feature is active. +/// +/// Syntax: +/// ``` +/// extern crate sysroot_crate or crates_io_crate; +/// ``` +// FIXME: Should this really be in `stdx`? +#[macro_export] +macro_rules! rustc_crates { + ( + $( + extern crate $sysroot_crate:ident or $crates_io_crate:ident; + )* + ) => { + ::std::cfg_select! { + feature = "in-rust-tree" => { + $( extern crate $sysroot_crate; )* + } + _ => { + $( extern crate $crates_io_crate as $sysroot_crate; )* + } + } + }; +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/syntax-bridge/src/lib.rs b/crates/syntax-bridge/src/lib.rs index 3e6e5f804e26..1bb7d9b91abd 100644 --- a/crates/syntax-bridge/src/lib.rs +++ b/crates/syntax-bridge/src/lib.rs @@ -15,10 +15,9 @@ use syntax::{ Parse, PreorderWithTokens, SmolStr, SyntaxElement, SyntaxKind::{self, *}, SyntaxNode, SyntaxToken, SyntaxTreeBuilder, T, TextRange, TextSize, WalkEvent, - ast::make::tokens::doc_comment, - format_smolstr, + ast::{CommentShape, make::tokens::doc_comment}, }; -use tt::{Punct, buffer::Cursor, token_to_literal}; +use tt::{Punct, buffer::Cursor}; pub mod prettify_macro_expansion; mod to_parser_input; @@ -86,10 +85,11 @@ pub mod dummy_test_span_utils { /// Doc comment desugaring differs between mbe and proc-macros. #[derive(Copy, Clone, PartialEq, Eq)] pub enum DocCommentDesugarMode { - /// Desugars doc comments as quoted raw strings - Mbe, - /// Desugars doc comments as quoted strings - ProcMacro, + /// Desugars doc comments as quoted raw strings. This is only used for macro-by-example inputs + /// (**not** MBE themselves, they should use [`Self::Keep`]). + DesugarMbeInput, + /// Keep doc comments as [`tt::DocComment`]. + Keep, } /// Converts a syntax tree to a [`tt::Subtree`] using the provided span map to populate the @@ -185,8 +185,7 @@ pub fn parse_to_token_tree( if lexed.errors().next().is_some() { return None; } - let mut conv = - RawConverter { lexed, anchor, pos: 0, ctx, mode: DocCommentDesugarMode::ProcMacro }; + let mut conv = RawConverter { lexed, anchor, pos: 0, ctx, mode: DocCommentDesugarMode::Keep }; Some(convert_tokens(&mut conv)) } @@ -200,8 +199,7 @@ pub fn parse_to_token_tree_static_span( if lexed.errors().next().is_some() { return None; } - let mut conv = - StaticRawConverter { lexed, pos: 0, span, mode: DocCommentDesugarMode::ProcMacro }; + let mut conv = StaticRawConverter { lexed, pos: 0, span, mode: DocCommentDesugarMode::Keep }; Some(convert_tokens(&mut conv)) } @@ -318,7 +316,7 @@ where k if k.is_literal() => { let text = token.to_text(conv); let span = conv.span_for(abs_range); - token_to_literal(&text, span).into() + tt::literal_from_str_or_err(&text, span).into() } LIFETIME_IDENT => { let apostrophe = tt::Leaf::from(tt::Punct { @@ -391,30 +389,20 @@ fn is_single_token_op(kind: SyntaxKind) -> bool { /// That is, strips leading `///` (or `/**`, etc) /// and strips the ending `*/` /// And then quote the string, which is needed to convert to `tt::Literal` -/// -/// Note that proc-macros desugar with string literals where as macro_rules macros desugar with raw string literals. -pub fn desugar_doc_comment_text(text: &str, mode: DocCommentDesugarMode) -> (Symbol, tt::LitKind) { - match mode { - DocCommentDesugarMode::Mbe => { - let mut num_of_hashes = 0; - let mut count = 0; - for ch in text.chars() { - count = match ch { - '"' => 1, - '#' if count > 0 => count + 1, - _ => 0, - }; - num_of_hashes = num_of_hashes.max(count); - } - - // Quote raw string with delimiters - (Symbol::intern(text), tt::LitKind::StrRaw(num_of_hashes)) - } - // Quote string with delimiters - DocCommentDesugarMode::ProcMacro => { - (Symbol::intern(&format_smolstr!("{}", text.escape_debug())), tt::LitKind::Str) - } +fn desugar_doc_comment_text_for_mbe(text: &str) -> (Symbol, tt::LitKind) { + let mut num_of_hashes = 0; + let mut count = 0; + for ch in text.chars() { + count = match ch { + '"' => 1, + '#' if count > 0 => count + 1, + _ => 0, + }; + num_of_hashes = num_of_hashes.max(count); } + + // Quote raw string with delimiters + (Symbol::intern(text), tt::LitKind::StrRaw(num_of_hashes)) } fn convert_doc_comment( @@ -424,35 +412,57 @@ fn convert_doc_comment( mode: DocCommentDesugarMode, builder: &mut tt::TopSubtreeBuilder, ) { - let mk_ident = |s: &str| { - tt::Leaf::from(tt::Ident { sym: Symbol::intern(s), span, is_raw: tt::IdentIsRaw::No }) - }; + match mode { + DocCommentDesugarMode::Keep => { + let doc_style = + if is_inner { tt::DocCommentStyle::Inner } else { tt::DocCommentStyle::Outer }; + let comment_style = match CommentShape::from_text(token.text()) { + CommentShape::Line => tt::CommentStyle::Line, + CommentShape::Block => tt::CommentStyle::Block, + }; + builder.push(tt::Leaf::DocComment(tt::DocComment::new( + token.text(), + span, + doc_style, + comment_style, + ))) + } + DocCommentDesugarMode::DesugarMbeInput => { + let mk_ident = |s: &str| { + tt::Leaf::from(tt::Ident { + sym: Symbol::intern(s), + span, + is_raw: tt::IdentIsRaw::No, + }) + }; - let mk_punct = - |c: char| tt::Leaf::from(tt::Punct { char: c, spacing: tt::Spacing::Alone, span }); + let mk_punct = + |c: char| tt::Leaf::from(tt::Punct { char: c, spacing: tt::Spacing::Alone, span }); - let mk_doc_literal = |token: &SyntaxToken| { - let text = token.text(); - let from_end = if text.starts_with("/*") && text.ends_with("*/") { 2 } else { 0 }; - let text = &text[3..text.len() - from_end]; + let mk_doc_literal = |token: &SyntaxToken| { + let text = token.text(); + let from_end = if text.starts_with("/*") && text.ends_with("*/") { 2 } else { 0 }; + let text = &text[3..text.len() - from_end]; - let (text, kind) = desugar_doc_comment_text(text, mode); - let lit = tt::Literal { text_and_suffix: text, span, kind, suffix_len: 0 }; + let (text, kind) = desugar_doc_comment_text_for_mbe(text); + let lit = tt::Literal { text_and_suffix: text, span, kind, suffix_len: 0 }; - tt::Leaf::from(lit) - }; + tt::Leaf::from(lit) + }; - // Make `doc="\" Comments\"" - let meta_tkns = [mk_ident("doc"), mk_punct('='), mk_doc_literal(token)]; + // Make `doc="\" Comments\"" + let meta_tkns = [mk_ident("doc"), mk_punct('='), mk_doc_literal(token)]; - // Make `#![]` - builder.push(mk_punct('#')); - if is_inner { - builder.push(mk_punct('!')); + // Make `#![]` + builder.push(mk_punct('#')); + if is_inner { + builder.push(mk_punct('!')); + } + builder.open(tt::DelimiterKind::Bracket, span); + builder.extend(meta_tkns); + builder.close(span); + } } - builder.open(tt::DelimiterKind::Bracket, span); - builder.extend(meta_tkns); - builder.close(span); } /// A raw token (straight from lexer) converter @@ -983,6 +993,19 @@ impl TtTreeSink<'_> { self.cursor.bump(); continue 'tokens; } + tt::Leaf::DocComment(doc_comment) => { + let text = doc_comment.text_with_comment_signs.as_str(); + self.buf.push_str(text); + self.text_pos += TextSize::of(text); + combined_span = match combined_span { + None => Some(doc_comment.span), + Some(prev_span) => { + Some(Self::merge_spans(prev_span, doc_comment.span)) + } + }; + self.cursor.bump(); + continue 'tokens; + } }, Some(tt::TokenTree::Subtree(subtree)) => { self.cursor.bump(); diff --git a/crates/syntax-bridge/src/prettify_macro_expansion.rs b/crates/syntax-bridge/src/prettify_macro_expansion.rs index c5e4aca95ee1..172414739a32 100644 --- a/crates/syntax-bridge/src/prettify_macro_expansion.rs +++ b/crates/syntax-bridge/src/prettify_macro_expansion.rs @@ -106,6 +106,9 @@ pub fn prettify_macro_expansion( AS_KW | DYN_KW | IMPL_KW | CONST_KW | MUT_KW | LET_KW | MATCH_KW => { mods.push(do_ws(after, tok)); } + INNER_DOC_COMMENT | OUTER_DOC_COMMENT => { + mods.push(do_nl(after, tok)); + } T![;] if is_next(|it| it != R_CURLY, true) => { mods.push(do_indent(after, tok, indent)); if tok.text_range().end() != syn.text_range().end() { diff --git a/crates/syntax-bridge/src/tests.rs b/crates/syntax-bridge/src/tests.rs index 691084d8b096..24f45d25fefe 100644 --- a/crates/syntax-bridge/src/tests.rs +++ b/crates/syntax-bridge/src/tests.rs @@ -17,7 +17,7 @@ fn check_punct_spacing(fixture: &str) { source_file.syntax(), DummyTestSpanMap, DUMMY, - DocCommentDesugarMode::Mbe, + DocCommentDesugarMode::Keep, ); let mut annotations: FxHashMap<_, _> = extract_annotations(fixture) .into_iter() diff --git a/crates/syntax-bridge/src/to_parser_input.rs b/crates/syntax-bridge/src/to_parser_input.rs index 851a4af86439..6198a60b8e17 100644 --- a/crates/syntax-bridge/src/to_parser_input.rs +++ b/crates/syntax-bridge/src/to_parser_input.rs @@ -84,6 +84,13 @@ pub fn to_parser_input( res.was_joint(); } } + tt::Leaf::DocComment(doc_comment) => { + let kind = match doc_comment.doc_style { + tt::DocCommentStyle::Inner => INNER_DOC_COMMENT, + tt::DocCommentStyle::Outer => OUTER_DOC_COMMENT, + }; + res.push(kind, ctx_edition(doc_comment.span.ctx)); + } } current.bump(); } diff --git a/crates/syntax/Cargo.toml b/crates/syntax/Cargo.toml index a9df1acdae9a..8bf7bb582942 100644 --- a/crates/syntax/Cargo.toml +++ b/crates/syntax/Cargo.toml @@ -40,3 +40,6 @@ in-rust-tree = [] [lints] workspace = true + +[package.metadata.cargo-machete] +ignored = ["ra-ap-rustc_lexer"] diff --git a/crates/syntax/src/lib.rs b/crates/syntax/src/lib.rs index dc7d7af7a2b9..8f46683da748 100644 --- a/crates/syntax/src/lib.rs +++ b/crates/syntax/src/lib.rs @@ -21,12 +21,12 @@ #![cfg_attr(feature = "in-rust-tree", feature(rustc_private))] -#[cfg(not(feature = "in-rust-tree"))] -extern crate ra_ap_rustc_lexer as rustc_lexer; #[cfg(feature = "in-rust-tree")] extern crate rustc_driver as _; -#[cfg(feature = "in-rust-tree")] -extern crate rustc_lexer; + +stdx::rustc_crates! { + extern crate rustc_lexer or ra_ap_rustc_lexer; +} mod parsing; mod ptr; diff --git a/crates/test-fixture/src/lib.rs b/crates/test-fixture/src/lib.rs index 63e76d049b72..47b1cc9115f7 100644 --- a/crates/test-fixture/src/lib.rs +++ b/crates/test-fixture/src/lib.rs @@ -1031,6 +1031,15 @@ impl ProcMacroExpander for ShortenProcMacroExpander { it.text_and_suffix = Symbol::empty(); it.suffix_len = 0; } + Leaf::DocComment(it) => { + it.text_with_comment_signs = + Symbol::intern(match (it.doc_style, it.comment_style) { + (tt::DocCommentStyle::Inner, tt::CommentStyle::Line) => "//!", + (tt::DocCommentStyle::Inner, tt::CommentStyle::Block) => "/*!*/", + (tt::DocCommentStyle::Outer, tt::CommentStyle::Line) => "///", + (tt::DocCommentStyle::Outer, tt::CommentStyle::Block) => "/***/", + }); + } Leaf::Punct(_) => {} Leaf::Ident(it) => { it.sym = Symbol::intern(&it.sym.as_str().chars().take(1).collect::()); diff --git a/crates/tt/Cargo.toml b/crates/tt/Cargo.toml index bd8f740b2f50..506063ac3cb4 100644 --- a/crates/tt/Cargo.toml +++ b/crates/tt/Cargo.toml @@ -14,16 +14,29 @@ doctest = false [dependencies] arrayvec.workspace = true -text-size.workspace = true -rustc-hash.workspace = true +text-size = { workspace = true, optional = true } +rustc-hash = { workspace = true, optional = true } -span = { path = "../span", version = "0.0", default-features = false } +span = { path = "../span", version = "0.0", default-features = false, optional = true } stdx.workspace = true intern.workspace = true ra-ap-rustc_lexer.workspace = true [features] +default = ["in-ra"] in-rust-tree = [] +# Inside rust-analyzer and not the proc macro server. Being in the proc macro server disables everything but the TokenTree types. +in-ra = [ + "dep:text-size", + "dep:rustc-hash", + "dep:span", +] [lints] workspace = true + +[package.metadata.rust-analyzer] +rustc_private = true + +[package.metadata.cargo-machete] +ignored = ["ra-ap-rustc_lexer"] diff --git a/crates/tt/src/leaf_types.rs b/crates/tt/src/leaf_types.rs new file mode 100644 index 000000000000..8036dfd6844c --- /dev/null +++ b/crates/tt/src/leaf_types.rs @@ -0,0 +1,505 @@ +//! Types that are shared between rust-analyzer and the proc macro server. + +use std::fmt; + +use arrayvec::ArrayString; +use intern::Symbol; + +#[cfg(feature = "in-ra")] +type DefaultSpan = span::Span; +#[cfg(not(feature = "in-ra"))] +pub enum DefaultSpan {} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +#[repr(u8)] +// The discriminants are important for `storage.rs` decoding. +pub enum IdentIsRaw { + No = 0, + Yes = 1, +} + +impl IdentIsRaw { + pub fn yes(self) -> bool { + matches!(self, IdentIsRaw::Yes) + } + pub fn no(&self) -> bool { + matches!(self, IdentIsRaw::No) + } + pub fn as_str(self) -> &'static str { + match self { + IdentIsRaw::No => "", + IdentIsRaw::Yes => "r#", + } + } + pub fn split_from_symbol(sym: &str) -> (Self, &str) { + if let Some(sym) = sym.strip_prefix("r#") { + (IdentIsRaw::Yes, sym) + } else { + (IdentIsRaw::No, sym) + } + } +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] +pub enum LitKind { + Byte, + Char, + Integer, // e.g. `1`, `1u8`, `1f32` + Float, // e.g. `1.`, `1.0`, `1e3f32` + Str, + StrRaw(u8), // raw string delimited by `n` hash symbols + ByteStr, + ByteStrRaw(u8), // raw byte string delimited by `n` hash symbols + CStr, + CStrRaw(u8), + Err(()), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[repr(u8)] +// The discriminants are important for decoding for `storage.rs`. +pub enum DocCommentStyle { + Inner = 0, + Outer = 1, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[repr(u8)] +// The discriminants are important for decoding for `storage.rs`. +pub enum CommentStyle { + Line = 0, + Block = 1, +} + +impl DocCommentStyle { + #[inline] + pub fn from_lexer(style: rustc_lexer::DocStyle) -> DocCommentStyle { + match style { + rustc_lexer::DocStyle::Outer => DocCommentStyle::Outer, + rustc_lexer::DocStyle::Inner => DocCommentStyle::Inner, + } + } +} + +/// We need a specific kind for doc comments and can't just desugar them since declarative macros ignore doc comments +/// in their matcher. Furthermore when the declarative macro is created from another macro they are not ignored, but +/// if the creating macro is a proc macro and it never touches the doc comment then they *are* ignored (in other words, +/// they are ignored while staying in the compiler's original representation, but not ignored when converted to the lossy +/// representation of the proc macro bridge). This means that they must be a property of the token tree and not just of +/// the AST (in rustc, the AST and the token tree and are the same, which is why this messy situation was created). +/// +/// See also . +/// +/// Note: not all doc comments are represented via this, in particular when passed to proc macros they are desugared +/// into `#[doc = "..."]`. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct DocComment { + /// Includes the `///` or `//!` or `/** */` or `/*! */`. + /// + /// We don't really need that information (as well as [`Self::comment_style`]), but it makes it easier to interoperate + /// with the parser (`syntax_bridge::to_parser_input`) and so we keep it. Therefore we also need to keep the [`CommentStyle`], + /// so we can strip them properly. + pub text_with_comment_signs: Symbol, + pub span: Span, + pub doc_style: DocCommentStyle, + pub comment_style: CommentStyle, +} + +impl DocComment { + #[inline] + pub fn new( + text: &str, + span: Span, + doc_style: DocCommentStyle, + comment_style: CommentStyle, + ) -> Self { + let text_with_comment_signs = + if comment_style == CommentStyle::Block && !text.ends_with("*/") { + // Fixup broken block comment, so it won't cause panics or errors later. This can happen from parser recovery. + Symbol::intern(&format!("{text}*/")) + } else { + Symbol::intern(text) + }; + Self { text_with_comment_signs, span, doc_style, comment_style } + } + + #[inline] + pub fn text(&self) -> &str { + let text_with_comment_signs = self.text_with_comment_signs.as_str(); + let strip_end = match self.comment_style { + CommentStyle::Block => 2, + CommentStyle::Line => 0, + }; + &text_with_comment_signs[3..text_with_comment_signs.len() - strip_end] + } + + /// The literal for `#[doc = "..."]` desugaring, as should be seen by proc macros. + pub fn literal_for_proc_macros(&self) -> Literal + where + Span: Copy, + { + Literal::new_no_suffix(&self.text().escape_debug().to_string(), self.span, LitKind::Str) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum Leaf { + Literal(Literal), + Punct(Punct), + Ident(Ident), + DocComment(DocComment), +} + +#[derive(Debug, Copy, Clone, PartialEq)] +pub struct DelimSpan { + pub open: Span, + pub close: Span, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct Delimiter { + pub open: Span, + pub close: Span, + pub kind: DelimiterKind, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[repr(u8)] +// The discriminants are important for decoding for `storage.rs`. +pub enum DelimiterKind { + Parenthesis = 0, + Brace = 1, + Bracket = 2, + Invisible = 3, +} + +impl DelimiterKind { + pub fn display_open_close(self) -> (&'static str, &'static str) { + match self { + DelimiterKind::Brace => ("{", "}"), + DelimiterKind::Bracket => ("[", "]"), + DelimiterKind::Parenthesis => ("(", ")"), + DelimiterKind::Invisible => ("", ""), + } + } + + pub fn debug_view(self) -> &'static str { + match self { + DelimiterKind::Invisible => "$$", + DelimiterKind::Parenthesis => "()", + DelimiterKind::Brace => "{}", + DelimiterKind::Bracket => "[]", + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Literal { + /// Escaped, text then suffix concatenated. + pub text_and_suffix: Symbol, + pub span: Span, + pub kind: LitKind, + pub suffix_len: u8, +} + +impl Literal { + #[inline] + pub fn text_and_suffix(&self) -> (&str, &str) { + let text_and_suffix = self.text_and_suffix.as_str(); + text_and_suffix.split_at(text_and_suffix.len() - usize::from(self.suffix_len)) + } + + pub fn text_and_suffix_symbols(&self) -> (Symbol, Option) { + if self.suffix_len == 0 { + (self.text_and_suffix.clone(), None) + } else { + let (text, suffix) = self.text_and_suffix(); + (Symbol::intern(text), Some(Symbol::intern(suffix))) + } + } + + #[inline] + pub fn text(&self) -> &str { + self.text_and_suffix().0 + } + + #[inline] + pub fn suffix(&self) -> &str { + self.text_and_suffix().1 + } + + pub fn new(text: &str, span: Span, kind: LitKind, suffix: &str) -> Self { + const MAX_INLINE_CAPACITY: usize = 30; + let text_and_suffix = if suffix.is_empty() { + Symbol::intern(text) + } else if (text.len() + suffix.len()) < MAX_INLINE_CAPACITY { + let mut text_and_suffix = ArrayString::::new(); + text_and_suffix.push_str(text); + text_and_suffix.push_str(suffix); + Symbol::intern(&text_and_suffix) + } else { + let mut text_and_suffix = String::with_capacity(text.len() + suffix.len()); + text_and_suffix.push_str(text); + text_and_suffix.push_str(suffix); + Symbol::intern(&text_and_suffix) + }; + + Self { text_and_suffix, span, kind, suffix_len: suffix.len().try_into().unwrap() } + } + + #[inline] + pub fn new_no_suffix(text: &str, span: Span, kind: LitKind) -> Self { + Self { text_and_suffix: Symbol::intern(text), span, kind, suffix_len: 0 } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct Punct { + pub char: char, + pub spacing: Spacing, + pub span: Span, +} + +/// Indicates whether a token can join with the following token to form a +/// compound token. Used for conversions to `proc_macro::Spacing`. Also used to +/// guide pretty-printing, which is where the `JointHidden` value (which isn't +/// part of `proc_macro::Spacing`) comes in useful. +// The discriminants are important for decoding for `storage.rs`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[repr(u8)] +pub enum Spacing { + /// The token cannot join with the following token to form a compound + /// token. + /// + /// In token streams parsed from source code, the compiler will use `Alone` + /// for any token immediately followed by whitespace, a non-doc comment, or + /// EOF. + /// + /// When constructing token streams within the compiler, use this for each + /// token that (a) should be pretty-printed with a space after it, or (b) + /// is the last token in the stream. (In the latter case the choice of + /// spacing doesn't matter because it is never used for the last token. We + /// arbitrarily use `Alone`.) + /// + /// Converts to `proc_macro::Spacing::Alone`, and + /// `proc_macro::Spacing::Alone` converts back to this. + Alone = 0, + + /// The token can join with the following token to form a compound token. + /// + /// In token streams parsed from source code, the compiler will use `Joint` + /// for any token immediately followed by punctuation (as determined by + /// `Token::is_punct`). + /// + /// When constructing token streams within the compiler, use this for each + /// token that (a) should be pretty-printed without a space after it, and + /// (b) is followed by a punctuation token. + /// + /// Converts to `proc_macro::Spacing::Joint`, and + /// `proc_macro::Spacing::Joint` converts back to this. + Joint = 1, + + /// The token can join with the following token to form a compound token, + /// but this will not be visible at the proc macro level. (This is what the + /// `Hidden` means; see below.) + /// + /// In token streams parsed from source code, the compiler will use + /// `JointHidden` for any token immediately followed by anything not + /// covered by the `Alone` and `Joint` cases: an identifier, lifetime, + /// literal, delimiter, doc comment. + /// + /// When constructing token streams, use this for each token that (a) + /// should be pretty-printed without a space after it, and (b) is followed + /// by a non-punctuation token. + /// + /// Converts to `proc_macro::Spacing::Alone`, but + /// `proc_macro::Spacing::Alone` converts back to `token::Spacing::Alone`. + /// Because of that, pretty-printing of `TokenStream`s produced by proc + /// macros is unavoidably uglier (with more whitespace between tokens) than + /// pretty-printing of `TokenStream`'s produced by other means (i.e. parsed + /// source code, internally constructed token streams, and token streams + /// produced by declarative macros). + JointHidden = 2, +} + +/// Identifier or keyword. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Ident { + pub sym: Symbol, + pub span: Span, + pub is_raw: IdentIsRaw, +} + +impl fmt::Display for Leaf { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Leaf::Ident(it) => fmt::Display::fmt(it, f), + Leaf::Literal(it) => fmt::Display::fmt(it, f), + Leaf::Punct(it) => fmt::Display::fmt(it, f), + Leaf::DocComment(it) => fmt::Display::fmt(it, f), + } + } +} + +impl fmt::Display for Ident { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.is_raw.as_str(), f)?; + fmt::Display::fmt(&self.sym, f) + } +} + +impl fmt::Display for Literal { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let (text, suffix) = self.text_and_suffix(); + match self.kind { + LitKind::Byte => write!(f, "b'{}'", text), + LitKind::Char => write!(f, "'{}'", text), + LitKind::Integer | LitKind::Float | LitKind::Err(_) => write!(f, "{}", text), + LitKind::Str => write!(f, "\"{}\"", text), + LitKind::ByteStr => write!(f, "b\"{}\"", text), + LitKind::CStr => write!(f, "c\"{}\"", text), + LitKind::StrRaw(num_of_hashes) => { + let num_of_hashes = num_of_hashes as usize; + write!(f, r#"r{0:# { + let num_of_hashes = num_of_hashes as usize; + write!(f, r#"br{0:# { + let num_of_hashes = num_of_hashes as usize; + write!(f, r#"cr{0:# fmt::Display for DocComment { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{}{}", + self.text_with_comment_signs, + // Print a newline after line comments so they can roundtrip (which is important for the proc macro server). + if self.comment_style == CommentStyle::Line { "\n" } else { "" }, + ) + } +} + +impl fmt::Display for Punct { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.char, f) + } +} + +impl Leaf { + pub fn print_debug(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Leaf::Literal(lit) => { + write!(f, "LITERAL {:?} {} {:#?}", lit.kind, lit.text_and_suffix, lit.span)?; + } + Leaf::Punct(punct) => { + write!( + f, + "PUNCT {} [{}] {:#?}", + punct.char, + if punct.spacing == Spacing::Alone { "alone" } else { "joint" }, + punct.span + )?; + } + Leaf::Ident(ident) => { + write!(f, "IDENT {}{} {:#?}", ident.is_raw.as_str(), ident.sym, ident.span)?; + } + Leaf::DocComment(doc) => { + write!( + f, + "DOC_COMMENT {:?} {:?} {} {:#?}", + doc.doc_style, doc.comment_style, doc.text_with_comment_signs, doc.span, + )?; + } + } + + Ok(()) + } +} + +pub fn literal_from_lexer( + text: &str, + span: Span, + kind: rustc_lexer::LiteralKind, + suffix_start: u32, +) -> Literal { + use rustc_lexer::LiteralKind; + + let (kind, start_offset, end_offset) = match kind { + LiteralKind::Int { .. } => (LitKind::Integer, 0, 0), + LiteralKind::Float { .. } => (LitKind::Float, 0, 0), + LiteralKind::Char { terminated } => (LitKind::Char, 1, terminated as usize), + LiteralKind::Byte { terminated } => (LitKind::Byte, 2, terminated as usize), + LiteralKind::Str { terminated } => (LitKind::Str, 1, terminated as usize), + LiteralKind::ByteStr { terminated } => (LitKind::ByteStr, 2, terminated as usize), + LiteralKind::CStr { terminated } => (LitKind::CStr, 2, terminated as usize), + LiteralKind::RawStr { n_hashes } => ( + LitKind::StrRaw(n_hashes.unwrap_or_default()), + 2 + n_hashes.unwrap_or_default() as usize, + 1 + n_hashes.unwrap_or_default() as usize, + ), + LiteralKind::RawByteStr { n_hashes } => ( + LitKind::ByteStrRaw(n_hashes.unwrap_or_default()), + 3 + n_hashes.unwrap_or_default() as usize, + 1 + n_hashes.unwrap_or_default() as usize, + ), + LiteralKind::RawCStr { n_hashes } => ( + LitKind::CStrRaw(n_hashes.unwrap_or_default()), + 3 + n_hashes.unwrap_or_default() as usize, + 1 + n_hashes.unwrap_or_default() as usize, + ), + }; + + let (lit, suffix) = text.split_at(suffix_start as usize); + let lit = &lit[start_offset..lit.len() - end_offset]; + let suffix = match suffix { + "" | "_" => "", + // ill-suffixed literals + _ if !matches!(kind, LitKind::Integer | LitKind::Float | LitKind::Err(_)) => { + return Literal::new_no_suffix(text, span, LitKind::Err(())); + } + suffix => suffix, + }; + + Literal::new(lit, span, kind, suffix) +} + +pub fn literal_from_str(text: &str, span: Span) -> Result, ()> { + use rustc_lexer::{LiteralKind, Token, TokenKind}; + + let mut tokens = rustc_lexer::tokenize(text, rustc_lexer::FrontmatterAllowed::No); + let minus_or_lit = tokens.next().unwrap_or(Token { kind: TokenKind::Eof, len: 0 }); + + let lit = if minus_or_lit.kind == TokenKind::Minus { + let lit = tokens.next().ok_or(())?; + if !matches!( + lit.kind, + TokenKind::Literal { kind: LiteralKind::Int { .. } | LiteralKind::Float { .. }, .. } + ) { + return Err(()); + } + lit + } else { + minus_or_lit + }; + + if tokens.next().is_some() { + return Err(()); + } + + let TokenKind::Literal { kind, suffix_start } = lit.kind else { return Err(()) }; + Ok(literal_from_lexer(text, span, kind, suffix_start)) +} + +pub fn literal_from_str_or_err(text: &str, span: Span) -> Literal { + literal_from_str(text, span) + .unwrap_or_else(|_| Literal::new_no_suffix(text, span, LitKind::Err(()))) +} diff --git a/crates/tt/src/lib.rs b/crates/tt/src/lib.rs index 2bc2b64cd4fd..c782e94ef3f0 100644 --- a/crates/tt/src/lib.rs +++ b/crates/tt/src/lib.rs @@ -8,812 +8,494 @@ #[cfg(feature = "in-rust-tree")] extern crate rustc_driver as _; -#[cfg(not(feature = "in-rust-tree"))] -extern crate ra_ap_rustc_lexer as rustc_lexer; -#[cfg(feature = "in-rust-tree")] -extern crate rustc_lexer; +stdx::rustc_crates! { + extern crate rustc_lexer or ra_ap_rustc_lexer; +} +#[cfg(feature = "in-ra")] pub mod buffer; +#[cfg(feature = "in-ra")] pub mod iter; +mod leaf_types; +#[cfg(feature = "in-ra")] mod storage; -use std::fmt; +#[cfg(feature = "in-ra")] +pub use self::in_ra::*; +pub use self::leaf_types::*; -use arrayvec::ArrayString; -use buffer::Cursor; -use intern::Symbol; -use stdx::{impl_from, itertools::Itertools as _}; +#[cfg(feature = "in-ra")] +mod in_ra { + use std::fmt; -pub use span::Span; -pub use text_size::{TextRange, TextSize}; + use intern::Symbol; + use stdx::impl_from; -use crate::storage::TokenTreesSlice; + pub use span::Span; + pub use text_size::{TextRange, TextSize}; -pub use self::iter::{TtElement, TtIter}; -pub use self::storage::{TopSubtree, TopSubtreeBuilder}; + use crate::{leaf_types::*, storage::TokenTreesSlice}; -pub const MAX_GLUED_PUNCT_LEN: usize = 3; + pub use crate::{ + buffer::Cursor, + iter::{TtElement, TtIter}, + storage::{TopSubtree, TopSubtreeBuilder}, + }; -#[derive(Clone, PartialEq, Debug)] -pub struct Lit { - pub kind: LitKind, - pub symbol: Symbol, - pub suffix: Option, -} + pub const MAX_GLUED_PUNCT_LEN: usize = 3; -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -#[repr(u8)] -// The discriminants are important for `storage.rs` decoding. -pub enum IdentIsRaw { - No = 0, - Yes = 1, -} -impl IdentIsRaw { - pub fn yes(self) -> bool { - matches!(self, IdentIsRaw::Yes) - } - pub fn no(&self) -> bool { - matches!(self, IdentIsRaw::No) + #[derive(Debug, Clone, PartialEq, Eq, Hash)] + pub enum TokenTree { + Leaf(Leaf), + Subtree(Subtree), } - pub fn as_str(self) -> &'static str { - match self { - IdentIsRaw::No => "", - IdentIsRaw::Yes => "r#", + impl_from!(Leaf, Subtree for TokenTree); + impl TokenTree { + pub fn first_span(&self) -> Span { + match self { + TokenTree::Leaf(l) => *l.span(), + TokenTree::Subtree(s) => s.delimiter.open, + } } } - pub fn split_from_symbol(sym: &str) -> (Self, &str) { - if let Some(sym) = sym.strip_prefix("r#") { - (IdentIsRaw::Yes, sym) - } else { - (IdentIsRaw::No, sym) + + impl Leaf { + pub fn span(&self) -> &Span { + match self { + Leaf::Literal(it) => &it.span, + Leaf::Punct(it) => &it.span, + Leaf::Ident(it) => &it.span, + Leaf::DocComment(it) => &it.span, + } } - } -} -#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] -pub enum LitKind { - Byte, - Char, - Integer, // e.g. `1`, `1u8`, `1f32` - Float, // e.g. `1.`, `1.0`, `1e3f32` - Str, - StrRaw(u8), // raw string delimited by `n` hash symbols - ByteStr, - ByteStrRaw(u8), // raw byte string delimited by `n` hash symbols - CStr, - CStrRaw(u8), - Err(()), -} + pub fn span_mut(&mut self) -> &mut Span { + match self { + Leaf::Literal(it) => &mut it.span, + Leaf::Punct(it) => &mut it.span, + Leaf::Ident(it) => &mut it.span, + Leaf::DocComment(it) => &mut it.span, + } + } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum TokenTree { - Leaf(Leaf), - Subtree(Subtree), -} -impl_from!(Leaf, Subtree for TokenTree); -impl TokenTree { - pub fn first_span(&self) -> Span { - match self { - TokenTree::Leaf(l) => *l.span(), - TokenTree::Subtree(s) => s.delimiter.open, + pub(crate) fn symbol(&self) -> Option<&Symbol> { + match self { + Leaf::Literal(Literal { text_and_suffix: symbol, .. }) + | Leaf::Ident(Ident { sym: symbol, .. }) + | Leaf::DocComment(DocComment { text_with_comment_signs: symbol, .. }) => { + Some(symbol) + } + Leaf::Punct(_) => None, + } } } -} + impl_from!(Literal, Punct, Ident, DocComment for Leaf); -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum Leaf { - Literal(Literal), - Punct(Punct), - Ident(Ident), -} - -impl Leaf { - pub fn span(&self) -> &Span { - match self { - Leaf::Literal(it) => &it.span, - Leaf::Punct(it) => &it.span, - Leaf::Ident(it) => &it.span, - } + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] + pub struct Subtree { + pub delimiter: Delimiter, + /// Number of following token trees that belong to this subtree, excluding this subtree. + pub len: u32, } - fn symbol(&self) -> Option<&Symbol> { - match self { - Leaf::Literal(Literal { text_and_suffix: symbol, .. }) - | Leaf::Ident(Ident { sym: symbol, .. }) => Some(symbol), - Leaf::Punct(_) => None, + impl Subtree { + pub fn usize_len(&self) -> usize { + self.len as usize } } -} -impl_from!(Literal, Punct, Ident for Leaf); - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct Subtree { - pub delimiter: Delimiter, - /// Number of following token trees that belong to this subtree, excluding this subtree. - pub len: u32, -} -impl Subtree { - pub fn usize_len(&self) -> usize { - self.len as usize + #[derive(Clone, Copy)] + pub struct TokenTreesView<'a> { + pub(crate) slice: TokenTreesSlice<'a>, + pub(crate) len: usize, } -} - -#[derive(Clone, Copy)] -pub struct TokenTreesView<'a> { - slice: TokenTreesSlice<'a>, - len: usize, -} -impl<'a> TokenTreesView<'a> { - #[inline] - pub fn empty() -> Self { - Self { slice: TokenTreesSlice::empty(), len: 0 } - } + impl<'a> TokenTreesView<'a> { + #[inline] + pub fn empty() -> Self { + Self { slice: TokenTreesSlice::empty(), len: 0 } + } - pub fn iter(&self) -> TtIter<'a> { - TtIter::new(*self) - } + pub fn iter(&self) -> TtIter<'a> { + TtIter::new(*self) + } - pub fn cursor(&self) -> Cursor<'a> { - Cursor::new(*self) - } + pub fn cursor(&self) -> Cursor<'a> { + Cursor::new(*self) + } - pub fn len(&self) -> usize { - self.len - } + pub fn len(&self) -> usize { + self.len + } - pub fn is_empty(&self) -> bool { - self.len() == 0 - } + pub fn is_empty(&self) -> bool { + self.len() == 0 + } - pub fn try_into_subtree(self) -> Option> { - let is_subtree = self.iter_flat_tokens().next().is_some_and( + pub fn try_into_subtree(self) -> Option> { + let is_subtree = self.iter_flat_tokens().next().is_some_and( |it| matches!(it, TokenTree::Subtree(subtree) if subtree.usize_len() == self.len - 1), ); - if is_subtree { Some(SubtreeView(self)) } else { None } - } + if is_subtree { Some(SubtreeView(self)) } else { None } + } - pub fn strip_invisible(self) -> TokenTreesView<'a> { - self.try_into_subtree().map(|subtree| subtree.strip_invisible()).unwrap_or(self) - } + pub fn strip_invisible(self) -> TokenTreesView<'a> { + self.try_into_subtree().map(|subtree| subtree.strip_invisible()).unwrap_or(self) + } - pub fn split( - self, - mut split_fn: impl FnMut(TtElement<'a>) -> bool, - ) -> impl Iterator> { - let mut subtree_iter = self.iter(); - let mut need_to_yield_even_if_empty = true; + pub fn split( + self, + mut split_fn: impl FnMut(TtElement<'a>) -> bool, + ) -> impl Iterator> { + let mut subtree_iter = self.iter(); + let mut need_to_yield_even_if_empty = true; - std::iter::from_fn(move || { - if subtree_iter.is_empty() && !need_to_yield_even_if_empty { - return None; - }; + std::iter::from_fn(move || { + if subtree_iter.is_empty() && !need_to_yield_even_if_empty { + return None; + }; - need_to_yield_even_if_empty = false; - let savepoint = subtree_iter.savepoint(); - let mut result = subtree_iter.from_savepoint(savepoint); - while let Some(tt) = subtree_iter.next() { - if split_fn(tt) { - need_to_yield_even_if_empty = true; - break; + need_to_yield_even_if_empty = false; + let savepoint = subtree_iter.savepoint(); + let mut result = subtree_iter.from_savepoint(savepoint); + while let Some(tt) = subtree_iter.next() { + if split_fn(tt) { + need_to_yield_even_if_empty = true; + break; + } + result = subtree_iter.from_savepoint(savepoint); } - result = subtree_iter.from_savepoint(savepoint); - } - Some(result) - }) - } + Some(result) + }) + } - pub fn first_span(&self) -> Option { - self.iter_flat_tokens().next().map(|it| it.first_span()) - } + pub fn first_span(&self) -> Option { + self.iter_flat_tokens().next().map(|it| it.first_span()) + } - /// Note: this is quite expensive, this needs to decode the whole view, - /// although it "tricks" by skipping subtrees (since we know their byte length). - pub fn last_span(&self) -> Option { - let mut iter = self.iter(); - loop { - match iter.last()? { - TtElement::Leaf(leaf) => return Some(*leaf.span()), - TtElement::Subtree(subtree, tt_iter) => { - if subtree.len == 0 { - return Some(subtree.delimiter.close); - } else { - iter = tt_iter; + /// Note: this is quite expensive, this needs to decode the whole view, + /// although it "tricks" by skipping subtrees (since we know their byte length). + pub fn last_span(&self) -> Option { + let mut iter = self.iter(); + loop { + match iter.last()? { + TtElement::Leaf(leaf) => return Some(*leaf.span()), + TtElement::Subtree(subtree, tt_iter) => { + if subtree.len == 0 { + return Some(subtree.delimiter.close); + } else { + iter = tt_iter; + } } } } } - } - pub fn iter_flat_tokens(&self) -> impl Iterator + use<'a> { - self.slice.iter().take(self.len) + pub fn iter_flat_tokens(&self) -> impl Iterator + use<'a> { + self.slice.iter().take(self.len) + } } -} -impl fmt::Debug for TokenTreesView<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let mut iter = self.iter(); - while let Some(tt) = iter.next() { - print_debug_token(f, 0, tt)?; - if !iter.is_empty() { - writeln!(f)?; + impl fmt::Debug for TokenTreesView<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut iter = self.iter(); + while let Some(tt) = iter.next() { + print_debug_token(f, 0, tt)?; + if !iter.is_empty() { + writeln!(f)?; + } } + Ok(()) } - Ok(()) } -} -impl fmt::Display for TokenTreesView<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - return token_trees_display(f, self.iter()); - - fn subtree_display( - subtree: &Subtree, - f: &mut fmt::Formatter<'_>, - iter: TtIter<'_>, - ) -> fmt::Result { - let (l, r) = match subtree.delimiter.kind { - DelimiterKind::Parenthesis => ("(", ")"), - DelimiterKind::Brace => ("{", "}"), - DelimiterKind::Bracket => ("[", "]"), - DelimiterKind::Invisible => ("", ""), - }; - f.write_str(l)?; - token_trees_display(f, iter)?; - f.write_str(r)?; - Ok(()) - } + impl fmt::Display for TokenTreesView<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + return token_trees_display(f, self.iter()); - fn token_trees_display(f: &mut fmt::Formatter<'_>, iter: TtIter<'_>) -> fmt::Result { - let mut needs_space = false; - for child in iter { - if needs_space { - f.write_str(" ")?; - } - needs_space = true; + fn subtree_display( + subtree: &Subtree, + f: &mut fmt::Formatter<'_>, + iter: TtIter<'_>, + ) -> fmt::Result { + let (l, r) = match subtree.delimiter.kind { + DelimiterKind::Parenthesis => ("(", ")"), + DelimiterKind::Brace => ("{", "}"), + DelimiterKind::Bracket => ("[", "]"), + DelimiterKind::Invisible => ("", ""), + }; + f.write_str(l)?; + token_trees_display(f, iter)?; + f.write_str(r)?; + Ok(()) + } - match child { - TtElement::Leaf(Leaf::Punct(p)) => { - needs_space = p.spacing == Spacing::Alone; - fmt::Display::fmt(&p, f)?; + fn token_trees_display(f: &mut fmt::Formatter<'_>, iter: TtIter<'_>) -> fmt::Result { + let mut needs_space = false; + for child in iter { + if needs_space { + f.write_str(" ")?; } - TtElement::Leaf(leaf) => fmt::Display::fmt(&leaf, f)?, - TtElement::Subtree(subtree, subtree_iter) => { - subtree_display(&subtree, f, subtree_iter)? + needs_space = true; + + match child { + TtElement::Leaf(Leaf::Punct(p)) => { + needs_space = p.spacing == Spacing::Alone; + fmt::Display::fmt(&p, f)?; + } + TtElement::Leaf(leaf) => fmt::Display::fmt(&leaf, f)?, + TtElement::Subtree(subtree, subtree_iter) => { + subtree_display(&subtree, f, subtree_iter)? + } } } + Ok(()) } - Ok(()) } } -} - -#[derive(Clone, Copy)] -// Invariant: always starts with `Subtree` that covers the entire thing. -pub struct SubtreeView<'a>(TokenTreesView<'a>); - -impl<'a> SubtreeView<'a> { - pub fn as_token_trees(self) -> TokenTreesView<'a> { - self.0 - } - - pub fn iter(&self) -> TtIter<'a> { - self.token_trees().iter() - } - pub fn top_subtree(&self) -> Subtree { - let Some(TokenTree::Subtree(subtree)) = self.0.iter_flat_tokens().next() else { - unreachable!("the first token tree is always the top subtree"); - }; - subtree - } + #[derive(Clone, Copy)] + // Invariant: always starts with `Subtree` that covers the entire thing. + pub struct SubtreeView<'a>(pub(crate) TokenTreesView<'a>); - pub fn strip_invisible(&self) -> TokenTreesView<'a> { - if self.top_subtree().delimiter.kind == DelimiterKind::Invisible { - self.token_trees() - } else { + impl<'a> SubtreeView<'a> { + pub fn as_token_trees(self) -> TokenTreesView<'a> { self.0 } - } - - pub fn token_trees(&self) -> TokenTreesView<'a> { - let mut result = self.0; - result.slice.advance(); - result.len -= 1; - result - } -} - -impl fmt::Debug for SubtreeView<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Debug::fmt(&self.0, f) - } -} -impl fmt::Display for SubtreeView<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(&self.0, f) - } -} - -#[derive(Debug, Copy, Clone, PartialEq)] -pub struct DelimSpan { - pub open: Span, - pub close: Span, -} + pub fn iter(&self) -> TtIter<'a> { + self.token_trees().iter() + } -impl DelimSpan { - pub fn from_single(sp: Span) -> Self { - DelimSpan { open: sp, close: sp } - } + pub fn top_subtree(&self) -> Subtree { + let Some(TokenTree::Subtree(subtree)) = self.0.iter_flat_tokens().next() else { + unreachable!("the first token tree is always the top subtree"); + }; + subtree + } - pub fn from_pair(open: Span, close: Span) -> Self { - DelimSpan { open, close } - } -} -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub struct Delimiter { - pub open: Span, - pub close: Span, - pub kind: DelimiterKind, -} + pub fn strip_invisible(&self) -> TokenTreesView<'a> { + if self.top_subtree().delimiter.kind == DelimiterKind::Invisible { + self.token_trees() + } else { + self.0 + } + } -impl Delimiter { - pub const fn invisible_spanned(span: Span) -> Self { - Delimiter { open: span, close: span, kind: DelimiterKind::Invisible } + pub fn token_trees(&self) -> TokenTreesView<'a> { + let mut result = self.0; + result.slice.advance(); + result.len -= 1; + result + } } - pub const fn invisible_delim_spanned(span: DelimSpan) -> Self { - Delimiter { open: span.open, close: span.close, kind: DelimiterKind::Invisible } + impl fmt::Debug for SubtreeView<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&self.0, f) + } } - pub fn delim_span(&self) -> DelimSpan { - DelimSpan { open: self.open, close: self.close } + impl fmt::Display for SubtreeView<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.0, f) + } } -} -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -#[repr(u8)] -// The discriminants are important for decoding for `storage.rs`. -pub enum DelimiterKind { - Parenthesis = 0, - Brace = 1, - Bracket = 2, - Invisible = 3, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Literal { - /// Escaped, text then suffix concatenated. - pub text_and_suffix: Symbol, - pub span: Span, - pub kind: LitKind, - pub suffix_len: u8, -} - -impl Literal { - #[inline] - pub fn text_and_suffix(&self) -> (&str, &str) { - let text_and_suffix = self.text_and_suffix.as_str(); - text_and_suffix.split_at(text_and_suffix.len() - usize::from(self.suffix_len)) - } + impl DelimSpan { + pub fn from_single(sp: Span) -> Self { + DelimSpan { open: sp, close: sp } + } - #[inline] - pub fn text(&self) -> &str { - self.text_and_suffix().0 + pub fn from_pair(open: Span, close: Span) -> Self { + DelimSpan { open, close } + } } - #[inline] - pub fn suffix(&self) -> &str { - self.text_and_suffix().1 - } + impl Delimiter { + pub const fn invisible_spanned(span: Span) -> Self { + Delimiter { open: span, close: span, kind: DelimiterKind::Invisible } + } - pub fn new(text: &str, span: Span, kind: LitKind, suffix: &str) -> Self { - const MAX_INLINE_CAPACITY: usize = 30; - let text_and_suffix = if suffix.is_empty() { - Symbol::intern(text) - } else if (text.len() + suffix.len()) < MAX_INLINE_CAPACITY { - let mut text_and_suffix = ArrayString::::new(); - text_and_suffix.push_str(text); - text_and_suffix.push_str(suffix); - Symbol::intern(&text_and_suffix) - } else { - let mut text_and_suffix = String::with_capacity(text.len() + suffix.len()); - text_and_suffix.push_str(text); - text_and_suffix.push_str(suffix); - Symbol::intern(&text_and_suffix) - }; - - Self { text_and_suffix, span, kind, suffix_len: suffix.len().try_into().unwrap() } - } + pub const fn invisible_delim_spanned(span: DelimSpan) -> Self { + Delimiter { open: span.open, close: span.close, kind: DelimiterKind::Invisible } + } - #[inline] - pub fn new_no_suffix(text: &str, span: Span, kind: LitKind) -> Self { - Self { text_and_suffix: Symbol::intern(text), span, kind, suffix_len: 0 } + pub fn delim_span(&self) -> DelimSpan { + DelimSpan { open: self.open, close: self.close } + } } -} - -pub fn token_to_literal(text: &str, span: Span) -> Literal { - use rustc_lexer::LiteralKind; - - let token = rustc_lexer::tokenize(text, rustc_lexer::FrontmatterAllowed::No).next_tuple(); - let Some((rustc_lexer::Token { - kind: rustc_lexer::TokenKind::Literal { kind, suffix_start }, - .. - },)) = token - else { - return Literal::new_no_suffix(text, span, LitKind::Err(())); - }; - - let (kind, start_offset, end_offset) = match kind { - LiteralKind::Int { .. } => (LitKind::Integer, 0, 0), - LiteralKind::Float { .. } => (LitKind::Float, 0, 0), - LiteralKind::Char { terminated } => (LitKind::Char, 1, terminated as usize), - LiteralKind::Byte { terminated } => (LitKind::Byte, 2, terminated as usize), - LiteralKind::Str { terminated } => (LitKind::Str, 1, terminated as usize), - LiteralKind::ByteStr { terminated } => (LitKind::ByteStr, 2, terminated as usize), - LiteralKind::CStr { terminated } => (LitKind::CStr, 2, terminated as usize), - LiteralKind::RawStr { n_hashes } => ( - LitKind::StrRaw(n_hashes.unwrap_or_default()), - 2 + n_hashes.unwrap_or_default() as usize, - 1 + n_hashes.unwrap_or_default() as usize, - ), - LiteralKind::RawByteStr { n_hashes } => ( - LitKind::ByteStrRaw(n_hashes.unwrap_or_default()), - 3 + n_hashes.unwrap_or_default() as usize, - 1 + n_hashes.unwrap_or_default() as usize, - ), - LiteralKind::RawCStr { n_hashes } => ( - LitKind::CStrRaw(n_hashes.unwrap_or_default()), - 3 + n_hashes.unwrap_or_default() as usize, - 1 + n_hashes.unwrap_or_default() as usize, - ), - }; - let (lit, suffix) = text.split_at(suffix_start as usize); - let lit = &lit[start_offset..lit.len() - end_offset]; - let suffix = match suffix { - "" | "_" => "", - // ill-suffixed literals - _ if !matches!(kind, LitKind::Integer | LitKind::Float | LitKind::Err(_)) => { - return Literal::new_no_suffix(text, span, LitKind::Err(())); + impl Ident { + pub fn new(text: &str, span: Span) -> Self { + // let raw_stripped = IdentIsRaw::split_from_symbol(text.as_ref()); + let (is_raw, text) = IdentIsRaw::split_from_symbol(text); + Ident { sym: Symbol::intern(text), span, is_raw } } - suffix => suffix, - }; - - Literal::new(lit, span, kind, suffix) -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct Punct { - pub char: char, - pub spacing: Spacing, - pub span: Span, -} - -/// Indicates whether a token can join with the following token to form a -/// compound token. Used for conversions to `proc_macro::Spacing`. Also used to -/// guide pretty-printing, which is where the `JointHidden` value (which isn't -/// part of `proc_macro::Spacing`) comes in useful. -// The discriminants are important for decoding for `storage.rs`. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[repr(u8)] -pub enum Spacing { - /// The token cannot join with the following token to form a compound - /// token. - /// - /// In token streams parsed from source code, the compiler will use `Alone` - /// for any token immediately followed by whitespace, a non-doc comment, or - /// EOF. - /// - /// When constructing token streams within the compiler, use this for each - /// token that (a) should be pretty-printed with a space after it, or (b) - /// is the last token in the stream. (In the latter case the choice of - /// spacing doesn't matter because it is never used for the last token. We - /// arbitrarily use `Alone`.) - /// - /// Converts to `proc_macro::Spacing::Alone`, and - /// `proc_macro::Spacing::Alone` converts back to this. - Alone = 0, - - /// The token can join with the following token to form a compound token. - /// - /// In token streams parsed from source code, the compiler will use `Joint` - /// for any token immediately followed by punctuation (as determined by - /// `Token::is_punct`). - /// - /// When constructing token streams within the compiler, use this for each - /// token that (a) should be pretty-printed without a space after it, and - /// (b) is followed by a punctuation token. - /// - /// Converts to `proc_macro::Spacing::Joint`, and - /// `proc_macro::Spacing::Joint` converts back to this. - Joint = 1, - - /// The token can join with the following token to form a compound token, - /// but this will not be visible at the proc macro level. (This is what the - /// `Hidden` means; see below.) - /// - /// In token streams parsed from source code, the compiler will use - /// `JointHidden` for any token immediately followed by anything not - /// covered by the `Alone` and `Joint` cases: an identifier, lifetime, - /// literal, delimiter, doc comment. - /// - /// When constructing token streams, use this for each token that (a) - /// should be pretty-printed without a space after it, and (b) is followed - /// by a non-punctuation token. - /// - /// Converts to `proc_macro::Spacing::Alone`, but - /// `proc_macro::Spacing::Alone` converts back to `token::Spacing::Alone`. - /// Because of that, pretty-printing of `TokenStream`s produced by proc - /// macros is unavoidably uglier (with more whitespace between tokens) than - /// pretty-printing of `TokenStream`'s produced by other means (i.e. parsed - /// source code, internally constructed token streams, and token streams - /// produced by declarative macros). - JointHidden = 2, -} - -/// Identifier or keyword. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Ident { - pub sym: Symbol, - pub span: Span, - pub is_raw: IdentIsRaw, -} - -impl Ident { - pub fn new(text: &str, span: Span) -> Self { - // let raw_stripped = IdentIsRaw::split_from_symbol(text.as_ref()); - let (is_raw, text) = IdentIsRaw::split_from_symbol(text); - Ident { sym: Symbol::intern(text), span, is_raw } } -} -fn print_debug_subtree( - f: &mut fmt::Formatter<'_>, - subtree: &Subtree, - level: usize, - iter: TtIter<'_>, -) -> fmt::Result { - let align = " ".repeat(level); - - let Delimiter { kind, open, close } = &subtree.delimiter; - let delim = match kind { - DelimiterKind::Invisible => "$$", - DelimiterKind::Parenthesis => "()", - DelimiterKind::Brace => "{}", - DelimiterKind::Bracket => "[]", - }; + fn print_debug_subtree( + f: &mut fmt::Formatter<'_>, + subtree: &Subtree, + level: usize, + iter: TtIter<'_>, + ) -> fmt::Result { + let Delimiter { kind, open, close } = &subtree.delimiter; + let delim = kind.debug_view(); + + write!(f, "SUBTREE {delim} ",)?; + write!(f, "{open:#?}")?; + write!(f, " ")?; + write!(f, "{close:#?}")?; + for child in iter { + writeln!(f)?; + print_debug_token(f, level + 1, child)?; + } - write!(f, "{align}SUBTREE {delim} ",)?; - write!(f, "{open:#?}")?; - write!(f, " ")?; - write!(f, "{close:#?}")?; - for child in iter { - writeln!(f)?; - print_debug_token(f, level + 1, child)?; + Ok(()) } - Ok(()) -} - -fn print_debug_token(f: &mut fmt::Formatter<'_>, level: usize, tt: TtElement<'_>) -> fmt::Result { - let align = " ".repeat(level); + fn print_debug_token( + f: &mut fmt::Formatter<'_>, + level: usize, + tt: TtElement<'_>, + ) -> fmt::Result { + write!(f, "{:indent$}", "", indent = level * 2)?; - match tt { - TtElement::Leaf(leaf) => match leaf { - Leaf::Literal(lit) => { - let (text, suffix) = lit.text_and_suffix(); - write!(f, "{}LITERAL {:?} {}{} {:#?}", align, lit.kind, text, suffix, lit.span)?; - } - Leaf::Punct(punct) => { - write!( - f, - "{}PUNCH {} [{}] {:#?}", - align, - punct.char, - if punct.spacing == Spacing::Alone { "alone" } else { "joint" }, - punct.span - )?; - } - Leaf::Ident(ident) => { - write!( - f, - "{}IDENT {}{} {:#?}", - align, - ident.is_raw.as_str(), - ident.sym, - ident.span - )?; + match tt { + TtElement::Leaf(leaf) => leaf.print_debug(f), + TtElement::Subtree(subtree, subtree_iter) => { + print_debug_subtree(f, &subtree, level, subtree_iter) } - }, - TtElement::Subtree(subtree, subtree_iter) => { - print_debug_subtree(f, &subtree, level, subtree_iter)?; } } - Ok(()) -} - -impl fmt::Debug for TopSubtree { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Debug::fmt(&self.view(), f) - } -} - -impl fmt::Display for TopSubtree { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(&self.view(), f) - } -} - -impl fmt::Display for Leaf { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Leaf::Ident(it) => fmt::Display::fmt(it, f), - Leaf::Literal(it) => fmt::Display::fmt(it, f), - Leaf::Punct(it) => fmt::Display::fmt(it, f), + impl fmt::Debug for TopSubtree { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&self.view(), f) } } -} - -impl fmt::Display for Ident { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(&self.is_raw.as_str(), f)?; - fmt::Display::fmt(&self.sym, f) - } -} -impl fmt::Display for Literal { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let (text, suffix) = self.text_and_suffix(); - match self.kind { - LitKind::Byte => write!(f, "b'{}'", text), - LitKind::Char => write!(f, "'{}'", text), - LitKind::Integer | LitKind::Float | LitKind::Err(_) => write!(f, "{}", text), - LitKind::Str => write!(f, "\"{}\"", text), - LitKind::ByteStr => write!(f, "b\"{}\"", text), - LitKind::CStr => write!(f, "c\"{}\"", text), - LitKind::StrRaw(num_of_hashes) => { - let num_of_hashes = num_of_hashes as usize; - write!(f, r#"r{0:# { - let num_of_hashes = num_of_hashes as usize; - write!(f, r#"br{0:# { - let num_of_hashes = num_of_hashes as usize; - write!(f, r#"cr{0:#) -> fmt::Result { - fmt::Display::fmt(&self.char, f) + impl fmt::Display for TopSubtree { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.view(), f) + } } -} -impl Subtree { - /// Count the number of tokens recursively - pub fn count(&self) -> usize { - self.usize_len() + impl Subtree { + /// Count the number of tokens recursively + pub fn count(&self) -> usize { + self.usize_len() + } } -} -pub fn pretty(tkns: TokenTreesView<'_>) -> String { - return pretty_impl(tkns.iter()); + pub fn pretty(tkns: TokenTreesView<'_>) -> String { + return pretty_impl(tkns.iter()); - fn tokentree_to_text(tkn: TtElement<'_>) -> String { - match tkn { - TtElement::Leaf(leaf) => { - format!("{}", leaf) - } - TtElement::Subtree(Subtree { delimiter, .. }, subtree_content) => { - let content = pretty_impl(subtree_content); - let (open, close) = match delimiter.kind { - DelimiterKind::Brace => ("{", "}"), - DelimiterKind::Bracket => ("[", "]"), - DelimiterKind::Parenthesis => ("(", ")"), - DelimiterKind::Invisible => ("", ""), - }; - format!("{open}{content}{close}") + fn tokentree_to_text(tkn: TtElement<'_>) -> String { + match tkn { + TtElement::Leaf(leaf) => { + format!("{}", leaf) + } + TtElement::Subtree(Subtree { delimiter, .. }, subtree_content) => { + let content = pretty_impl(subtree_content); + let (open, close) = delimiter.kind.display_open_close(); + format!("{open}{content}{close}") + } } } - } - fn pretty_impl(tkns: TtIter<'_>) -> String { - let mut last = String::new(); - let mut last_to_joint = true; - - for tkn in tkns { - last = - [last, tokentree_to_text(tkn.clone())].join(if last_to_joint { "" } else { " " }); - last_to_joint = false; - if let TtElement::Leaf(Leaf::Punct(Punct { spacing, .. })) = tkn - && spacing == Spacing::Joint - { - last_to_joint = true; + fn pretty_impl(tkns: TtIter<'_>) -> String { + let mut last = String::new(); + let mut last_to_joint = true; + + for tkn in tkns { + last = [last, tokentree_to_text(tkn.clone())].join(if last_to_joint { + "" + } else { + " " + }); + last_to_joint = false; + if let TtElement::Leaf(Leaf::Punct(Punct { spacing, .. })) = tkn + && spacing == Spacing::Joint + { + last_to_joint = true; + } } + last } - last } -} -#[derive(Debug)] -pub enum TransformTtAction<'a> { - Keep, - ReplaceWith(TokenTreesView<'a>), -} - -impl TransformTtAction<'_> { - #[inline] - pub fn remove() -> Self { - Self::ReplaceWith(TokenTreesView::empty()) + #[derive(Debug)] + pub enum TransformTtAction<'a> { + Keep, + ReplaceWith(TokenTreesView<'a>), } -} -/// This function takes a token tree, and calls `callback` with each token tree in it. -/// Then it does what the callback says: keeps the tt or replaces it with a (possibly empty) -/// tts view. -pub fn transform_tt<'b>( - tt: &mut TopSubtree, - mut callback: impl FnMut(&TokenTree) -> TransformTtAction<'b>, -) { - let mut tt_vec = tt.as_token_trees().iter_flat_tokens().collect::>(); - - // We need to keep a stack of the currently open subtrees, because we need to update - // them if we change the number of items in them. - let mut subtrees_stack = Vec::new(); - let mut i = 0; - while i < tt_vec.len() { - 'pop_finished_subtrees: while let Some(&subtree_idx) = subtrees_stack.last() { - let TokenTree::Subtree(subtree) = &tt_vec[subtree_idx] else { - unreachable!("non-subtree on subtrees stack"); - }; - if i >= subtree_idx + 1 + subtree.usize_len() { - subtrees_stack.pop(); - } else { - break 'pop_finished_subtrees; - } + impl TransformTtAction<'_> { + #[inline] + pub fn remove() -> Self { + Self::ReplaceWith(TokenTreesView::empty()) } + } - let current = &tt_vec[i]; - let action = callback(current); - match action { - TransformTtAction::Keep => { - // This cannot be shared with the replaced case, because then we may push the same subtree - // twice, and will update it twice which will lead to errors. - if let TokenTree::Subtree(_) = current { - subtrees_stack.push(i); + /// This function takes a token tree, and calls `callback` with each token tree in it. + /// Then it does what the callback says: keeps the tt or replaces it with a (possibly empty) + /// tts view. + pub fn transform_tt<'b>( + tt: &mut TopSubtree, + mut callback: impl FnMut(&TokenTree) -> TransformTtAction<'b>, + ) { + let mut tt_vec = tt.as_token_trees().iter_flat_tokens().collect::>(); + + // We need to keep a stack of the currently open subtrees, because we need to update + // them if we change the number of items in them. + let mut subtrees_stack = Vec::new(); + let mut i = 0; + while i < tt_vec.len() { + 'pop_finished_subtrees: while let Some(&subtree_idx) = subtrees_stack.last() { + let TokenTree::Subtree(subtree) = &tt_vec[subtree_idx] else { + unreachable!("non-subtree on subtrees stack"); + }; + if i >= subtree_idx + 1 + subtree.usize_len() { + subtrees_stack.pop(); + } else { + break 'pop_finished_subtrees; } - - i += 1; } - TransformTtAction::ReplaceWith(replacement) => { - let old_len = 1 + match current { - TokenTree::Leaf(_) => 0, - TokenTree::Subtree(subtree) => subtree.usize_len(), - }; - let len_diff = replacement.len() as i64 - old_len as i64; - tt_vec.splice(i..i + old_len, replacement.iter_flat_tokens()); - // Skip the newly inserted replacement, we don't want to visit it. - i += replacement.len(); - - for &subtree_idx in &subtrees_stack { - let TokenTree::Subtree(subtree) = &mut tt_vec[subtree_idx] else { - unreachable!("non-subtree on subtrees stack"); + + let current = &tt_vec[i]; + let action = callback(current); + match action { + TransformTtAction::Keep => { + // This cannot be shared with the replaced case, because then we may push the same subtree + // twice, and will update it twice which will lead to errors. + if let TokenTree::Subtree(_) = current { + subtrees_stack.push(i); + } + + i += 1; + } + TransformTtAction::ReplaceWith(replacement) => { + let old_len = 1 + match current { + TokenTree::Leaf(_) => 0, + TokenTree::Subtree(subtree) => subtree.usize_len(), }; - subtree.len = (i64::from(subtree.len) + len_diff).try_into().unwrap(); + let len_diff = replacement.len() as i64 - old_len as i64; + tt_vec.splice(i..i + old_len, replacement.iter_flat_tokens()); + // Skip the newly inserted replacement, we don't want to visit it. + i += replacement.len(); + + for &subtree_idx in &subtrees_stack { + let TokenTree::Subtree(subtree) = &mut tt_vec[subtree_idx] else { + unreachable!("non-subtree on subtrees stack"); + }; + subtree.len = (i64::from(subtree.len) + len_diff).try_into().unwrap(); + } } } } - } - *tt = TopSubtree::from_serialized(tt_vec); + *tt = TopSubtree::from_serialized(tt_vec); + } } diff --git a/crates/tt/src/storage.rs b/crates/tt/src/storage.rs index ba7a661b5e8a..c11a2e95bf1a 100644 --- a/crates/tt/src/storage.rs +++ b/crates/tt/src/storage.rs @@ -17,8 +17,9 @@ use rustc_hash::FxHashMap; use span::{Span, SpanAnchor, SyntaxContext, TextRange, TextSize}; use crate::{ - DelimSpan, Delimiter, DelimiterKind, Ident, IdentIsRaw, Leaf, LitKind, Literal, Punct, Spacing, - Subtree, SubtreeView, TokenTree, TokenTreesView, TtIter, + CommentStyle, DelimSpan, Delimiter, DelimiterKind, DocComment, DocCommentStyle, Ident, + IdentIsRaw, Leaf, LitKind, Literal, Punct, Spacing, Subtree, SubtreeView, TokenTree, + TokenTreesView, TtIter, }; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -340,19 +341,12 @@ unsafe fn encode<'a>( unsafe { match tt { TokenTree::Leaf(Leaf::Punct(Punct { char, spacing, span })) => { - if char.is_ascii() { - let spacing = spacing as u8; - let span_extra = 0b1 | (u32::from(spacing) & 0b10); - let char = ((char as u8) << 1) | (spacing & 0b1); - ptr.write::(char); - ptr = encode_span(ptr, &span, span_parts_map, span_extra, false); - } else { - let mut control_byte = 0b110; - control_byte |= (spacing as u8) << 3; - ptr.write::(char); - ptr.write::(control_byte); - ptr = encode_span(ptr, &span, span_parts_map, 0b00, false); - } + debug_assert!(char.is_ascii(), "non-ascii puncts should be impossible"); + let spacing = spacing as u8; + let span_extra = 0b1 | (u32::from(spacing) & 0b10); + let char = ((char as u8) << 1) | (spacing & 0b1); + ptr.write::(char); + ptr = encode_span(ptr, &span, span_parts_map, span_extra, false); } TokenTree::Leaf(Leaf::Ident(Ident { sym, span, is_raw })) => { ptr = encode_symbol(ptr, &sym, is_raw as u32, symbols_map); @@ -399,6 +393,19 @@ unsafe fn encode<'a>( } ptr = encode_span(ptr, &span, span_parts_map, 0b00, false); } + TokenTree::Leaf(Leaf::DocComment(DocComment { + text_with_comment_signs, + span, + doc_style, + comment_style, + })) => { + let mut control_byte = 0b110; + control_byte |= (doc_style as u8) << 3; + control_byte |= (comment_style as u8) << 4; + ptr = encode_symbol(ptr, &text_with_comment_signs, 0, symbols_map); + ptr.write::(control_byte); + ptr = encode_span(ptr, &span, span_parts_map, 0b00, false); + } TokenTree::Subtree(Subtree { delimiter, len }) => { let open_span_parts_index = span_parts_map[&CompressedSpanPart::from_span(&delimiter.open)] as u32; @@ -957,13 +964,22 @@ unsafe fn decode<'a>( }) } 0b110 => { - cold_path(); - - // Non-ASCII punct. Extremely rare but technically possible. - let spacing = transmute::(control_byte_extra_data as u8); - let char = ptr.read::(); - - TokenTree::Leaf(Leaf::Punct(Punct { char, spacing, span })) + // A doc comment. + let doc_style = + transmute::((control_byte_extra_data & 0b1) as u8); + let comment_style = + transmute::((control_byte_extra_data >> 1) as u8); + let text_with_comment_signs_first_byte = ptr.read::(); + let text_with_comment_signs; + (ptr, text_with_comment_signs) = + decode_symbol(ptr, text_with_comment_signs_first_byte, symbols); + + TokenTree::Leaf(Leaf::DocComment(DocComment { + text_with_comment_signs, + span, + doc_style, + comment_style, + })) } 0b011 | 0b100 => { // Literal, format 1: the 6 bits remaining from `control_byte` decide the kind and the suffix len from a constant set