From 909e1af8ee28629e82144b1220018cba13f05f43 Mon Sep 17 00:00:00 2001 From: konstin Date: Wed, 23 Aug 2023 13:53:02 +0200 Subject: [PATCH] Formatter: Show preceding, following and enclosing nodes of comments, Attempt 2 This is a new attempt after https://github.com/astral-sh/ruff/pull/6337 **Summary** I used to always add dbg! for the preceding, following and enclosing node. With this change --print-comments can do this instead. ```python match foo: # a # b case [1, 2, * # c rest]: pass # d case [1, 2, * #e _]: pass case [ 1, 2, *rest, ]: pass ``` Additional `--print-comments` Output: ``` 11..14 Some((ExprName, 6..9)) Some((MatchCase, 27..68)) (StmtMatch, 0..186) "# a" 19..22 Some((ExprName, 6..9)) Some((MatchCase, 27..68)) (StmtMatch, 0..186) "# b" 41..44 None None (PatternMatchStar, 39..53) "# c" 73..76 Some((MatchCase, 27..68)) Some((MatchCase, 81..118)) (StmtMatch, 0..186) "# d" 95..97 None None (PatternMatchStar, 93..103) "#e" ``` **Test Plan** n/a --- crates/ruff_python_formatter/src/cli.rs | 25 +++++ .../ruff_python_formatter/src/comments/mod.rs | 13 ++- .../src/comments/visitor.rs | 95 +++++++++++++------ 3 files changed, 101 insertions(+), 32 deletions(-) diff --git a/crates/ruff_python_formatter/src/cli.rs b/crates/ruff_python_formatter/src/cli.rs index fff146fdb91cd5..bad83e26124485 100644 --- a/crates/ruff_python_formatter/src/cli.rs +++ b/crates/ruff_python_formatter/src/cli.rs @@ -6,10 +6,12 @@ use anyhow::{bail, Context, Result}; use clap::{command, Parser, ValueEnum}; use ruff_formatter::SourceCode; +use ruff_python_ast::Ranged; use ruff_python_index::CommentRangesBuilder; use ruff_python_parser::lexer::lex; use ruff_python_parser::{parse_tokens, Mode}; +use crate::comments::Comments; use crate::{format_node, PyFormatOptions}; #[derive(ValueEnum, Clone, Debug)] @@ -64,6 +66,29 @@ pub fn format_and_debug_print(input: &str, cli: &Cli, source_type: &Path) -> Res println!("{}", formatted.document().display(SourceCode::new(input))); } if cli.print_comments { + // Print preceding, following and enclosing nodes + let decorated_comments = Comments::collect_decorated_comments( + &python_ast, + SourceCode::new(input), + &comment_ranges, + ); + for comment in decorated_comments { + println!( + "{:?} {:?} {:?} {:?} {:?}", + comment.slice().range(), + comment + .preceding_node() + .map(|node| (node.kind(), node.range())), + comment + .following_node() + .map(|node| (node.kind(), node.range())), + ( + comment.enclosing_node().kind(), + comment.enclosing_node().range() + ), + comment.slice().text(SourceCode::new(input)), + ); + } println!( "{:#?}", formatted.context().comments().debug(SourceCode::new(input)) diff --git a/crates/ruff_python_formatter/src/comments/mod.rs b/crates/ruff_python_formatter/src/comments/mod.rs index d46014ae29711f..508285261cf567 100644 --- a/crates/ruff_python_formatter/src/comments/mod.rs +++ b/crates/ruff_python_formatter/src/comments/mod.rs @@ -108,7 +108,7 @@ use ruff_python_trivia::PythonWhitespace; use crate::comments::debug::{DebugComment, DebugComments}; use crate::comments::map::{LeadingDanglingTrailing, MultiMap}; use crate::comments::node_key::NodeRefEqualityKey; -use crate::comments::visitor::CommentsVisitor; +use crate::comments::visitor::{collect_comments, get_comments_map, DecoratedComment}; mod debug; pub(crate) mod format; @@ -324,12 +324,21 @@ impl<'a> Comments<'a> { let map = if comment_ranges.is_empty() { CommentsMap::new() } else { - CommentsVisitor::new(source_code, comment_ranges).visit(root) + get_comments_map(root, source_code, comment_ranges) }; Self::new(map) } + /// Extracts the comments from the AST. + pub(crate) fn collect_decorated_comments( + root: &'a Mod, + source_code: SourceCode<'a>, + comment_ranges: &'a CommentRanges, + ) -> Vec> { + collect_comments(root, source_code, comment_ranges) + } + /// Returns `true` if the given `node` has any [leading comments](self#leading-comments). #[inline] pub(crate) fn has_leading(&self, node: T) -> bool diff --git a/crates/ruff_python_formatter/src/comments/visitor.rs b/crates/ruff_python_formatter/src/comments/visitor.rs index f779820a35a407..203f6576c50d72 100644 --- a/crates/ruff_python_formatter/src/comments/visitor.rs +++ b/crates/ruff_python_formatter/src/comments/visitor.rs @@ -1,3 +1,4 @@ +use std::fmt::Debug; use std::iter::Peekable; use ruff_python_ast::{Mod, Ranged, Stmt}; @@ -17,20 +18,46 @@ use crate::comments::node_key::NodeRefEqualityKey; use crate::comments::placement::place_comment; use crate::comments::{CommentLinePosition, CommentsMap, SourceComment}; +/// Assign comments to nodes through an AST visitor and [`place_comment`]. +pub(super) fn get_comments_map<'a>( + root: &'a Mod, + source_code: SourceCode<'a>, + comment_ranges: &'a CommentRanges, +) -> CommentsMap<'a> { + let mut builder = CommentsBuilder::default(); + CommentsVisitor::new(source_code, comment_ranges, &mut builder).visit(root); + builder.finish() +} + +/// Collect the preceding, following and enclosing node for each comment before applying +/// [`place_comment`] for debugging. +pub(super) fn collect_comments<'a>( + root: &'a Mod, + source_code: SourceCode<'a>, + comment_ranges: &'a CommentRanges, +) -> Vec> { + let mut collector = CommentsCollector::default(); + CommentsVisitor::new(source_code, comment_ranges, &mut collector).visit(root); + collector.comments +} + /// Visitor extracting the comments from an AST. -#[derive(Debug, Clone)] -pub(crate) struct CommentsVisitor<'a> { - builder: CommentsBuilder<'a>, +struct CommentsVisitor<'a, 'builder> { + builder: &'builder mut (dyn CommentsBuilderTrait<'a> + 'a), source_code: SourceCode<'a>, parents: Vec>, preceding_node: Option>, comment_ranges: Peekable>, } -impl<'a> CommentsVisitor<'a> { - pub(crate) fn new(source_code: SourceCode<'a>, comment_ranges: &'a CommentRanges) -> Self { +impl<'a, 'builder> CommentsVisitor<'a, 'builder> { + fn new( + source_code: SourceCode<'a>, + comment_ranges: &'a CommentRanges, + builder: &'builder mut (dyn CommentsBuilderTrait<'a> + 'a), + ) -> Self { Self { - builder: CommentsBuilder::default(), + builder, source_code, parents: Vec::new(), preceding_node: None, @@ -38,10 +65,8 @@ impl<'a> CommentsVisitor<'a> { } } - pub(super) fn visit(mut self, root: &'a Mod) -> CommentsMap<'a> { + fn visit(mut self, root: &'a Mod) { self.visit_mod(root); - - self.finish() } // Try to skip the subtree if @@ -52,13 +77,9 @@ impl<'a> CommentsVisitor<'a> { .peek() .map_or(true, |next_comment| next_comment.start() >= node_end) } - - fn finish(self) -> CommentsMap<'a> { - self.builder.finish() - } } -impl<'ast> PreorderVisitor<'ast> for CommentsVisitor<'ast> { +impl<'ast> PreorderVisitor<'ast> for CommentsVisitor<'ast, '_> { fn enter_node(&mut self, node: AnyNodeRef<'ast>) -> TraversalSignal { let node_range = node.range(); @@ -82,10 +103,8 @@ impl<'ast> PreorderVisitor<'ast> for CommentsVisitor<'ast> { slice: self.source_code.slice(*comment_range), }; - self.builder.add_comment(place_comment( - comment, - &Locator::new(self.source_code.as_str()), - )); + self.builder + .add_comment(comment, &Locator::new(self.source_code.as_str())); self.comment_ranges.next(); } @@ -125,10 +144,8 @@ impl<'ast> PreorderVisitor<'ast> for CommentsVisitor<'ast> { slice: self.source_code.slice(*comment_range), }; - self.builder.add_comment(place_comment( - comment, - &Locator::new(self.source_code.as_str()), - )); + self.builder + .add_comment(comment, &Locator::new(self.source_code.as_str())); self.comment_ranges.next(); } @@ -177,7 +194,7 @@ fn text_position(comment_range: TextRange, source_code: SourceCode) -> CommentLi /// /// Used by [`CommentStyle::place_comment`] to determine if this should become a [leading](self#leading-comments), [dangling](self#dangling-comments), or [trailing](self#trailing-comments) comment. #[derive(Debug, Clone)] -pub(super) struct DecoratedComment<'a> { +pub(crate) struct DecoratedComment<'a> { enclosing: AnyNodeRef<'a>, preceding: Option>, following: Option>, @@ -203,7 +220,7 @@ impl<'a> DecoratedComment<'a> { /// /// The enclosing node is the list expression and not the name `b` because /// `a` and `b` are children of the list expression and `comment` is between the two nodes. - pub(super) fn enclosing_node(&self) -> AnyNodeRef<'a> { + pub(crate) fn enclosing_node(&self) -> AnyNodeRef<'a> { self.enclosing } @@ -213,7 +230,7 @@ impl<'a> DecoratedComment<'a> { } /// Returns the slice into the source code. - pub(super) fn slice(&self) -> &SourceCodeSlice { + pub(crate) fn slice(&self) -> &SourceCodeSlice { &self.slice } @@ -257,7 +274,7 @@ impl<'a> DecoratedComment<'a> { /// /// Returns `Some(a)` because `a` is the preceding node of `comment`. The presence of the `,` token /// doesn't change that. - pub(super) fn preceding_node(&self) -> Option> { + pub(crate) fn preceding_node(&self) -> Option> { self.preceding } @@ -315,7 +332,7 @@ impl<'a> DecoratedComment<'a> { /// /// Returns `None` because `comment` is enclosed inside the parenthesized expression and it has no children /// following `# comment`. - pub(super) fn following_node(&self) -> Option> { + pub(crate) fn following_node(&self) -> Option> { self.following } @@ -512,13 +529,29 @@ impl<'a> CommentPlacement<'a> { } } +pub(super) trait CommentsBuilderTrait<'a> { + fn add_comment(&mut self, placement: DecoratedComment<'a>, locator: &Locator); +} + +#[derive(Debug, Default)] +struct CommentsCollector<'a> { + comments: Vec>, +} + +impl<'a> CommentsBuilderTrait<'a> for CommentsCollector<'a> { + fn add_comment(&mut self, placement: DecoratedComment<'a>, _: &Locator) { + self.comments.push(placement); + } +} + #[derive(Clone, Debug, Default)] struct CommentsBuilder<'a> { comments: CommentsMap<'a>, } -impl<'a> CommentsBuilder<'a> { - fn add_comment(&mut self, placement: CommentPlacement<'a>) { +impl<'a> CommentsBuilderTrait<'a> for CommentsBuilder<'a> { + fn add_comment(&mut self, comment: DecoratedComment<'a>, locator: &Locator) { + let placement = place_comment(comment, locator); match placement { CommentPlacement::Leading { node, comment } => { self.push_leading_comment(node, comment); @@ -577,8 +610,10 @@ impl<'a> CommentsBuilder<'a> { } } } +} - fn finish(self) -> CommentsMap<'a> { +impl<'a> CommentsBuilder<'a> { + pub(crate) fn finish(self) -> CommentsMap<'a> { self.comments }