Fix comments being removed before fmt:off/on blocks (#4845)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: cobalt <[email protected]>

Author: 3 people

CHANGES.md

@@ -15,6 +15,9 @@
 
 <!-- Changes that affect Black's stable style -->
 
+- Fix bug where comments preceding `# fmt: off`/`# fmt: on` blocks were incorrectly
+  removed, particularly affecting Jupytext's `# %% [markdown]` comments (#4845)
+
 ### Preview style
 
 <!-- Changes that affect Black's preview style -->

src/black/comments.py

@@ -281,9 +281,18 @@ def _handle_comment_only_fmt_block(
     if hidden_value.endswith("\n"):
         hidden_value = hidden_value[:-1]
 
-    # Build the standalone comment prefix
+    # Build the standalone comment prefix - preserve all content before fmt:off
+    # including any comments that precede it
+    if fmt_off_idx == 0:
+        # No comments before fmt:off, use previous_consumed
+        pre_fmt_off_consumed = previous_consumed
+    else:
+        # Use the consumed position of the last comment before fmt:off
+        # This preserves all comments and content before the fmt:off directive
+        pre_fmt_off_consumed = all_comments[fmt_off_idx - 1].consumed
+
     standalone_comment_prefix = (
-        original_prefix[:previous_consumed] + "\n" * comment.newlines
+        original_prefix[:pre_fmt_off_consumed] + "\n" * comment.newlines
     )
 
     fmt_off_prefix = original_prefix.split(comment.value)[0]
@@ -326,6 +335,15 @@ def convert_one_fmt_off_pair(
     Returns True if a pair was converted.
     """
     for leaf in node.leaves():
+        # Skip STANDALONE_COMMENT nodes that were created by fmt:off/on processing
+        # to avoid reprocessing them in subsequent iterations
+        if (
+            leaf.type == STANDALONE_COMMENT
+            and hasattr(leaf, "fmt_pass_converted_first_leaf")
+            and leaf.fmt_pass_converted_first_leaf is None
+        ):
+            continue
+
         previous_consumed = 0
         for comment in list_comments(leaf.prefix, is_endmarker=False, mode=mode):
             should_process, is_fmt_off, is_fmt_skip = _should_process_fmt_comment(

src/black/linegen.py

@@ -17,7 +17,13 @@
     get_leaves_inside_matching_brackets,
     max_delimiter_priority_in_atom,
 )
-from black.comments import FMT_OFF, FMT_ON, generate_comments, list_comments
+from black.comments import (
+    FMT_OFF,
+    FMT_ON,
+    _contains_fmt_directive,
+    generate_comments,
+    list_comments,
+)
 from black.lines import (
     Line,
     RHSResult,
@@ -405,7 +411,23 @@ def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]:
         else:
             is_fmt_off_block = False
         if is_fmt_off_block:
-            # This is a fmt:off/on block from normalize_fmt_off - append directly
+            # This is a fmt:off/on block from normalize_fmt_off - we still need
+            # to process any prefix comments (like markdown comments) but append
+            # the fmt block itself directly to preserve its formatting
+
+            # Only process prefix comments if there actually is a prefix with comments
+            if leaf.prefix and any(
+                line.strip().startswith("#")
+                and not _contains_fmt_directive(line.strip())
+                for line in leaf.prefix.split("\n")
+            ):
+                for comment in generate_comments(leaf, mode=self.mode):
+                    yield from self.line()
+                    self.current_line.append(comment)
+                    yield from self.line()
+                # Clear the prefix since we've processed it as comments above
+                leaf.prefix = ""
+
             self.current_line.append(leaf)
             yield from self.line()
         else:

tests/data/cases/jupytext_markdown_fmt.py

@@ -0,0 +1,24 @@
+# Test that Jupytext markdown comments are preserved before fmt:off/on blocks
+# %% [markdown]
+
+# fmt: off
+# fmt: on
+
+# Also test with other comments
+# Some comment
+# %% [markdown]
+# Another comment
+
+# fmt: off
+x = 1
+# fmt: on
+
+# Test multiple markdown comments
+# %% [markdown]
+# First markdown
+# %% [code]
+# Code cell
+
+# fmt: off
+y = 2
+# fmt: on