Rebase _Pyink_ to _Black_ v23.10.0.

PiperOrigin-RevId: 575878450

pyproject.toml

@@ -42,7 +42,7 @@ dependencies = [
   "platformdirs>=2",
   "tomli>=1.1.0; python_version < '3.11'",
   "typing_extensions>=4.0.1; python_version < '3.11'",
-  "black==23.9.1",
+  "black==23.10.0",
 ]
 dynamic = ["version"]
 

src/pyink/cache.py

@@ -1,4 +1,5 @@
 """Caching of formatted files with feature-based invalidation."""
+
 import hashlib
 import os
 import pickle
@@ -36,8 +37,9 @@ def get_cache_dir() -> Path:
     repeated calls.
     """
     # NOTE: Function mostly exists as a clean way to test getting the cache directory.
-    default_cache_dir = user_cache_dir("pyink", version=__version__)
+    default_cache_dir = user_cache_dir("pyink")
     cache_dir = Path(os.environ.get("PYINK_CACHE_DIR", default_cache_dir))
+    cache_dir = cache_dir / __version__
     return cache_dir
 
 

src/pyink/concurrency.py

@@ -9,6 +9,7 @@
 import os
 import signal
 import sys
+import traceback
 from concurrent.futures import Executor, ProcessPoolExecutor, ThreadPoolExecutor
 from multiprocessing import Manager
 from pathlib import Path
@@ -170,8 +171,10 @@ async def schedule_formatting(
             src = tasks.pop(task)
             if task.cancelled():
                 cancelled.append(task)
-            elif task.exception():
-                report.failed(src, str(task.exception()))
+            elif exc := task.exception():
+                if report.verbose:
+                    traceback.print_exception(type(exc), exc, exc.__traceback__)
+                report.failed(src, str(exc))
             else:
                 changed = Changed.YES if task.result() else Changed.NO
                 # If the file was written back or was successfully checked as

src/pyink/debug.py

@@ -1,5 +1,5 @@
-from dataclasses import dataclass
-from typing import Iterator, TypeVar, Union
+from dataclasses import dataclass, field
+from typing import Any, Iterator, List, TypeVar, Union
 
 from pyink.nodes import Visitor
 from pyink.output import out
@@ -14,26 +14,33 @@
 @dataclass
 class DebugVisitor(Visitor[T]):
     tree_depth: int = 0
+    list_output: List[str] = field(default_factory=list)
+    print_output: bool = True
+
+    def out(self, message: str, *args: Any, **kwargs: Any) -> None:
+        self.list_output.append(message)
+        if self.print_output:
+            out(message, *args, **kwargs)
 
     def visit_default(self, node: LN) -> Iterator[T]:
         indent = " " * (2 * self.tree_depth)
         if isinstance(node, Node):
             _type = type_repr(node.type)
-            out(f"{indent}{_type}", fg="yellow")
+            self.out(f"{indent}{_type}", fg="yellow")
             self.tree_depth += 1
             for child in node.children:
                 yield from self.visit(child)
 
             self.tree_depth -= 1
-            out(f"{indent}/{_type}", fg="yellow", bold=False)
+            self.out(f"{indent}/{_type}", fg="yellow", bold=False)
         else:
             _type = token.tok_name.get(node.type, str(node.type))
-            out(f"{indent}{_type}", fg="blue", nl=False)
+            self.out(f"{indent}{_type}", fg="blue", nl=False)
             if node.prefix:
                 # We don't have to handle prefixes for `Node` objects since
                 # that delegates to the first child anyway.
-                out(f" {node.prefix!r}", fg="green", bold=False, nl=False)
-            out(f" {node.value!r}", fg="blue", bold=False)
+                self.out(f" {node.prefix!r}", fg="green", bold=False, nl=False)
+            self.out(f" {node.value!r}", fg="blue", bold=False)
 
     @classmethod
     def show(cls, code: Union[str, Leaf, Node]) -> None:

src/pyink/linegen.py

@@ -1,6 +1,7 @@
 """
 Generating lines of code.
 """
+
 import sys
 from dataclasses import replace
 from enum import Enum, auto
@@ -428,6 +429,24 @@ def visit_factor(self, node: Node) -> Iterator[Line]:
             node.insert_child(index, Node(syms.atom, [lpar, operand, rpar]))
         yield from self.visit_default(node)
 
+    def visit_tname(self, node: Node) -> Iterator[Line]:
+        """
+        Add potential parentheses around types in function parameter lists to be made
+        into real parentheses in case the type hint is too long to fit on a line
+        Examples:
+        def foo(a: int, b: float = 7): ...
+
+        ->
+
+        def foo(a: (int), b: (float) = 7): ...
+        """
+        if Preview.parenthesize_long_type_hints in self.mode:
+            assert len(node.children) == 3
+            if maybe_make_parens_invisible_in_atom(node.children[2], parent=node):
+                wrap_in_parentheses(node, node.children[2], visible=False)
+
+        yield from self.visit_default(node)
+
     def visit_STRING(self, leaf: Leaf) -> Iterator[Line]:
         if (
             Preview.hex_codes_in_unicode_sequences in self.mode
@@ -550,7 +569,14 @@ def __post_init__(self) -> None:
         self.visit_except_clause = partial(v, keywords={"except"}, parens={"except"})
         self.visit_with_stmt = partial(v, keywords={"with"}, parens={"with"})
         self.visit_classdef = partial(v, keywords={"class"}, parens=Ø)
-        self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS)
+
+        # When this is moved out of preview, add ":" directly to ASSIGNMENTS in nodes.py
+        if Preview.parenthesize_long_type_hints in self.mode:
+            assignments = ASSIGNMENTS | {":"}
+        else:
+            assignments = ASSIGNMENTS
+        self.visit_expr_stmt = partial(v, keywords=Ø, parens=assignments)
+
         self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"})
         if not self.mode.is_pyink:
             self.visit_import_from = partial(v, keywords=Ø, parens={"import"})
@@ -563,6 +589,17 @@ def __post_init__(self) -> None:
         self.visit_case_block = self.visit_match_case
 
 
+def _hugging_power_ops_line_to_string(
+    line: Line,
+    features: Collection[Feature],
+    mode: Mode,
+) -> Optional[str]:
+    try:
+        return line_to_string(next(hug_power_op(line, features, mode)))
+    except CannotTransform:
+        return None
+
+
 def transform_line(
     line: Line, mode: Mode, features: Collection[Feature] = ()
 ) -> Iterator[Line]:
@@ -578,6 +615,14 @@ def transform_line(
 
     line_str = line_to_string(line)
 
+    # We need the line string when power operators are hugging to determine if we should
+    # split the line. Default to line_str, if no power operator are present on the line.
+    line_str_hugging_power_ops = (
+        (_hugging_power_ops_line_to_string(line, features, mode) or line_str)
+        if Preview.fix_power_op_line_length in mode
+        else line_str
+    )
+
     ll = mode.line_length
     sn = mode.string_normalization
     preferred_quote = mode.preferred_quote
@@ -600,7 +645,7 @@ def transform_line(
         and not line.should_split_rhs
         and not line.magic_trailing_comma
         and (
-            is_line_short_enough(line, mode=mode, line_str=line_str)
+            is_line_short_enough(line, mode=mode, line_str=line_str_hugging_power_ops)
             or line.contains_unsplittable_type_ignore()
         )
         and not (line.inside_brackets and line.contains_standalone_comments())
@@ -610,7 +655,7 @@ def transform_line(
             transformers = [string_merge, string_paren_strip]
         else:
             transformers = []
-    elif line.is_def:
+    elif line.is_def and not should_split_funcdef_with_rhs(line, mode):
         transformers = [left_hand_split]
     else:
 
@@ -689,6 +734,40 @@ def _rhs(
         yield line
 
 
+def should_split_funcdef_with_rhs(line: Line, mode: Mode) -> bool:
+    """If a funcdef has a magic trailing comma in the return type, then we should first
+    split the line with rhs to respect the comma.
+    """
+    if Preview.respect_magic_trailing_comma_in_return_type not in mode:
+        return False
+
+    return_type_leaves: List[Leaf] = []
+    in_return_type = False
+
+    for leaf in line.leaves:
+        if leaf.type == token.COLON:
+            in_return_type = False
+        if in_return_type:
+            return_type_leaves.append(leaf)
+        if leaf.type == token.RARROW:
+            in_return_type = True
+
+    # using `bracket_split_build_line` will mess with whitespace, so we duplicate a
+    # couple lines from it.
+    result = Line(mode=line.mode, depth=line.depth)
+    leaves_to_track = get_leaves_inside_matching_brackets(return_type_leaves)
+    for leaf in return_type_leaves:
+        result.append(
+            leaf,
+            preformatted=True,
+            track_bracket=id(leaf) in leaves_to_track,
+        )
+
+    # we could also return true if the line is too long, and the return type is longer
+    # than the param list. Or if `should_split_rhs` returns True.
+    return result.magic_trailing_comma is not None
+
+
 class _BracketSplitComponent(Enum):
     head = auto()
     body = auto()
@@ -1477,7 +1556,7 @@ def maybe_make_parens_invisible_in_atom(
     Returns whether the node should itself be wrapped in invisible parentheses.
     """
     if (
-        node.type != syms.atom
+        node.type not in (syms.atom, syms.expr)
         or is_empty_tuple(node)
         or is_one_tuple(node)
         or (is_yield(node) and parent.type != syms.expr_stmt)
@@ -1501,6 +1580,7 @@ def maybe_make_parens_invisible_in_atom(
             syms.except_clause,
             syms.funcdef,
             syms.with_stmt,
+            syms.tname,
             # these ones aren't useful to end users, but they do please fuzzers
             syms.for_stmt,
             syms.del_stmt,

src/pyink/lines.py

@@ -213,11 +213,16 @@ def is_class_paren_empty(self) -> bool:
     @property
     def is_triple_quoted_string(self) -> bool:
         """Is the line a triple quoted string?"""
-        return (
-            bool(self)
-            and self.leaves[0].type == token.STRING
-            and self.leaves[0].value.startswith(('"""', "'''"))
-        )
+        if not self or self.leaves[0].type != token.STRING:
+            return False
+        value = self.leaves[0].value
+        if value.startswith(('"""', "'''")):
+            return True
+        if Preview.accept_raw_docstrings in self.mode and value.startswith(
+            ("r'''", 'r"""', "R'''", 'R"""')
+        ):
+            return True
+        return False
 
     @property
     def opens_block(self) -> bool:
@@ -584,6 +589,15 @@ def maybe_empty_lines(self, current_line: Line) -> LinesBlock:
             if self.previous_line is None
             else before - previous_after
         )
+        if (
+            Preview.module_docstring_newlines in current_line.mode
+            and self.previous_block
+            and self.previous_block.previous_block is None
+            and len(self.previous_block.original_line.leaves) == 1
+            and self.previous_block.original_line.is_triple_quoted_string
+        ):
+            before = 1
+
         block = LinesBlock(
             mode=self.mode,
             previous_block=self.previous_block,

src/pyink/mode.py

@@ -180,11 +180,16 @@ class Preview(Enum):
     # for https://github.com/psf/black/issues/3117 to be fixed.
     string_processing = auto()
     parenthesize_conditional_expressions = auto()
+    parenthesize_long_type_hints = auto()
+    respect_magic_trailing_comma_in_return_type = auto()
     skip_magic_trailing_comma_in_subscript = auto()
     wrap_long_dict_values_in_parens = auto()
     wrap_multiple_context_managers_in_parens = auto()
     dummy_implementations = auto()
     walrus_subscript = auto()
+    module_docstring_newlines = auto()
+    accept_raw_docstrings = auto()
+    fix_power_op_line_length = auto()
 
 
 class Deprecated(UserWarning):

src/pyink/numerics.py

@@ -1,6 +1,7 @@
 """
 Formatting numeric literals.
 """
+
 from blib2to3.pytree import Leaf
 
 

src/pyink/parsing.py

@@ -1,9 +1,10 @@
 """
 Parse Python code and perform AST validation.
 """
+
 import ast
 import sys
-from typing import Final, Iterable, Iterator, List, Set, Tuple
+from typing import Iterable, Iterator, List, Set, Tuple
 
 from pyink.mode import VERSION_TO_FEATURES, Feature, TargetVersion, supports_feature
 from pyink.nodes import syms
@@ -14,8 +15,6 @@
 from blib2to3.pgen2.tokenize import TokenError
 from blib2to3.pytree import Leaf, Node
 
-PY2_HINT: Final = "Python 2 support was removed in version 22.0."
-
 
 class InvalidInput(ValueError):
     """Raised when input source code fails all parse attempts."""
@@ -26,9 +25,9 @@ def get_grammars(target_versions: Set[TargetVersion]) -> List[Grammar]:
         # No target_version specified, so try all grammars.
         return [
             # Python 3.7-3.9
-            pygram.python_grammar_no_print_statement_no_exec_statement_async_keywords,
+            pygram.python_grammar_async_keywords,
             # Python 3.0-3.6
-            pygram.python_grammar_no_print_statement_no_exec_statement,
+            pygram.python_grammar,
             # Python 3.10+
             pygram.python_grammar_soft_keywords,
         ]
@@ -39,12 +38,10 @@ def get_grammars(target_versions: Set[TargetVersion]) -> List[Grammar]:
         target_versions, Feature.ASYNC_IDENTIFIERS
     ) and not supports_feature(target_versions, Feature.PATTERN_MATCHING):
         # Python 3.7-3.9
-        grammars.append(
-            pygram.python_grammar_no_print_statement_no_exec_statement_async_keywords
-        )
+        grammars.append(pygram.python_grammar_async_keywords)
     if not supports_feature(target_versions, Feature.ASYNC_KEYWORDS):
         # Python 3.0-3.6
-        grammars.append(pygram.python_grammar_no_print_statement_no_exec_statement)
+        grammars.append(pygram.python_grammar)
     if any(Feature.PATTERN_MATCHING in VERSION_TO_FEATURES[v] for v in target_versions):
         # Python 3.10+
         grammars.append(pygram.python_grammar_soft_keywords)
@@ -89,14 +86,6 @@ def lib2to3_parse(src_txt: str, target_versions: Iterable[TargetVersion] = ()) -
         # Choose the latest version when raising the actual parsing error.
         assert len(errors) >= 1
         exc = errors[max(errors)]
-
-        if matches_grammar(src_txt, pygram.python_grammar) or matches_grammar(
-            src_txt, pygram.python_grammar_no_print_statement
-        ):
-            original_msg = exc.args[0]
-            msg = f"{original_msg}\n{PY2_HINT}"
-            raise InvalidInput(msg) from None
-
         raise exc from None
 
     if isinstance(result, Leaf):

src/pyink/report.py

@@ -1,6 +1,7 @@
 """
 Summarize Black runs to users.
 """
+
 from dataclasses import dataclass
 from enum import Enum
 from pathlib import Path

src/pyink/rusty.py

@@ -2,6 +2,7 @@
 
 See https://doc.rust-lang.org/book/ch09-00-error-handling.html.
 """
+
 from typing import Generic, TypeVar, Union
 
 T = TypeVar("T")