From d84c837bd6b02b1b89c788f4f7ff48fb60559f72 Mon Sep 17 00:00:00 2001 From: Tushar Sadhwani Date: Thu, 1 Aug 2024 00:35:22 +0530 Subject: [PATCH 1/6] fix: respect braces better in f-string parsing --- src/blib2to3/pgen2/tokenize.py | 4 ++-- tests/data/cases/pep_701.py | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/blib2to3/pgen2/tokenize.py b/src/blib2to3/pgen2/tokenize.py index 28972a9bd78..d4b6bc1ddfe 100644 --- a/src/blib2to3/pgen2/tokenize.py +++ b/src/blib2to3/pgen2/tokenize.py @@ -136,8 +136,8 @@ def _combinations(*l: str) -> Set[str]: ) # beginning of a single quoted f-string. must not end with `{{` or `\N{` -SingleLbrace = r"(?:\\N{|\\.|{{|[^'\\{])*(? Date: Thu, 1 Aug 2024 00:40:34 +0530 Subject: [PATCH 2/6] de-duplicate the two regexes --- src/blib2to3/pgen2/tokenize.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/blib2to3/pgen2/tokenize.py b/src/blib2to3/pgen2/tokenize.py index d4b6bc1ddfe..ae26617a4bd 100644 --- a/src/blib2to3/pgen2/tokenize.py +++ b/src/blib2to3/pgen2/tokenize.py @@ -136,8 +136,8 @@ def _combinations(*l: str) -> Set[str]: ) # beginning of a single quoted f-string. must not end with `{{` or `\N{` -SingleLbrace = r"(?:\\N{|\\[^{]|{{|[^'\\{])*(? Set[str]: _string_middle_double = r'(?:[^\n"\\]|\\.)*' # FSTRING_MIDDLE and LBRACE, must not end with a `{{` or `\N{` -_fstring_middle_single = r"(?:\\N{|\\[^{]|{{|[^\n'{\\])*(? Date: Fri, 2 Aug 2024 00:14:53 +0530 Subject: [PATCH 3/6] fix the regex some more --- src/blib2to3/pgen2/tokenize.py | 8 ++++---- tests/data/cases/pep_701.py | 4 ++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/blib2to3/pgen2/tokenize.py b/src/blib2to3/pgen2/tokenize.py index ae26617a4bd..5407bf69495 100644 --- a/src/blib2to3/pgen2/tokenize.py +++ b/src/blib2to3/pgen2/tokenize.py @@ -136,12 +136,12 @@ def _combinations(*l: str) -> Set[str]: ) # beginning of a single quoted f-string. must not end with `{{` or `\N{` -SingleLbrace = r"(?:\\N{|\\[^{]|{{|[^\n'\\{])*(? Date: Fri, 2 Aug 2024 00:15:28 +0530 Subject: [PATCH 4/6] newline at the end --- tests/data/cases/pep_701.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data/cases/pep_701.py b/tests/data/cases/pep_701.py index 871b0fd6b4f..9acee951e71 100644 --- a/tests/data/cases/pep_701.py +++ b/tests/data/cases/pep_701.py @@ -273,4 +273,4 @@ f"{1}\{{" f"{2} foo \{{[\}}" f"\{3}" -rf"\{"a"}" \ No newline at end of file +rf"\{"a"}" From a415fc14ddbd6a33dafdaf5addd8b77bd8d1c18d Mon Sep 17 00:00:00 2001 From: Tushar Sadhwani Date: Fri, 2 Aug 2024 00:17:50 +0530 Subject: [PATCH 5/6] Add changelog entry --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index e3e37484a59..16fb1c9d7d8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -39,6 +39,8 @@ - Fix bug with Black incorrectly parsing empty lines with a backslash (#4343) +- Fix bugs with black's tokenizer not handling `\{` inside f-strings very well (#4422) + ### Performance From c8ff6e8372cfd7a864604381e550208a8a511e5b Mon Sep 17 00:00:00 2001 From: Tushar Sadhwani Date: Fri, 2 Aug 2024 00:18:01 +0530 Subject: [PATCH 6/6] capitalization --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 16fb1c9d7d8..ebbe92ec63e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -39,7 +39,7 @@ - Fix bug with Black incorrectly parsing empty lines with a backslash (#4343) -- Fix bugs with black's tokenizer not handling `\{` inside f-strings very well (#4422) +- Fix bugs with Black's tokenizer not handling `\{` inside f-strings very well (#4422) ### Performance