Fix and add additional tests

.gitignore

@@ -4,3 +4,4 @@
 .*.swo
 .DS_Store
 .tox
+.coverage

.travis.yml

@@ -8,12 +8,10 @@ matrix:
   - python: 3.6
   - python: 3.7
   - python: 3.8
+  - python: 3.9
 install:
-# python 2.6 and 3.3 compatibility
-- if [[ $TRAVIS_PYTHON_VERSION == 2.6 ]]; then pip install pycparser==2.14 cryptography==2.1.4; fi
-- if [[ $TRAVIS_PYTHON_VERSION == 3.3 ]]; then pip install virtualenv==15.1.0 setuptools==39.2.0; fi
-- pip install tox-travis
-- pip install coveralls
+- pip install -U importlib-metadata setuptools
+- pip install tox-travis coveralls
 script:
 - tox
 after_success:
@@ -26,4 +24,4 @@ deploy:
   on:
     tags: true
     branch: master
-    python: '3.6'
+    python: '3.9'

astmonkey/tests/test_visitors.py

@@ -69,6 +69,8 @@ class TestSourceGeneratorNodeVisitor(object):
     EMPTY_FUNC = FUNC_DEF + EOL + INDENT + PASS
     SINGLE_LINE_DOCSTRING = "''' This is a single line docstring.'''"
     MULTI_LINE_DOCSTRING = "''' This is a multi line docstring." + EOL + EOL + 'Further description...' + EOL + "'''"
+    EMBEDDED_QUOTE_DOCSTRING = "'''this is \"double quotes\" inside single quotes'''"
+    DOC_FUNC = FUNC_DEF + EOL + INDENT + MULTI_LINE_DOCSTRING
     LINE_CONT = '\\'
 
     roundtrip_testdata = [
@@ -80,6 +82,7 @@ class TestSourceGeneratorNodeVisitor(object):
         '(a, b) = enumerate(c)',
         SIMPLE_ASSIGN + EOL + SIMPLE_ASSIGN,
         SIMPLE_ASSIGN + EOL + EOL + SIMPLE_ASSIGN,
+        EMBEDDED_QUOTE_DOCSTRING,
         EOL + SIMPLE_ASSIGN,
         EOL + EOL + SIMPLE_ASSIGN,
         'x = \'string assign\'',
@@ -162,6 +165,7 @@ class TestSourceGeneratorNodeVisitor(object):
         'try:' + EOL + INDENT + PASS + EOL + 'finally:' + EOL + INDENT + PASS,
         'try:' + EOL + INDENT + PASS + EOL + 'except Y:' + EOL + INDENT + PASS + EOL + 'except Z:' + EOL + INDENT + PASS,
         'try:' + EOL + INDENT + PASS + EOL + 'except Y:' + EOL + INDENT + PASS + EOL + 'else:' + EOL + INDENT + PASS,
+        'try:' + EOL + INDENT + PASS + EOL + 'except Y:' + EOL + INDENT + PASS + EOL + 'else:' + EOL + INDENT + PASS + EOL + 'finally:' + EOL + INDENT + PASS,
 
         # del
         'del x',
@@ -186,8 +190,8 @@ class TestSourceGeneratorNodeVisitor(object):
 
         # slice
         'x[y:z:q]',
-        'x[1:2,3:4]',
-        'x[:2,:2]',
+        'x[1:2, 3:4]',
+        'x[:2, :2]',
         'x[1:2]',
         'x[::2]',
 
@@ -204,6 +208,9 @@ class TestSourceGeneratorNodeVisitor(object):
 
         # decorator
         '@x(y)' + EOL + EMPTY_FUNC,
+        '@x(y)' + EOL + DOC_FUNC,
+        '@x(y)' + EOL + '@x(y)' + EOL + EMPTY_FUNC,
+        '@x(y)' + EOL + '@x(y)' + EOL + DOC_FUNC,
 
         # call
         'f(a)',
@@ -353,6 +360,13 @@ class TestSourceGeneratorNodeVisitor(object):
             "f'{x!r}'",
             "f'{x!s}'",
             "f'{x!a}'",
+            # annotated assignment
+            "a: int = 1",
+            # dubious annotated assignment of slice
+            "x[:]: None = ()",
+            # Async list literals & generators
+            "[i async for i in some_iterable]",
+            "(i async for i in some_iterable)",
         ]
 
     if utils.check_version(from_inclusive=(3, 8)):
@@ -373,6 +387,12 @@ class TestSourceGeneratorNodeVisitor(object):
         'b\'\'\'byte string' + EOL + 'next line' + EOL + '\'\'\'',
         r'r"""\a\b\f\n\r\t\v"""',
         'if x:' + EOL + INDENT + PASS + EOL + 'else:' + EOL + INDENT + 'if x:' + EOL + INDENT + INDENT + PASS,
+
+        # strings
+        '''s = 'this is \\'single quotes\\' inside single quotes' ''',
+        '''s = 'this is "double quotes" inside single quotes' ''',
+        '''s = "this is 'single quotes' inside double quotes" ''',
+        '''s = "this is \\"double quotes\\" inside double quotes" ''',
     ]
 
     if utils.check_version(from_inclusive=(3, 6)):
@@ -387,6 +407,7 @@ def test_codegen_roundtrip(self, source):
         """Check if converting code into AST and converting it back to code yields the same code."""
         node = ast.parse(source)
         generated = visitors.to_source(node)
+        print((source, generated))
         assert source == generated
 
     @pytest.mark.parametrize("source", semantic_testdata)

astmonkey/visitors.py

@@ -340,7 +340,7 @@ def decorators(self, node):
             for decorator in node.decorator_list:
                 self.write('@')
                 self.visit(decorator)
-            self.write_newline()
+                self.write_newline()
 
     def visit(self, node):
         self.correct_line_number(node)
@@ -678,7 +678,16 @@ def visit_UnaryOp(self, node):
     def visit_Subscript(self, node):
         self.visit(node.value)
         with self.inside('[', ']'):
-            self.visit(node.slice)
+            if isinstance(node.slice, ast.Tuple):
+                idx = -1
+                for idx, item in enumerate(node.slice.elts):
+                    if idx:
+                        self.write(', ')
+                    self.visit(item)
+                if not idx:
+                    self.write(',')
+            else:
+                self.visit(node.slice)
 
     def visit_Slice(self, node):
         self.slice_lower(node)
@@ -703,7 +712,7 @@ def slice_lower(self, node):
     def visit_ExtSlice(self, node):
         for idx, item in enumerate(node.dims):
             if idx:
-                self.write(',')
+                self.write(', ')
             self.visit(item)
 
     def visit_Yield(self, node):
@@ -768,6 +777,8 @@ def visit_alias(self, node):
             self.write(' as ' + node.asname)
 
     def visit_comprehension(self, node):
+        if getattr(node, 'is_async', 0):
+            self.write(' async')
         self.write(' for ')
         self.visit(node.target)
         self.write(' in ')
@@ -900,10 +911,10 @@ def visit_Try(self, node):
         self.body(node.body)
         if node.handlers:
             self.try_handlers(node)
-        if node.finalbody:
-            self.final_body(node)
         if node.orelse:
             self.or_else(node)
+        if node.finalbody:
+            self.final_body(node)
 
     def with_body(self, node, prefixes=[]):
         self._prefixes(prefixes)
@@ -1007,11 +1018,18 @@ def visit_JoinedStr(self, node):
 
     def visit_FormattedValue(self, node):
         if self._is_node_args_valid(node, 'value'):
+            self.add_missing_lines(node.value, True, True)
             with self.inside('{', '}'):
                 self.visit(node.value)
                 if node.conversion != -1:
                     self.write('!%c' % (node.conversion,))
 
+    def visit_AnnAssign(self, node):
+        self.visit(node.target)
+        self.write(': ')
+        self.visit(node.annotation)
+        self.write(' = ')
+        self.visit(node.value)
 
 class SourceGeneratorNodeVisitorPython38(SourceGeneratorNodeVisitorPython36):
     __python_version__ = (3, 8)

setup.py

@@ -3,8 +3,8 @@
 
 import astmonkey
 
-with open('README.rst') as f:
-    long_description = f.read()
+with open('README.rst', 'rb') as f:
+    long_description = f.read().decode('utf-8', 'ignore')
 
 setup(
     name='astmonkey',
@@ -18,14 +18,15 @@
     long_description=long_description,
     classifiers=[
         'Intended Audience :: Developers',
-        'Programming Language :: Python :: 2.6',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
         'License :: OSI Approved :: Apache Software License'
     ]
 )

tox.ini

@@ -1,12 +1,11 @@
 [tox]
 envlist =
     coverage-erase
-    test-py{27,34,35,36,37,38}
+    test-py{37,38,39}
     coverage-report
 [testenv]
 deps=
     coverage
-    test-py26: unittest2
     test: pydot
     test: pytest
     test: pytest-cov