From bb9b319f292170990232533635de8a4f96e74ce9 Mon Sep 17 00:00:00 2001 From: Dennis Burke Date: Mon, 11 Jul 2016 13:50:16 -0400 Subject: [PATCH 1/9] bugfix for issue #30 --- tests/test_github_issues.py | 6 ++++++ textile/utils.py | 12 ++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/test_github_issues.py b/tests/test_github_issues.py index fa14237e..20448c7f 100644 --- a/tests/test_github_issues.py +++ b/tests/test_github_issues.py @@ -79,3 +79,9 @@ def parseWapProfile(self, url): \t

Of course there’s a lot more error handling to do (and useful data to glean off the XML), but being able to cut through all the usual parsing crap is immensely gratifying.

""") assert result == expect + +def test_github_issue_30(): + text ='"Tëxtíle (Tëxtíle)":http://lala.com' + result = textile.textile(text) + expect = '\t

Tëxtíle

' + assert result == expect diff --git a/textile/utils.py b/textile/utils.py index d917e9e4..72d48b1e 100644 --- a/textile/utils.py +++ b/textile/utils.py @@ -57,17 +57,17 @@ def generate_tag(tag, content, attributes=None): if not tag: return content # FIXME: Kind of an ugly hack. There *must* be a cleaner way. I tried - # adding text by assigning it to a.text. That results in non-ascii text - # being html-entity encoded. Not bad, but not entirely matching - # php-textile either. + # adding text by assigning it to element_tag.text. That results in + # non-ascii text being html-entity encoded. Not bad, but not entirely + # matching php-textile either. try: - element_tag = ElementTree.tostringlist(element, encoding=enc, - method='html') + element_tag = [v.decode(enc) for v in ElementTree.tostringlist(element, + encoding=enc, method='html')] element_tag.insert(len(element_tag) - 1, content) element_text = ''.join(element_tag) except AttributeError: # Python 2.6 doesn't have the tostringlist method, so we have to treat - # it different. + # it differently. element_tag = ElementTree.tostring(element, encoding=enc) element_text = re.sub(r"<\?xml version='1.0' encoding='UTF-8'\?>\n", '', element_tag) From d1aa498b6579928466c8222bd649f56955b655cb Mon Sep 17 00:00:00 2001 From: Dennis Burke Date: Mon, 11 Jul 2016 16:16:48 -0400 Subject: [PATCH 2/9] try this on for size: Drop support for Py26. --- .travis.yml | 1 - README.textile | 2 +- setup.py | 2 -- textile/utils.py | 19 ++++--------------- 4 files changed, 5 insertions(+), 19 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7f40ce9b..3ecc5903 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,6 @@ env: - REQUIREMENTS=true - REQUIREMENTS=false python: - - "2.6" - "2.7" - "3.2" - "3.3" diff --git a/README.textile b/README.textile index e645e341..e02bf805 100644 --- a/README.textile +++ b/README.textile @@ -43,4 +43,4 @@ h2. Usage h3. Notes: -* Active development supports Python 2.6 or later (including Python 3.2+). +* Active development supports Python 2.7 or later (including Python 3.2+). diff --git a/setup.py b/setup.py index 897f07dd..b2fa4660 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,6 @@ def get_version(): 'Topic :: Software Development :: Libraries :: Python Modules', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.2', @@ -37,7 +36,6 @@ def get_version(): keywords='textile,text,html markup', install_requires=['six',], extras_require={ - ':python_version=="2.6"': ['ordereddict>=1.1'], 'develop': ['regex', 'pytest', 'pytest-cov'], }, setup_requires=['pytest-runner'], diff --git a/textile/utils.py b/textile/utils.py index 72d48b1e..22cee8f0 100644 --- a/textile/utils.py +++ b/textile/utils.py @@ -60,21 +60,10 @@ def generate_tag(tag, content, attributes=None): # adding text by assigning it to element_tag.text. That results in # non-ascii text being html-entity encoded. Not bad, but not entirely # matching php-textile either. - try: - element_tag = [v.decode(enc) for v in ElementTree.tostringlist(element, - encoding=enc, method='html')] - element_tag.insert(len(element_tag) - 1, content) - element_text = ''.join(element_tag) - except AttributeError: - # Python 2.6 doesn't have the tostringlist method, so we have to treat - # it differently. - element_tag = ElementTree.tostring(element, encoding=enc) - element_text = re.sub(r"<\?xml version='1.0' encoding='UTF-8'\?>\n", - '', element_tag) - if content != six.text_type(' /'): - element_text = element_text.rstrip(' />') - element_text = six.text_type('{0}>{1}').format(six.text_type( - element_text), content, tag) + element_tag = ElementTree.tostringlist(element, encoding=enc, + method='html') + element_tag.insert(len(element_tag) - 1, content) + element_text = ''.join(element_tag) return element_text def has_raw_text(text): From 5fb0cdd6f9c4a4aaed8b0b29556a31623a204916 Mon Sep 17 00:00:00 2001 From: Dennis Burke Date: Mon, 11 Jul 2016 16:25:19 -0400 Subject: [PATCH 3/9] drop more py26 code and don't revert the fix for #30. --- textile/core.py | 5 +---- textile/utils.py | 9 +++------ 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/textile/core.py b/textile/core.py index 5cac161b..f3300ba2 100644 --- a/textile/core.py +++ b/textile/core.py @@ -31,10 +31,7 @@ from textile.objects import Block, Table -try: - from collections import OrderedDict -except ImportError: - from ordereddict import OrderedDict +from collections import OrderedDict from six.moves import urllib urlparse, urlsplit, urlunsplit, quote, unquote = (urllib.parse.urlparse, diff --git a/textile/utils.py b/textile/utils.py index 22cee8f0..022cda4d 100644 --- a/textile/utils.py +++ b/textile/utils.py @@ -10,10 +10,7 @@ urlparse = urllib.parse.urlparse HTMLParser = html_parser.HTMLParser -try: - from collections import OrderedDict -except ImportError: - from ordereddict import OrderedDict +from collections import OrderedDict from xml.etree import ElementTree @@ -60,8 +57,8 @@ def generate_tag(tag, content, attributes=None): # adding text by assigning it to element_tag.text. That results in # non-ascii text being html-entity encoded. Not bad, but not entirely # matching php-textile either. - element_tag = ElementTree.tostringlist(element, encoding=enc, - method='html') + element_tag = [v.decode(enc) for v in ElementTree.tostringlist(element, encoding=enc, + method='html')] element_tag.insert(len(element_tag) - 1, content) element_text = ''.join(element_tag) return element_text From 038a548414acd64920997430ad5d2e02651ab977 Mon Sep 17 00:00:00 2001 From: Dennis Burke Date: Mon, 11 Jul 2016 16:50:42 -0400 Subject: [PATCH 4/9] six don't fail me now. --- textile/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/textile/utils.py b/textile/utils.py index 022cda4d..d6057d5b 100644 --- a/textile/utils.py +++ b/textile/utils.py @@ -57,8 +57,10 @@ def generate_tag(tag, content, attributes=None): # adding text by assigning it to element_tag.text. That results in # non-ascii text being html-entity encoded. Not bad, but not entirely # matching php-textile either. - element_tag = [v.decode(enc) for v in ElementTree.tostringlist(element, encoding=enc, - method='html')] + element_tag = ElementTree.tostringlist(element, encoding=enc, + method='html') + if six.PY2: + element_tag = [v.decode('utf8') for v in element_tag] element_tag.insert(len(element_tag) - 1, content) element_text = ''.join(element_tag) return element_text From 4cca67d0bcab2c7bef3fa7c22bed38a5075b1dcb Mon Sep 17 00:00:00 2001 From: Dennis Burke Date: Tue, 12 Jul 2016 10:04:43 -0400 Subject: [PATCH 5/9] one last vestige of py2.6 --- textile/objects/block.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/textile/objects/block.py b/textile/objects/block.py index 7b46bc1f..8efc89c8 100644 --- a/textile/objects/block.py +++ b/textile/objects/block.py @@ -1,7 +1,5 @@ -try: - from collections import OrderedDict -except ImportError: - from ordereddict import OrderedDict +from collections import OrderedDict + try: import regex as re except ImportError: From 2f848a40ce7701c72254eb770dc1d718924ede9b Mon Sep 17 00:00:00 2001 From: Dennis Burke Date: Tue, 12 Jul 2016 11:36:38 -0400 Subject: [PATCH 6/9] I've rethought py2.6; giving it one last try. --- .travis.yml | 1 + README.textile | 2 +- setup.py | 2 ++ textile/__init__.py | 12 ++++++++++++ textile/core.py | 6 +++++- textile/objects/block.py | 6 ++++-- textile/utils.py | 28 +++++++++++++++++++++------- 7 files changed, 46 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3ecc5903..7f40ce9b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ env: - REQUIREMENTS=true - REQUIREMENTS=false python: + - "2.6" - "2.7" - "3.2" - "3.3" diff --git a/README.textile b/README.textile index e02bf805..e645e341 100644 --- a/README.textile +++ b/README.textile @@ -43,4 +43,4 @@ h2. Usage h3. Notes: -* Active development supports Python 2.7 or later (including Python 3.2+). +* Active development supports Python 2.6 or later (including Python 3.2+). diff --git a/setup.py b/setup.py index b2fa4660..897f07dd 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,7 @@ def get_version(): 'Topic :: Software Development :: Libraries :: Python Modules', 'Programming Language :: Python', 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.2', @@ -36,6 +37,7 @@ def get_version(): keywords='textile,text,html markup', install_requires=['six',], extras_require={ + ':python_version=="2.6"': ['ordereddict>=1.1'], 'develop': ['regex', 'pytest', 'pytest-cov'], }, setup_requires=['pytest-runner'], diff --git a/textile/__init__.py b/textile/__init__.py index e7ea6659..ca6d5c38 100644 --- a/textile/__init__.py +++ b/textile/__init__.py @@ -1,8 +1,20 @@ from __future__ import unicode_literals +import sys +import warnings + from .core import textile, textile_restricted, Textile from .version import VERSION __all__ = ['textile', 'textile_restricted'] __version__ = VERSION + + +if sys.version_info[:2] == (2, 6): + warnings.warn( + "Python 2.6 is no longer supported by the Python core team, please " + "upgrade your Python. A future version of cryptography will drop " + "support for Python 2.6", + DeprecationWarning + ) diff --git a/textile/core.py b/textile/core.py index f3300ba2..79351450 100644 --- a/textile/core.py +++ b/textile/core.py @@ -31,7 +31,10 @@ from textile.objects import Block, Table -from collections import OrderedDict +try: + from collections import OrderedDict +except ImportError: + from ordereddict import OrderedDict from six.moves import urllib urlparse, urlsplit, urlunsplit, quote, unquote = (urllib.parse.urlparse, @@ -871,6 +874,7 @@ def _casesdefault(c, pop, popped, url_chars, counts, pre): url = self.shelveURL(self.encode_url(urlunsplit(uri_parts))) attributes = parse_attributes(atts) if title: + title = title.encode('utf8') attributes['title'] = title attributes['href'] = url if self.rel: diff --git a/textile/objects/block.py b/textile/objects/block.py index 8efc89c8..7b46bc1f 100644 --- a/textile/objects/block.py +++ b/textile/objects/block.py @@ -1,5 +1,7 @@ -from collections import OrderedDict - +try: + from collections import OrderedDict +except ImportError: + from ordereddict import OrderedDict try: import regex as re except ImportError: diff --git a/textile/utils.py b/textile/utils.py index d6057d5b..8cd04ab1 100644 --- a/textile/utils.py +++ b/textile/utils.py @@ -10,7 +10,10 @@ urlparse = urllib.parse.urlparse HTMLParser = html_parser.HTMLParser -from collections import OrderedDict +try: + from collections import OrderedDict +except ImportError: + from ordereddict import OrderedDict from xml.etree import ElementTree @@ -47,22 +50,33 @@ def generate_tag(tag, content, attributes=None): content are strings, the attributes argument is a dictionary. As a convenience, if the content is ' /', a self-closing tag is generated.""" content = six.text_type(content) - element = ElementTree.Element(tag, attrib=attributes) enc = 'unicode' if six.PY2: enc = 'UTF-8' + attributes = OrderedDict((k, v.decode('utf8')) for k, v in attributes.items()) if not tag: return content + element = ElementTree.Element(tag, attrib=attributes) # FIXME: Kind of an ugly hack. There *must* be a cleaner way. I tried # adding text by assigning it to element_tag.text. That results in # non-ascii text being html-entity encoded. Not bad, but not entirely # matching php-textile either. - element_tag = ElementTree.tostringlist(element, encoding=enc, - method='html') - if six.PY2: + try: + element_tag = ElementTree.tostringlist(element, encoding=enc, + method='html') element_tag = [v.decode('utf8') for v in element_tag] - element_tag.insert(len(element_tag) - 1, content) - element_text = ''.join(element_tag) + element_tag.insert(len(element_tag) - 1, content) + element_text = ''.join(element_tag) + except AttributeError: + # Python 2.6 doesn't have the tostringlist method, so we have to treat + # it differently. + element_tag = ElementTree.tostring(element, encoding=enc) + element_text = re.sub(r"<\?xml version='1.0' encoding='UTF-8'\?>\n", + '', element_tag) + if content != six.text_type(' /'): + element_text = element_text.rstrip(' />') + element_text = six.text_type('{0}>{1}').format(six.text_type( + element_text), content, tag) return element_text def has_raw_text(text): From aa4d6439294444ff521f540e2dafb87f44e8a90d Mon Sep 17 00:00:00 2001 From: Dennis Burke Date: Tue, 12 Jul 2016 14:58:15 -0400 Subject: [PATCH 7/9] taking another stab at this. --- textile/core.py | 2 +- textile/utils.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/textile/core.py b/textile/core.py index 79351450..c21f9f61 100644 --- a/textile/core.py +++ b/textile/core.py @@ -874,7 +874,7 @@ def _casesdefault(c, pop, popped, url_chars, counts, pre): url = self.shelveURL(self.encode_url(urlunsplit(uri_parts))) attributes = parse_attributes(atts) if title: - title = title.encode('utf8') + title = six.text_type(title) attributes['title'] = title attributes['href'] = url if self.rel: diff --git a/textile/utils.py b/textile/utils.py index 8cd04ab1..f62824c2 100644 --- a/textile/utils.py +++ b/textile/utils.py @@ -53,7 +53,7 @@ def generate_tag(tag, content, attributes=None): enc = 'unicode' if six.PY2: enc = 'UTF-8' - attributes = OrderedDict((k, v.decode('utf8')) for k, v in attributes.items()) + attributes = OrderedDict((k, six.text_type(v)) for k, v in attributes.items()) if not tag: return content element = ElementTree.Element(tag, attrib=attributes) @@ -64,7 +64,8 @@ def generate_tag(tag, content, attributes=None): try: element_tag = ElementTree.tostringlist(element, encoding=enc, method='html') - element_tag = [v.decode('utf8') for v in element_tag] + if six.PY2: + element_tag = [v.decode('utf8') for v in element_tag] element_tag.insert(len(element_tag) - 1, content) element_text = ''.join(element_tag) except AttributeError: From 93f5382fb0f5d09d968149f2de08e94c889917f4 Mon Sep 17 00:00:00 2001 From: Dennis Burke Date: Tue, 12 Jul 2016 15:42:57 -0400 Subject: [PATCH 8/9] self.shelve to the rescue? --- textile/core.py | 3 +-- textile/utils.py | 3 --- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/textile/core.py b/textile/core.py index c21f9f61..52d70f17 100644 --- a/textile/core.py +++ b/textile/core.py @@ -874,8 +874,7 @@ def _casesdefault(c, pop, popped, url_chars, counts, pre): url = self.shelveURL(self.encode_url(urlunsplit(uri_parts))) attributes = parse_attributes(atts) if title: - title = six.text_type(title) - attributes['title'] = title + attributes['title'] = self.shelve(title) attributes['href'] = url if self.rel: attributes['rel'] = self.rel diff --git a/textile/utils.py b/textile/utils.py index f62824c2..89e35e81 100644 --- a/textile/utils.py +++ b/textile/utils.py @@ -53,7 +53,6 @@ def generate_tag(tag, content, attributes=None): enc = 'unicode' if six.PY2: enc = 'UTF-8' - attributes = OrderedDict((k, six.text_type(v)) for k, v in attributes.items()) if not tag: return content element = ElementTree.Element(tag, attrib=attributes) @@ -64,8 +63,6 @@ def generate_tag(tag, content, attributes=None): try: element_tag = ElementTree.tostringlist(element, encoding=enc, method='html') - if six.PY2: - element_tag = [v.decode('utf8') for v in element_tag] element_tag.insert(len(element_tag) - 1, content) element_text = ''.join(element_tag) except AttributeError: From e7763b3c7406fb0daf44d492692341ae32e68bf7 Mon Sep 17 00:00:00 2001 From: Dennis Burke Date: Tue, 12 Jul 2016 16:18:39 -0400 Subject: [PATCH 9/9] quick fixes, bump version number, and changelog. --- CHANGELOG.textile | 4 ++++ textile/__init__.py | 4 ++-- textile/core.py | 3 +++ textile/version.py | 2 +- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.textile b/CHANGELOG.textile index 06f8e8da..9f8ffcd8 100644 --- a/CHANGELOG.textile +++ b/CHANGELOG.textile @@ -1,5 +1,9 @@ h1. Textile Changelog +h2. Version 2.3.3 +* Bugfix: Unicode in URL titles no longer break everything ("#30":https://github.com/textile/python-textile/issues/30) +* Display DeprecationWarning when using textile on Python 2.6. + h2. Version 2.3.2 * Bugfix: properly handle @":"@ as text, not a link. diff --git a/textile/__init__.py b/textile/__init__.py index ca6d5c38..c019f413 100644 --- a/textile/__init__.py +++ b/textile/__init__.py @@ -14,7 +14,7 @@ if sys.version_info[:2] == (2, 6): warnings.warn( "Python 2.6 is no longer supported by the Python core team, please " - "upgrade your Python. A future version of cryptography will drop " - "support for Python 2.6", + "upgrade your Python. A future version of textile will drop support " + "for Python 2.6", DeprecationWarning ) diff --git a/textile/core.py b/textile/core.py index 52d70f17..7cc7439c 100644 --- a/textile/core.py +++ b/textile/core.py @@ -874,6 +874,9 @@ def _casesdefault(c, pop, popped, url_chars, counts, pre): url = self.shelveURL(self.encode_url(urlunsplit(uri_parts))) attributes = parse_attributes(atts) if title: + # if the title contains unicode data, it is annoying to get Python + # 2.6 and all the latter versions working properly. But shelving + # the title is a quick and dirty solution. attributes['title'] = self.shelve(title) attributes['href'] = url if self.rel: diff --git a/textile/version.py b/textile/version.py index a5ac6164..47cb28fc 100644 --- a/textile/version.py +++ b/textile/version.py @@ -1 +1 @@ -VERSION = '2.3.2' +VERSION = '2.3.3'