From 2fe81595d6eb89572939ba0343e98ea048ee7849 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Tue, 11 Jul 2023 00:38:22 -0400 Subject: [PATCH 01/91] Update for release. --- RELEASE.txt | 74 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 43 insertions(+), 31 deletions(-) diff --git a/RELEASE.txt b/RELEASE.txt index 36376782..4eef1345 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -33,8 +33,10 @@ Roundup release checklist: CHANGES.txt (set date for version as well) roundup/__init__.py website/www/index.txt (current stable version, release highlights) - website/www/conf.py (update copyright, version from __init__.py) - scripts/Docker/Docker update value of org.opencontainers.image.version + website/www/conf.py (update copyright, version auto-set from + roundup/__init__.py) + scripts/Docker/Dockerfile update value of + org.opencontainers.image.version 3a. Update license end date in COPYING.txt 3b. Update doc/acknowledgements.txt (add section for release, churn contributers etc.). (Use hg churn -c -r ####..####) @@ -128,6 +130,26 @@ Roundup release checklist: with all available Python versions. 11a. (TBD how to test wheel binary distribution before uploading.) +11b. Generate GPG signature file + + cd dist + gpg --detach-sign --armor -u 1F2DD0CB756A76D8 .tar.gz + + you should be prompted to use the roundup release key. If not you + can add --local=roundup-devel@lists.sourceforge.net. + This will create a file by the name .tar.gz.asc. + + Move file to website/www/signature directory + + mv .tar.gz.asc ../webite/www/signature/. + hg add ../website/www/signature/.tar.gz.asc + # commiting the file will be done in step 12 + cd .. + + Add a link to the signature to doc/security.txt. Add a new link + to the start of the signature list in doc/security.txt (look for + the word multicol). + 12. Assuming all is well commit and tag the release in the version-control system. a) hg commit ... # commit any edits from steps 1-5 @@ -137,32 +159,11 @@ Roundup release checklist: 13. Upload source distribution to PyPI - requires you sign up for a pypi account and be added as a maintainer to roundup. Ask existing - maintainer for access. You can do this two ways: - - python3 setup.py sdist upload --repository pypi + maintainer for access. Do this using twine (pip install twine). - which rebuilds the source distribution tarball and uploads it. - This means that you have uploaded something that is not tested, - also it does not have a gpg signature. It should be the same as - the tarball you tested but.... - - A better way to do this is to use twine (pip install twine). - You need to sign the tarball. This can be done with: - - cd dist - gpg --detach-sign --armor -u 1F2DD0CB756A76D8 .tgz - - you should be prompted to use the roundup release key. If not you - can add --local=roundup-devel@lists.sourceforge.net. - This will create a file by the name .tgz.asc. The original directions used twine to upload the tarball and the signature, but as of May 2023, PyPI no longer accepts signature - files. - - So we publish the signature as part of the website. Move the file - to the website/www/signatures directory. Commit the .asc signature - file to mercurial. Add a new list item at the start of the - signature list in doc/security.txt (look for the word multicol). + files. So we publish the signature as part of the website. Use twine to upload the distribution tarball. E.G. @@ -178,6 +179,15 @@ Roundup release checklist: the gpg asc files and place the .whl.asc in the signature directory. + Another way to upload is to use: + + python3 setup.py sdist upload --repository pypi + + BUT this rebuilds the source distribution tarball and uploads it. + This means that you have uploaded something that is not tested. + Also the metadata in the file changes and will not match the GPG + signature you commited in step 12. So use twine. + 14. Refresh website. website/README.txt https://www.roundup-tracker.org/ should state that the stable @@ -205,9 +215,10 @@ Roundup release checklist: 17a. install docker 17b. run: (issues, how to release a version e.g. to update alpine for security issues. Currently thinking that release tag is - rounduptracker/roundup:2.2.0-1, -2 etc? Then add a tag + rounduptracker/roundup:2.2.0-1, -2 etc. Then add a tag rounduptracker/roundup:2.2.0 that moves to always tag - the latest -N release??) + the latest -N release. Also roundup:latest points to the + newest -N for the newest roundup version.) docker build -t rounduptracker/roundup:2.2.0 \ --build-arg="source=pypi" -f scripts/Docker/Dockerfile . @@ -303,13 +314,14 @@ $ gpg --edit-key 411E354B5D1AF26125D621221F2DD0CB756A76D8 > save [ saves both keys, will need the private key and passphrase ] -EXPORT NEW KEY -============== +EXPORT NEW PUBLIC KEY +===================== $ gpg --export -a roundup-devel@lists.sourceforge.net >> \ tools/roundup.public.pgp.key -then edit roundup.public.pgp.key keeping only the last key stat starts +then edit roundup.public.pgp.key keeping only the last key that starts with: -----BEGIN PGP PUBLIC KEY BLOCK----- -Commmit new key to mercurial. +and add back the preamble that describes where to find doc for +it. Commmit new key to mercurial. From cf91c6af961ee43d8d05cc8c3fa4d0172c0d0c61 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Wed, 12 Jul 2023 22:59:49 -0400 Subject: [PATCH 02/91] Updates for 2.3.0 release. --- CHANGES.txt | 2 +- RELEASE.txt | 6 ++-- doc/acknowledgements.txt | 2 +- doc/announcement.txt | 32 +++++++++++-------- doc/security.txt | 1 + locale/de.po | 20 ++++++------ locale/en.po | 20 ++++++------ locale/es.po | 20 ++++++------ locale/fr.po | 20 ++++++------ locale/hu.po | 20 ++++++------ locale/it.po | 20 ++++++------ locale/ja.po | 20 ++++++------ locale/lt.po | 20 ++++++------ locale/nb.po | 20 ++++++------ locale/roundup.pot | 20 ++++++------ locale/ru.po | 20 ++++++------ locale/zh_CN.po | 20 ++++++------ locale/zh_TW.po | 20 ++++++------ roundup/__init__.py | 2 +- scripts/Docker/Dockerfile | 2 +- setup.py | 4 +-- website/www/index.txt | 11 ++++--- .../www/signatures/roundup-2.3.0.tar.gz.asc | 16 ++++++++++ 23 files changed, 181 insertions(+), 157 deletions(-) create mode 100644 website/www/signatures/roundup-2.3.0.tar.gz.asc diff --git a/CHANGES.txt b/CHANGES.txt index 69a4811e..75fbf2ba 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -12,7 +12,7 @@ v2.7.2 or later are required to run newer releases of Roundup. Roundup 2.0 supports Python 3.4 and later. Roundup 2.1.0 supports python 3.6 or newer (3.4/3.5 might work, but they are not tested). -2023-xx-yy 2.3.0 +2023-07-13 2.3.0 Fixed: diff --git a/RELEASE.txt b/RELEASE.txt index 4eef1345..3e216814 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -103,7 +103,7 @@ Roundup release checklist: added and removed files. Last release e.g. 1.5.1 where tip is what would become 1.6) E.G. - hg status --rev 2.0.0:tip | sed -ne 's/^A //p' | while read i ; \ + hg status --rev 2.2.0:tip | sed -ne 's/^A //p' | while read i ; \ do echo $i; grep "$i" roundup.egg-info/SOURCES.txt; done | \ uniq -c | sort -rn @@ -114,7 +114,7 @@ Roundup release checklist: (Note: files under website/ shouldn't be in the manifest.) 10a: Check for removed files still in manifest: - hg status --rev 2.0.0:tip | sed -ne 's/^R //p' | while read i ; \ + hg status --rev 2.2.0:tip | sed -ne 's/^R //p' | while read i ; \ do echo $i; grep "$i" roundup.egg-info/SOURCES.txt; done | \ uniq -c | sort -n @@ -141,7 +141,7 @@ Roundup release checklist: Move file to website/www/signature directory - mv .tar.gz.asc ../webite/www/signature/. + mv .tar.gz.asc ../webite/www/signatures/. hg add ../website/www/signature/.tar.gz.asc # commiting the file will be done in step 12 cd .. diff --git a/doc/acknowledgements.txt b/doc/acknowledgements.txt index b1593b78..e5e307d4 100644 --- a/doc/acknowledgements.txt +++ b/doc/acknowledgements.txt @@ -28,7 +28,7 @@ Release Manager: John Rouillard Developer activity by changesets:: - rouilj@ieee.org 636 ***************************************************** + rouilj@ieee.org 722 **************************************************** rsc@runtux.com 14 * Other contributers diff --git a/doc/announcement.txt b/doc/announcement.txt index fc456a0d..1d3c8f17 100644 --- a/doc/announcement.txt +++ b/doc/announcement.txt @@ -1,5 +1,5 @@ -I'm proud to release version 2.3.0b2 of the Roundup issue -tracker. This release is a bugfix and minor feature +I'm proud to release version 2.3.0 of the Roundup issue +tracker. This release is a bugfix and feature release, so make sure to read `docs/upgrading.txt `_ to bring your tracker up to date. @@ -24,10 +24,11 @@ then unpack and test/install the tarball. Also:: Among the notable improvements from the 2.2.0 release are: -* Dockerfile demo mode implemented. +* Dockerfile demo mode implemented. This allows quick evaluation as + well as the ability to spin up a configured tracker to customise. * SQLite backends can use WAL mode to reduce blocking between readers - and writers. + and writers improving concurrent use. * Redis can be used for session database with SQLite and dbm backends. Provides a major performance improvement. @@ -38,13 +39,13 @@ Among the notable improvements from the 2.2.0 release are: * Postgres full text index can now be enabled. * Modifications to in-reply-to threading when there are multiple - matches. + matches resulting in more predictable handling of messages. * Many updates to documentation to make it scannable, useful and work on mobile. * Admin documentation includes a section on setting up Content - Security Policy (CSP) + Security Policy (CSP) to better secure your Roundup trackers. * REST now allows rate limiting headers to be accessed by client JavaScript. @@ -52,7 +53,8 @@ Among the notable improvements from the 2.2.0 release are: * Default number of rounds for PBKDF2 updated to 2M to account for improvements in password crackers and CPU power. -* Support PBKDF2 with SHA512 for password storage +* Support PBKDF2 with SHA512 for password storage to improve + resistance to password crackers. * Deprecate SSHA password hash function. @@ -60,13 +62,14 @@ Among the notable improvements from the 2.2.0 release are: incurred by reindexing. * roundup-admin can list available templates and their installed - locations. + locations. This is useful when installing via pip or in a docker + container as supporting files are not stored in the usual locations + like /usr/share/roundup. -* Crash fixes in detector handling, configuration handling, fix for - sorting of multilinks. +* Crash fixes in detector handling The file CHANGES.txt has a detailed list of feature additions and -bug fixes (52) for each release. The most recent changes from +bug fixes (53) for each release. The most recent changes from there are at the end of this announcement. Also see the information in doc/upgrading.txt. @@ -148,7 +151,7 @@ and supports four database back-ends (anydbm, sqlite, mysql and postgresql). Recent Changes ============== -From 2.2.0 to 2.3.0b2 +From 2.2.0 to 2.3.0 Fixed: ------ @@ -207,7 +210,7 @@ Fixed: 'Access-Control-Allow-Credentials' when not matching '*'. Fixes security issue with rest when using '*'. (John Rouillard) - issue2551263: In REST response expose rate limiting, sunset, allow - HTTP headers to calling javascript. (John Rouillard) + HTTP headers to calling JavaScript. (John Rouillard) - issue2551257: When downloading an attached (user supplied file), make sure that an 'X-Content-Type-Options: nosniff' header is sent. (John Rouillard) @@ -298,3 +301,6 @@ Features: reindex the first 1000 issues while reporting any missing issues in the range. Also completion progress is reported when indexing a specific class. +- doc updates: add explanation for SQL code in 1.3.3->1.4.0 upgrade. + document schema table in rdbms backends and how to dump/extract + version from them. (John Rouillard) diff --git a/doc/security.txt b/doc/security.txt index 0f8b95e7..1f7e588b 100644 --- a/doc/security.txt +++ b/doc/security.txt @@ -102,6 +102,7 @@ with 1.6.0 have been moved and are linked below: .. rst-class:: multicol +* `2.3.0 <../signatures/roundup-2.3.0.tar.gz.asc>`_ * `2.3.0b2 <../signatures/roundup-2.3.0b2.tar.gz.asc>`_ * `2.2.0 <../signatures/roundup-2.2.0.tar.gz.asc>`_ * `2.1.0 <../signatures/roundup-2.1.0.tar.gz.asc>`_ diff --git a/locale/de.po b/locale/de.po index 8ca1e96a..5df9cf00 100644 --- a/locale/de.po +++ b/locale/de.po @@ -5,9 +5,9 @@ # msgid "" msgstr "" -"Project-Id-Version: Roundup 2.3.0b2\n" +"Project-Id-Version: Roundup 2.3.0\n" "Report-Msgid-Bugs-To: roundup-devel@lists.sourceforge.net\n" -"POT-Creation-Date: 2023-05-29 20:29-0400\n" +"POT-Creation-Date: 2023-07-12 22:51-0400\n" "PO-Revision-Date: 2016-04-11 09:13+0200\n" "Last-Translator: Tobias Herp \n" "Language-Team: German Translators \n" @@ -2520,38 +2520,38 @@ msgid "" "property %(property)s: %(value)r is an invalid date interval (%(errormsg)s)" msgstr "Eigenschaft %s: %r ist kein gültiges Datumsintervall (%s)" -#: ../roundup/hyperdb.py:429 +#: ../roundup/hyperdb.py:434 #, fuzzy, python-format msgid "property %(property)s: %(value)r is not a number" msgstr "Eigenschaft %s: %r ist keine Zahl" -#: ../roundup/hyperdb.py:443 +#: ../roundup/hyperdb.py:448 #, fuzzy, python-format msgid "property %(property)s: %(value)r is not an integer" msgstr "Eigenschaft %s: %r ist keine Zahl" -#: ../roundup/hyperdb.py:465 +#: ../roundup/hyperdb.py:470 #, python-format msgid "\"%s\" not a node designator" msgstr "\"%s\" ist kein gültiger Bezeichner" -#: ../roundup/hyperdb.py:1494 ../roundup/hyperdb.py:1502 -#: ../roundup/hyperdb.py:1494:1502 +#: ../roundup/hyperdb.py:1499 ../roundup/hyperdb.py:1507 +#: ../roundup/hyperdb.py:1499:1507 #, python-format msgid "Not a property name: %s" msgstr "Keine Eigenschaft: %s" -#: ../roundup/hyperdb.py:1979 +#: ../roundup/hyperdb.py:1984 #, fuzzy, python-format msgid "property %(property)s: %(value)r is not a %(classname)s." msgstr "Der Wert \"%(value)s\" ist nicht in der Liste für \"%(propname)s\"" -#: ../roundup/hyperdb.py:1985 +#: ../roundup/hyperdb.py:1990 #, python-format msgid "you may only enter ID values for property %s" msgstr "Sie können für die Eigenschaft %s nur IDs eingeben" -#: ../roundup/hyperdb.py:2020 +#: ../roundup/hyperdb.py:2025 #, fuzzy, python-format msgid "%(property)r is not a property of %(classname)s" msgstr "%r ist keine Eigenschaft von %s" diff --git a/locale/en.po b/locale/en.po index 58614781..7eebd1fd 100644 --- a/locale/en.po +++ b/locale/en.po @@ -9,9 +9,9 @@ # msgid "" msgstr "" -"Project-Id-Version: Roundup 2.3.0b2\n" +"Project-Id-Version: Roundup 2.3.0\n" "Report-Msgid-Bugs-To: roundup-devel@lists.sourceforge.net\n" -"POT-Creation-Date: 2023-05-29 20:29-0400\n" +"POT-Creation-Date: 2023-07-12 22:51-0400\n" "PO-Revision-Date: 2004-11-20 13:47+0200\n" "Last-Translator: Not applicable\n" "Language-Team: English\n" @@ -2042,38 +2042,38 @@ msgid "" "property %(property)s: %(value)r is an invalid date interval (%(errormsg)s)" msgstr "" -#: ../roundup/hyperdb.py:429 +#: ../roundup/hyperdb.py:434 #, python-format msgid "property %(property)s: %(value)r is not a number" msgstr "" -#: ../roundup/hyperdb.py:443 +#: ../roundup/hyperdb.py:448 #, python-format msgid "property %(property)s: %(value)r is not an integer" msgstr "" -#: ../roundup/hyperdb.py:465 +#: ../roundup/hyperdb.py:470 #, python-format msgid "\"%s\" not a node designator" msgstr "" -#: ../roundup/hyperdb.py:1494 ../roundup/hyperdb.py:1502 -#: ../roundup/hyperdb.py:1494:1502 +#: ../roundup/hyperdb.py:1499 ../roundup/hyperdb.py:1507 +#: ../roundup/hyperdb.py:1499:1507 #, python-format msgid "Not a property name: %s" msgstr "" -#: ../roundup/hyperdb.py:1979 +#: ../roundup/hyperdb.py:1984 #, python-format msgid "property %(property)s: %(value)r is not a %(classname)s." msgstr "" -#: ../roundup/hyperdb.py:1985 +#: ../roundup/hyperdb.py:1990 #, python-format msgid "you may only enter ID values for property %s" msgstr "" -#: ../roundup/hyperdb.py:2020 +#: ../roundup/hyperdb.py:2025 #, python-format msgid "%(property)r is not a property of %(classname)s" msgstr "" diff --git a/locale/es.po b/locale/es.po index 4f2e7d22..79ee0af6 100644 --- a/locale/es.po +++ b/locale/es.po @@ -5,9 +5,9 @@ # msgid "" msgstr "" -"Project-Id-Version: Roundup 2.3.0b2\n" +"Project-Id-Version: Roundup 2.3.0\n" "Report-Msgid-Bugs-To: roundup-devel@lists.sourceforge.net\n" -"POT-Creation-Date: 2023-05-29 20:29-0400\n" +"POT-Creation-Date: 2023-07-12 22:51-0400\n" "PO-Revision-Date: 2013-10-31 10:45+0100\n" "Last-Translator: Ramiro Morales \n" "Language-Team: Spanish Translators \n" @@ -2539,44 +2539,44 @@ msgid "" "property %(property)s: %(value)r is an invalid date interval (%(errormsg)s)" msgstr "" -#: ../roundup/hyperdb.py:429 +#: ../roundup/hyperdb.py:434 #, fuzzy, python-format msgid "property %(property)s: %(value)r is not a number" msgstr "" "propiedad \"%(propname)s\": \"%(value)s\" no se encuentra en este momento en " "la lista" -#: ../roundup/hyperdb.py:443 +#: ../roundup/hyperdb.py:448 #, fuzzy, python-format msgid "property %(property)s: %(value)r is not an integer" msgstr "" "propiedad \"%(propname)s\": \"%(value)s\" no se encuentra en este momento en " "la lista" -#: ../roundup/hyperdb.py:465 +#: ../roundup/hyperdb.py:470 #, python-format msgid "\"%s\" not a node designator" msgstr "" -#: ../roundup/hyperdb.py:1494 ../roundup/hyperdb.py:1502 -#: ../roundup/hyperdb.py:1494:1502 +#: ../roundup/hyperdb.py:1499 ../roundup/hyperdb.py:1507 +#: ../roundup/hyperdb.py:1499:1507 #, python-format msgid "Not a property name: %s" msgstr "" -#: ../roundup/hyperdb.py:1979 +#: ../roundup/hyperdb.py:1984 #, fuzzy, python-format msgid "property %(property)s: %(value)r is not a %(classname)s." msgstr "" "propiedad \"%(propname)s\": \"%(value)s\" no se encuentra en este momento en " "la lista" -#: ../roundup/hyperdb.py:1985 +#: ../roundup/hyperdb.py:1990 #, python-format msgid "you may only enter ID values for property %s" msgstr "" -#: ../roundup/hyperdb.py:2020 +#: ../roundup/hyperdb.py:2025 #, python-format msgid "%(property)r is not a property of %(classname)s" msgstr "" diff --git a/locale/fr.po b/locale/fr.po index 164ea071..5d2d4bff 100644 --- a/locale/fr.po +++ b/locale/fr.po @@ -8,9 +8,9 @@ # msgid "" msgstr "" -"Project-Id-Version: Roundup 2.3.0b2\n" +"Project-Id-Version: Roundup 2.3.0\n" "Report-Msgid-Bugs-To: roundup-devel@lists.sourceforge.net\n" -"POT-Creation-Date: 2023-05-29 20:29-0400\n" +"POT-Creation-Date: 2023-07-12 22:51-0400\n" "PO-Revision-Date: 2013-10-31 12:19+0100\n" "Last-Translator: Stephane Raimbault \n" "Language-Team: GNOME French Team \n" @@ -2565,44 +2565,44 @@ msgid "" "property %(property)s: %(value)r is an invalid date interval (%(errormsg)s)" msgstr "" -#: ../roundup/hyperdb.py:429 +#: ../roundup/hyperdb.py:434 #, fuzzy, python-format msgid "property %(property)s: %(value)r is not a number" msgstr "" "proprit %(propname)s: %(value)s n'est pas actuellement dans la " "liste" -#: ../roundup/hyperdb.py:443 +#: ../roundup/hyperdb.py:448 #, fuzzy, python-format msgid "property %(property)s: %(value)r is not an integer" msgstr "" "proprit %(propname)s: %(value)s n'est pas actuellement dans la " "liste" -#: ../roundup/hyperdb.py:465 +#: ../roundup/hyperdb.py:470 #, python-format msgid "\"%s\" not a node designator" msgstr "" -#: ../roundup/hyperdb.py:1494 ../roundup/hyperdb.py:1502 -#: ../roundup/hyperdb.py:1494:1502 +#: ../roundup/hyperdb.py:1499 ../roundup/hyperdb.py:1507 +#: ../roundup/hyperdb.py:1499:1507 #, python-format msgid "Not a property name: %s" msgstr "" -#: ../roundup/hyperdb.py:1979 +#: ../roundup/hyperdb.py:1984 #, fuzzy, python-format msgid "property %(property)s: %(value)r is not a %(classname)s." msgstr "" "proprit %(propname)s: %(value)s n'est pas actuellement dans la " "liste" -#: ../roundup/hyperdb.py:1985 +#: ../roundup/hyperdb.py:1990 #, python-format msgid "you may only enter ID values for property %s" msgstr "" -#: ../roundup/hyperdb.py:2020 +#: ../roundup/hyperdb.py:2025 #, python-format msgid "%(property)r is not a property of %(classname)s" msgstr "" diff --git a/locale/hu.po b/locale/hu.po index 3ae2fb05..007ae412 100644 --- a/locale/hu.po +++ b/locale/hu.po @@ -6,9 +6,9 @@ # kilo aka Gabor Kmetyko , 2007. msgid "" msgstr "" -"Project-Id-Version: Roundup 2.3.0b2\n" +"Project-Id-Version: Roundup 2.3.0\n" "Report-Msgid-Bugs-To: roundup-devel@lists.sourceforge.net\n" -"POT-Creation-Date: 2023-05-29 20:29-0400\n" +"POT-Creation-Date: 2023-07-12 22:51-0400\n" "PO-Revision-Date: 2013-10-31 12:19+0100\n" "Last-Translator: kilo aka Gabor Kmetyko \n" "Language-Team: Hungarian\n" @@ -2205,39 +2205,39 @@ msgid "" "property %(property)s: %(value)r is an invalid date interval (%(errormsg)s)" msgstr "" -#: ../roundup/hyperdb.py:429 +#: ../roundup/hyperdb.py:434 #, fuzzy, python-format msgid "property %(property)s: %(value)r is not a number" msgstr "\"%(propname)s\" tulajdonság: \"%(value)s\" jelenleg nincs a listában" -#: ../roundup/hyperdb.py:443 +#: ../roundup/hyperdb.py:448 #, fuzzy, python-format msgid "property %(property)s: %(value)r is not an integer" msgstr "\"%(propname)s\" tulajdonság: \"%(value)s\" jelenleg nincs a listában" -#: ../roundup/hyperdb.py:465 +#: ../roundup/hyperdb.py:470 #, python-format msgid "\"%s\" not a node designator" msgstr "" # ../roundup/hyperdb.py:949:957 -#: ../roundup/hyperdb.py:1494 ../roundup/hyperdb.py:1502 -#: ../roundup/hyperdb.py:1494:1502 +#: ../roundup/hyperdb.py:1499 ../roundup/hyperdb.py:1507 +#: ../roundup/hyperdb.py:1499:1507 #, python-format msgid "Not a property name: %s" msgstr "" -#: ../roundup/hyperdb.py:1979 +#: ../roundup/hyperdb.py:1984 #, fuzzy, python-format msgid "property %(property)s: %(value)r is not a %(classname)s." msgstr "\"%(propname)s\" tulajdonság: \"%(value)s\" jelenleg nincs a listában" -#: ../roundup/hyperdb.py:1985 +#: ../roundup/hyperdb.py:1990 #, python-format msgid "you may only enter ID values for property %s" msgstr "" -#: ../roundup/hyperdb.py:2020 +#: ../roundup/hyperdb.py:2025 #, python-format msgid "%(property)r is not a property of %(classname)s" msgstr "" diff --git a/locale/it.po b/locale/it.po index ec2a9eb0..a1b1f3db 100644 --- a/locale/it.po +++ b/locale/it.po @@ -5,9 +5,9 @@ # msgid "" msgstr "" -"Project-Id-Version: Roundup 2.3.0b2\n" +"Project-Id-Version: Roundup 2.3.0\n" "Report-Msgid-Bugs-To: roundup-devel@lists.sourceforge.net\n" -"POT-Creation-Date: 2023-05-29 20:29-0400\n" +"POT-Creation-Date: 2023-07-12 22:51-0400\n" "PO-Revision-Date: 2013-10-31 12:20+0100\n" "Last-Translator: Marco Ghidinelli \n" "Language-Team: italian \n" @@ -2131,41 +2131,41 @@ msgid "" "property %(property)s: %(value)r is an invalid date interval (%(errormsg)s)" msgstr "" -#: ../roundup/hyperdb.py:429 +#: ../roundup/hyperdb.py:434 #, fuzzy, python-format msgid "property %(property)s: %(value)r is not a number" msgstr "" "la proprietà \"%(propname)s\": \"%(value)s\" non è al momento nella lista" -#: ../roundup/hyperdb.py:443 +#: ../roundup/hyperdb.py:448 #, fuzzy, python-format msgid "property %(property)s: %(value)r is not an integer" msgstr "" "la proprietà \"%(propname)s\": \"%(value)s\" non è al momento nella lista" -#: ../roundup/hyperdb.py:465 +#: ../roundup/hyperdb.py:470 #, python-format msgid "\"%s\" not a node designator" msgstr "" -#: ../roundup/hyperdb.py:1494 ../roundup/hyperdb.py:1502 -#: ../roundup/hyperdb.py:1494:1502 +#: ../roundup/hyperdb.py:1499 ../roundup/hyperdb.py:1507 +#: ../roundup/hyperdb.py:1499:1507 #, python-format msgid "Not a property name: %s" msgstr "" -#: ../roundup/hyperdb.py:1979 +#: ../roundup/hyperdb.py:1984 #, fuzzy, python-format msgid "property %(property)s: %(value)r is not a %(classname)s." msgstr "" "la proprietà \"%(propname)s\": \"%(value)s\" non è al momento nella lista" -#: ../roundup/hyperdb.py:1985 +#: ../roundup/hyperdb.py:1990 #, python-format msgid "you may only enter ID values for property %s" msgstr "" -#: ../roundup/hyperdb.py:2020 +#: ../roundup/hyperdb.py:2025 #, python-format msgid "%(property)r is not a property of %(classname)s" msgstr "" diff --git a/locale/ja.po b/locale/ja.po index f7a7a928..c7e8a799 100644 --- a/locale/ja.po +++ b/locale/ja.po @@ -5,9 +5,9 @@ # msgid "" msgstr "" -"Project-Id-Version: Roundup 2.3.0b2\n" +"Project-Id-Version: Roundup 2.3.0\n" "Report-Msgid-Bugs-To: roundup-devel@lists.sourceforge.net\n" -"POT-Creation-Date: 2023-05-29 20:29-0400\n" +"POT-Creation-Date: 2023-07-12 22:51-0400\n" "PO-Revision-Date: 2013-10-31 12:20+0100\n" "Last-Translator: Yasushi Iwata \n" "Language-Team: Yasushi Iwata \n" @@ -2054,38 +2054,38 @@ msgid "" "property %(property)s: %(value)r is an invalid date interval (%(errormsg)s)" msgstr "" -#: ../roundup/hyperdb.py:429 +#: ../roundup/hyperdb.py:434 #, fuzzy, python-format msgid "property %(property)s: %(value)r is not a number" msgstr "プロパティ \"%(propname)s\": \"%(value)s\" がリストの中にありません" -#: ../roundup/hyperdb.py:443 +#: ../roundup/hyperdb.py:448 #, fuzzy, python-format msgid "property %(property)s: %(value)r is not an integer" msgstr "プロパティ \"%(propname)s\": \"%(value)s\" がリストの中にありません" -#: ../roundup/hyperdb.py:465 +#: ../roundup/hyperdb.py:470 #, python-format msgid "\"%s\" not a node designator" msgstr "" -#: ../roundup/hyperdb.py:1494 ../roundup/hyperdb.py:1502 -#: ../roundup/hyperdb.py:1494:1502 +#: ../roundup/hyperdb.py:1499 ../roundup/hyperdb.py:1507 +#: ../roundup/hyperdb.py:1499:1507 #, python-format msgid "Not a property name: %s" msgstr "" -#: ../roundup/hyperdb.py:1979 +#: ../roundup/hyperdb.py:1984 #, fuzzy, python-format msgid "property %(property)s: %(value)r is not a %(classname)s." msgstr "プロパティ \"%(propname)s\": \"%(value)s\" がリストの中にありません" -#: ../roundup/hyperdb.py:1985 +#: ../roundup/hyperdb.py:1990 #, python-format msgid "you may only enter ID values for property %s" msgstr "" -#: ../roundup/hyperdb.py:2020 +#: ../roundup/hyperdb.py:2025 #, python-format msgid "%(property)r is not a property of %(classname)s" msgstr "" diff --git a/locale/lt.po b/locale/lt.po index 18e13d64..731fba3e 100644 --- a/locale/lt.po +++ b/locale/lt.po @@ -5,9 +5,9 @@ # msgid "" msgstr "" -"Project-Id-Version: Roundup 2.3.0b2\n" +"Project-Id-Version: Roundup 2.3.0\n" "Report-Msgid-Bugs-To: roundup-devel@lists.sourceforge.net\n" -"POT-Creation-Date: 2023-05-29 20:29-0400\n" +"POT-Creation-Date: 2023-07-12 22:51-0400\n" "PO-Revision-Date: 2013-10-31 12:21+0100\n" "Last-Translator: Nerijus Baliunas \n" "Language-Team: \n" @@ -2497,38 +2497,38 @@ msgid "" "property %(property)s: %(value)r is an invalid date interval (%(errormsg)s)" msgstr "" -#: ../roundup/hyperdb.py:429 +#: ../roundup/hyperdb.py:434 #, fuzzy, python-format msgid "property %(property)s: %(value)r is not a number" msgstr "parinkties \"%(propname)s\": \"%(value)s\" nėra sąraše" -#: ../roundup/hyperdb.py:443 +#: ../roundup/hyperdb.py:448 #, fuzzy, python-format msgid "property %(property)s: %(value)r is not an integer" msgstr "parinkties \"%(propname)s\": \"%(value)s\" nėra sąraše" -#: ../roundup/hyperdb.py:465 +#: ../roundup/hyperdb.py:470 #, python-format msgid "\"%s\" not a node designator" msgstr "" -#: ../roundup/hyperdb.py:1494 ../roundup/hyperdb.py:1502 -#: ../roundup/hyperdb.py:1494:1502 +#: ../roundup/hyperdb.py:1499 ../roundup/hyperdb.py:1507 +#: ../roundup/hyperdb.py:1499:1507 #, python-format msgid "Not a property name: %s" msgstr "" -#: ../roundup/hyperdb.py:1979 +#: ../roundup/hyperdb.py:1984 #, fuzzy, python-format msgid "property %(property)s: %(value)r is not a %(classname)s." msgstr "parinkties \"%(propname)s\": \"%(value)s\" nėra sąraše" -#: ../roundup/hyperdb.py:1985 +#: ../roundup/hyperdb.py:1990 #, python-format msgid "you may only enter ID values for property %s" msgstr "" -#: ../roundup/hyperdb.py:2020 +#: ../roundup/hyperdb.py:2025 #, python-format msgid "%(property)r is not a property of %(classname)s" msgstr "" diff --git a/locale/nb.po b/locale/nb.po index f2edf9c9..415d130e 100644 --- a/locale/nb.po +++ b/locale/nb.po @@ -5,9 +5,9 @@ # msgid "" msgstr "" -"Project-Id-Version: Roundup 2.3.0b2\n" +"Project-Id-Version: Roundup 2.3.0\n" "Report-Msgid-Bugs-To: roundup-devel@lists.sourceforge.net\n" -"POT-Creation-Date: 2023-05-29 20:29-0400\n" +"POT-Creation-Date: 2023-07-12 22:51-0400\n" "PO-Revision-Date: 2013-10-31 12:21+0100\n" "Last-Translator: Christian Aastorp \n" "Language-Team: Norwegian Bokmal \n" @@ -2466,38 +2466,38 @@ msgid "" "property %(property)s: %(value)r is an invalid date interval (%(errormsg)s)" msgstr "egenskap %s: %r er et ugyldig datointervalll (%s)" -#: ../roundup/hyperdb.py:429 +#: ../roundup/hyperdb.py:434 #, fuzzy, python-format msgid "property %(property)s: %(value)r is not a number" msgstr "egenskap %s: %r er ikke et nummer" -#: ../roundup/hyperdb.py:443 +#: ../roundup/hyperdb.py:448 #, fuzzy, python-format msgid "property %(property)s: %(value)r is not an integer" msgstr "egenskap %s: %r er ikke et nummer" -#: ../roundup/hyperdb.py:465 +#: ../roundup/hyperdb.py:470 #, python-format msgid "\"%s\" not a node designator" msgstr "\"%s\" ikke en node-benevnelse" -#: ../roundup/hyperdb.py:1494 ../roundup/hyperdb.py:1502 -#: ../roundup/hyperdb.py:1494:1502 +#: ../roundup/hyperdb.py:1499 ../roundup/hyperdb.py:1507 +#: ../roundup/hyperdb.py:1499:1507 #, python-format msgid "Not a property name: %s" msgstr "Ikke et navn på egenskap: %s" -#: ../roundup/hyperdb.py:1979 +#: ../roundup/hyperdb.py:1984 #, fuzzy, python-format msgid "property %(property)s: %(value)r is not a %(classname)s." msgstr "egenskapen \"%(propname)s\": \"%(value)s\" er ikke i listen nå" -#: ../roundup/hyperdb.py:1985 +#: ../roundup/hyperdb.py:1990 #, python-format msgid "you may only enter ID values for property %s" msgstr "du kan bare oppgi ID-verdier for egenskap %s" -#: ../roundup/hyperdb.py:2020 +#: ../roundup/hyperdb.py:2025 #, fuzzy, python-format msgid "%(property)r is not a property of %(classname)s" msgstr "%r er ikke en egenskap ved %s" diff --git a/locale/roundup.pot b/locale/roundup.pot index df007592..15846f5e 100644 --- a/locale/roundup.pot +++ b/locale/roundup.pot @@ -6,9 +6,9 @@ #, fuzzy msgid "" msgstr "" -"Project-Id-Version: Roundup 2.3.0b2\n" +"Project-Id-Version: Roundup 2.3.0\n" "Report-Msgid-Bugs-To: roundup-devel@lists.sourceforge.net\n" -"POT-Creation-Date: 2023-05-31 19:36-0400\n" +"POT-Creation-Date: 2023-07-12 22:51-0400\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -2029,38 +2029,38 @@ msgid "" "property %(property)s: %(value)r is an invalid date interval (%(errormsg)s)" msgstr "" -#: ../roundup/hyperdb.py:429 +#: ../roundup/hyperdb.py:434 #, python-format msgid "property %(property)s: %(value)r is not a number" msgstr "" -#: ../roundup/hyperdb.py:443 +#: ../roundup/hyperdb.py:448 #, python-format msgid "property %(property)s: %(value)r is not an integer" msgstr "" -#: ../roundup/hyperdb.py:465 +#: ../roundup/hyperdb.py:470 #, python-format msgid "\"%s\" not a node designator" msgstr "" -#: ../roundup/hyperdb.py:1494 ../roundup/hyperdb.py:1502 -#: ../roundup/hyperdb.py:1494:1502 +#: ../roundup/hyperdb.py:1499 ../roundup/hyperdb.py:1507 +#: ../roundup/hyperdb.py:1499:1507 #, python-format msgid "Not a property name: %s" msgstr "" -#: ../roundup/hyperdb.py:1979 +#: ../roundup/hyperdb.py:1984 #, python-format msgid "property %(property)s: %(value)r is not a %(classname)s." msgstr "" -#: ../roundup/hyperdb.py:1985 +#: ../roundup/hyperdb.py:1990 #, python-format msgid "you may only enter ID values for property %s" msgstr "" -#: ../roundup/hyperdb.py:2020 +#: ../roundup/hyperdb.py:2025 #, python-format msgid "%(property)r is not a property of %(classname)s" msgstr "" diff --git a/locale/ru.po b/locale/ru.po index 89aa14d8..ae238991 100644 --- a/locale/ru.po +++ b/locale/ru.po @@ -5,9 +5,9 @@ # msgid "" msgstr "" -"Project-Id-Version: Roundup 2.3.0b2\n" +"Project-Id-Version: Roundup 2.3.0\n" "Report-Msgid-Bugs-To: roundup-devel@lists.sourceforge.net\n" -"POT-Creation-Date: 2023-05-29 20:29-0400\n" +"POT-Creation-Date: 2023-07-12 22:51-0400\n" "PO-Revision-Date: 2013-10-31 12:21+0100\n" "Last-Translator: alexander smishlajev \n" "Language-Team: Russian\n" @@ -2506,38 +2506,38 @@ msgid "" "property %(property)s: %(value)r is an invalid date interval (%(errormsg)s)" msgstr "" -#: ../roundup/hyperdb.py:429 +#: ../roundup/hyperdb.py:434 #, fuzzy, python-format msgid "property %(property)s: %(value)r is not a number" msgstr " \"%(propname)s\": \"%(value)s\" " -#: ../roundup/hyperdb.py:443 +#: ../roundup/hyperdb.py:448 #, fuzzy, python-format msgid "property %(property)s: %(value)r is not an integer" msgstr " \"%(propname)s\": \"%(value)s\" " -#: ../roundup/hyperdb.py:465 +#: ../roundup/hyperdb.py:470 #, python-format msgid "\"%s\" not a node designator" msgstr "" -#: ../roundup/hyperdb.py:1494 ../roundup/hyperdb.py:1502 -#: ../roundup/hyperdb.py:1494:1502 +#: ../roundup/hyperdb.py:1499 ../roundup/hyperdb.py:1507 +#: ../roundup/hyperdb.py:1499:1507 #, python-format msgid "Not a property name: %s" msgstr "" -#: ../roundup/hyperdb.py:1979 +#: ../roundup/hyperdb.py:1984 #, fuzzy, python-format msgid "property %(property)s: %(value)r is not a %(classname)s." msgstr " \"%(propname)s\": \"%(value)s\" " -#: ../roundup/hyperdb.py:1985 +#: ../roundup/hyperdb.py:1990 #, python-format msgid "you may only enter ID values for property %s" msgstr "" -#: ../roundup/hyperdb.py:2020 +#: ../roundup/hyperdb.py:2025 #, python-format msgid "%(property)r is not a property of %(classname)s" msgstr "" diff --git a/locale/zh_CN.po b/locale/zh_CN.po index f80aa32a..98f533a1 100644 --- a/locale/zh_CN.po +++ b/locale/zh_CN.po @@ -6,9 +6,9 @@ # msgid "" msgstr "" -"Project-Id-Version: Roundup 2.3.0b2\n" +"Project-Id-Version: Roundup 2.3.0\n" "Report-Msgid-Bugs-To: roundup-devel@lists.sourceforge.net\n" -"POT-Creation-Date: 2023-05-29 20:29-0400\n" +"POT-Creation-Date: 2023-07-12 22:51-0400\n" "PO-Revision-Date: 2013-10-31 12:22+0100\n" "Last-Translator: Cheer Xiao \n" "Language-Team: Chinese Simplified \n" @@ -2370,38 +2370,38 @@ msgid "" "property %(property)s: %(value)r is an invalid date interval (%(errormsg)s)" msgstr "" -#: ../roundup/hyperdb.py:429 +#: ../roundup/hyperdb.py:434 #, fuzzy, python-format msgid "property %(property)s: %(value)r is not a number" msgstr "属性 \"%(propname)s\": \"%(value)s\" 当前不在列表中" -#: ../roundup/hyperdb.py:443 +#: ../roundup/hyperdb.py:448 #, fuzzy, python-format msgid "property %(property)s: %(value)r is not an integer" msgstr "属性 \"%(propname)s\": \"%(value)s\" 当前不在列表中" -#: ../roundup/hyperdb.py:465 +#: ../roundup/hyperdb.py:470 #, python-format msgid "\"%s\" not a node designator" msgstr "" -#: ../roundup/hyperdb.py:1494 ../roundup/hyperdb.py:1502 -#: ../roundup/hyperdb.py:1494:1502 +#: ../roundup/hyperdb.py:1499 ../roundup/hyperdb.py:1507 +#: ../roundup/hyperdb.py:1499:1507 #, python-format msgid "Not a property name: %s" msgstr "不是属性名: %s" -#: ../roundup/hyperdb.py:1979 +#: ../roundup/hyperdb.py:1984 #, fuzzy, python-format msgid "property %(property)s: %(value)r is not a %(classname)s." msgstr "属性 \"%(propname)s\": \"%(value)s\" 当前不在列表中" -#: ../roundup/hyperdb.py:1985 +#: ../roundup/hyperdb.py:1990 #, python-format msgid "you may only enter ID values for property %s" msgstr "" -#: ../roundup/hyperdb.py:2020 +#: ../roundup/hyperdb.py:2025 #, python-format msgid "%(property)r is not a property of %(classname)s" msgstr "" diff --git a/locale/zh_TW.po b/locale/zh_TW.po index 0c083da7..5e8fe743 100644 --- a/locale/zh_TW.po +++ b/locale/zh_TW.po @@ -5,9 +5,9 @@ # msgid "" msgstr "" -"Project-Id-Version: Roundup 2.3.0b2\n" +"Project-Id-Version: Roundup 2.3.0\n" "Report-Msgid-Bugs-To: roundup-devel@lists.sourceforge.net\n" -"POT-Creation-Date: 2023-05-29 20:29-0400\n" +"POT-Creation-Date: 2023-07-12 22:51-0400\n" "PO-Revision-Date: 2013-10-31 12:23+0100\n" "Last-Translator: Fred Lin \n" "Language-Team: Chinese Traditional \n" @@ -2372,38 +2372,38 @@ msgid "" "property %(property)s: %(value)r is an invalid date interval (%(errormsg)s)" msgstr "" -#: ../roundup/hyperdb.py:429 +#: ../roundup/hyperdb.py:434 #, fuzzy, python-format msgid "property %(property)s: %(value)r is not a number" msgstr "屬性 \"%(propname)s\": \"%(value)s\" 當前不在列表中" -#: ../roundup/hyperdb.py:443 +#: ../roundup/hyperdb.py:448 #, fuzzy, python-format msgid "property %(property)s: %(value)r is not an integer" msgstr "屬性 \"%(propname)s\": \"%(value)s\" 當前不在列表中" -#: ../roundup/hyperdb.py:465 +#: ../roundup/hyperdb.py:470 #, python-format msgid "\"%s\" not a node designator" msgstr "" -#: ../roundup/hyperdb.py:1494 ../roundup/hyperdb.py:1502 -#: ../roundup/hyperdb.py:1494:1502 +#: ../roundup/hyperdb.py:1499 ../roundup/hyperdb.py:1507 +#: ../roundup/hyperdb.py:1499:1507 #, fuzzy, python-format msgid "Not a property name: %s" msgstr "不是日期格式：%s" -#: ../roundup/hyperdb.py:1979 +#: ../roundup/hyperdb.py:1984 #, fuzzy, python-format msgid "property %(property)s: %(value)r is not a %(classname)s." msgstr "屬性 \"%(propname)s\": \"%(value)s\" 當前不在列表中" -#: ../roundup/hyperdb.py:1985 +#: ../roundup/hyperdb.py:1990 #, python-format msgid "you may only enter ID values for property %s" msgstr "" -#: ../roundup/hyperdb.py:2020 +#: ../roundup/hyperdb.py:2025 #, python-format msgid "%(property)r is not a property of %(classname)s" msgstr "" diff --git a/roundup/__init__.py b/roundup/__init__.py index 6afbe762..c83b7c96 100644 --- a/roundup/__init__.py +++ b/roundup/__init__.py @@ -67,6 +67,6 @@ ''' __docformat__ = 'restructuredtext' -__version__ = '2.3.0b2' +__version__ = '2.3.0' # vim: set filetype=python ts=4 sw=4 et si diff --git a/scripts/Docker/Dockerfile b/scripts/Docker/Dockerfile index 4a6a09ff..f13938d0 100644 --- a/scripts/Docker/Dockerfile +++ b/scripts/Docker/Dockerfile @@ -200,7 +200,7 @@ ARG pip_mod LABEL "org.opencontainers.image.vendor"="Roundup Issue Tracker Team" \ "org.opencontainers.image.title"="Roundup Issue Tracker" \ "org.opencontainers.image.description"="Roundup Issue Tracker with multi-backend support installed via $source with python version $pythonversion" \ - "org.opencontainers.image.version"="2.2.0" \ + "org.opencontainers.image.version"="2.3.0" \ "org.opencontainers.image.authors"="roundup-devel@lists.sourceforge.net" \ "org.opencontainers.image.licenses"="MIT AND ZPL-2.0 AND Python-2.0" \ "org.opencontainers.image.documentation"="https://www.roundup-tracker.org/docs/installation.html" \ diff --git a/setup.py b/setup.py index d920b76d..d53e69f2 100755 --- a/setup.py +++ b/setup.py @@ -198,8 +198,8 @@ def main(): long_description_content_type='text/x-rst', url='https://www.roundup-tracker.org', download_url='https://pypi.org/project/roundup', - classifiers=[#'Development Status :: 5 - Production/Stable', - 'Development Status :: 4 - Beta', + classifiers=['Development Status :: 5 - Production/Stable', + #'Development Status :: 4 - Beta', #'Development Status :: 3 - Alpha', 'Environment :: Console', 'Environment :: Web Environment', diff --git a/website/www/index.txt b/website/www/index.txt index b3cd0377..6497efa5 100644 --- a/website/www/index.txt +++ b/website/www/index.txt @@ -77,8 +77,8 @@ on the winning design from Ka-Ping Yee in the Software Carpentry It is designed to be customised so you can "track your issues your way". -The current stable version of Roundup is 2.3.0b2. It is a bug fix and -feature release for the 2.2.0 release. +The current stable version of Roundup is 2.3.0. It fixes bugs and +and adds features compared to the 2.2.0 release. It runs with Python 2.7.12+ or 3.6+. @@ -110,7 +110,7 @@ Some improvements from the 2.2.0 release are: * Postgres full text index can now be enabled. * Modifications to in-reply-to threading when there are multiple - matches results in more predictable handling of messages. + matches resulting in more predictable handling of messages. * Many updates to documentation to make it scannable, useful and work on mobile. @@ -137,9 +137,10 @@ Some improvements from the 2.2.0 release are: container as supporting files are not stored in the usual locations like /usr/share/roundup. -* Crash fixes in detector handling and configuration handling. +* Crash fixes in detector handling, configuration handling, fix for + sorting of multilinks. -More info on the 51 changes can be found in the `change notes`_. +More info on the 53 changes can be found in the `change notes`_. Roundup Use Cases ================= diff --git a/website/www/signatures/roundup-2.3.0.tar.gz.asc b/website/www/signatures/roundup-2.3.0.tar.gz.asc new file mode 100644 index 00000000..7d52046a --- /dev/null +++ b/website/www/signatures/roundup-2.3.0.tar.gz.asc @@ -0,0 +1,16 @@ +-----BEGIN PGP SIGNATURE----- + +iQIzBAABCgAdFiEEQR41S10a8mEl1iEiHy3Qy3VqdtgFAmSvZ/oACgkQHy3Qy3Vq +dtiO0xAAwvXEHRrpxH1MnkADBoNOXsNa+LVDA33x4+VFs1f7rstvkcnrT+YkD1bS +UdCPfhc3pTLwylxW/FJOqJP5GF6LYh/aXZpyHgOYi1FVvROio24rs/2mq1/8hM08 +VQQgmr3+SY97Wp8M7ysYbUhzRln75A1uZuXWCbXqTFpNwJEMwhxR1nthyrVJxP4X +SCKXJqM1CQLXfozpQ45s82VNlHXWsypko24rG1zXr31OoSQtFwTx9hfabmZlu17x +bVgdECLoMfBqy8on+SxX++33ndq56XLSO7tiYUC1IsgR3UJrOQFzbYzTvSZ1zQpz +1N8hUEQg/VFWTdLaHCkM3GioGR1PkJuP+mm5xMpb4wLAvV6nkvMLrf1DS1Vofx4A +E50TOzh9RFqCrW8+ghCTcuIrq6cx+Uj90QY6ktbJZySG31lsd6HjtVbNcs12ZnAt +uA2ZJ0KFsd1k8nNgc5CnPGCnosp9LZBerL53xIr+aGO8T7AA0es/4n3mG5szFA2E +/4c87SOIssERgseKnzyDoIMHnlKLvaLfR5QWs6zcs04opBZ2NMCPb8t7Ddo695iH +QQidFzRhoXGLDrbN1MeZroFCvlfDXI2eR/bWnszPOox0354FlDU4LmuPLkqZEGgJ +wu2i5Ogmgqpiji0NPqUtllPIokkOBbznjhAbrPHMN5+y94U3pR0= +=luer +-----END PGP SIGNATURE----- From 173ea7c69368fe25d421a588667780b17d637ef9 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Wed, 12 Jul 2023 23:00:25 -0400 Subject: [PATCH 03/91] Update for 2.3.0 release --- doc/installation.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/installation.txt b/doc/installation.txt index 61810e3a..a55f7011 100644 --- a/doc/installation.txt +++ b/doc/installation.txt @@ -2009,7 +2009,7 @@ tags when 2.3.0 is released will be: security guarantees that using a sha256 sum does. In addition to the release tags, there may be one or more -development tags available. All tags will start with `devel`. For +development tags available. All tags will include `devel`. For example: ``rounduptracker/roundup:2.3.0b1-devel``, ``rounduptracker/roundup:devel`` From 2745910af186b172e7416807819aa44b8cb23f1d Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Wed, 12 Jul 2023 23:01:03 -0400 Subject: [PATCH 04/91] Added tag 2.3.0 for changeset 913a73b9fab5 --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index d11cec54..9147c881 100644 --- a/.hgtags +++ b/.hgtags @@ -142,3 +142,4 @@ c90104abe508e3886917243e4acd069c8ef7a1a4 2.2.0 0000000000000000000000000000000000000000 2.2.0 239d9542b02062c56f88fd1de8b87c4d88d700ad 2.2.0 51fc06fabcee043db116e2fbdcdcf5e86b67ed3d 2.3.0b2 +913a73b9fab58e9c7e43e1fad379b68cae6ee3ae 2.3.0 From 392b0cc84bc96bade7a651d330f3321acd1f85c1 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Thu, 13 Jul 2023 00:09:23 -0400 Subject: [PATCH 05/91] 3.12b4 is causing failing tests. Disable for now. --- .github/workflows/ci-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 0bf377de..8b996d22 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -65,9 +65,9 @@ jobs: include: # example: if 3.12 fails the jobs still succeeds - - python-version: 3.12 - os: ubuntu-22.04 - experimental: [true] + #- python-version: 3.12 + # os: ubuntu-22.04 + # experimental: [true] # 3.6 not available on new 22.04 runners, so run on 20.04 ubuntu - python-version: 3.6 os: ubuntu-20.04 From dd2cf0fe051d01b1eba7f94e5428c2d345fd0919 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Thu, 13 Jul 2023 00:10:44 -0400 Subject: [PATCH 06/91] skip travisci build [skip travis] save cycles. --- .github/workflows/ci-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 8b996d22..4d80a969 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -68,6 +68,7 @@ jobs: #- python-version: 3.12 # os: ubuntu-22.04 # experimental: [true] + # 3.6 not available on new 22.04 runners, so run on 20.04 ubuntu - python-version: 3.6 os: ubuntu-20.04 From b8b9cd12488c8d30068fb4034120ef4e0bd9bc3d Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Thu, 13 Jul 2023 19:42:13 -0400 Subject: [PATCH 07/91] issue2551284 - python 3.12b4 breaks email address parsing I disabled testing under 3.12 because it as breaking the build. Apparently the experimental setting wasn't working right. Try tweaking syntax to see if I can get a full run with a broken 3.12 and still pass testing. --- .github/workflows/ci-test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 4d80a969..2f64d178 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -34,7 +34,7 @@ jobs: # run the finalizer for coveralls even if one or more # experimental matrix runs fail. - # continue-on-error: ${{ matrix.experimental }} + continue-on-error: ${{ matrix.experimental }} #runs-on: ubuntu-latest # use below if running on multiple OS's. @@ -65,9 +65,9 @@ jobs: include: # example: if 3.12 fails the jobs still succeeds - #- python-version: 3.12 - # os: ubuntu-22.04 - # experimental: [true] + - python-version: 3.12 + os: ubuntu-22.04 + experimental: true # 3.6 not available on new 22.04 runners, so run on 20.04 ubuntu - python-version: 3.6 From 35f8da0a7353c2d84af0aee6fae05d029405f6f6 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Thu, 13 Jul 2023 19:43:13 -0400 Subject: [PATCH 08/91] gratuitous commit to skip travis build. forgot [skip travis] on last commit. --- .github/workflows/ci-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 2f64d178..688461a4 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -65,7 +65,7 @@ jobs: include: # example: if 3.12 fails the jobs still succeeds - - python-version: 3.12 + - python-version: 3.12 os: ubuntu-22.04 experimental: true From a0f8705f442706404b98c866e9872b9902a860e4 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Thu, 13 Jul 2023 19:49:47 -0400 Subject: [PATCH 09/91] gratuitous commit to skip travis build. run failed. Didn't get out of the gate. Try making true an array/list. [skip travis] --- .github/workflows/ci-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 688461a4..86cbc0a1 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -67,7 +67,7 @@ jobs: # example: if 3.12 fails the jobs still succeeds - python-version: 3.12 os: ubuntu-22.04 - experimental: true + experimental: [true] # 3.6 not available on new 22.04 runners, so run on 20.04 ubuntu - python-version: 3.6 From db02cc8832659f09b2a35d041d00dc8f8eb04c6a Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Thu, 13 Jul 2023 19:55:51 -0400 Subject: [PATCH 10/91] try without any lists run failed. Didn't get out of the gate. Try making true a bollean always despite https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstrategyfail-fast example of: jobs: test: runs-on: ubuntu-latest continue-on-error: ${{ matrix.experimental }} strategy: fail-fast: true matrix: version: [6, 7, 8] experimental: [false] include: - version: 9 experimental: true as error annotation from github is: Error when evaluating 'continue-on-error' for job 'test'. .github/workflows/ci-test.yml (Line: 37, Col: 24): A sequence was not expected [skip travis] --- .github/workflows/ci-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 86cbc0a1..1e0ff8fc 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -61,13 +61,13 @@ jobs: os: [ubuntu-latest, ubuntu-20.04] # if the ones above fail. fail the build - experimental: [false] + experimental: false include: # example: if 3.12 fails the jobs still succeeds - python-version: 3.12 os: ubuntu-22.04 - experimental: [true] + experimental: true # 3.6 not available on new 22.04 runners, so run on 20.04 ubuntu - python-version: 3.6 From 102c974f6e8e01db08f0acaef2875a4ff69c43ed Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Thu, 13 Jul 2023 20:08:56 -0400 Subject: [PATCH 11/91] another try. list of strings for toplevel. [skip travis] --- .github/workflows/ci-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 1e0ff8fc..b6ac5be4 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -61,7 +61,7 @@ jobs: os: [ubuntu-latest, ubuntu-20.04] # if the ones above fail. fail the build - experimental: false + experimental: [ "false" ] include: # example: if 3.12 fails the jobs still succeeds From d7a0f7c84e6acd60678052fbbc0a558ff426fe98 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Thu, 13 Jul 2023 20:11:41 -0400 Subject: [PATCH 12/91] try unquoted string. This is what I had originally. If this fails, I'll comment out the continue on error again. --- .github/workflows/ci-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index b6ac5be4..11b548e4 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -61,7 +61,7 @@ jobs: os: [ubuntu-latest, ubuntu-20.04] # if the ones above fail. fail the build - experimental: [ "false" ] + experimental: [ false ] include: # example: if 3.12 fails the jobs still succeeds From 75dc03aaf8b567ee99d0058fa0f3b7653f867a01 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Thu, 13 Jul 2023 20:14:02 -0400 Subject: [PATCH 13/91] one last try move continue-on-error after all matrix defs rather than before. [skip travis] --- .github/workflows/ci-test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 11b548e4..b38b38cd 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -32,10 +32,6 @@ jobs: test: name: CI build test - # run the finalizer for coveralls even if one or more - # experimental matrix runs fail. - continue-on-error: ${{ matrix.experimental }} - #runs-on: ubuntu-latest # use below if running on multiple OS's. runs-on: ${{ matrix.os }} @@ -77,6 +73,10 @@ jobs: # skip all python versions on 20.04 except explicitly included - os: ubuntu-20.04 + # run the finalizer for coveralls even if one or more + # experimental matrix runs fail. + continue-on-error: ${{ matrix.experimental }} + env: # get colorized pytest output even without a controlling tty PYTEST_ADDOPTS: "--color=yes" From bb1b59ab21379d7ff58eb1a6474e5e36c4f2ed5b Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Thu, 13 Jul 2023 20:16:44 -0400 Subject: [PATCH 14/91] I give up. Moving it after the strategy produces: Error when evaluating 'continue-on-error' for job 'test'. .github/workflows/ci-test.yml (Line: 78, Col: 24): Unexpected value '' Comment out 3.12 run, continue directive and push. --- .github/workflows/ci-test.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index b38b38cd..c1db4234 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -61,9 +61,9 @@ jobs: include: # example: if 3.12 fails the jobs still succeeds - - python-version: 3.12 - os: ubuntu-22.04 - experimental: true + #- python-version: 3.12 + # os: ubuntu-22.04 + # experimental: true # 3.6 not available on new 22.04 runners, so run on 20.04 ubuntu - python-version: 3.6 @@ -75,7 +75,9 @@ jobs: # run the finalizer for coveralls even if one or more # experimental matrix runs fail. - continue-on-error: ${{ matrix.experimental }} + # moving it above strategy produces unexpected value false + # moving it below (here) produces unexpected value ''. + # continue-on-error: ${{ matrix.experimental }} env: # get colorized pytest output even without a controlling tty From e70c785eda205ab1564a98946b16572b08aa2d41 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Thu, 13 Jul 2023 23:47:43 -0400 Subject: [PATCH 15/91] - issue2551103 - add pragma 'display_protected' to roundup-admin. If setting is true, print protected attributes like id, activity, actor... when using display or specification subcommands. --- CHANGES.txt | 10 ++++++++++ roundup/admin.py | 36 +++++++++++++++++++++++++++++++----- test/test_admin.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 5 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 75fbf2ba..22fa6127 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -12,6 +12,16 @@ v2.7.2 or later are required to run newer releases of Roundup. Roundup 2.0 supports Python 3.4 and later. Roundup 2.1.0 supports python 3.6 or newer (3.4/3.5 might work, but they are not tested). +2024-XX-YY 2.4.0 + +Fixed: + +Features: + +- issue2551103 - add pragma 'display_protected' to roundup-admin. If + true, print protected attributes like id, activity, actor... + when using display or specification subcommands. (John Rouillard) + 2023-07-13 2.3.0 Fixed: diff --git a/roundup/admin.py b/roundup/admin.py index fec3e9c1..f46eff00 100644 --- a/roundup/admin.py +++ b/roundup/admin.py @@ -114,7 +114,8 @@ def __init__(self): } self.settings_help = { 'display_protected': - _("Have 'display designator' show protected fields: creator. NYI"), + _("Have 'display designator' show protected fields: " + "creator, id etc."), 'indexer_backend': _("Set indexer to use when running 'reindex' NYI"), @@ -522,10 +523,18 @@ def do_display(self, args): cl = self.get_class(classname) # display the values - keys = sorted(cl.properties) + normal_props = sorted(cl.properties) + if self.settings['display_protected']: + keys = sorted(cl.getprops()) + else: + keys = normal_props + for key in keys: value = cl.get(nodeid, key) - print(_('%(key)s: %(value)s') % locals()) + # prepend * for protected propeties else just indent + # with space. + protected = "*" if key not in normal_props else ' ' + print(_('%(protected)s%(key)s: %(value)s') % locals()) def do_export(self, args, export_files=True): ''"""Usage: export [[-]class[,class]] export_dir @@ -1479,6 +1488,19 @@ def do_pragma(self, args): will show all settings and their current values. If verbose is enabled hidden settings and descriptions will be shown. """ + """ + The following are to be implemented: + + indexer - Not Implemented - set indexer to use for + reindex. Use when changing indexer backends. + + exportfiles={true|false} - Not Implemented - If true + (default) export/import db tables and files. If + False, export/import just database tables, not + files. Use for faster database migration. + Replaces exporttables/importtables with + exportfiles=false then export/import + """ if len(args) < 1: raise UsageError(_('Not enough arguments supplied')) @@ -1798,8 +1820,12 @@ def do_specification(self, args): # get the key property keyprop = cl.getkey() - for key in cl.properties: - value = cl.properties[key] + if self.settings['display_protected']: + properties = cl.getprops() + else: + properties = cl.properties + for key in properties: + value = properties[key] if keyprop == key: sys.stdout.write(_('%(key)s: %(value)s (key property)\n') % locals()) diff --git a/test/test_admin.py b/test/test_admin.py index f8203a42..4b0179e1 100644 --- a/test/test_admin.py +++ b/test/test_admin.py @@ -1032,6 +1032,26 @@ def testPragma(self): expected = 'Error: Internal error: pragma can not handle values of type: float' self.assertIn(expected, out) + + # ----- + inputs = iter(["pragma display_protected=yes", + "display user1", + "quit"]) + AdminTool.my_input = lambda _self, _prompt: next(inputs) + + self.install_init() + self.admin=AdminTool() + sys.argv=['main', '-i', self.dirname] + + with captured_output() as (out, err): + ret = self.admin.main() + + out = out.getvalue().strip() + + print(ret) + expected = '\n*creation: ' + self.assertIn(expected, out) + # ----- AdminTool.my_input = orig_input @@ -1479,6 +1499,7 @@ def testSpecification(self): 'timezone: ', 'password: ', ] + with captured_output() as (out, err): sys.argv=['main', '-i', self.dirname, 'specification', 'user'] @@ -1488,6 +1509,28 @@ def testSpecification(self): print(outlist) self.assertEqual(sorted(outlist), sorted(spec)) + # ----- + inputs = iter(["pragma display_protected=1", "spec user", "quit"]) + AdminTool.my_input = lambda _self, _prompt: next(inputs) + + self.install_init() + self.admin=AdminTool() + sys.argv=['main', '-i', self.dirname] + + with captured_output() as (out, err): + ret = self.admin.main() + + # strip greeting and help text lines + outlist = out.getvalue().strip().split('\n')[2:] + + protected = [ 'id: ', + 'creation: ', + 'activity: ', + 'creator: ', + 'actor: '] + print(outlist) + self.assertEqual(sorted(outlist), sorted(spec + protected)) + def testRetireRestore(self): ''' Note the tests will fail if you run this under pdb. the context managers capture the pdb prompts and this screws From 00ce3c7e2469fa805236a278f0bf8228cb84482d Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Fri, 14 Jul 2023 00:09:47 -0400 Subject: [PATCH 16/91] initialize indexer_backend pragma from config. Prep for someday allowing roundup-admin to reindex using a different indexer than the running roundup installation. This allows the existing install to use one indexer while preparing to move to another indexer using roundup-admin. --- roundup/admin.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/roundup/admin.py b/roundup/admin.py index f46eff00..d3d8b558 100644 --- a/roundup/admin.py +++ b/roundup/admin.py @@ -1490,10 +1490,6 @@ def do_pragma(self, args): """ """ The following are to be implemented: - - indexer - Not Implemented - set indexer to use for - reindex. Use when changing indexer backends. - exportfiles={true|false} - Not Implemented - If true (default) export/import db tables and files. If False, export/import just database tables, not @@ -2046,6 +2042,8 @@ def run_command(self, args): print("Reopening tracker") tracker = roundup.instance.open(self.tracker_home) self.tracker = tracker + self.settings['indexer_backend'] = self.tracker.config['INDEXER'] + except ValueError as message: # noqa: F841 self.tracker_home = '' print(_("Error: Couldn't open tracker: %(message)s") % locals()) From 6777f520e408224aed6ff4e1eea0d8bda8d49a5b Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Fri, 14 Jul 2023 00:10:42 -0400 Subject: [PATCH 17/91] update help text on display_protected pragma mention specification command. --- roundup/admin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roundup/admin.py b/roundup/admin.py index d3d8b558..6605adcf 100644 --- a/roundup/admin.py +++ b/roundup/admin.py @@ -114,8 +114,8 @@ def __init__(self): } self.settings_help = { 'display_protected': - _("Have 'display designator' show protected fields: " - "creator, id etc."), + _("Have 'display designator' and 'specification class' show " + "protected fields: creator, id etc."), 'indexer_backend': _("Set indexer to use when running 'reindex' NYI"), From b36fda8aecf845c5654aece2f7aab2669dea8ca1 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Fri, 14 Jul 2023 00:30:44 -0400 Subject: [PATCH 18/91] Add -P pragma=value command line option to roundup-admin. To set pragmas when using non-interactive mode, or set on command line when going into interactive mode. Also changed specification test to use command line pragma setting rather than interactive. This tests the -P option without having to run an extra test. Docs updated as well. --- CHANGES.txt | 2 ++ doc/admin_guide.txt | 2 ++ roundup/admin.py | 6 +++++- share/man/man1/roundup-admin.1 | 4 ++++ test/test_admin.py | 9 +++------ 5 files changed, 16 insertions(+), 7 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 22fa6127..e7b03e16 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -21,6 +21,8 @@ Features: - issue2551103 - add pragma 'display_protected' to roundup-admin. If true, print protected attributes like id, activity, actor... when using display or specification subcommands. (John Rouillard) +- add -P pragma=value command line option to roundup-admin. Allows + setting pragmas when using non-interactive mode. (John Rouillard) 2023-07-13 2.3.0 diff --git a/doc/admin_guide.txt b/doc/admin_guide.txt index 44e40c91..8b6a32ef 100644 --- a/doc/admin_guide.txt +++ b/doc/admin_guide.txt @@ -1248,6 +1248,8 @@ The basic usage is:: -S -- when outputting lists of data, string-separate them -s -- when outputting lists of data, space-separate them. Same as '-S " "'. + -P pragma=value -- Set a pragma on command line rather than interactively. + Can be used multiple times. -V -- be verbose when importing -v -- report Roundup and Python versions (and quit) diff --git a/roundup/admin.py b/roundup/admin.py index 6605adcf..90b7be0d 100644 --- a/roundup/admin.py +++ b/roundup/admin.py @@ -173,6 +173,8 @@ def usage(self, message=''): -S -- when outputting lists of data, string-separate them -s -- when outputting lists of data, space-separate them. Same as '-S " "'. + -P pragma=value -- Set a pragma on command line rather than interactively. + Can be used multiple times. -V -- be verbose when importing -v -- report Roundup and Python versions (and quit) @@ -2114,7 +2116,7 @@ def interactive(self): def main(self): try: - opts, args = getopt.getopt(sys.argv[1:], 'i:u:hcdsS:vV') + opts, args = getopt.getopt(sys.argv[1:], 'i:u:hcdP:sS:vV') except getopt.GetoptError as e: self.usage(str(e)) return 1 @@ -2160,6 +2162,8 @@ def main(self): self.separator = ' ' elif opt == '-d': self.print_designator = 1 + elif opt == '-P': + self.do_pragma([arg]) elif opt == '-u': login_opt = arg.split(':') self.name = login_opt[0] diff --git a/share/man/man1/roundup-admin.1 b/share/man/man1/roundup-admin.1 index f4dac62b..a7570b95 100644 --- a/share/man/man1/roundup-admin.1 +++ b/share/man/man1/roundup-admin.1 @@ -29,6 +29,10 @@ When outputting lists of data, separate items with given string. When outputting lists of data, space-separate them. Same as \fB-S " "\fP. .TP +\fB-P pragma=value\fP +Set a pragma on the command line. Multiple \fB-P\fP options can be +specified to set multiple pragmas. +.TP \fB-V\fP Be verbose when importing data. .TP diff --git a/test/test_admin.py b/test/test_admin.py index 4b0179e1..fa4dc5da 100644 --- a/test/test_admin.py +++ b/test/test_admin.py @@ -1510,18 +1510,15 @@ def testSpecification(self): self.assertEqual(sorted(outlist), sorted(spec)) # ----- - inputs = iter(["pragma display_protected=1", "spec user", "quit"]) - AdminTool.my_input = lambda _self, _prompt: next(inputs) - self.install_init() self.admin=AdminTool() - sys.argv=['main', '-i', self.dirname] with captured_output() as (out, err): + sys.argv=['main', '-i', self.dirname, '-P', + 'display_protected=1', 'specification', 'user'] ret = self.admin.main() - # strip greeting and help text lines - outlist = out.getvalue().strip().split('\n')[2:] + outlist = out.getvalue().strip().split('\n') protected = [ 'id: ', 'creation: ', From 30bdd31e5c11608714a5c94cacbda0106ffdbf6a Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Fri, 14 Jul 2023 20:34:30 -0400 Subject: [PATCH 19/91] issue685275 - show retired/unretire commands For roundup-admin, add pragma show_retired [no, only, both] to control showing retired items in list and table commands: no - do not show retired only - only show retired both - show retired and unretired (active) items Also sort results of Class::getnodeids() in back_anydbm.py. Anydbm Class::list() expicitly sorts the returned values and a test depends on the order of the returned items. I can't find any docs that say Class::list() sorts and there is no explicit sort in the rdbms_common.py implementation. It looks like the natural order returned in the rdbms case for these methods is sorted. However the test fails for the anydbm case if I don't sort the results of back_anydbm.py:Class::getnodeids() to match back_anydbm.py:Class::list(). This also fixes a spelling error in comment. --- roundup/admin.py | 36 +++++++++++++++++++++++-------- roundup/backends/back_anydbm.py | 2 ++ test/test_admin.py | 38 +++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 9 deletions(-) diff --git a/roundup/admin.py b/roundup/admin.py index 90b7be0d..6d8c35cd 100644 --- a/roundup/admin.py +++ b/roundup/admin.py @@ -107,7 +107,8 @@ def __init__(self): 'display_protected': False, 'indexer_backend': "as set in config.ini", '_reopen_tracker': False, - 'show_retired': False, + 'show_retired': "no", + '_retired_val': False, 'verbose': False, '_inttest': 3, '_floattest': 3.5, @@ -123,7 +124,8 @@ def __init__(self): '_reopen_tracker': _("Force reopening of tracker when running each command."), - 'show_retired': _("Show retired items in table, list etc. NYI"), + 'show_retired': _("Show retired items in table, list etc. One of 'no', 'only', 'both'"), + '_retired_val': _("internal mapping for show_retired."), 'verbose': _("Enable verbose output: tracing, descriptions..."), '_inttest': "Integer valued setting. For testing only.", @@ -533,7 +535,7 @@ def do_display(self, args): for key in keys: value = cl.get(nodeid, key) - # prepend * for protected propeties else just indent + # prepend * for protected properties else just indent # with space. protected = "*" if key not in normal_props else ' ' print(_('%(protected)s%(key)s: %(value)s') % locals()) @@ -1296,6 +1298,9 @@ def do_list(self, args): raise UsageError(_('Too many arguments supplied')) if len(args) < 1: raise UsageError(_('Not enough arguments supplied')) + + retired = self.settings['_retired_val'] + classname = args[0] # get the class @@ -1311,7 +1316,7 @@ def do_list(self, args): if len(args) == 2: # create a list of propnames since user specified propname proplist = [] - for nodeid in cl.list(): + for nodeid in cl.getnodeids(retired=retired): try: proplist.append(cl.get(nodeid, propname)) except KeyError: @@ -1321,9 +1326,9 @@ def do_list(self, args): else: # create a list of index id's since user didn't specify # otherwise - print(self.separator.join(cl.list())) + print(self.separator.join(cl.getnodeids(retired=retired))) else: - for nodeid in cl.list(): + for nodeid in cl.getnodeids(retired=retired): try: value = cl.get(nodeid, propname) except KeyError: @@ -1544,7 +1549,18 @@ def do_pragma(self, args): '%(value)s.') % {"setting": setting, "value": value}) value = _val elif type(self.settings[setting]) is str: - pass + if setting == "show_retired": + if value not in ["no", "only", "both"]: + raise UsageError(_( + 'Incorrect value for setting %(setting)s: ' + '%(value)s. Should be no, both, or only.') % { + "setting": setting, "value": value}) + if value == "both": + self.settings['_retired_val'] = None + elif value == "only": # numerical value not boolean + self.settings['_retired_val'] = True + else: # numerical value not boolean + self.settings['_retired_val'] = False else: raise UsageError(_('Internal error: pragma can not handle ' 'values of type: %s') % @@ -1863,6 +1879,8 @@ def do_table(self, args): raise UsageError(_('Not enough arguments supplied')) classname = args[0] + retired = self.settings['_retired_val'] + # get the class cl = self.get_class(classname) @@ -1903,7 +1921,7 @@ def do_table(self, args): else: # this is going to be slow maxlen = len(spec) - for nodeid in cl.list(): + for nodeid in cl.getnodeids(retired=retired): curlen = len(str(cl.get(nodeid, spec))) if curlen > maxlen: maxlen = curlen @@ -1914,7 +1932,7 @@ def do_table(self, args): for name, width in props])) # and the table data - for nodeid in cl.list(): + for nodeid in cl.getnodeids(retired=retired): table_columns = [] for name, width in props: if name != 'id': diff --git a/roundup/backends/back_anydbm.py b/roundup/backends/back_anydbm.py index 008ebd0f..f2284ad1 100644 --- a/roundup/backends/back_anydbm.py +++ b/roundup/backends/back_anydbm.py @@ -1707,6 +1707,8 @@ def getnodeids(self, db=None, retired=None): finally: if must_close: db.close() + + res.sort() return res num_re = re.compile(r'^\d+$') diff --git a/test/test_admin.py b/test/test_admin.py index fa4dc5da..d3ce4745 100644 --- a/test/test_admin.py +++ b/test/test_admin.py @@ -1612,6 +1612,44 @@ def testRetireRestore(self): expected="1: admin\n 2: anonymous\n 3: user1" self.assertEqual(out, expected) + # test show_retired pragma three cases: + # no - no retired items + # only - only retired items + # both - all items + + # verify that user4 only is listed + self.admin=AdminTool() + with captured_output() as (out, err): + sys.argv=['main', '-i', self.dirname, '-P', + 'show_retired=only', 'list', 'user'] + ret = self.admin.main() + out = out.getvalue().strip() + print(out) + expected="4: user1" + self.assertEqual(out, expected) + + # verify that all users are shown + self.admin=AdminTool() + with captured_output() as (out, err): + sys.argv=['main', '-i', self.dirname, '-P', + 'show_retired=both', 'list', 'user'] + ret = self.admin.main() + out = out.getvalue().strip() + print(out) + expected="1: admin\n 2: anonymous\n 3: user1\n 4: user1" + self.assertEqual(out, expected) + + + # verify that active users + self.admin=AdminTool() + with captured_output() as (out, err): + sys.argv=['main', '-i', self.dirname, '-P', + 'show_retired=no', 'list', 'user'] + ret = self.admin.main() + out = out.getvalue().strip() + print(out) + expected="1: admin\n 2: anonymous\n 3: user1" + self.assertEqual(out, expected) def testTable(self): From bd6a7b91d0c028387f0f4f422c781e51debb8bd0 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Fri, 14 Jul 2023 21:56:27 -0400 Subject: [PATCH 20/91] Fix test where postgres returned items in different order --- test/test_admin.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_admin.py b/test/test_admin.py index d3ce4745..87aa2246 100644 --- a/test/test_admin.py +++ b/test/test_admin.py @@ -1634,10 +1634,10 @@ def testRetireRestore(self): sys.argv=['main', '-i', self.dirname, '-P', 'show_retired=both', 'list', 'user'] ret = self.admin.main() - out = out.getvalue().strip() + out_list = sorted(out.getvalue().strip().split("\n")) print(out) - expected="1: admin\n 2: anonymous\n 3: user1\n 4: user1" - self.assertEqual(out, expected) + expected_list=sorted("1: admin\n 2: anonymous\n 3: user1\n 4: user1".split("\n")) + self.assertEqual(out_list, expected_list) # verify that active users From 13ec5210fe0512ec8f7072ff64004a5e0a87f913 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Fri, 14 Jul 2023 22:07:23 -0400 Subject: [PATCH 21/91] issue685275 - show retired/unretired items in roundup-admin add pragma display_header to print headers for display command. Header displays designator and retired/active status. Add doc of pragma to affected commands. Add test for code paths. --- roundup/admin.py | 8 ++++++++ share/man/man1/roundup-admin.1 | 12 +++++++++++- test/test_admin.py | 23 +++++++++++++++++++++-- 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/roundup/admin.py b/roundup/admin.py index 6d8c35cd..1f253867 100644 --- a/roundup/admin.py +++ b/roundup/admin.py @@ -104,6 +104,7 @@ def __init__(self): self.db_uncommitted = False self.force = None self.settings = { + 'display_header': False, 'display_protected': False, 'indexer_backend': "as set in config.ini", '_reopen_tracker': False, @@ -114,6 +115,10 @@ def __init__(self): '_floattest': 3.5, } self.settings_help = { + 'display_header': + _("Have 'display designator[,designator*]' show header inside " + " []'s before items. Includes retired/active status."), + 'display_protected': _("Have 'display designator' and 'specification class' show " "protected fields: creator, id etc."), @@ -533,6 +538,9 @@ def do_display(self, args): else: keys = normal_props + if self.settings['display_header']: + status = "retired" if cl.is_retired(nodeid) else "active" + print('\n[%s (%s)]' % (designator, status)) for key in keys: value = cl.get(nodeid, key) # prepend * for protected properties else just indent diff --git a/share/man/man1/roundup-admin.1 b/share/man/man1/roundup-admin.1 index a7570b95..2755d1db 100644 --- a/share/man/man1/roundup-admin.1 +++ b/share/man/man1/roundup-admin.1 @@ -76,7 +76,9 @@ command. .TP \fBdisplay\fP \fIdesignator[,designator]*\fP This lists the properties and their associated values for the given -node. +node. The pragma \fBdisplay_header\fP can be used to add a header +between designators that includes the active/retired status of +the item. .TP \fBexport\fP \fI[[-]class[,class]] export_dir\fP Export the database to colon-separated-value files. @@ -182,6 +184,10 @@ property, alphabetically. With \fB-c\fP, \fB-S\fP or \fB-s\fP print a list of item id's if no property specified. If property specified, print list of that property for every class instance. + +The pragma \fBshow_retired\fP can be used to print only retired items +or to print retired and active items. The default is to print only +active items. .TP \fBmigrate\fP Update a tracker's database to be compatible with the Roundup @@ -272,6 +278,10 @@ Lists the names, location and description of all known templates. Lists all instances of the given class. If the properties are not specified, all properties are displayed. By default, the column widths are the width of the largest value. + +The pragma \fBshow_retired\fP can be used to print only retired items +or to print retired and active items. The default is to print only +active items. .TP \fBupdateconfig\fP \fI\fP This is used when updating software. It merges the \fBconfig.ini\fP diff --git a/test/test_admin.py b/test/test_admin.py index 87aa2246..2489acac 100644 --- a/test/test_admin.py +++ b/test/test_admin.py @@ -1639,8 +1639,7 @@ def testRetireRestore(self): expected_list=sorted("1: admin\n 2: anonymous\n 3: user1\n 4: user1".split("\n")) self.assertEqual(out_list, expected_list) - - # verify that active users + # verify that active users are shown self.admin=AdminTool() with captured_output() as (out, err): sys.argv=['main', '-i', self.dirname, '-P', @@ -1651,6 +1650,26 @@ def testRetireRestore(self): expected="1: admin\n 2: anonymous\n 3: user1" self.assertEqual(out, expected) + # test display headers for retired/active + self.admin=AdminTool() + with captured_output() as (out, err): + sys.argv=['main', '-i', self.dirname, '-P', + 'display_header=yes', 'display', 'user3,user4'] + ret = self.admin.main() + out = out.getvalue().strip() + print(out) + self.assertIn("[user3 (active)]\n", out) + self.assertIn( "[user4 (retired)]\n", out) + + # test that there are no headers + self.admin=AdminTool() + with captured_output() as (out, err): + sys.argv=['main', '-i', self.dirname, 'display', 'user3,user4'] + ret = self.admin.main() + out = out.getvalue().strip() + print(out) + self.assertNotIn("user3", out) + self.assertNotIn("user4", out) def testTable(self): ''' Note the tests will fail if you run this under pdb. From 22c2fecc6d4def95839020e4694e4329193134ee Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 16 Jul 2023 17:59:42 -0400 Subject: [PATCH 22/91] Add tests for <= >= and check exception is raised for python3. --- roundup/anypy/cmp_.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/roundup/anypy/cmp_.py b/roundup/anypy/cmp_.py index 0b17aae0..56740bbf 100644 --- a/roundup/anypy/cmp_.py +++ b/roundup/anypy/cmp_.py @@ -71,6 +71,9 @@ def __gt__(self, other): def _test(): + import sys + _py3 = sys.version_info[0] > 2 + Comp = NoneAndDictComparable assert Comp(None) < Comp(0) @@ -82,6 +85,13 @@ def _test(): assert not Comp({}) < Comp(None) assert not Comp((0, 0)) < Comp((0, None)) + try: + not Comp("") < Comp((0, None)) + if _py3: + assert False, "Incompatible types are reporting comparable." + except TypeError: + pass + assert Comp((0, 0)) < Comp((0, 0, None)) assert Comp((0, None, None)) < Comp((0, 0, 0)) @@ -90,6 +100,12 @@ def _test(): assert not Comp(1) < Comp(0) assert not Comp(0) > Comp(0) + + assert Comp(0) <= Comp(1) + assert Comp(1) >= Comp(0) + assert not Comp(1) <= Comp(0) + assert Comp(0) >= Comp(0) + assert Comp({0: None}) < Comp({0: 0}) assert Comp({0: 0}) < Comp({0: 1}) From 3ed8e160a0189ee4474340baf94a3020a48d6490 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 16 Jul 2023 18:01:20 -0400 Subject: [PATCH 23/91] Update changes.txt --- CHANGES.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index e7b03e16..bf29d19d 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -23,6 +23,10 @@ Features: when using display or specification subcommands. (John Rouillard) - add -P pragma=value command line option to roundup-admin. Allows setting pragmas when using non-interactive mode. (John Rouillard) +- issue685275 - add pragma show_retired to control display of retired + items when using list/table. Add pragma display_header to print + headers for display command. Header displays designator and + retired/active status. 2023-07-13 2.3.0 From 2efd0d7ed4543743b00c9463fdab13661dd144d0 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 16 Jul 2023 20:12:45 -0400 Subject: [PATCH 24/91] Fix race condition that results in missing Retry-After header when rate limit exceeded. --- roundup/rest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/roundup/rest.py b/roundup/rest.py index bc90dba8..6fe62e5f 100644 --- a/roundup/rest.py +++ b/roundup/rest.py @@ -2090,6 +2090,7 @@ def dispatch(self, method, uri, input): # to the client. We treat update as sole # source of truth for exceeded rate limits. retry_after = 1 + self.client.setHeader('Retry-After', '1') msg = _("Api rate limits exceeded. Please wait: %s seconds.") % retry_after output = self.error_obj(429, msg, source="ApiRateLimiter") From fc5c06d9be4e4a5cfd0fae8f924825507255ee56 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Tue, 18 Jul 2023 16:55:47 -0400 Subject: [PATCH 25/91] Bump coverallsapp/github-action from 2.2.0 to 2.2.1 - https://github.com/roundup-tracker/roundup/pull/44 --- .github/workflows/ci-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index c1db4234..06d15378 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -257,7 +257,7 @@ jobs: - name: Upload coverage to Coveralls # python 2.7 and 3.6 versions of coverage can't produce lcov files. if: matrix.python-version != '2.7' && matrix.python-version != '3.6' - uses: coverallsapp/github-action@c7885c00cb7ec0b8f9f5ff3f53cddb980f7a4412 # master + uses: coverallsapp/github-action@95b1a2355bd0e526ad2fd62da9fd386ad4c98474 # master with: github-token: ${{ secrets.GITHUB_TOKEN }} path-to-lcov: coverage.lcov @@ -293,7 +293,7 @@ jobs: steps: - name: Coveralls Finished - uses: coverallsapp/github-action@c7885c00cb7ec0b8f9f5ff3f53cddb980f7a4412 # master + uses: coverallsapp/github-action@95b1a2355bd0e526ad2fd62da9fd386ad4c98474 # master with: github-token: ${{ secrets.github_token }} parallel-finished: true From 2c1455309accff706e583a7622098a7ccb32fdf3 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Tue, 18 Jul 2023 16:57:34 -0400 Subject: [PATCH 26/91] Bump coverallsapp/github-action from 2.2.0 to 2.2.1 - https://github.com/roundup-tracker/roundup/pull/45 --- .github/workflows/ci-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 06d15378..f55a0f89 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -97,7 +97,7 @@ jobs: # Setup version of Python to use - name: Set Up Python ${{ matrix.python-version }} - uses: actions/setup-python@bd6b4b6205c4dbad673328db7b31b7fab9e241c0 # v4.6.1 + uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 with: python-version: ${{ matrix.python-version }} allow-prereleases: true From 6559d55bb089e984388692f10db45e302bfc5761 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Tue, 18 Jul 2023 23:18:09 -0400 Subject: [PATCH 27/91] test: Modify testRestRateLimit test to report when system is too slow. Add an explicit check on the runtime and an error message that reports that the runtime was exceeded for the test to complete as written. testRestRateLimit requires that it finish within 3 seconds. Otherwise the number of remaining requests in the rate limit does not decrease on every call. If disk I/O is high, the anydbm version of this test can take > 3 seconds and result in a failed test. My other alternative was to measure the runtime and adjust the test to match the values that are returned. This seems like too much work and is unlikely to be an issue outside of a developers under powered system. --- test/rest_common.py | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/test/rest_common.py b/test/rest_common.py index 5ed5a5df..91b87b21 100644 --- a/test/rest_common.py +++ b/test/rest_common.py @@ -1076,8 +1076,12 @@ def testPagination(self): def testRestRateLimit(self): - self.db.config['WEB_API_CALLS_PER_INTERVAL'] = 20 - self.db.config['WEB_API_INTERVAL_IN_SEC'] = 60 + calls_per_interval = 20 + interval_sec = 60 + wait_time_str = str(int(interval_sec/calls_per_interval)) + + self.db.config['WEB_API_CALLS_PER_INTERVAL'] = calls_per_interval + self.db.config['WEB_API_INTERVAL_IN_SEC'] = interval_sec # Otk code never passes through the # retry loop. Not sure why but I can force it @@ -1090,17 +1094,22 @@ def testRestRateLimit(self): # sqlite or anydbm. So don't need to exercise code. pass - print("Now realtime start:", datetime.utcnow()) + start_time = datetime.utcnow() # don't set an accept header; json should be the default # use up all our allowed api calls - for i in range(20): - # i is 0 ... 19 + for i in range(calls_per_interval): + # i is 0 ... calls_per_interval self.client_error_message = [] self.server.client.env.update({'REQUEST_METHOD': 'GET'}) results = self.server.dispatch('GET', "/rest/data/user/%s/realname"%self.joeid, self.empty_form) + loop_time = datetime.utcnow() + self.assertLess((loop_time-start_time).total_seconds(), + int(wait_time_str), + "Test system is too slow to complete test as configured") + # is successful self.assertEqual(self.server.client.response_code, 200) # does not have Retry-After header as we have @@ -1136,11 +1145,12 @@ def testRestRateLimit(self): 59, delta=5) self.assertEqual( str(self.server.client.additional_headers["Retry-After"]), - "3") # check as string + wait_time_str) # check as string print("Reset:", self.server.client.additional_headers["X-RateLimit-Reset"]) print("Now realtime pre-sleep:", datetime.utcnow()) - sleep(3.1) # sleep as requested so we can do another login + # sleep as requested so we can do another login + sleep(float(wait_time_str) + 0.1) print("Now realtime post-sleep:", datetime.utcnow()) # this should succeed @@ -1181,11 +1191,13 @@ def testRestRateLimit(self): self.assertEqual(self.server.client.response_code, 429) self.assertEqual( str(self.server.client.additional_headers["Retry-After"]), - "3") # check as string + wait_time_str) # check as string json_dict = json.loads(b2s(results)) - self.assertEqual(json_dict['error']['msg'], - "Api rate limits exceeded. Please wait: 3 seconds.") + self.assertEqual( + json_dict['error']['msg'], + "Api rate limits exceeded. Please wait: %s seconds." % + wait_time_str) # reset rest params self.db.config['WEB_API_CALLS_PER_INTERVAL'] = 0 From 21aae943529b1bb136b9e02d47a58e53c626a5c1 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Wed, 19 Jul 2023 20:37:45 -0400 Subject: [PATCH 28/91] fix(api): - issue2551063 - Rest/Xmlrpc interfaces needs failed login protection. Failed API login rate limiting with expiring lockout added. --- CHANGES.txt | 4 + doc/rest.txt | 33 +++++ doc/upgrading.txt | 36 ++++++ doc/xmlrpc.txt | 8 ++ roundup/cgi/actions.py | 208 +++++++++++++++++++++++++----- roundup/cgi/client.py | 33 ++++- roundup/cgi/wsgi_handler.py | 9 ++ roundup/configuration.py | 29 ++++- roundup/exceptions.py | 4 + test/test_actions.py | 8 +- test/test_config.py | 25 ++++ test/test_liveserver.py | 251 +++++++++++++++++++++++++++++------- 12 files changed, 552 insertions(+), 96 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index bf29d19d..3c41495a 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -16,6 +16,10 @@ python 3.6 or newer (3.4/3.5 might work, but they are not tested). Fixed: +- issue2551063 - Rest/Xmlrpc interfaces needs failed login protection. + Failed API login rate limiting with expiring lockout added. (John + Rouillard) + Features: - issue2551103 - add pragma 'display_protected' to roundup-admin. If diff --git a/doc/rest.txt b/doc/rest.txt index 8cf16911..1dae4507 100644 --- a/doc/rest.txt +++ b/doc/rest.txt @@ -77,6 +77,39 @@ that is not hosted at the same origin as Roundup, you must permit the origin using the ``allowed_api_origins`` setting in ``config.ini``. +Rate Limiting API Failed Logins +------------------------------- + +To make brute force password guessing harder, the REST API has an +invalid login rate limiter. It rate limits the number of failed login +attempts with an invalid user or password. Valid login attempts are +managed by the normal API rate limiter. The rate limiter is a GCRA +leaky bucket variant. All APIs (REST/XMLRPC) share the same rate +limiter. The rate limiter for the HTML/web interface is not shared by +the API failed login rate limiter. + +It is configured by settings in config.ini. Setting +``api_failed_login_limit`` to a non-zero value enabled the limiter. +Setting it to 0 disables the limiter (not suggested). If a user fails +to log in more than ``api_failed_login_limit`` times in +``api_failed_login_interval_in_sec`` seconds, a 429 HTTP error will be +returned. The error also tell the user how long they must wait to try +to log in again. + +When a 429 error is returned, the account is locked until enough time +has passed +(``api_failed_login_interval_in_sec/api_failed_login_limit`` seconds) +to make one additional login token available. Any attempt to log in +while it is locked will fail. This is true even if a the correct +password is supplied for a locked account. This means a brute force +attempt can't try more than one password every +``api_failed_login_interval_in_sec/api_failed_login_limit`` seconds on +average. + +The default values allow up to 4 attempts to login before delaying the +user by 2.5 minutes (150 seconds). At this time there is no supported +method to reset the rate limiter. + Rate Limiting the API --------------------- diff --git a/doc/upgrading.txt b/doc/upgrading.txt index 13318aeb..9c195d62 100644 --- a/doc/upgrading.txt +++ b/doc/upgrading.txt @@ -92,6 +92,42 @@ Contents: .. index:: Upgrading; 2.2.0 to 2.3.0 +Migrating from 2.3.0 to 2.4.0 +============================= + +Update your ``config.ini`` (required) +------------------------------------- + +Upgrade tracker's config.ini file. Use:: + + roundup-admin -i /path/to/tracker updateconfig newconfig.ini + +to generate a new ini file preserving all your settings. +You can then merge any local comments from the tracker's +``config.ini`` to ``newconfig.ini`` and replace +``config.ini`` with ``newconfig.ini``. + +``updateconfig`` will tell you if it is changing old default +values or if a value must be changed manually. + +This will insert the bad API login rate limiting settings. + +Bad Login Rate Limiting and Locking (info) +------------------------------------------ + +Brute force logins have been rate limited in the HTML web interface +for a while. This was not the case with the API interfaces. + +This release introduces rate limiting for invalid REST or XMLRPC API +logins. As with the web interface, users who have hit the rate limit +have their accounts locked until after the recommended delay time has +passed. See `information on configuring the API rate limits`_ for +details. + +.. _`information on configuring the API rate limits`: rest.html#rate-limiting-api-failed-logins + +.. index:: Upgrading; 2.2.0 to 2.3.0 + Migrating from 2.2.0 to 2.3.0 ============================= diff --git a/doc/xmlrpc.txt b/doc/xmlrpc.txt index fa98852b..873155aa 100644 --- a/doc/xmlrpc.txt +++ b/doc/xmlrpc.txt @@ -88,6 +88,14 @@ issues. Patches with tests to roundup to use defusedxml are welcome. be passed in cleartext unless the server is proxied behind another server (such as Apache or lighttpd) that provides SSL. +Rate Limiting Failed Logins +--------------------------- + +See the `rest documentation +`_ for rate limiting failed +logins on the API. The XML-RPC uses the same method as the REST API. +Rate limiting is shared between the XMLRPC and REST APIs. + Client API ========== The server currently implements seven methods/commands. Each method diff --git a/roundup/cgi/actions.py b/roundup/cgi/actions.py index cd3e3475..73f9d079 100644 --- a/roundup/cgi/actions.py +++ b/roundup/cgi/actions.py @@ -12,7 +12,7 @@ from roundup.anypy.strings import StringIO from roundup.cgi import exceptions, templating from roundup.cgi.timestamp import Timestamped -from roundup.exceptions import Reject, RejectRaw +from roundup.exceptions import RateLimitExceeded, Reject, RejectRaw from roundup.i18n import _ from roundup.mailgw import uidFromAddress from roundup.rate_limit import Gcra, RateLimit @@ -26,6 +26,8 @@ class Action: + loginLimit = None + def __init__(self, client): self.client = client self.form = client.form @@ -37,8 +39,6 @@ def __init__(self, client): self.base = client.base self.user = client.user self.context = templating.context(client) - self.loginLimit = RateLimit(client.db.config.WEB_LOGIN_ATTEMPTS_MIN, - timedelta(seconds=60)) def handle(self): """Action handler procedure""" @@ -149,7 +149,7 @@ def examine_url(self, url): if not allowed_pattern.match(parsed_url_tuple.fragment): raise ValueError(self._("Fragment component (%(url_fragment)s) in %(url)s is not properly escaped") % info) - return(urllib_.urlunparse(parsed_url_tuple)) + return urllib_.urlunparse(parsed_url_tuple) name = '' permissionType = None @@ -1298,36 +1298,6 @@ def handle(self): redirect_url_tuple.fragment)) try: - # Implement rate limiting of logins by login name. - # Use prefix to prevent key collisions maybe?? - # set client.db.config.WEB_LOGIN_ATTEMPTS_MIN to 0 - # to disable - if self.client.db.config.WEB_LOGIN_ATTEMPTS_MIN: # if 0 - off - rlkey = "LOGIN-" + self.client.user - limit = self.loginLimit - gcra = Gcra() - otk = self.client.db.Otk - try: - val = otk.getall(rlkey) - gcra.set_tat_as_string(rlkey, val['tat']) - except KeyError: - # ignore if tat not set, it's 1970-1-1 by default. - pass - # see if rate limit exceeded and we need to reject the attempt - reject = gcra.update(rlkey, limit) - - # Calculate a timestamp that will make OTK expire the - # unused entry 1 hour in the future - ts = otk.lifetime(3600) - otk.set(rlkey, tat=gcra.get_tat_as_string(rlkey), - __timestamp=ts) - otk.commit() - - if reject: - # User exceeded limits: find out how long to wait - status = gcra.status(rlkey, limit) - raise Reject(_("Logins occurring too fast. Please wait: %s seconds.") % status['Retry-After']) - self.verifyLogin(self.client.user, password) except exceptions.LoginError as err: self.client.make_user_anonymous() @@ -1347,6 +1317,28 @@ def handle(self): raise exceptions.Redirect(redirect_url) # if no __came_from, send back to base url with error return + except RateLimitExceeded as err: + self.client.make_user_anonymous() + for arg in err.args: + self.client.add_error_message(arg) + + if '__came_from' in self.form: + # usually web only. If API uses this they will get + # confused as the 429 isn't returned. Without this + # a web user will get redirected back to the home + # page rather than stay on the page where they tried + # to login. + # set a new error message + query['@error_message'] = err.args + redirect_url = urllib_.urlunparse( + (redirect_url_tuple.scheme, + redirect_url_tuple.netloc, + redirect_url_tuple.path, + redirect_url_tuple.params, + urllib_.urlencode(list(sorted(query.items())), doseq=True), + redirect_url_tuple.fragment)) + raise exceptions.Redirect(redirect_url) + raise # now we're OK, re-open the database for real, using the user self.client.opendb(self.client.user) @@ -1370,7 +1362,139 @@ def handle(self): raise exceptions.Redirect(redirect_url) - def verifyLogin(self, username, password): + def rateLimitLogin(self, username, is_api=False, update=True): + """Implement rate limiting of logins by login name. + + username - username attempting to log in. May or may not + be valid. + is_api - set to False for login via html page + set to "xmlrpc" for xmlrpc api + set to "rest" for rest api + update - if False will raise RateLimitExceeded without + updating the stored value. Default is True + which updates stored value. Used to deny access + when user successfully logs in but the user + doesn't have a valid attempt available. + + Login rates for a user are split based on html vs api + login. + + Set self.client.db.config.WEB_LOGIN_ATTEMPTS_MIN to 0 + to disable for web interface. Set + self.client.db.config.API_LOGIN_ATTEMPTS to 0 + to disable for web interface. + + By setting LoginAction.limitLogin, the admin can override + the HTML web page rate limiter if they need to change the + interval from 1 minute. + """ + config = self.client.db.config + + if not is_api: + # HTML web login. Period is fixed at 1 minute. + # Override by setting self.loginLimit. Yech. + allowed_attempts = config.WEB_LOGIN_ATTEMPTS_MIN + per_period = 60 + rlkey = "LOGIN-" + username + else: + # api login. Both Rest and XMLRPC use this. + allowed_attempts = config.WEB_API_FAILED_LOGIN_LIMIT + per_period = config.WEB_API_FAILED_LOGIN_INTERVAL_IN_SEC + rlkey = "LOGIN-API" + username + + if not allowed_attempts: # if allowed_attempt == 0 - off + return + + if self.loginLimit and not is_api: + # provide a way for user (via interfaces.py) to + # change the interval on the html login limit. + limit = self.loginLimit + else: + limit = RateLimit(allowed_attempts, + timedelta(seconds=per_period)) + + gcra = Gcra() + otk = self.client.db.Otk + + try: + val = otk.getall(rlkey) + gcra.set_tat_as_string(rlkey, val['tat']) + except KeyError: + # ignore if tat not set, tat is 1970-1-1 by default. + pass + + # see if rate limit exceeded and we need to reject + # the attempt + reject = gcra.update(rlkey, limit) + + # Calculate a timestamp that will make OTK expire the + # unused entry in twice the period defined for the + # rate limiter. + if update: + ts = otk.lifetime(per_period*2) + otk.set(rlkey, tat=gcra.get_tat_as_string(rlkey), + __timestamp=ts) + otk.commit() + + # User exceeded limits: find out how long to wait + limitStatus = gcra.status(rlkey, limit) + + if not reject: + return + + for header, value in limitStatus.items(): + self.client.setHeader(header, value) + + # User exceeded limits: tell humans how long to wait + # Headers above will do the right thing for api + # aware clients. + try: + retry_after = limitStatus['Retry-After'] + except KeyError: + # handle race condition. If the time between + # the call to grca.update and grca.status + # is sufficient to reload the bucket by 1 + # item, Retry-After will be missing from + # limitStatus. So report a 1 second delay back + # to the client. We treat update as sole + # source of truth for exceeded rate limits. + retry_after = 1 + self.client.setHeader('Retry-After', '1') + + # make rate limiting headers available to javascript + # even for non-api calls. + self.client.setHeader( + "Access-Control-Expose-Headers", + ", ".join([ + "X-RateLimit-Limit", + "X-RateLimit-Remaining", + "X-RateLimit-Reset", + "X-RateLimit-Limit-Period", + "Retry-After" + ]) + ) + + raise RateLimitExceeded(_("Logins occurring too fast. Please wait: %s seconds.") % retry_after) + + def verifyLogin(self, username, password, is_api=False): + """Authenticate the user with rate limits. + + All logins (valid and failing) from a web page calling the + LoginAction method are rate limited to the config.ini + configured value in 1 minute. (Interval can be changed see + rateLimitLogin method.) + + API logins are only rate limited if they fail. Successful + api logins are rate limited using the + api_calls_per_interval and api_interval_in_sec settings in + config.ini. + + Once a user receives a rate limit notice, they must + wait the recommended time to try again as the account is + locked out for the recommended time. If a user tries to + log in while locked out, they will get a 429 rejection + even if the username and password are correct. + """ # make sure the user exists try: # Note: lookup only searches non-retired items. @@ -1381,10 +1505,24 @@ def verifyLogin(self, username, password): # delay caused by checking password only on valid # users. _discard = self.verifyPassword("2", password) # noqa: F841 + + # limit logins to an acceptable rate. Do it for + # invalid usernames so attempts to break + # an invalid user will also be rate limited. + self.rateLimitLogin(username, is_api=is_api) + + # this is not hit if rate limit is exceeded raise exceptions.LoginError(self._('Invalid login')) + # if we are rate limited and the user tries again, + # reject the login. update=false so we don't count + # a potentially valid login. + self.rateLimitLogin(username, is_api=is_api, update=False) + # verify the password if not self.verifyPassword(self.client.userid, password): + self.rateLimitLogin(username, is_api=is_api) + # this is not hit if rate limit is exceeded raise exceptions.LoginError(self._('Invalid login')) # Determine whether the user has permission to log in. diff --git a/roundup/cgi/client.py b/roundup/cgi/client.py index e8adb645..1d618fb3 100644 --- a/roundup/cgi/client.py +++ b/roundup/cgi/client.py @@ -43,8 +43,8 @@ class SysCallError(Exception): Redirect, SendFile, SendStaticFile, SeriousError) from roundup.cgi.form_parser import FormParser -from roundup.exceptions import LoginError, Reject, RejectRaw, \ - Unauthorised, UsageError +from roundup.exceptions import LoginError, RateLimitExceeded, Reject, \ + RejectRaw, Unauthorised, UsageError from roundup.mailer import Mailer, MessageSendError @@ -572,7 +572,7 @@ def handle_xmlrpc(self): self.determine_language() # Open the database as the correct user. try: - self.determine_user() + self.determine_user(is_api="xmlrpc") self.db.tx_Source = "xmlrpc" self.db.i18n = self.translator except LoginError as msg: @@ -583,6 +583,14 @@ def handle_xmlrpc(self): self.setHeader("Content-Length", str(len(output))) self.write(s2b(output)) return + except RateLimitExceeded as msg: + output = xmlrpc_.client.dumps( + xmlrpc_.client.Fault(429, "%s" % msg), + allow_none=True) + self.setHeader("Content-Type", "text/xml") + self.setHeader("Content-Length", str(len(output))) + self.write(s2b(output)) + return if not self.db.security.hasPermission('Xmlrpc Access', self.userid): output = xmlrpc_.client.dumps( @@ -655,13 +663,19 @@ def handle_rest(self): # Open the database as the correct user. # TODO: add everything to RestfulDispatcher try: - self.determine_user() + self.determine_user(is_api="rest") self.db.tx_Source = "rest" self.db.i18n = self.translator except LoginError as err: output = s2b("Invalid Login - %s" % str(err)) self.reject_request(output, status=http_.client.UNAUTHORIZED) return + except RateLimitExceeded as err: + output = s2b("%s" % str(err)) + # PYTHON2:FIXME http_.client.TOO_MANY_REQUESTS missing + # python2 so use numeric code. + self.reject_request(output, status=429) + return # verify Origin is allowed on all requests including GET. # If a GET, missing origin is allowed (i.e. same site GET request) @@ -854,6 +868,8 @@ def inner_main(self): except SysCallError: # OpenSSL.SSL.SysCallError is similar to IOError above pass + except RateLimitExceeded: + raise except SeriousError as message: self.write_html(str(message)) @@ -917,6 +933,9 @@ def inner_main(self): except FormError as e: self.add_error_message(self._('Form Error: ') + str(e)) self.write_html(self.renderContext()) + except RateLimitExceeded as e: + self.add_error_message(str(e)) + self.write_html(self.renderContext()) except IOError: # IOErrors here are due to the client disconnecting before # receiving the reply. @@ -1110,7 +1129,7 @@ def authenticate_bearer_token(self, challenge): return(token) - def determine_user(self): + def determine_user(self, is_api=False): """Determine who the user is""" self.opendb('admin') @@ -1171,8 +1190,8 @@ def determine_user(self): # So we set the user to anonymous first. self.make_user_anonymous() login = self.get_action_class('login')(self) - login.verifyLogin(username, password) - except LoginError: + login.verifyLogin(username, password, is_api=is_api) + except (LoginError, RateLimitExceeded): self.make_user_anonymous() raise user = username diff --git a/roundup/cgi/wsgi_handler.py b/roundup/cgi/wsgi_handler.py index ecb095fb..0542bce0 100644 --- a/roundup/cgi/wsgi_handler.py +++ b/roundup/cgi/wsgi_handler.py @@ -21,6 +21,15 @@ BaseHTTPRequestHandler = http_.server.BaseHTTPRequestHandler DEFAULT_ERROR_MESSAGE = http_.server.DEFAULT_ERROR_MESSAGE +try: + # python2 is missing this definition + http_.server.BaseHTTPRequestHandler.responses[429] +except KeyError: + http_.server.BaseHTTPRequestHandler.responses[429] = ( + 'Too Many Requests', + 'The user has sent too many requests in ' + 'a given amount of time ("rate limiting")' + ) class Headers(object): """ Idea more or less stolen from the 'apache.py' in same directory. diff --git a/roundup/configuration.py b/roundup/configuration.py index b1fffc02..9957bb06 100644 --- a/roundup/configuration.py +++ b/roundup/configuration.py @@ -652,6 +652,21 @@ def str2value(self, value): except ValueError: raise OptionValueError(self, value, "Integer number required") +class IntegerNumberGtZeroOption(Option): + + """Integer numbers greater than zero.""" + + def str2value(self, value): + try: + v = int(value) + if v < 1: + raise OptionValueError(self, value, + "Integer number greater than zero required") + return v + except OptionValueError: + raise # pass through subclass + except ValueError: + raise OptionValueError(self, value, "Integer number required") class OctalNumberOption(Option): @@ -1247,13 +1262,23 @@ def str2value(self, value): "calls will be made available. If set to 360 and\n" "api_intervals_in_sec is set to 3600, the 361st call in\n" "10 seconds results in a 429 error to the caller. It\n" - "tells them to wait 10 seconds (360/3600) before making\n" + "tells them to wait 10 seconds (3600/360) before making\n" "another api request. A value of 0 turns off rate\n" "limiting in the API. Tune this as needed. See rest\n" "documentation for more info.\n"), - (IntegerNumberGeqZeroOption, 'api_interval_in_sec', "3600", + (IntegerNumberGtZeroOption, 'api_interval_in_sec', "3600", "Defines the interval in seconds over which an api client can\n" "make api_calls_per_interval api calls. Tune this as needed.\n"), + (IntegerNumberGeqZeroOption, 'api_failed_login_limit', "4", + "Limit login failure to the API per api_failed_login_interval_in_sec\n" + "seconds.\n" + "A value of 0 turns off failed login rate\n" + "limiting in the API. You should not disable this. See rest\n" + "documentation for more info.\n"), + (IntegerNumberGtZeroOption, 'api_failed_login_interval_in_sec', "600", + "Defines the interval in seconds over which api login failures\n" + "are recorded. It allows api_failed_login_limit login failures\n" + "in this time interval. Tune this as needed.\n"), (CsrfSettingOption, 'csrf_enforce_token', "yes", """How do we deal with @csrf fields in posted forms. Set this to 'required' to block the post and notify diff --git a/roundup/exceptions.py b/roundup/exceptions.py index a90d927e..ee4c8279 100644 --- a/roundup/exceptions.py +++ b/roundup/exceptions.py @@ -12,6 +12,10 @@ class LoginError(RoundupException): pass +class RateLimitExceeded(Exception): + pass + + class Unauthorised(RoundupException): pass diff --git a/test/test_actions.py b/test/test_actions.py index 41161ffc..32494c35 100644 --- a/test/test_actions.py +++ b/test/test_actions.py @@ -7,7 +7,7 @@ from roundup.cgi.actions import * from roundup.cgi.client import add_message from roundup.cgi.exceptions import Redirect, Unauthorised, SeriousError, FormError -from roundup.exceptions import Reject +from roundup.exceptions import RateLimitExceeded, Reject from roundup.anypy.cmp_ import NoneAndDictComparable from time import sleep @@ -459,7 +459,8 @@ def testLoginRateLimit(self): self.client._error_message = [] self.assertLoginLeavesMessages(['Invalid login']) - self.assertRaisesMessage(Reject, LoginAction(self.client).handle, + self.assertRaisesMessage(RateLimitExceeded, + LoginAction(self.client).handle, 'Logins occurring too fast. Please wait: 3 seconds.') sleep(3) # sleep as requested so we can do another login @@ -467,7 +468,8 @@ def testLoginRateLimit(self): self.assertLoginLeavesMessages(['Invalid login']) # this is expected # and make sure we need to wait another three seconds - self.assertRaisesMessage(Reject, LoginAction(self.client).handle, + self.assertRaisesMessage(RateLimitExceeded, + LoginAction(self.client).handle, 'Logins occurring too fast. Please wait: 3 seconds.') def testLoginRateLimitOff(self): diff --git a/test/test_config.py b/test/test_config.py index 044102ec..b8e75af0 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -301,6 +301,31 @@ def testFloatAndInt_with_update_option(self): self.assertEqual("3", config._get_option('WEB_LOGIN_ATTEMPTS_MIN')._value2str(3.00)) + def testIntegerNumberGtZeroOption(self): + + config = configuration.CoreConfig() + + # Update existing IntegerNumberGeqZeroOption to IntegerNumberOption + config.update_option('WEB_LOGIN_ATTEMPTS_MIN', + configuration.IntegerNumberGtZeroOption, + "1", description="new desc") + + self.assertEqual(None, + config._get_option('WEB_LOGIN_ATTEMPTS_MIN').set("1")) + + # -1 is not allowed + self.assertRaises(configuration.OptionValueError, + config._get_option('WEB_LOGIN_ATTEMPTS_MIN').set, "-1") + + # but can't float this + self.assertRaises(configuration.OptionValueError, + config._get_option('WEB_LOGIN_ATTEMPTS_MIN').set, "2.4") + + # but can't float this + self.assertRaises(configuration.OptionValueError, + config._get_option('WEB_LOGIN_ATTEMPTS_MIN').set, "0.5") + + def testOriginHeader(self): config = configuration.CoreConfig() diff --git a/test/test_liveserver.py b/test/test_liveserver.py index 39a72174..f5b72f13 100644 --- a/test/test_liveserver.py +++ b/test/test_liveserver.py @@ -7,6 +7,7 @@ from roundup.cgi.wsgi_handler import RequestDispatcher from .wsgi_liveserver import LiveServerTestCase from . import db_test_base +from time import sleep from wsgiref.validate import validator @@ -616,55 +617,6 @@ def test_rest_endpoint_attribute_options(self): self.assertEqual(f.status_code, 404) - def DISABLEtest_rest_login_rate_limit(self): - """login rate limit applies to api endpoints. Only failure - logins count though. So log in 10 times in a row - to verify that valid username/passwords aren't limited. - - FIXME: client.py does not implement this. Also need a live - server instance that has - - cls.db.config.WEB_LOGIN_ATTEMPTS_MIN = 4 - - not 0. - """ - - for i in range(10): - # use basic auth for rest endpoint - - f = requests.options(self.url_base() + '/rest/data', - auth=('admin', 'sekrit'), - headers = {'content-type': "", - 'Origin': "http://localhost:9001",} - ) - print(f.status_code) - print(f.headers) - - self.assertEqual(f.status_code, 204) - expected = { 'Access-Control-Allow-Origin': '*', - 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-Requested-With, X-HTTP-Method-Override', - 'Allow': 'OPTIONS, GET', - 'Access-Control-Allow-Methods': 'HEAD, OPTIONS, GET, POST, PUT, DELETE, PATCH', - 'Access-Control-Allow-Credentials': 'true', - } - - for i in range(10): - # use basic auth for rest endpoint - - f = requests.options(self.url_base() + '/rest/data', - auth=('admin', 'ekrit'), - headers = {'content-type': "", - 'Origin': "http://localhost:9001",} - ) - print(i, f.status_code) - print(f.headers) - print(f.text) - - if (i < 3): # assuming limit is 4. - self.assertEqual(f.status_code, 401) - else: - self.assertEqual(f.status_code, 429) - def test_ims(self): ''' retreive the user_utils.js file with old and new if-modified-since timestamps. @@ -1361,3 +1313,204 @@ def test_native_fts(self): # use a ts: search as well so it only works on postgres_fts indexer f = requests.get(self.url_base() + "?@search_text=ts:RESULT") self.assertIn("foo bar RESULT", f.text) + +class TestApiRateLogin(WsgiSetup): + """Class to run test in BaseTestCases with the cache_tracker + feature flag enabled when starting the wsgi server + """ + + backend = 'sqlite' + + @classmethod + def setup_class(cls): + '''All tests in this class use the same roundup instance. + This instance persists across all tests. + Create the tracker dir here so that it is ready for the + create_app() method to be called. + + cribbed from WsgiSetup::setup_class + ''' + + # tests in this class. + # set up and open a tracker + cls.instance = db_test_base.setupTracker(cls.dirname, cls.backend) + + # open the database + cls.db = cls.instance.open('admin') + + # add a user without edit access for status. + cls.db.user.create(username="fred", roles='User', + password=password.Password('sekrit'), address='fred@example.com') + + # set the url the test instance will run at. + cls.db.config['TRACKER_WEB'] = "http://localhost:9001/" + # set up mailhost so errors get reported to debuging capture file + cls.db.config.MAILHOST = "localhost" + cls.db.config.MAIL_HOST = "localhost" + cls.db.config.MAIL_DEBUG = "../_test_tracker_mail.log" + + # added to enable csrf forgeries/CORS to be tested + cls.db.config.WEB_CSRF_ENFORCE_HEADER_ORIGIN = "required" + cls.db.config.WEB_ALLOWED_API_ORIGINS = "https://client.com" + cls.db.config['WEB_CSRF_ENFORCE_HEADER_X-REQUESTED-WITH'] = "required" + + # set login failure api limits + cls.db.config.WEB_API_FAILED_LOGIN_LIMIT = 4 + cls.db.config.WEB_API_FAILED_LOGIN_INTERVAL_IN_SEC = 12 + + # enable static precompressed files + cls.db.config.WEB_USE_PRECOMPRESSED_FILES = 1 + + cls.db.config.save() + + cls.db.commit() + cls.db.close() + + # re-open the database to get the updated INDEXER + cls.db = cls.instance.open('admin') + + result = cls.db.issue.create(title="foo bar RESULT") + + # add a message to allow retrieval + result = cls.db.msg.create(author = "1", + content = "a message foo bar RESULT", + date=rdate.Date(), + messageid="test-msg-id") + + cls.db.commit() + cls.db.close() + + # Force locale config to find locales in checkout not in + # installed directories + cls.backup_domain = i18n.DOMAIN + cls.backup_locale_dirs = i18n.LOCALE_DIRS + i18n.LOCALE_DIRS = ['locale'] + i18n.DOMAIN = '' + + def test_rest_login_RateLimit(self): + """login rate limit applies to api endpoints. Only failure + logins count though. So log in 10 times in a row + to verify that valid username/passwords aren't limited. + """ + + # verify that valid logins are not counted against the limit. + for i in range(10): + # use basic auth for rest endpoint + + request_headers = {'content-type': "", + 'Origin': "http://localhost:9001",} + f = requests.options(self.url_base() + '/rest/data', + auth=('admin', 'sekrit'), + headers=request_headers + ) + #print(f.status_code) + #print(f.headers) + #print(f.text) + + self.assertEqual(f.status_code, 204) + + # Save time. check headers only for final response. + headers_expected = { + 'Access-Control-Allow-Origin': request_headers['Origin'], + 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-Requested-With, X-HTTP-Method-Override', + 'Allow': 'OPTIONS, GET', + 'Access-Control-Allow-Methods': 'OPTIONS, GET', + 'Access-Control-Allow-Credentials': 'true', + } + + for header in headers_expected.keys(): + self.assertEqual(f.headers[header], + headers_expected[header]) + + + # first 3 logins should report 401 then the rest should report + # 429 + headers_expected = { + 'Content-Type': 'text/plain' + } + + for i in range(10): + # use basic auth for rest endpoint + + f = requests.options(self.url_base() + '/rest/data', + auth=('admin', 'ekrit'), + headers = {'content-type': "", + 'Origin': "http://localhost:9001",} + ) + + if (i < 4): # assuming limit is 4. + for header in headers_expected.keys(): + self.assertEqual(f.headers[header], + headers_expected[header]) + self.assertEqual(f.status_code, 401) + else: + self.assertEqual(f.status_code, 429) + + headers_expected = { 'Content-Type': 'text/plain', + 'X-RateLimit-Limit': '4', + 'X-RateLimit-Limit-Period': '12', + 'X-RateLimit-Remaining': '0', + 'Retry-After': '3', + 'Access-Control-Expose-Headers': + ('X-RateLimit-Limit, ' + 'X-RateLimit-Remaining, ' + 'X-RateLimit-Reset, ' + 'X-RateLimit-Limit-Period, ' + 'Retry-After'), + 'Content-Length': '50'} + + for header in headers_expected.keys(): + self.assertEqual(f.headers[header], + headers_expected[header]) + + self.assertAlmostEqual(float(f.headers['X-RateLimit-Reset']), + 10.0, delta=3, + msg="limit reset not within 3 seconds of 10") + + # test lockout this is a valid login but should be rejected + # with 429. + f = requests.options(self.url_base() + '/rest/data', + auth=('admin', 'sekrit'), + headers = {'content-type': "", + 'Origin': "http://localhost:9001",} + ) + self.assertEqual(f.status_code, 429) + + for header in headers_expected.keys(): + self.assertEqual(f.headers[header], + headers_expected[header]) + + + sleep(4) + # slept long enough to get a login slot. Should work with + # 200 return code. + f = requests.get(self.url_base() + '/rest/data', + auth=('admin', 'sekrit'), + headers = {'content-type': "", + 'Origin': "http://localhost:9001",} + ) + self.assertEqual(f.status_code, 200) + print(i, f.status_code) + print(f.headers) + print(f.text) + + headers_expected = { + 'Content-Type': 'application/json', + 'Vary': 'Origin, Accept-Encoding', + 'Access-Control-Expose-Headers': + ( 'X-RateLimit-Limit, ' + 'X-RateLimit-Remaining, ' + 'X-RateLimit-Reset, ' + 'X-RateLimit-Limit-Period, ' + 'Retry-After, ' + 'Sunset, ' + 'Allow'), + 'Access-Control-Allow-Origin': 'http://localhost:9001', + 'Access-Control-Allow-Credentials': 'true', + 'Allow': 'OPTIONS, GET, POST, PUT, DELETE, PATCH', + 'Content-Length': '167', + 'Content-Encoding': 'gzip'} + + for header in headers_expected.keys(): + self.assertEqual(f.headers[header], + headers_expected[header]) From 8ad545efea168c7857761faf7dfce7d09c2ed4c3 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Wed, 19 Jul 2023 21:12:56 -0400 Subject: [PATCH 29/91] test: fix test_rest_login_RateLimit CI has different char numbers It looks like the json returned has different spacing when pretty printed. Under CI, I get 157 chars under python2 and 161 under python3. On local development, I get 167. The data is the same in all three environments. Change test to load json data structure and compare against a dict that matches the returned data. Also remove encoding type, not critical to test. --- test/test_liveserver.py | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/test/test_liveserver.py b/test/test_liveserver.py index f5b72f13..f4f3e87b 100644 --- a/test/test_liveserver.py +++ b/test/test_liveserver.py @@ -1507,10 +1507,39 @@ def test_rest_login_RateLimit(self): 'Allow'), 'Access-Control-Allow-Origin': 'http://localhost:9001', 'Access-Control-Allow-Credentials': 'true', - 'Allow': 'OPTIONS, GET, POST, PUT, DELETE, PATCH', - 'Content-Length': '167', - 'Content-Encoding': 'gzip'} + 'Allow': 'OPTIONS, GET, POST, PUT, DELETE, PATCH' + } for header in headers_expected.keys(): self.assertEqual(f.headers[header], headers_expected[header]) + + expected_data = { + "status": { + "link": "http://localhost:9001/rest/data/status" + }, + "keyword": { + "link": "http://localhost:9001/rest/data/keyword" + }, + "priority": { + "link": "http://localhost:9001/rest/data/priority" + }, + "user": { + "link": "http://localhost:9001/rest/data/user" + }, + "file": { + "link": "http://localhost:9001/rest/data/file" + }, + "msg": { + "link": "http://localhost:9001/rest/data/msg" + }, + "query": { + "link": "http://localhost:9001/rest/data/query" + }, + "issue": { + "link": "http://localhost:9001/rest/data/issue" + } + } + + json_dict = json.loads(f.text) + self.assertEqual(json_dict['data'], expected_data) From abd416c6520075b4708f45b5beb417f82402b1e9 Mon Sep 17 00:00:00 2001 From: Marcus Preisch Date: Thu, 20 Jul 2023 20:08:28 -0400 Subject: [PATCH 30/91] fix(i18n): issue2551184 - improve i18n handling Apply patch to make sure that the dates tests use the locale files in the deployed tracker and not other roundup files. --- CHANGES.txt | 3 +++ test/test_dates.py | 12 ++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 3c41495a..0dbf10b0 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -19,6 +19,9 @@ Fixed: - issue2551063 - Rest/Xmlrpc interfaces needs failed login protection. Failed API login rate limiting with expiring lockout added. (John Rouillard) +- issue2551184 - improve i18n handling. Patch to test to make sure it + uses the test tracker's locale files and not other locale + files. (Marcus Priesch) Features: diff --git a/test/test_dates.py b/test/test_dates.py index ea38de28..7638c1f6 100644 --- a/test/test_dates.py +++ b/test/test_dates.py @@ -47,10 +47,14 @@ def setUp(self): self.old_gettext_ = i18n.gettext self.old_ngettext_ = i18n.ngettext - i18n.gettext = i18n.get_translation(language='C').gettext - i18n.degettext = i18n.get_translation(language='de').gettext - i18n.ngettext = i18n.get_translation(language='C').ngettext - i18n.dengettext = i18n.get_translation(language='de').ngettext + i18n.gettext = i18n.get_translation( + language='C', tracker_home=".").gettext + i18n.degettext = i18n.get_translation( + language='de', tracker_home=".").gettext + i18n.ngettext = i18n.get_translation( + language='C', tracker_home=".").ngettext + i18n.dengettext = i18n.get_translation( + language='de', tracker_home=".").ngettext def tearDown(self): i18n.gettext = self.old_gettext_ From 25de331914e5590ed9dc0a098885b8e220fd88ab Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sat, 22 Jul 2023 22:05:44 -0400 Subject: [PATCH 31/91] Fix typo in index reference --- doc/reference.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/reference.txt b/doc/reference.txt index 11183771..daf73c45 100644 --- a/doc/reference.txt +++ b/doc/reference.txt @@ -483,7 +483,7 @@ Section **mailgw** parts of the multipart/alternative are ignored. The default is to keep all parts and attach them to the issue. -.. index:: config.ini; sections php +.. index:: config.ini; sections pgp Section **pgp** OpenPGP mail processing options From 4e160544c11967e465018a5121e1fae9e24fc774 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 23 Jul 2023 16:11:23 -0400 Subject: [PATCH 32/91] prevent python\nline 1\nline 2\n The code was mapping < etc back to < and > and confusing the parser as to where the tag really started. It inserpreted the real pre tag as data and inserted a newline. --- test/html_norm.py | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/test/html_norm.py b/test/html_norm.py index 3c186145..980cfdb8 100644 --- a/test/html_norm.py +++ b/test/html_norm.py @@ -40,7 +40,7 @@ class NormalizingHtmlParser(HTMLParser): Note that using this rewrites all attributes parsed by HTMLParser into attr="value" form even though HTMLParser accepts other - attribute specifiction forms. + attribute specification forms. """ debug = False # set to true to enable more verbose output @@ -63,7 +63,7 @@ def handle_starttag(self, tag, attrs): if self.debug: print(" attr:", attr) self.current_normalized_string += ' %s="%s"' % attr - self.current_normalized_string += ">" + self.current_normalized_string += ">\n" if tag == 'pre': self.preserve_data = True @@ -83,26 +83,11 @@ def handle_data(self, data): data = " ".join(data.strip().split()) if data: - self.current_normalized_string += "\n%s" % data + self.current_normalized_string += "%s" % data def handle_comment(self, data): print("Comment :", data) - def handle_entityref(self, name): - c = chr(name2codepoint[name]) - if self.debug: print("Named ent:", c) - - self.current_normalized_string += "%s" % c - - def handle_charref(self, name): - if name.startswith('x'): - c = chr(int(name[1:], 16)) - else: - c = chr(int(name)) - if self.debug: print("Num ent :", c) - - self.current_normalized_string += "%s" % c - def handle_decl(self, data): print("Decl :", data) From c6652fe1301e7abd09466e3263969484eccacb0a Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 23 Jul 2023 16:50:35 -0400 Subject: [PATCH 33/91] Support markdown2 2.4.10, 2.4.8- and exclude 2.4.9 Handle these changes to markdown2 version 2.4.9 broke links like (issue1)[issue1]: raise error if used Version 2.4.10 changed how filtering of schemes is done: adapt to new method Mail url's in markdown are formatted [label](mailto:user@something.com). The markdown format wrapper uses the plain text formatter to turn issue1 and user@something.com into markdown formatted strings to be htmlized by the markdown formatters. However when the plain text formatter saw (mailto:user@something.com) it made it (mailto:). This is broken as the enamil address shouldn't have the angle brackets. By modifying the email pattern to include an optional mailto:, all three markdown formatters do the right thing and I don't end up with href="" in the link. --- CHANGES.txt | 3 ++ roundup/cgi/templating.py | 49 ++++++++++++++++++-- test/test_templating.py | 97 ++++++++++++++++++++++++++++++++++----- 3 files changed, 132 insertions(+), 17 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 0dbf10b0..cf38a9c9 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -22,6 +22,9 @@ Fixed: - issue2551184 - improve i18n handling. Patch to test to make sure it uses the test tracker's locale files and not other locale files. (Marcus Priesch) +- issue2551283 - fail if version 2.4.9 of markdown2 is used, it's + broken. Support version 2.4.10 with its new schema filtering + method and 2.4.8 and earlier. (John Rouillard) Features: diff --git a/roundup/cgi/templating.py b/roundup/cgi/templating.py index aa35de68..ed25bd7e 100644 --- a/roundup/cgi/templating.py +++ b/roundup/cgi/templating.py @@ -61,10 +61,49 @@ def _import_markdown2(): import markdown2 import re - class Markdown(markdown2.Markdown): - # don't allow disabled protocols in links - _safe_protocols = re.compile('(?!' + ':|'.join([ - re.escape(s) for s in _disable_url_schemes]) + # Note: version 2.4.9 does not work with Roundup as it breaks + # [issue1](issue1) formatted links. + + # Versions 2.4.8 and 2.4.10 use different methods to filter + # allowed schemes. 2.4.8 uses a pre-compiled regexp while + # 2.4.10 uses a regexp string that it compiles. + + markdown2_vi = markdown2.__version_info__ + if markdown2_vi > (2, 4, 9): + # Create the filtering regexp. + # Allowed default is same as what hyper_re supports. + + # pathed_schemes are terminated with :// + pathed_schemes = [ 'http', 'https', 'ftp', 'ftps' ] + # non_pathed are terminated with a : + non_pathed_schemes = [ "mailto" ] + + for disabled in _disable_url_schemes: + try: + pathed_schemes.remove(disabled) + except ValueError: # if disabled not in list + pass + try: + non_pathed_schemes.remove(disabled) + except ValueError: + pass + + re_list = [] + for scheme in pathed_schemes: + re_list.append(r'(?:%s)://' % scheme) + for scheme in non_pathed_schemes: + re_list.append(r'(?:%s):' % scheme) + + enabled_schemes = r"|".join(re_list) + class Markdown(markdown2.Markdown): + _safe_protocols = enabled_schemes + elif markdown2_vi == (2, 4, 9): + raise RuntimeError("Unsupported version - markdown2 v2.4.9\n") + else: + class Markdown(markdown2.Markdown): + # don't allow disabled protocols in links + _safe_protocols = re.compile('(?!' + ':|'.join([ + re.escape(s) for s in _disable_url_schemes]) + ':)', re.IGNORECASE) def _extras(config): @@ -1639,7 +1678,7 @@ class StringHTMLProperty(HTMLProperty): (:[\d]{1,5})? # port (/[\w\-$.+!*(),;:@&=?/~\\#%]*)? # path etc. )| - (?P[-+=%/\w\.]+@[\w\.\-]+)| + (?P(?:mailto:)?[-+=%/\w\.]+@[\w\.\-]+)| (?P(?P[A-Za-z_]+)(\s*)(?P\d+)(?P\#[^][\#%^{}"<>\s]+)?) )''', re.X | re.I) protocol_re = re.compile('^(ht|f)tp(s?)://', re.I) diff --git a/test/test_templating.py b/test/test_templating.py index e1a41041..79aac84e 100644 --- a/test/test_templating.py +++ b/test/test_templating.py @@ -11,6 +11,8 @@ import pytest from .pytest_patcher import mark_class +from markdown2 import __version_info__ as md2__version_info__ + if ReStructuredText: skip_rst = lambda func, *args, **kwargs: func else: @@ -774,8 +776,12 @@ def test_string_markdown(self): self.assertEqual(p.markdown().strip(), u2s(u'

A string with <br> embedded \u00df

')) def test_string_markdown_link(self): - p = StringHTMLProperty(self.client, 'test', '1', None, 'test', u2s(u'A link ')) - self.assertEqual(p.markdown().strip(), u2s(u'

A link http://localhost

')) + p = StringHTMLProperty(self.client, 'test', '1', None, 'test', + u2s(u'A link ')) + m = p.markdown().strip() + m = self.mangleMarkdown2(m) + + self.assertEqual( u2s(u'

A link http://localhost

'), m) def test_string_markdown_link_item(self): """ The link formats for the different markdown engines changes. @@ -783,29 +789,78 @@ def test_string_markdown_link_item(self): is different. So most tests check for a substring that indicates success rather than the entire returned string. """ - p = StringHTMLProperty(self.client, 'test', '1', None, 'test', u2s(u'An issue1 link')) + p = StringHTMLProperty(self.client, 'test', '1', None, 'test', + u2s(u'An issue1 link')) self.assertIn( u2s(u'href="issue1"'), p.markdown().strip()) # just verify that plain linking is working self.assertIn( u2s(u'href="issue1"'), p.plain(hyperlink=1)) - p = StringHTMLProperty(self.client, 'test', '1', None, 'test', u2s(u'An [issue1](issue1) link')) + p = StringHTMLProperty(self.client, 'test', '1', None, 'test', + u2s(u'An [issue1](issue1) link')) self.assertIn( u2s(u'href="issue1"'), p.markdown().strip()) # just verify that plain linking is working self.assertIn( u2s(u'href="issue1"'), p.plain(hyperlink=1)) - p = StringHTMLProperty(self.client, 'test', '1', None, 'test', u2s(u'An [issue1](https://example.com/issue1) link')) - self.assertIn( u2s(u'href="https://example.com/issue1"'), p.markdown().strip()) + p = StringHTMLProperty( + self.client, 'test', '1', None, 'test', + u2s(u'An [issue1](https://example.com/issue1) link')) + self.assertIn( u2s(u'href="https://example.com/issue1"'), + p.markdown().strip()) + + p = StringHTMLProperty(self.client, 'test', '1', None, 'test', + u2s(u'An [issu1](#example) link')) + self.assertIn( u2s(u'href="#example"'), p.markdown().strip()) + + p = StringHTMLProperty(self.client, 'test', '1', None, 'test', + u2s(u'An [issu1](/example) link')) + self.assertIn( u2s(u'href="/example"'), p.markdown().strip()) + + p = StringHTMLProperty(self.client, 'test', '1', None, 'test', + u2s(u'An [issu1](./example) link')) + self.assertIn( u2s(u'href="./example"'), p.markdown().strip()) - p = StringHTMLProperty(self.client, 'test', '1', None, 'test', u2s(u'An [issue1] (https://example.com/issue1) link')) + p = StringHTMLProperty(self.client, 'test', '1', None, 'test', + u2s(u'An [issu1](../example) link')) + self.assertIn( u2s(u'href="../example"'), p.markdown().strip()) + + p = StringHTMLProperty( + self.client, 'test', '1', None, 'test', + u2s(u'A [wuarchive_ftp](ftp://www.wustl.gov/file) link')) + self.assertIn( u2s(u'href="ftp://www.wustl.gov/file"'), + p.markdown().strip()) + + p = StringHTMLProperty( + self.client, 'test', '1', None, 'test', + u2s(u'An [issue1] (https://example.com/issue1) link')) self.assertIn( u2s(u'href="issue1"'), p.markdown().strip()) if type(self) == MistuneTestCase: # mistune makes the https url into a real link - self.assertIn( u2s(u'href="https://example.com/issue1"'), p.markdown().strip()) + self.assertIn( u2s(u'href="https://example.com/issue1"'), + p.markdown().strip()) else: # the other two engines leave the parenthesized url as is. - self.assertIn( u2s(u' (https://example.com/issue1) link'), p.markdown().strip()) + self.assertIn( u2s(u' (https://example.com/issue1) link'), + p.markdown().strip()) - def test_string_markdown_link(self): + p = StringHTMLProperty(self.client, 'test', '1', None, 'test', + u2s(u'An [issu1](.../example) link')) + if (isinstance(self, Markdown2TestCase) and + md2__version_info__ > (2, 4, 9)): + # markdown2 > 2.4.9 handles this differently + self.assertIn( u2s(u'href="#"'), p.markdown().strip()) + else: + self.assertIn( u2s(u'href=".../example"'), p.markdown().strip()) + + p = StringHTMLProperty(self.client, 'test', '1', None, 'test', + u2s(u'A [phone](tel:0016175555555) link')) + if (isinstance(self, Markdown2TestCase) and + md2__version_info__ > (2, 4, 9)): + self.assertIn(u2s(u'href="#"'), p.markdown().strip()) + else: + self.assertIn( u2s(u'href="tel:0016175555555"'), + p.markdown().strip()) + + def test_string_email_markdown_link(self): # markdown2 and markdown escape the email address try: from html import unescape as html_unescape @@ -813,12 +868,30 @@ def test_string_markdown_link(self): from HTMLParser import HTMLParser html_unescape = HTMLParser().unescape - p = StringHTMLProperty(self.client, 'test', '1', None, 'test', u2s(u'A link ')) + p = StringHTMLProperty(self.client, 'test', '1', None, 'test', + u2s(u'A link ')) m = html_unescape(p.markdown().strip()) m = self.mangleMarkdown2(m) self.assertEqual(m, u2s(u'

A link cmeerw@example.com

')) + p = StringHTMLProperty( + self.client, 'test', '1', None, 'test', + u2s(u'An bare email baduser@daemons.com link')) + m = self.mangleMarkdown2(html_unescape(p.markdown().strip())) + self.assertIn( u2s(u'href="mailto:baduser@daemons.com"'), + m) + + p = StringHTMLProperty( + self.client, 'test', '1', None, 'test', + u2s(u'An [email_url](mailto:baduser@daemons.com) link')) + m = self.mangleMarkdown2(html_unescape(p.markdown().strip())) + + if isinstance(self, MistuneTestCase): + self.assertIn('email_url', m) + else: + self.assertIn('email_url', m) + def test_string_markdown_javascript_link(self): # make sure we don't get a "javascript:" link p = StringHTMLProperty(self.client, 'test', '1', None, 'test', u2s(u'')) @@ -866,7 +939,7 @@ def test_string_markdown_code_block_attribute(self): if type(self) == MistuneTestCase: self.assertEqual(m, parser.normalize('

embedded code block <pre>

\n
line 1\nline 2\n
\n

new </pre> paragraph

')) elif type(self) == MarkdownTestCase: - self.assertEqual(m, parser.normalize('

embedded code block <pre>

\n
line 1\nline 2\n
\n

new </pre> paragraph

')) + self.assertEqual(m.replace('class="python"','class="language-python"'), parser.normalize('

embedded code block <pre>

\n
line 1\nline 2\n
\n

new </pre> paragraph

')) else: expected_result = parser.normalize('

embedded code block <pre>

\n
line 1\nline 2\n
\n

new </pre> paragraph

') self.assertEqual(m, expected_result) From f94862e731d7da26037372dc0f2a3687f766745c Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 23 Jul 2023 16:55:16 -0400 Subject: [PATCH 34/91] label mistune as python3 only support. Earlier mistunes that support python2 have a different set of exports/symbols. --- doc/installation.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/installation.txt b/doc/installation.txt index a55f7011..c8f3f344 100644 --- a/doc/installation.txt +++ b/doc/installation.txt @@ -258,7 +258,7 @@ docutils markdown, markdown2 or mistune To use markdown rendering you need to have the markdown_, markdown2_ (2.4.9 known to be broken, 2.3.3 known to work), - or mistune_ (v0.8.4 tested) package installed. + or mistune_ (v0.8.4 tested; python3 only) package installed. zstd, brotli To have roundup compress the returned data using one of these From 08433d7f0ac3630ce64a2e0c4ecb8803a9d09ac9 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 23 Jul 2023 20:51:29 -0400 Subject: [PATCH 35/91] Add new test new email body unchanged is no. verify that sig is stipped. --- test/test_mailgw.py | 62 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/test/test_mailgw.py b/test/test_mailgw.py index 0857648a..4774a327 100644 --- a/test/test_mailgw.py +++ b/test/test_mailgw.py @@ -3333,6 +3333,68 @@ def testEmailQuotingNewIsFollowup(self): self.assertEqual(content, '''This is a followup''') self.assertEqual(summary, '''This is a followup''') + fourthquotingtest = '''Content-Type: text/plain; + charset="iso-8859-1" +From: richard +To: issue_tracker@your.tracker.email.domain.example +Message-Id: +In-Reply-To: +Subject: Re: [issue1] Testing... + +Blah blah wrote: +> Blah bklaskdfj sdf asdf jlaskdf skj sdkfjl asdf +> skdjlkjsdfalsdkfjasdlfkj dlfksdfalksd fj +> + +This is a followup + +> mumble mumble but +> more mumble mumble + +I see your mubble and raise you a mumble. + +But I also have a full house which beats your +>mumbler + +so I win. +''' + def testEmailBodyUnchangedNewIsNo(self): + """verify that only the signature is stripped""" + + mysig = "\n--\nmy sig\n\n" + self.instance.config.EMAIL_LEAVE_BODY_UNCHANGED = 'no' + # create the message, remove the prefix from subject + testmessage=self.fourthquotingtest.replace(" Re: [issue1]", "") + mysig + print(testmessage) + print("\n======\n") + nodeid = self._handle_mail(testmessage) + + msgs = self.db.issue.get(nodeid, 'messages') + # validate content and summary + content = self.db.msg.get(msgs[0], 'content') + print(content) + self.assertIn(content, '''Blah blah wrote: +> Blah bklaskdfj sdf asdf jlaskdf skj sdkfjl asdf +> skdjlkjsdfalsdkfjasdlfkj dlfksdfalksd fj +> + +This is a followup + +> mumble mumble but +> more mumble mumble + +I see your mubble and raise you a mumble. + +But I also have a full house which beats your +>mumbler + +so I win. +''' +) + + summary = self.db.msg.get(msgs[0], 'summary') + self.assertEqual(summary, '''This is a followup''') + def testEmailBodyUnchangedNewIsYes(self): mysig = "--\nmy sig\n\n" self.instance.config.EMAIL_LEAVE_BODY_UNCHANGED = 'yes' From 4bfaf893e5be9c8169ce8220a3b40b6644b73fdd Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 23 Jul 2023 20:54:16 -0400 Subject: [PATCH 36/91] Update changelog for last markdown2 support change. --- CHANGES.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index cf38a9c9..a0d10366 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -22,9 +22,9 @@ Fixed: - issue2551184 - improve i18n handling. Patch to test to make sure it uses the test tracker's locale files and not other locale files. (Marcus Priesch) -- issue2551283 - fail if version 2.4.9 of markdown2 is used, it's - broken. Support version 2.4.10 with its new schema filtering - method and 2.4.8 and earlier. (John Rouillard) +- issue2551283 - fail if version 2.4.9 of markdown2 is used, it broke + [issue1](issue1) style links. Support markdown2 2.4.8 and earlier + and 2.4.10 with its new schema filtering method. (John Rouillard) Features: From f385ea792c72797e5bee7ddff152ba2d6ee39046 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 23 Jul 2023 22:04:27 -0400 Subject: [PATCH 37/91] Fix tests The testEmailBodyUnchangedNewIsYes was using a signature (sig) that didn't have a blank line before it. Therefore the sig wasn't recognised as a message section and wasn't a candidate to be removed even if EMAIL_LEAVE_BODY_UNCHANGED was set to no. Appended a newline before the signature block so testing with self.instance.config.EMAIL_LEAVE_BODY_UNCHANGED = 'no' will cause test to fail and setting it to yes (the test condition) will cause test to pass by keeping signature. Make same sig change to testEmailBodyUnchangedFollowupIsYes for same reason. Add new tests for other permutations: testEmailBodyUnchangedNewIsNew - keep sig testEmailBodyUnchangedFollowupIsNew - remove sig testEmailBodyUnchangedFollowupIsNo - remove sig --- test/test_mailgw.py | 69 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 2 deletions(-) diff --git a/test/test_mailgw.py b/test/test_mailgw.py index 4774a327..d3b4d533 100644 --- a/test/test_mailgw.py +++ b/test/test_mailgw.py @@ -3333,6 +3333,30 @@ def testEmailQuotingNewIsFollowup(self): self.assertEqual(content, '''This is a followup''') self.assertEqual(summary, '''This is a followup''') + def testEmailBodyUnchangedNewIsNew(self): + mysig = "\n--\nmy sig\n\n" + self.instance.config.EMAIL_LEAVE_BODY_UNCHANGED = 'new' + # create the message, remove the prefix from subject + testmessage=self.firstquotingtest.replace(" Re: [issue1]", "") + mysig + nodeid = self._handle_mail(testmessage) + + msgs = self.db.issue.get(nodeid, 'messages') + # validate content and summary + content = self.db.msg.get(msgs[0], 'content') + self.assertEqual(content, '''Blah blah wrote: +> Blah bklaskdfj sdf asdf jlaskdf skj sdkfjl asdf +> skdjlkjsdfalsdkfjasdlfkj dlfksdfalksd fj +> + +This is a followup\n''' + mysig[:-2]) + # the :-2 requrement to strip the trailing newlines is probably a bug + # somewhere mailgw has right content maybe trailing \n are stripped by + # msg or something. + + summary = self.db.msg.get(msgs[0], 'summary') + self.assertEqual(summary, '''This is a followup''') + + fourthquotingtest = '''Content-Type: text/plain; charset="iso-8859-1" From: richard @@ -3396,7 +3420,7 @@ def testEmailBodyUnchangedNewIsNo(self): self.assertEqual(summary, '''This is a followup''') def testEmailBodyUnchangedNewIsYes(self): - mysig = "--\nmy sig\n\n" + mysig = "\n--\nmy sig\n\n" self.instance.config.EMAIL_LEAVE_BODY_UNCHANGED = 'yes' # create the message, remove the prefix from subject testmessage=self.firstquotingtest.replace(" Re: [issue1]", "") + mysig @@ -3418,8 +3442,49 @@ def testEmailBodyUnchangedNewIsYes(self): summary = self.db.msg.get(msgs[0], 'summary') self.assertEqual(summary, '''This is a followup''') + def testEmailBodyUnchangedFollowupIsNew(self): + mysig = "\n--\nmy sig\n\n" + self.instance.config.EMAIL_LEAVE_BODY_UNCHANGED = 'new' + + # create issue1 that we can followup on + self.doNewIssue() + testmessage=self.firstquotingtest + mysig + nodeid = self._handle_mail(testmessage) + msgs = self.db.issue.get(nodeid, 'messages') + # validate content and summary + content = self.db.msg.get(msgs[1], 'content') + self.assertEqual(content, '''Blah blah wrote: +> Blah bklaskdfj sdf asdf jlaskdf skj sdkfjl asdf +> skdjlkjsdfalsdkfjasdlfkj dlfksdfalksd fj +> + +This is a followup''') + + summary = self.db.msg.get(msgs[1], 'summary') + self.assertEqual(summary, '''This is a followup''') + + def testEmailBodyUnchangedFollowupIsNo(self): + mysig = "\n--\nmy sig\n\n" + self.instance.config.EMAIL_LEAVE_BODY_UNCHANGED = 'No' + # create the message, remove the prefix from subject + self.doNewIssue() + testmessage=self.firstquotingtest + mysig + nodeid = self._handle_mail(testmessage) + + msgs = self.db.issue.get(nodeid, 'messages') + # validate content and summary + content = self.db.msg.get(msgs[1], 'content') + self.assertEqual(content, '''Blah blah wrote: +> Blah bklaskdfj sdf asdf jlaskdf skj sdkfjl asdf +> skdjlkjsdfalsdkfjasdlfkj dlfksdfalksd fj +> + +This is a followup''') + summary = self.db.msg.get(msgs[1], 'summary') + self.assertEqual(summary, '''This is a followup''') + def testEmailBodyUnchangedFollowupIsYes(self): - mysig = "--\nmy sig\n\n" + mysig = "\n--\nmy sig\n\n" self.instance.config.EMAIL_LEAVE_BODY_UNCHANGED = 'yes' # create issue1 that we can followup on From 039d90ced3593800ba95e62a5a300a0b903fe9db Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 23 Jul 2023 23:18:26 -0400 Subject: [PATCH 38/91] flake8 fix indents. --- roundup/admin.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/roundup/admin.py b/roundup/admin.py index 1f253867..2d551564 100644 --- a/roundup/admin.py +++ b/roundup/admin.py @@ -1614,9 +1614,9 @@ def do_reindex(self, args, desre=designator_re, desrng=designator_rng): cl.index(str(item)) except IndexError: print(_('no such item "%(class)s%(id)s"') % { - 'class': r.group(1), - 'id': item}) - + 'class': r.group(1), + 'id': item}) + else: cl = self.get_class(arg) # Bad class raises UsageError self.db.reindex(arg, show_progress=True) @@ -2040,7 +2040,7 @@ def run_command(self, args): ret = function(args[1:]) return ret except UsageError as message: # noqa F841 - return self.usageError_feedback(message, function) + return self.usageError_feedback(message, function) # make sure we have a tracker_home while not self.tracker_home: @@ -2054,12 +2054,12 @@ def run_command(self, args): try: return self.do_initialise(self.tracker_home, args) except UsageError as message: # noqa: F841 - return self.usageError_feedback(message, function) + return self.usageError_feedback(message, function) elif command == 'install': try: return self.do_install(self.tracker_home, args) except UsageError as message: # noqa: F841 - return self.usageError_feedback(message, function) + return self.usageError_feedback(message, function) # get the tracker try: From 5f56a30079ec669913457964c59ca857aac3d424 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 23 Jul 2023 23:19:03 -0400 Subject: [PATCH 39/91] flake8: add extra blank lines --- roundup/configuration.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roundup/configuration.py b/roundup/configuration.py index 9957bb06..2ec27e6a 100644 --- a/roundup/configuration.py +++ b/roundup/configuration.py @@ -652,6 +652,7 @@ def str2value(self, value): except ValueError: raise OptionValueError(self, value, "Integer number required") + class IntegerNumberGtZeroOption(Option): """Integer numbers greater than zero.""" @@ -668,6 +669,7 @@ def str2value(self, value): except ValueError: raise OptionValueError(self, value, "Integer number required") + class OctalNumberOption(Option): """Octal Integer numbers""" From 837f16e164f421584dcc002aaa6cc3622a38cc05 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 23 Jul 2023 23:21:57 -0400 Subject: [PATCH 40/91] flake8: correct continutation line indent --- roundup/password.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roundup/password.py b/roundup/password.py index 9be96dd7..223f9a0d 100644 --- a/roundup/password.py +++ b/roundup/password.py @@ -398,7 +398,7 @@ class Password(JournalPassword): deprecated_schemes = ["SSHA", "SHA", "MD5", "crypt", "plaintext"] experimental_schemes = ["PBKDF2S5"] known_schemes = ["PBKDF2"] + experimental_schemes + \ - deprecated_schemes + deprecated_schemes def __init__(self, plaintext=None, scheme=None, encrypted=None, strict=False, config=None): From 81d41027dfbbdb4152f522f44d19062b11cf75e5 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 23 Jul 2023 23:23:17 -0400 Subject: [PATCH 41/91] flake8: add space between raise and ( --- roundup/rest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roundup/rest.py b/roundup/rest.py index 6fe62e5f..3827e273 100644 --- a/roundup/rest.py +++ b/roundup/rest.py @@ -567,7 +567,7 @@ def transitive_props(self, class_name, props): 'Multilink Traversal not allowed: %s' % p) # Now we have the classname in cn and the prop name in pn. if not self.db.security.hasPermission('View', uid, cn, pn): - raise(Unauthorised + raise (Unauthorised ('User does not have permission on "%s.%s"' % (cn, pn))) try: From 2be6460603cbaee9ebfb58a5a81673f914960c25 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 23 Jul 2023 23:23:38 -0400 Subject: [PATCH 42/91] flake8: remove extra blank line --- roundup/anypy/cmp_.py | 1 - 1 file changed, 1 deletion(-) diff --git a/roundup/anypy/cmp_.py b/roundup/anypy/cmp_.py index 56740bbf..639d951c 100644 --- a/roundup/anypy/cmp_.py +++ b/roundup/anypy/cmp_.py @@ -100,7 +100,6 @@ def _test(): assert not Comp(1) < Comp(0) assert not Comp(0) > Comp(0) - assert Comp(0) <= Comp(1) assert Comp(1) >= Comp(0) assert not Comp(1) <= Comp(0) From 9873a5ea679909a7e51ab8e7e0bbccc9a97365d8 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 23 Jul 2023 23:25:37 -0400 Subject: [PATCH 43/91] flake8: add space between return, del and ( --- roundup/cgi/client.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/roundup/cgi/client.py b/roundup/cgi/client.py index 1d618fb3..37565924 100644 --- a/roundup/cgi/client.py +++ b/roundup/cgi/client.py @@ -531,9 +531,9 @@ def main(self): # strip HTTP_PROXY issue2550925 in case # PROXY header is set. if 'HTTP_PROXY' in self.env: - del(self.env['HTTP_PROXY']) + del (self.env['HTTP_PROXY']) if 'HTTP_PROXY' in os.environ: - del(os.environ['HTTP_PROXY']) + del (os.environ['HTTP_PROXY']) xmlrpc_enabled = self.instance.config.WEB_ENABLE_XMLRPC rest_enabled = self.instance.config.WEB_ENABLE_REST @@ -1127,7 +1127,7 @@ def authenticate_bearer_token(self, challenge): self.make_user_anonymous() raise LoginError(str(err)) - return(token) + return (token) def determine_user(self, is_api=False): """Determine who the user is""" @@ -2762,7 +2762,7 @@ def setHeader(self, header, value): """ if value is None: try: - del(self.additional_headers[header]) + del (self.additional_headers[header]) except KeyError: pass else: @@ -2783,7 +2783,7 @@ def header(self, headers=None, response=None): headers['Content-Type'] = 'text/html; charset=utf-8' if response in [204, 304]: # has no body so no content-type - del(headers['Content-Type']) + del (headers['Content-Type']) headers = list(headers.items()) From 5511d1a95bdf1b21c9e0505115dd5715b19114b4 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 23 Jul 2023 23:26:03 -0400 Subject: [PATCH 44/91] flake8: whitespace fixes --- roundup/cgi/templating.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/roundup/cgi/templating.py b/roundup/cgi/templating.py index ed25bd7e..788ac03a 100644 --- a/roundup/cgi/templating.py +++ b/roundup/cgi/templating.py @@ -69,14 +69,14 @@ def _import_markdown2(): # 2.4.10 uses a regexp string that it compiles. markdown2_vi = markdown2.__version_info__ - if markdown2_vi > (2, 4, 9): + if markdown2_vi > (2, 4, 9): # Create the filtering regexp. # Allowed default is same as what hyper_re supports. # pathed_schemes are terminated with :// - pathed_schemes = [ 'http', 'https', 'ftp', 'ftps' ] + pathed_schemes = ['http', 'https', 'ftp', 'ftps'] # non_pathed are terminated with a : - non_pathed_schemes = [ "mailto" ] + non_pathed_schemes = ["mailto"] for disabled in _disable_url_schemes: try: @@ -95,6 +95,7 @@ def _import_markdown2(): re_list.append(r'(?:%s):' % scheme) enabled_schemes = r"|".join(re_list) + class Markdown(markdown2.Markdown): _safe_protocols = enabled_schemes elif markdown2_vi == (2, 4, 9): @@ -1895,7 +1896,7 @@ def rst(self, hyperlink=1): # causing a KeyError. So see if we removed it (and entered # it into valid_schemes). If we didn't raise KeyError. try: - del(schemes[sch]) + del (schemes[sch]) self.valid_schemes[sch] = True except KeyError: if sch in self.valid_schemes: From 0410aac1089ab92b9b2e6b7b4b7136c0f75530e7 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 23 Jul 2023 23:27:46 -0400 Subject: [PATCH 45/91] flake8: disable warning on import of version_check not used also add comment. import version_check will exit if it fails. --- roundup/scripts/roundup_admin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roundup/scripts/roundup_admin.py b/roundup/scripts/roundup_admin.py index 72a4b4f7..47473712 100644 --- a/roundup/scripts/roundup_admin.py +++ b/roundup/scripts/roundup_admin.py @@ -33,8 +33,8 @@ # --/ -# python version check -from roundup import version_check +# python version check - import exits if version invalid +from roundup import version_check # noqa: F401 # import the admin tool guts and make it go from roundup.admin import AdminTool From 401e179a38cad0252a7fb8fa9799f9e90e86e7e3 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 23 Jul 2023 23:28:12 -0400 Subject: [PATCH 46/91] flake8: add space between del and ( --- roundup/scripts/roundup_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roundup/scripts/roundup_server.py b/roundup/scripts/roundup_server.py index cfb81205..7f034612 100644 --- a/roundup/scripts/roundup_server.py +++ b/roundup/scripts/roundup_server.py @@ -451,7 +451,7 @@ def inner_run_cgi(self): env[h] = self.headers.get(h, None) # if header is MISSING if env[h] is None: - del(env[h]) + del (env[h]) env['SCRIPT_NAME'] = '' env['SERVER_NAME'] = self.server.server_name env['SERVER_PORT'] = str(self.server.server_port) From 6d309325757fa71f1d39e2747503944e68c80fd2 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 23 Jul 2023 23:40:12 -0400 Subject: [PATCH 47/91] flake8: rename loop variable in 'for sendto in sendto:' Flake8 reported 'B020 Found for loop that reassigns the iterable it is iterating with each iterable value.' Renamed loop variable to to_addr. There is a similar construct with a loop over bcc_sendto with a 'bcc' loop variable. So I assume the loop varaible can be chnaged w/o issue. Codecov shows all the affected lines are being tested and the tests I ran with testmon that should cover that code all passed. We shall see if a full CI run passes. --- CHANGES.txt | 2 ++ roundup/roundupdb.py | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index a0d10366..dd71acd6 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -25,6 +25,8 @@ Fixed: - issue2551283 - fail if version 2.4.9 of markdown2 is used, it broke [issue1](issue1) style links. Support markdown2 2.4.8 and earlier and 2.4.10 with its new schema filtering method. (John Rouillard) +- multiple flake8 fixes (John Rouillard) +- rename loop variable in 'for sendto in sendto:' (John Rouillard) Features: diff --git a/roundup/roundupdb.py b/roundup/roundupdb.py index 1b83ff5c..47a1b3de 100644 --- a/roundup/roundupdb.py +++ b/roundup/roundupdb.py @@ -577,7 +577,7 @@ def send_message(self, issueid, msgid, note, sendto, from_address=None, # can't fiddle the recipients in the message ... worth testing # and/or fixing some day first = True - for sendto in sendto: + for to_addr in sendto: # create the message mailer = Mailer(self.db.config) @@ -721,18 +721,18 @@ def send_message(self, issueid, msgid, note, sendto, from_address=None, message.set_payload(body, message.get_charset()) if crypt: - send_msg = self.encrypt_to(message, sendto) + send_msg = self.encrypt_to(message, to_addr) else: send_msg = message - mailer.set_message_attributes(send_msg, sendto, subject, author) + mailer.set_message_attributes(send_msg, to_addr, subject, author) if crypt: send_msg['Message-Id'] = message['Message-Id'] send_msg['Reply-To'] = message['Reply-To'] if message.get('In-Reply-To'): send_msg['In-Reply-To'] = message['In-Reply-To'] - if sendto: - mailer.smtp_send(sendto, send_msg.as_string()) + if to_addr: + mailer.smtp_send(to_addr, send_msg.as_string()) if first: if crypt: # send individual bcc mails, otherwise receivers can From 1df93cb2d78d20a2f261c249d79d482b417d4e5a Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 24 Jul 2023 00:13:31 -0400 Subject: [PATCH 48/91] Handle error id markdown2 not defined. --- test/test_templating.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/test_templating.py b/test/test_templating.py index 79aac84e..5043d53f 100644 --- a/test/test_templating.py +++ b/test/test_templating.py @@ -11,7 +11,10 @@ import pytest from .pytest_patcher import mark_class -from markdown2 import __version_info__ as md2__version_info__ +try: + from markdown2 import __version_info__ as md2__version_info__ +except ImportError: + md2__version_info__ = (0,0,0) if ReStructuredText: skip_rst = lambda func, *args, **kwargs: func From 6b27b285c4a6f7b768b19869f0b88d847ec9d90f Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 24 Jul 2023 00:35:50 -0400 Subject: [PATCH 49/91] only run TestPostgresWsgiServer if ostgresl is available. --- test/test_liveserver.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/test_liveserver.py b/test/test_liveserver.py index f4f3e87b..0d7ae02c 100644 --- a/test/test_liveserver.py +++ b/test/test_liveserver.py @@ -8,6 +8,7 @@ from .wsgi_liveserver import LiveServerTestCase from . import db_test_base from time import sleep +from .test_postresql import skip_postgresql from wsgiref.validate import validator @@ -1227,6 +1228,7 @@ def create_app(self): # doesn't support the max bytes to read argument. return RequestDispatcher(self.dirname, feature_flags=ff) +@skip_postgresql class TestPostgresWsgiServer(BaseTestCases, WsgiSetup): """Class to run all test in BaseTestCases with the cache_tracker feature flag enabled when starting the wsgi server From ac47d43958f41b9457f082da23f94d2405589056 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 24 Jul 2023 00:37:36 -0400 Subject: [PATCH 50/91] fix typo --- test/test_liveserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_liveserver.py b/test/test_liveserver.py index 0d7ae02c..7493800e 100644 --- a/test/test_liveserver.py +++ b/test/test_liveserver.py @@ -8,7 +8,7 @@ from .wsgi_liveserver import LiveServerTestCase from . import db_test_base from time import sleep -from .test_postresql import skip_postgresql +from .test_postgresql import skip_postgresql from wsgiref.validate import validator From c730f967ba011be30da558be35251a82c9f5b49a Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 24 Jul 2023 16:55:22 -0400 Subject: [PATCH 51/91] Convert cgi.escape to use html_escape from roundup.anypy.html Update for vendoring of cgi. --- frontends/roundup.cgi | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frontends/roundup.cgi b/frontends/roundup.cgi index 11188981..eed7bf7a 100755 --- a/frontends/roundup.cgi +++ b/frontends/roundup.cgi @@ -20,6 +20,7 @@ from __future__ import print_function from roundup import version_check from roundup.i18n import _ +from roundup.anypy.html import html_escape from roundup.anypy.strings import s2b, StringIO import sys, time @@ -181,7 +182,7 @@ def main(out, err): request.send_response(404) request.send_header('Content-Type', 'text/html') request.end_headers() - out.write(s2b('Not found: %s'%cgi.escape(client.path))) + out.write(s2b('Not found: %s'%html_escape(client.path))) else: from roundup.anypy import urllib_ @@ -196,7 +197,7 @@ def main(out, err): w(s2b(_('
  • %(tracker_name)s\n')%{ 'tracker_url': os.environ['SCRIPT_NAME']+'/'+ urllib_.quote(tracker), - 'tracker_name': cgi.escape(tracker)})) + 'tracker_name': html_escape(tracker)})) w(s2b(_(''))) # From dcf055bcf934705da22f00c69e308dc3be6e4855 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 24 Jul 2023 17:16:36 -0400 Subject: [PATCH 52/91] docs: document dependencies required for testing --- doc/installation.txt | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/doc/installation.txt b/doc/installation.txt index c8f3f344..26a68058 100644 --- a/doc/installation.txt +++ b/doc/installation.txt @@ -2213,10 +2213,13 @@ Problems? Testing your Python... .. note:: The ``run_tests.py`` script is not packaged in Roundup's source - distribution anymore. You should install pytest using your - distributions package manger or using pip/pip2/pip3 to install - pytest for your python version. See the `administration guide`_ - for details. + distribution anymore. You should install: + * pytest, + * requests, and + * mock + using your distributions package manger or using pip/pip2/pip3 to + install pytest for your python version. See the `administration + guide`_ for details. Remember to have a database user 'rounduptest' prepared (with password 'rounduptest'). This user From e4204381600aeb12039652ea01c1712a7b2e2e48 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 24 Jul 2023 17:19:39 -0400 Subject: [PATCH 53/91] docs: document dependencies required for testing fix formating. --- doc/installation.txt | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/doc/installation.txt b/doc/installation.txt index 26a68058..fba7c5e5 100644 --- a/doc/installation.txt +++ b/doc/installation.txt @@ -2214,12 +2214,14 @@ Problems? Testing your Python... .. note:: The ``run_tests.py`` script is not packaged in Roundup's source distribution anymore. You should install: - * pytest, - * requests, and - * mock + + * pytest, + * requests, and + * mock + using your distributions package manger or using pip/pip2/pip3 to - install pytest for your python version. See the `administration - guide`_ for details. + install pytest etc. for your Python version. See the + `administration guide`_ for details. Remember to have a database user 'rounduptest' prepared (with password 'rounduptest'). This user From f37b3464c78b1d35150f5887c7b27dca2ef9b8df Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 24 Jul 2023 17:49:58 -0400 Subject: [PATCH 54/91] fix: issue2551193 - Fix roundup for removal of cgi and cgitb ... standard python modules (and FieldStorage/MiniFieldStorage). Vendor cgi.py and modify imports. Details: roundup/anypy/cgi_.py import that accesses a working cgi.py. All imports dealing with cgi now use cgi_. roundup/anypy/vendored/cgi.py vendored version 2.6 of cgi.py from: https://pypi.org/project/legacy-cgi/ CHANGES.txt change note added COPYING.txt added license for cgi.py doc/rest.txt change example to use cgi_ doc/upgrading.txt doc removal and how to rework local code using cgi.py. frontends/roundup.cgi remove unneeded cgi import roundup/cgi/actions.py roundup/cgi/apache.py roundup/cgi/client.py roundup/cgi/templating.py roundup/cgi/TAL/TALGenerator.py test/db_test_base.py test/rest_common.py test/test_cgi.py remove import cgi and replace with from roundup.anypy.cgi_ import cgi test/test_actions.py test/test_templating.py modify import to get *FieldStorage test/test_admin.py test/test_hyperdbvals.py test/test_xmlrpc.py remove unneeded cgi import --- CHANGES.txt | 7 + COPYING.txt | 56 ++ doc/rest.txt | 2 +- doc/upgrading.txt | 15 + frontends/roundup.cgi | 2 +- roundup/anypy/cgi_.py | 8 + roundup/anypy/vendored/cgi.py | 1009 +++++++++++++++++++++++++++++++ roundup/cgi/TAL/TALGenerator.py | 2 +- roundup/cgi/actions.py | 2 +- roundup/cgi/apache.py | 2 +- roundup/cgi/client.py | 2 +- roundup/cgi/templating.py | 2 +- test/db_test_base.py | 3 +- test/rest_common.py | 2 +- test/test_actions.py | 2 +- test/test_admin.py | 2 +- test/test_cgi.py | 3 +- test/test_hyperdbvals.py | 2 +- test/test_templating.py | 2 +- test/test_xmlrpc.py | 2 +- 20 files changed, 1112 insertions(+), 15 deletions(-) create mode 100644 roundup/anypy/cgi_.py create mode 100755 roundup/anypy/vendored/cgi.py diff --git a/CHANGES.txt b/CHANGES.txt index dd71acd6..651a6d93 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -27,6 +27,13 @@ Fixed: and 2.4.10 with its new schema filtering method. (John Rouillard) - multiple flake8 fixes (John Rouillard) - rename loop variable in 'for sendto in sendto:' (John Rouillard) +- issue2551193 - Fix roundup for removal of cgi and cgitb standard + python modules (and FieldStorage/MiniFieldStorage). Replaced imports + from cgi to use roundup.anypy.cgi_ which will load the system cgi + unless it is missing. Then it will load roundup.anypy.vendored.cgi + and make *FieldStroage symbols available. Roundp uses its own + cgitb.py and not the system cgitb.py. It looks like it's the + precursor to the system cgitb.py. (John Rouillard) Features: diff --git a/COPYING.txt b/COPYING.txt index b5a84f2c..4877758f 100644 --- a/COPYING.txt +++ b/COPYING.txt @@ -113,3 +113,59 @@ accompanying credits file. Note link for http://www.zope.com/Marks is dead. + +Vendored cgi.py module +---------------------- + +This module is licensed under the Python Software Foundation License +Version 2 as it was extracted from the 3.12 Python distribution. + +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 +-------------------------------------------- + +1. This LICENSE AGREEMENT is between the Python Software Foundation + ("PSF"), and the Individual or Organization ("Licensee") accessing + and otherwise using this software ("Python") in source or binary + form and its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF + hereby grants Licensee a nonexclusive, royalty-free, world-wide + license to reproduce, analyze, test, perform and/or display + publicly, prepare derivative works, distribute, and otherwise use + Python alone or in any derivative version, provided, however, that + PSF's License Agreement and PSF's notice of copyright, i.e., + "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, + 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, + 2020, 2021, 2022 Python Software Foundation; All Rights Reserved" + are retained in Python alone or in any derivative version prepared + by Licensee. + +3. In the event Licensee prepares a derivative work that is based on + or incorporates Python or any part thereof, and wants to make the + derivative work available to others as provided herein, then + Licensee hereby agrees to include in any such work a brief summary + of the changes made to Python. + +4. PSF is making Python available to Licensee on an "AS IS" basis. + PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY + WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY + REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY + PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY + THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON + FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A + RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR + ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material + breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any + relationship of agency, partnership, or joint venture between PSF + and Licensee. This License Agreement does not grant permission to + use PSF trademarks or trade name in a trademark sense to endorse or + promote products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Python, Licensee agrees + to be bound by the terms and conditions of this License Agreement. diff --git a/doc/rest.txt b/doc/rest.txt index 1dae4507..9faad0a6 100644 --- a/doc/rest.txt +++ b/doc/rest.txt @@ -1791,7 +1791,7 @@ section this code adds a new endpoint `/data/@permission/Developer` that returns a list of users with the developer role:: from roundup.rest import Routing, RestfulInstance - from cgi import MiniFieldStorage + from roundup.anypy.cgi_ import MiniFieldStorage class RestfulInstance(object): diff --git a/doc/upgrading.txt b/doc/upgrading.txt index 9c195d62..31e04632 100644 --- a/doc/upgrading.txt +++ b/doc/upgrading.txt @@ -126,6 +126,21 @@ details. .. _`information on configuring the API rate limits`: rest.html#rate-limiting-api-failed-logins +Removal of cgi.py from Python (info) +------------------------------------ + +The ``cgi.py`` module will be `removed starting with Python 3.13 +`_. Roundup now `vendors a copy +`_ of ``cgi.py`` and makes it +and its storage objects available by importing from:: + + from roundup.anypy.cgi_ import cgi + from roundup.anypy.cgi_ import FieldStorage, MiniFieldStorage + +It is unlikey that you will care unless you have done some expert +level Roundup customization. If you have, use one of the imports above +if you plan on running on Python 3.13 (expected in 2024) or newer. + .. index:: Upgrading; 2.2.0 to 2.3.0 Migrating from 2.2.0 to 2.3.0 diff --git a/frontends/roundup.cgi b/frontends/roundup.cgi index eed7bf7a..2d106740 100755 --- a/frontends/roundup.cgi +++ b/frontends/roundup.cgi @@ -73,7 +73,7 @@ LOG = DevNull() # Set up the error handler # try: - import traceback, cgi + import traceback from roundup.cgi import cgitb except: print("Content-Type: text/plain\n") diff --git a/roundup/anypy/cgi_.py b/roundup/anypy/cgi_.py new file mode 100644 index 00000000..f39f00d4 --- /dev/null +++ b/roundup/anypy/cgi_.py @@ -0,0 +1,8 @@ +try: + # used for python2 and python 3 < 3.13 + import cgi as cgi + from cgi import FieldStorage, MiniFieldStorage +except ImportError: + # use for python3 >= 3.13 + from roundup.anypy.vendored import cgi + from roundup.anypy.vendored.cgi import FieldStorage, MiniFieldStorage diff --git a/roundup/anypy/vendored/cgi.py b/roundup/anypy/vendored/cgi.py new file mode 100755 index 00000000..22897a14 --- /dev/null +++ b/roundup/anypy/vendored/cgi.py @@ -0,0 +1,1009 @@ +#! /usr/local/bin/python + +# NOTE: the above "/usr/local/bin/python" is NOT a mistake. It is +# intentionally NOT "/usr/bin/env python". On many systems +# (e.g. Solaris), /usr/local/bin is not in $PATH as passed to CGI +# scripts, and /usr/local/bin is the default directory where Python is +# installed, so /usr/bin/env would be unable to find python. Granted, +# binary installations by Linux vendors often install Python in +# /usr/bin. So let those vendors patch cgi.py to match their choice +# of installation. + +"""Support module for CGI (Common Gateway Interface) scripts. + +This module defines a number of utilities for use by CGI scripts +written in Python. + +The global variable maxlen can be set to an integer indicating the maximum size +of a POST request. POST requests larger than this size will result in a +ValueError being raised during parsing. The default value of this variable is 0, +meaning the request size is unlimited. +""" + +# History +# ------- +# +# Michael McLay started this module. Steve Majewski changed the +# interface to SvFormContentDict and FormContentDict. The multipart +# parsing was inspired by code submitted by Andreas Paepcke. Guido van +# Rossum rewrote, reformatted and documented the module and is currently +# responsible for its maintenance. +# + +__version__ = "2.6" + + +# Imports +# ======= + +from io import StringIO, BytesIO, TextIOWrapper +from collections.abc import Mapping +import sys +import os +import urllib.parse +from email.parser import FeedParser +from email.message import Message +import html +import locale +import tempfile +import warnings + +__all__ = ["MiniFieldStorage", "FieldStorage", "parse", "parse_multipart", + "parse_header", "test", "print_exception", "print_environ", + "print_form", "print_directory", "print_arguments", + "print_environ_usage"] + +# Logging support +# =============== + +logfile = "" # Filename to log to, if not empty +logfp = None # File object to log to, if not None + +def initlog(*allargs): + """Write a log message, if there is a log file. + + Even though this function is called initlog(), you should always + use log(); log is a variable that is set either to initlog + (initially), to dolog (once the log file has been opened), or to + nolog (when logging is disabled). + + The first argument is a format string; the remaining arguments (if + any) are arguments to the % operator, so e.g. + log("%s: %s", "a", "b") + will write "a: b" to the log file, followed by a newline. + + If the global logfp is not None, it should be a file object to + which log data is written. + + If the global logfp is None, the global logfile may be a string + giving a filename to open, in append mode. This file should be + world writable!!! If the file can't be opened, logging is + silently disabled (since there is no safe place where we could + send an error message). + + """ + global log, logfile, logfp + warnings.warn("cgi.log() is deprecated as of 3.10. Use logging instead", + DeprecationWarning, stacklevel=2) + if logfile and not logfp: + try: + logfp = open(logfile, "a", encoding="locale") + except OSError: + pass + if not logfp: + log = nolog + else: + log = dolog + log(*allargs) + +def dolog(fmt, *args): + """Write a log message to the log file. See initlog() for docs.""" + logfp.write(fmt%args + "\n") + +def nolog(*allargs): + """Dummy function, assigned to log when logging is disabled.""" + pass + +def closelog(): + """Close the log file.""" + global log, logfile, logfp + logfile = '' + if logfp: + logfp.close() + logfp = None + log = initlog + +log = initlog # The current logging function + + +# Parsing functions +# ================= + +# Maximum input we will accept when REQUEST_METHOD is POST +# 0 ==> unlimited input +maxlen = 0 + +def parse(fp=None, environ=os.environ, keep_blank_values=0, + strict_parsing=0, separator='&'): + """Parse a query in the environment or from a file (default stdin) + + Arguments, all optional: + + fp : file pointer; default: sys.stdin.buffer + + environ : environment dictionary; default: os.environ + + keep_blank_values: flag indicating whether blank values in + percent-encoded forms should be treated as blank strings. + A true value indicates that blanks should be retained as + blank strings. The default false value indicates that + blank values are to be ignored and treated as if they were + not included. + + strict_parsing: flag indicating what to do with parsing errors. + If false (the default), errors are silently ignored. + If true, errors raise a ValueError exception. + + separator: str. The symbol to use for separating the query arguments. + Defaults to &. + """ + if fp is None: + fp = sys.stdin + + # field keys and values (except for files) are returned as strings + # an encoding is required to decode the bytes read from self.fp + if hasattr(fp,'encoding'): + encoding = fp.encoding + else: + encoding = 'latin-1' + + # fp.read() must return bytes + if isinstance(fp, TextIOWrapper): + fp = fp.buffer + + if not 'REQUEST_METHOD' in environ: + environ['REQUEST_METHOD'] = 'GET' # For testing stand-alone + if environ['REQUEST_METHOD'] == 'POST': + ctype, pdict = parse_header(environ['CONTENT_TYPE']) + if ctype == 'multipart/form-data': + return parse_multipart(fp, pdict, separator=separator) + elif ctype == 'application/x-www-form-urlencoded': + clength = int(environ['CONTENT_LENGTH']) + if maxlen and clength > maxlen: + raise ValueError('Maximum content length exceeded') + qs = fp.read(clength).decode(encoding) + else: + qs = '' # Unknown content-type + if 'QUERY_STRING' in environ: + if qs: qs = qs + '&' + qs = qs + environ['QUERY_STRING'] + elif sys.argv[1:]: + if qs: qs = qs + '&' + qs = qs + sys.argv[1] + environ['QUERY_STRING'] = qs # XXX Shouldn't, really + elif 'QUERY_STRING' in environ: + qs = environ['QUERY_STRING'] + else: + if sys.argv[1:]: + qs = sys.argv[1] + else: + qs = "" + environ['QUERY_STRING'] = qs # XXX Shouldn't, really + return urllib.parse.parse_qs(qs, keep_blank_values, strict_parsing, + encoding=encoding, separator=separator) + + +def parse_multipart(fp, pdict, encoding="utf-8", errors="replace", separator='&'): + """Parse multipart input. + + Arguments: + fp : input file + pdict: dictionary containing other parameters of content-type header + encoding, errors: request encoding and error handler, passed to + FieldStorage + + Returns a dictionary just like parse_qs(): keys are the field names, each + value is a list of values for that field. For non-file fields, the value + is a list of strings. + """ + # RFC 2046, Section 5.1 : The "multipart" boundary delimiters are always + # represented as 7bit US-ASCII. + boundary = pdict['boundary'].decode('ascii') + ctype = "multipart/form-data; boundary={}".format(boundary) + headers = Message() + headers.set_type(ctype) + try: + headers['Content-Length'] = pdict['CONTENT-LENGTH'] + except KeyError: + pass + fs = FieldStorage(fp, headers=headers, encoding=encoding, errors=errors, + environ={'REQUEST_METHOD': 'POST'}, separator=separator) + return {k: fs.getlist(k) for k in fs} + +def _parseparam(s): + while s[:1] == ';': + s = s[1:] + end = s.find(';') + while end > 0 and (s.count('"', 0, end) - s.count('\\"', 0, end)) % 2: + end = s.find(';', end + 1) + if end < 0: + end = len(s) + f = s[:end] + yield f.strip() + s = s[end:] + +def parse_header(line): + """Parse a Content-type like header. + + Return the main content-type and a dictionary of options. + + """ + parts = _parseparam(';' + line) + key = parts.__next__() + pdict = {} + for p in parts: + i = p.find('=') + if i >= 0: + name = p[:i].strip().lower() + value = p[i+1:].strip() + if len(value) >= 2 and value[0] == value[-1] == '"': + value = value[1:-1] + value = value.replace('\\\\', '\\').replace('\\"', '"') + pdict[name] = value + return key, pdict + + +# Classes for field storage +# ========================= + +class MiniFieldStorage: + + """Like FieldStorage, for use when no file uploads are possible.""" + + # Dummy attributes + filename = None + list = None + type = None + file = None + type_options = {} + disposition = None + disposition_options = {} + headers = {} + + def __init__(self, name, value): + """Constructor from field name and value.""" + self.name = name + self.value = value + # self.file = StringIO(value) + + def __repr__(self): + """Return printable representation.""" + return "MiniFieldStorage(%r, %r)" % (self.name, self.value) + + +class FieldStorage: + + """Store a sequence of fields, reading multipart/form-data. + + This class provides naming, typing, files stored on disk, and + more. At the top level, it is accessible like a dictionary, whose + keys are the field names. (Note: None can occur as a field name.) + The items are either a Python list (if there's multiple values) or + another FieldStorage or MiniFieldStorage object. If it's a single + object, it has the following attributes: + + name: the field name, if specified; otherwise None + + filename: the filename, if specified; otherwise None; this is the + client side filename, *not* the file name on which it is + stored (that's a temporary file you don't deal with) + + value: the value as a *string*; for file uploads, this + transparently reads the file every time you request the value + and returns *bytes* + + file: the file(-like) object from which you can read the data *as + bytes* ; None if the data is stored a simple string + + type: the content-type, or None if not specified + + type_options: dictionary of options specified on the content-type + line + + disposition: content-disposition, or None if not specified + + disposition_options: dictionary of corresponding options + + headers: a dictionary(-like) object (sometimes email.message.Message or a + subclass thereof) containing *all* headers + + The class is subclassable, mostly for the purpose of overriding + the make_file() method, which is called internally to come up with + a file open for reading and writing. This makes it possible to + override the default choice of storing all files in a temporary + directory and unlinking them as soon as they have been opened. + + """ + def __init__(self, fp=None, headers=None, outerboundary=b'', + environ=os.environ, keep_blank_values=0, strict_parsing=0, + limit=None, encoding='utf-8', errors='replace', + max_num_fields=None, separator='&'): + """Constructor. Read multipart/* until last part. + + Arguments, all optional: + + fp : file pointer; default: sys.stdin.buffer + (not used when the request method is GET) + Can be : + 1. a TextIOWrapper object + 2. an object whose read() and readline() methods return bytes + + headers : header dictionary-like object; default: + taken from environ as per CGI spec + + outerboundary : terminating multipart boundary + (for internal use only) + + environ : environment dictionary; default: os.environ + + keep_blank_values: flag indicating whether blank values in + percent-encoded forms should be treated as blank strings. + A true value indicates that blanks should be retained as + blank strings. The default false value indicates that + blank values are to be ignored and treated as if they were + not included. + + strict_parsing: flag indicating what to do with parsing errors. + If false (the default), errors are silently ignored. + If true, errors raise a ValueError exception. + + limit : used internally to read parts of multipart/form-data forms, + to exit from the reading loop when reached. It is the difference + between the form content-length and the number of bytes already + read + + encoding, errors : the encoding and error handler used to decode the + binary stream to strings. Must be the same as the charset defined + for the page sending the form (content-type : meta http-equiv or + header) + + max_num_fields: int. If set, then __init__ throws a ValueError + if there are more than n fields read by parse_qsl(). + + """ + method = 'GET' + self.keep_blank_values = keep_blank_values + self.strict_parsing = strict_parsing + self.max_num_fields = max_num_fields + self.separator = separator + if 'REQUEST_METHOD' in environ: + method = environ['REQUEST_METHOD'].upper() + self.qs_on_post = None + if method == 'GET' or method == 'HEAD': + if 'QUERY_STRING' in environ: + qs = environ['QUERY_STRING'] + elif sys.argv[1:]: + qs = sys.argv[1] + else: + qs = "" + qs = qs.encode(locale.getpreferredencoding(), 'surrogateescape') + fp = BytesIO(qs) + if headers is None: + headers = {'content-type': + "application/x-www-form-urlencoded"} + if headers is None: + headers = {} + if method == 'POST': + # Set default content-type for POST to what's traditional + headers['content-type'] = "application/x-www-form-urlencoded" + if 'CONTENT_TYPE' in environ: + headers['content-type'] = environ['CONTENT_TYPE'] + if 'QUERY_STRING' in environ: + self.qs_on_post = environ['QUERY_STRING'] + if 'CONTENT_LENGTH' in environ: + headers['content-length'] = environ['CONTENT_LENGTH'] + else: + if not (isinstance(headers, (Mapping, Message))): + raise TypeError("headers must be mapping or an instance of " + "email.message.Message") + self.headers = headers + if fp is None: + self.fp = sys.stdin.buffer + # self.fp.read() must return bytes + elif isinstance(fp, TextIOWrapper): + self.fp = fp.buffer + else: + if not (hasattr(fp, 'read') and hasattr(fp, 'readline')): + raise TypeError("fp must be file pointer") + self.fp = fp + + self.encoding = encoding + self.errors = errors + + if not isinstance(outerboundary, bytes): + raise TypeError('outerboundary must be bytes, not %s' + % type(outerboundary).__name__) + self.outerboundary = outerboundary + + self.bytes_read = 0 + self.limit = limit + + # Process content-disposition header + cdisp, pdict = "", {} + if 'content-disposition' in self.headers: + cdisp, pdict = parse_header(self.headers['content-disposition']) + self.disposition = cdisp + self.disposition_options = pdict + self.name = None + if 'name' in pdict: + self.name = pdict['name'] + self.filename = None + if 'filename' in pdict: + self.filename = pdict['filename'] + self._binary_file = self.filename is not None + + # Process content-type header + # + # Honor any existing content-type header. But if there is no + # content-type header, use some sensible defaults. Assume + # outerboundary is "" at the outer level, but something non-false + # inside a multi-part. The default for an inner part is text/plain, + # but for an outer part it should be urlencoded. This should catch + # bogus clients which erroneously forget to include a content-type + # header. + # + # See below for what we do if there does exist a content-type header, + # but it happens to be something we don't understand. + if 'content-type' in self.headers: + ctype, pdict = parse_header(self.headers['content-type']) + elif self.outerboundary or method != 'POST': + ctype, pdict = "text/plain", {} + else: + ctype, pdict = 'application/x-www-form-urlencoded', {} + self.type = ctype + self.type_options = pdict + if 'boundary' in pdict: + self.innerboundary = pdict['boundary'].encode(self.encoding, + self.errors) + else: + self.innerboundary = b"" + + clen = -1 + if 'content-length' in self.headers: + try: + clen = int(self.headers['content-length']) + except ValueError: + pass + if maxlen and clen > maxlen: + raise ValueError('Maximum content length exceeded') + self.length = clen + if self.limit is None and clen >= 0: + self.limit = clen + + self.list = self.file = None + self.done = 0 + if ctype == 'application/x-www-form-urlencoded': + self.read_urlencoded() + elif ctype[:10] == 'multipart/': + self.read_multi(environ, keep_blank_values, strict_parsing) + else: + self.read_single() + + def __del__(self): + try: + self.file.close() + except AttributeError: + pass + + def __enter__(self): + return self + + def __exit__(self, *args): + self.file.close() + + def __repr__(self): + """Return a printable representation.""" + return "FieldStorage(%r, %r, %r)" % ( + self.name, self.filename, self.value) + + def __iter__(self): + return iter(self.keys()) + + def __getattr__(self, name): + if name != 'value': + raise AttributeError(name) + if self.file: + self.file.seek(0) + value = self.file.read() + self.file.seek(0) + elif self.list is not None: + value = self.list + else: + value = None + return value + + def __getitem__(self, key): + """Dictionary style indexing.""" + if self.list is None: + raise TypeError("not indexable") + found = [] + for item in self.list: + if item.name == key: found.append(item) + if not found: + raise KeyError(key) + if len(found) == 1: + return found[0] + else: + return found + + def getvalue(self, key, default=None): + """Dictionary style get() method, including 'value' lookup.""" + if key in self: + value = self[key] + if isinstance(value, list): + return [x.value for x in value] + else: + return value.value + else: + return default + + def getfirst(self, key, default=None): + """ Return the first value received.""" + if key in self: + value = self[key] + if isinstance(value, list): + return value[0].value + else: + return value.value + else: + return default + + def getlist(self, key): + """ Return list of received values.""" + if key in self: + value = self[key] + if isinstance(value, list): + return [x.value for x in value] + else: + return [value.value] + else: + return [] + + def keys(self): + """Dictionary style keys() method.""" + if self.list is None: + raise TypeError("not indexable") + return list(set(item.name for item in self.list)) + + def __contains__(self, key): + """Dictionary style __contains__ method.""" + if self.list is None: + raise TypeError("not indexable") + return any(item.name == key for item in self.list) + + def __len__(self): + """Dictionary style len(x) support.""" + return len(self.keys()) + + def __bool__(self): + if self.list is None: + raise TypeError("Cannot be converted to bool.") + return bool(self.list) + + def read_urlencoded(self): + """Internal: read data in query string format.""" + qs = self.fp.read(self.length) + if not isinstance(qs, bytes): + raise ValueError("%s should return bytes, got %s" \ + % (self.fp, type(qs).__name__)) + qs = qs.decode(self.encoding, self.errors) + if self.qs_on_post: + qs += '&' + self.qs_on_post + query = urllib.parse.parse_qsl( + qs, self.keep_blank_values, self.strict_parsing, + encoding=self.encoding, errors=self.errors, + max_num_fields=self.max_num_fields, separator=self.separator) + self.list = [MiniFieldStorage(key, value) for key, value in query] + self.skip_lines() + + FieldStorageClass = None + + def read_multi(self, environ, keep_blank_values, strict_parsing): + """Internal: read a part that is itself multipart.""" + ib = self.innerboundary + if not valid_boundary(ib): + raise ValueError('Invalid boundary in multipart form: %r' % (ib,)) + self.list = [] + if self.qs_on_post: + query = urllib.parse.parse_qsl( + self.qs_on_post, self.keep_blank_values, self.strict_parsing, + encoding=self.encoding, errors=self.errors, + max_num_fields=self.max_num_fields, separator=self.separator) + self.list.extend(MiniFieldStorage(key, value) for key, value in query) + + klass = self.FieldStorageClass or self.__class__ + first_line = self.fp.readline() # bytes + if not isinstance(first_line, bytes): + raise ValueError("%s should return bytes, got %s" \ + % (self.fp, type(first_line).__name__)) + self.bytes_read += len(first_line) + + # Ensure that we consume the file until we've hit our inner boundary + while (first_line.strip() != (b"--" + self.innerboundary) and + first_line): + first_line = self.fp.readline() + self.bytes_read += len(first_line) + + # Propagate max_num_fields into the sub class appropriately + max_num_fields = self.max_num_fields + if max_num_fields is not None: + max_num_fields -= len(self.list) + + while True: + parser = FeedParser() + hdr_text = b"" + while True: + data = self.fp.readline() + hdr_text += data + if not data.strip(): + break + if not hdr_text: + break + # parser takes strings, not bytes + self.bytes_read += len(hdr_text) + parser.feed(hdr_text.decode(self.encoding, self.errors)) + headers = parser.close() + + # Some clients add Content-Length for part headers, ignore them + if 'content-length' in headers: + del headers['content-length'] + + limit = None if self.limit is None \ + else self.limit - self.bytes_read + part = klass(self.fp, headers, ib, environ, keep_blank_values, + strict_parsing, limit, + self.encoding, self.errors, max_num_fields, self.separator) + + if max_num_fields is not None: + max_num_fields -= 1 + if part.list: + max_num_fields -= len(part.list) + if max_num_fields < 0: + raise ValueError('Max number of fields exceeded') + + self.bytes_read += part.bytes_read + self.list.append(part) + if part.done or self.bytes_read >= self.length > 0: + break + self.skip_lines() + + def read_single(self): + """Internal: read an atomic part.""" + if self.length >= 0: + self.read_binary() + self.skip_lines() + else: + self.read_lines() + self.file.seek(0) + + bufsize = 8*1024 # I/O buffering size for copy to file + + def read_binary(self): + """Internal: read binary data.""" + self.file = self.make_file() + todo = self.length + if todo >= 0: + while todo > 0: + data = self.fp.read(min(todo, self.bufsize)) # bytes + if not isinstance(data, bytes): + raise ValueError("%s should return bytes, got %s" + % (self.fp, type(data).__name__)) + self.bytes_read += len(data) + if not data: + self.done = -1 + break + self.file.write(data) + todo = todo - len(data) + + def read_lines(self): + """Internal: read lines until EOF or outerboundary.""" + if self._binary_file: + self.file = self.__file = BytesIO() # store data as bytes for files + else: + self.file = self.__file = StringIO() # as strings for other fields + if self.outerboundary: + self.read_lines_to_outerboundary() + else: + self.read_lines_to_eof() + + def __write(self, line): + """line is always bytes, not string""" + if self.__file is not None: + if self.__file.tell() + len(line) > 1000: + self.file = self.make_file() + data = self.__file.getvalue() + self.file.write(data) + self.__file = None + if self._binary_file: + # keep bytes + self.file.write(line) + else: + # decode to string + self.file.write(line.decode(self.encoding, self.errors)) + + def read_lines_to_eof(self): + """Internal: read lines until EOF.""" + while 1: + line = self.fp.readline(1<<16) # bytes + self.bytes_read += len(line) + if not line: + self.done = -1 + break + self.__write(line) + + def read_lines_to_outerboundary(self): + """Internal: read lines until outerboundary. + Data is read as bytes: boundaries and line ends must be converted + to bytes for comparisons. + """ + next_boundary = b"--" + self.outerboundary + last_boundary = next_boundary + b"--" + delim = b"" + last_line_lfend = True + _read = 0 + while 1: + + if self.limit is not None and 0 <= self.limit <= _read: + break + line = self.fp.readline(1<<16) # bytes + self.bytes_read += len(line) + _read += len(line) + if not line: + self.done = -1 + break + if delim == b"\r": + line = delim + line + delim = b"" + if line.startswith(b"--") and last_line_lfend: + strippedline = line.rstrip() + if strippedline == next_boundary: + break + if strippedline == last_boundary: + self.done = 1 + break + odelim = delim + if line.endswith(b"\r\n"): + delim = b"\r\n" + line = line[:-2] + last_line_lfend = True + elif line.endswith(b"\n"): + delim = b"\n" + line = line[:-1] + last_line_lfend = True + elif line.endswith(b"\r"): + # We may interrupt \r\n sequences if they span the 2**16 + # byte boundary + delim = b"\r" + line = line[:-1] + last_line_lfend = False + else: + delim = b"" + last_line_lfend = False + self.__write(odelim + line) + + def skip_lines(self): + """Internal: skip lines until outer boundary if defined.""" + if not self.outerboundary or self.done: + return + next_boundary = b"--" + self.outerboundary + last_boundary = next_boundary + b"--" + last_line_lfend = True + while True: + line = self.fp.readline(1<<16) + self.bytes_read += len(line) + if not line: + self.done = -1 + break + if line.endswith(b"--") and last_line_lfend: + strippedline = line.strip() + if strippedline == next_boundary: + break + if strippedline == last_boundary: + self.done = 1 + break + last_line_lfend = line.endswith(b'\n') + + def make_file(self): + """Overridable: return a readable & writable file. + + The file will be used as follows: + - data is written to it + - seek(0) + - data is read from it + + The file is opened in binary mode for files, in text mode + for other fields + + This version opens a temporary file for reading and writing, + and immediately deletes (unlinks) it. The trick (on Unix!) is + that the file can still be used, but it can't be opened by + another process, and it will automatically be deleted when it + is closed or when the current process terminates. + + If you want a more permanent file, you derive a class which + overrides this method. If you want a visible temporary file + that is nevertheless automatically deleted when the script + terminates, try defining a __del__ method in a derived class + which unlinks the temporary files you have created. + + """ + if self._binary_file: + return tempfile.TemporaryFile("wb+") + else: + return tempfile.TemporaryFile("w+", + encoding=self.encoding, newline = '\n') + + +# Test/debug code +# =============== + +def test(environ=os.environ): + """Robust test CGI script, usable as main program. + + Write minimal HTTP headers and dump all information provided to + the script in HTML form. + + """ + print("Content-type: text/html") + print() + sys.stderr = sys.stdout + try: + form = FieldStorage() # Replace with other classes to test those + print_directory() + print_arguments() + print_form(form) + print_environ(environ) + print_environ_usage() + def f(): + exec("testing print_exception() -- italics?") + def g(f=f): + f() + print("

    What follows is a test, not an actual exception:

    ") + g() + except: + print_exception() + + print("

    Second try with a small maxlen...

    ") + + global maxlen + maxlen = 50 + try: + form = FieldStorage() # Replace with other classes to test those + print_directory() + print_arguments() + print_form(form) + print_environ(environ) + except: + print_exception() + +def print_exception(type=None, value=None, tb=None, limit=None): + if type is None: + type, value, tb = sys.exc_info() + import traceback + print() + print("

    Traceback (most recent call last):

    ") + list = traceback.format_tb(tb, limit) + \ + traceback.format_exception_only(type, value) + print("
    %s%s
    " % ( + html.escape("".join(list[:-1])), + html.escape(list[-1]), + )) + del tb + +def print_environ(environ=os.environ): + """Dump the shell environment as HTML.""" + keys = sorted(environ.keys()) + print() + print("

    Shell Environment:

    ") + print("
    ") + for key in keys: + print("
    ", html.escape(key), "
    ", html.escape(environ[key])) + print("
    ") + print() + +def print_form(form): + """Dump the contents of a form as HTML.""" + keys = sorted(form.keys()) + print() + print("

    Form Contents:

    ") + if not keys: + print("

    No form fields.") + print("

    ") + for key in keys: + print("
    " + html.escape(key) + ":", end=' ') + value = form[key] + print("" + html.escape(repr(type(value))) + "") + print("
    " + html.escape(repr(value))) + print("
    ") + print() + +def print_directory(): + """Dump the current directory as HTML.""" + print() + print("

    Current Working Directory:

    ") + try: + pwd = os.getcwd() + except OSError as msg: + print("OSError:", html.escape(str(msg))) + else: + print(html.escape(pwd)) + print() + +def print_arguments(): + print() + print("

    Command Line Arguments:

    ") + print() + print(sys.argv) + print() + +def print_environ_usage(): + """Dump a list of environment variables used by CGI as HTML.""" + print(""" +

    These environment variables could have been set:

    +
      +
    • AUTH_TYPE +
    • CONTENT_LENGTH +
    • CONTENT_TYPE +
    • DATE_GMT +
    • DATE_LOCAL +
    • DOCUMENT_NAME +
    • DOCUMENT_ROOT +
    • DOCUMENT_URI +
    • GATEWAY_INTERFACE +
    • LAST_MODIFIED +
    • PATH +
    • PATH_INFO +
    • PATH_TRANSLATED +
    • QUERY_STRING +
    • REMOTE_ADDR +
    • REMOTE_HOST +
    • REMOTE_IDENT +
    • REMOTE_USER +
    • REQUEST_METHOD +
    • SCRIPT_NAME +
    • SERVER_NAME +
    • SERVER_PORT +
    • SERVER_PROTOCOL +
    • SERVER_ROOT +
    • SERVER_SOFTWARE +
    +In addition, HTTP headers sent by the server may be passed in the +environment as well. Here are some common variable names: +
      +
    • HTTP_ACCEPT +
    • HTTP_CONNECTION +
    • HTTP_HOST +
    • HTTP_PRAGMA +
    • HTTP_REFERER +
    • HTTP_USER_AGENT +
    +""") + + +# Utilities +# ========= + +def valid_boundary(s): + import re + if isinstance(s, bytes): + _vb_pattern = b"^[ -~]{0,200}[!-~]$" + else: + _vb_pattern = "^[ -~]{0,200}[!-~]$" + return re.match(_vb_pattern, s) + +# Invoke mainline +# =============== + +# Call test() when this file is run as a script (not imported as a module) +if __name__ == '__main__': + test() diff --git a/roundup/cgi/TAL/TALGenerator.py b/roundup/cgi/TAL/TALGenerator.py index ae74f556..eb69679e 100644 --- a/roundup/cgi/TAL/TALGenerator.py +++ b/roundup/cgi/TAL/TALGenerator.py @@ -16,7 +16,6 @@ """ import re -import cgi from . import TALDefs @@ -25,6 +24,7 @@ from .TALDefs import parseSubstitution from .TranslationContext import TranslationContext, DEFAULT_DOMAIN +from roundup.anypy.cgi_ import cgi from roundup.anypy.html import html_escape I18N_REPLACE = 1 diff --git a/roundup/cgi/actions.py b/roundup/cgi/actions.py index 73f9d079..32d1ea88 100644 --- a/roundup/cgi/actions.py +++ b/roundup/cgi/actions.py @@ -1,4 +1,3 @@ -import cgi import codecs import csv import re @@ -8,6 +7,7 @@ from roundup import hyperdb, token_r, date, password from roundup.actions import Action as BaseAction from roundup.anypy import urllib_ +from roundup.anypy.cgi_ import cgi from roundup.anypy.html import html_escape from roundup.anypy.strings import StringIO from roundup.cgi import exceptions, templating diff --git a/roundup/cgi/apache.py b/roundup/cgi/apache.py index 2ca96a35..3a15ab9e 100644 --- a/roundup/cgi/apache.py +++ b/roundup/cgi/apache.py @@ -18,13 +18,13 @@ # is included. Look for this url below. It is not tested, but # we assume it's safe and syntax it seems ok. -import cgi import os import threading from mod_python import apache import roundup.instance +from roundup.anypy.cgi_ import cgi from roundup.cgi import TranslationService diff --git a/roundup/cgi/client.py b/roundup/cgi/client.py index 37565924..7b3f8558 100644 --- a/roundup/cgi/client.py +++ b/roundup/cgi/client.py @@ -4,7 +4,6 @@ import base64 import binascii -import cgi import codecs import email.utils import errno @@ -25,6 +24,7 @@ class SysCallError(Exception): pass +from roundup.anypy.cgi_ import cgi import roundup.anypy.email_ # noqa: F401 -- patches for email library code import roundup.anypy.random_ as random_ # quality of random checked below diff --git a/roundup/cgi/templating.py b/roundup/cgi/templating.py index 788ac03a..2a84902e 100644 --- a/roundup/cgi/templating.py +++ b/roundup/cgi/templating.py @@ -20,7 +20,6 @@ __docformat__ = 'restructuredtext' import calendar -import cgi import csv import os.path import re @@ -28,6 +27,7 @@ from roundup import hyperdb, date, support from roundup.anypy import urllib_ +from roundup.anypy.cgi_ import cgi from roundup.anypy.html import html_escape from roundup.anypy.strings import is_us, us2s, s2u, u2s, StringIO from roundup.cgi import TranslationService, ZTUtils diff --git a/test/db_test_base.py b/test/db_test_base.py index 6d6ee480..4b04584d 100644 --- a/test/db_test_base.py +++ b/test/db_test_base.py @@ -24,7 +24,8 @@ # python2 and deplricated in 3 from base64 import encodestring as base64_encode -import logging, cgi +import logging +from roundup.anypy.cgi_ import cgi from . import gpgmelib from email import message_from_string diff --git a/test/rest_common.py b/test/rest_common.py index 91b87b21..b165dad6 100644 --- a/test/rest_common.py +++ b/test/rest_common.py @@ -3,11 +3,11 @@ import os import shutil import errno -import cgi from time import sleep from datetime import datetime, timedelta from roundup.test.tx_Source_detector import init as tx_Source_init +from roundup.anypy.cgi_ import cgi try: from datetime import timezone diff --git a/test/test_actions.py b/test/test_actions.py index 32494c35..aa00fc25 100644 --- a/test/test_actions.py +++ b/test/test_actions.py @@ -1,6 +1,5 @@ from __future__ import print_function import unittest, copy -from cgi import FieldStorage, MiniFieldStorage from roundup import hyperdb from roundup.date import Date, Interval @@ -9,6 +8,7 @@ from roundup.cgi.exceptions import Redirect, Unauthorised, SeriousError, FormError from roundup.exceptions import RateLimitExceeded, Reject +from roundup.anypy.cgi_ import FieldStorage, MiniFieldStorage from roundup.anypy.cmp_ import NoneAndDictComparable from time import sleep from datetime import datetime diff --git a/test/test_admin.py b/test/test_admin.py index 2489acac..8fb554ce 100644 --- a/test/test_admin.py +++ b/test/test_admin.py @@ -6,7 +6,7 @@ from __future__ import print_function import fileinput -import unittest, os, shutil, errno, sys, difflib, cgi, re +import unittest, os, shutil, errno, sys, difflib, re from roundup.admin import AdminTool diff --git a/test/test_cgi.py b/test/test_cgi.py index eba5e0c4..ad115dca 100644 --- a/test/test_cgi.py +++ b/test/test_cgi.py @@ -9,11 +9,12 @@ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. from __future__ import print_function -import unittest, os, shutil, errno, sys, difflib, cgi, re, io +import unittest, os, shutil, errno, sys, difflib, re, io import pytest import copy +from roundup.anypy.cgi_ import cgi from roundup.cgi import client, actions, exceptions from roundup.cgi.exceptions import FormError, NotFound, Redirect from roundup.exceptions import UsageError, Reject diff --git a/test/test_hyperdbvals.py b/test/test_hyperdbvals.py index 52d182bb..6c8af65c 100644 --- a/test/test_hyperdbvals.py +++ b/test/test_hyperdbvals.py @@ -8,7 +8,7 @@ # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -import unittest, os, shutil, errno, sys, difflib, cgi, re +import unittest, os, shutil, errno, sys, difflib, re from hashlib import sha1 from mock import Mock diff --git a/test/test_templating.py b/test/test_templating.py index 5043d53f..deb06791 100644 --- a/test/test_templating.py +++ b/test/test_templating.py @@ -1,8 +1,8 @@ from __future__ import print_function import unittest import time -from cgi import FieldStorage, MiniFieldStorage +from roundup.anypy.cgi_ import FieldStorage, MiniFieldStorage from roundup.cgi.templating import * from .test_actions import MockNull, true from .html_norm import NormalizingHtmlParser diff --git a/test/test_xmlrpc.py b/test/test_xmlrpc.py index a49c4785..a68c1870 100644 --- a/test/test_xmlrpc.py +++ b/test/test_xmlrpc.py @@ -5,7 +5,7 @@ # from __future__ import print_function -import unittest, os, shutil, errno, sys, difflib, cgi, re +import unittest, os, shutil, errno, sys, difflib, re from roundup.anypy import xmlrpc_ MultiCall = xmlrpc_.client.MultiCall From 0f72704305428571e21aa66e98dba8cdcde07b22 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 24 Jul 2023 18:05:48 -0400 Subject: [PATCH 55/91] test: do not run jinja2 demo test if jinja2 missing. --- test/test_demo.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/test_demo.py b/test/test_demo.py index 2a58f0ed..ea7ae89d 100644 --- a/test/test_demo.py +++ b/test/test_demo.py @@ -1,3 +1,4 @@ +import pytest import unittest import os, sys, shutil @@ -23,6 +24,14 @@ def captured_output(): finally: sys.stdout, sys.stderr = old_out, old_err +try: + import jinja2 + skip_jinja2 = lambda func, *args, **kwargs: func +except ImportError: + from .pytest_patcher import mark_class + skip_jinja2 = mark_class(pytest.mark.skip( + reason='Skipping Jinja2 tests: jinja2 library not available')) + class TestDemo(unittest.TestCase): def setUp(self): self.home = os.path.abspath('_test_demo') @@ -73,6 +82,7 @@ def testDemoClassic(self): def testDemoMinimal(self): self.run_install_demo('../templates/minimal', db="sqlite") + @skip_jinja2 def testDemoJinja(self): self.run_install_demo('jinja2', db="anydbm") From dc50e34fc8e2e72276c9c1e31fd14e0a19d2d313 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 24 Jul 2023 18:07:11 -0400 Subject: [PATCH 56/91] test: fix mising / in directory spec. --- test/test_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_config.py b/test/test_config.py index b8e75af0..5e1736eb 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -224,7 +224,7 @@ def testConfigSave(self): self.startdir = os.getcwd() - self.dirname = os.getcwd() + '_test_config' + self.dirname = os.getcwd() + '/_test_config' os.mkdir(self.dirname) try: From 5901a1634ae54a106e18b3c89e968b9c191928ea Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 24 Jul 2023 21:16:41 -0400 Subject: [PATCH 57/91] test: fix failure under cygwin python caused by line endings reading config.ini files under cygwin python results in \r\n terminated lines which do not compare properly with the success conditions. replace \r\n with \n when required. --- test/test_admin.py | 7 +++++++ test/test_demo.py | 18 ++++++++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/test/test_admin.py b/test/test_admin.py index 8fb554ce..50bc87bc 100644 --- a/test/test_admin.py +++ b/test/test_admin.py @@ -62,6 +62,13 @@ def find_in_file(filename, regexp): with open(filename) as f: contents = f.read() + try: + # handle text files with \r\n line endings + contents.index("\r") + contents = contents.replace("\r\n", "\n") + except ValueError: + pass + m = re.search(regexp, contents, re.MULTILINE) if not m: return False diff --git a/test/test_demo.py b/test/test_demo.py index ea7ae89d..6842e81b 100644 --- a/test/test_demo.py +++ b/test/test_demo.py @@ -50,7 +50,14 @@ def run_install_demo(self, template, db="anydbm"): # verify that db was set properly by reading config with open(self.home + "/config.ini", "r") as f: - config_lines = f.readlines() + config_lines = f.read().replace("\r\n", "\n") + + try: + # handle text files with \r\n line endings + config_lines.index("\r") + config_lines = config_lines.replace("\r\n", "\n") + except ValueError: + pass self.assertIn("backend = %s\n"%db, config_lines) @@ -88,7 +95,14 @@ def testDemoJinja(self): # verify that template was set to jinja2 by reading config with open(self.home + "/config.ini", "r") as f: - config_lines = f.readlines() + config_lines = f.read() + + try: + # handle text files with \r\n line endings + config_lines.index("\r") + config_lines = config_lines.replace("\r\n", "\n") + except ValueError: + pass self.assertIn("template_engine = jinja2\n", config_lines) From 5fcd56c3540fe623ca74c375b5a26e68d77d7a53 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 24 Jul 2023 21:24:07 -0400 Subject: [PATCH 58/91] test: limit search for \r to first 100 bytes. Don't scan entire file. just look at 100 bytes which should include all of the first line. --- test/test_admin.py | 2 +- test/test_demo.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_admin.py b/test/test_admin.py index 50bc87bc..47044dc9 100644 --- a/test/test_admin.py +++ b/test/test_admin.py @@ -64,7 +64,7 @@ def find_in_file(filename, regexp): try: # handle text files with \r\n line endings - contents.index("\r") + contents.index("\r", 0, 100) contents = contents.replace("\r\n", "\n") except ValueError: pass diff --git a/test/test_demo.py b/test/test_demo.py index 6842e81b..19aaefc7 100644 --- a/test/test_demo.py +++ b/test/test_demo.py @@ -54,7 +54,7 @@ def run_install_demo(self, template, db="anydbm"): try: # handle text files with \r\n line endings - config_lines.index("\r") + config_lines.index("\r", 0, 100) config_lines = config_lines.replace("\r\n", "\n") except ValueError: pass @@ -99,7 +99,7 @@ def testDemoJinja(self): try: # handle text files with \r\n line endings - config_lines.index("\r") + config_lines.index("\r", 0, 100) config_lines = config_lines.replace("\r\n", "\n") except ValueError: pass From 272bc7ba3411a2438791fcd8c9f37025e7d0d458 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Tue, 25 Jul 2023 16:30:10 -0400 Subject: [PATCH 59/91] fix: issue2551278 - datetime.datetime.utcnow deprecation. Replace calls with equivalent that produces timezone aware dates rather than naive dates. Also some flake8 fixes for test/rest_common.py. --- CHANGES.txt | 3 + roundup/anypy/datetime_.py | 12 +++ roundup/date.py | 4 +- roundup/rate_limit.py | 6 +- test/rest_common.py | 194 +++++++++++++++++++------------------ 5 files changed, 123 insertions(+), 96 deletions(-) create mode 100644 roundup/anypy/datetime_.py diff --git a/CHANGES.txt b/CHANGES.txt index 651a6d93..0f38e0b0 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -34,6 +34,9 @@ Fixed: and make *FieldStroage symbols available. Roundp uses its own cgitb.py and not the system cgitb.py. It looks like it's the precursor to the system cgitb.py. (John Rouillard) +- issue2551278 - datetime.datetime.utcnow deprecation. Replace + calls with equivalent that produces timezone aware dates rather than + naive dates. (John Rouillard) Features: diff --git a/roundup/anypy/datetime_.py b/roundup/anypy/datetime_.py new file mode 100644 index 00000000..a5d5e4a3 --- /dev/null +++ b/roundup/anypy/datetime_.py @@ -0,0 +1,12 @@ +# https://issues.roundup-tracker.org/issue2551278 +# datetime.utcnow deprecated +try: + from datetime import now, UTC + + def utcnow(): + return now(UTC) +except ImportError: + import datetime + + def utcnow(): + return datetime.datetime.utcnow() diff --git a/roundup/date.py b/roundup/date.py index dba41214..fcfbe351 100644 --- a/roundup/date.py +++ b/roundup/date.py @@ -24,6 +24,8 @@ import datetime import re +from roundup.anypy.datetime_ import utcnow + try: import pytz except ImportError: @@ -376,7 +378,7 @@ def __init__(self, spec='.', offset=0, add_granularity=False, def now(self): """ To be able to override for testing """ - return datetime.datetime.utcnow() + return utcnow() def set(self, spec, offset=0, date_re=date_re, serialised_re=serialised_date_re, add_granularity=False): diff --git a/roundup/rate_limit.py b/roundup/rate_limit.py index 06e6e0a8..5ccf3b26 100644 --- a/roundup/rate_limit.py +++ b/roundup/rate_limit.py @@ -6,6 +6,8 @@ from datetime import timedelta, datetime +from roundup.anypy.datetime_ import utcnow + class RateLimit: # pylint: disable=too-few-public-methods def __init__(self, count, period): @@ -50,7 +52,7 @@ def update(self, key, limit, testonly=False): '''Determine if the item associated with the key should be rejected given the RateLimit limit. ''' - now = datetime.utcnow() + now = utcnow() tat = max(self.get_tat(key), now) separation = (tat - now).total_seconds() max_interval = limit.period.total_seconds() - limit.inverse @@ -88,7 +90,7 @@ def status(self, key, limit): ) # status of current limit as of now - now = datetime.utcnow() + now = utcnow() current_count = int((limit.period - (tat - now)).total_seconds() / limit.inverse) diff --git a/test/rest_common.py b/test/rest_common.py index b165dad6..d7f8de34 100644 --- a/test/rest_common.py +++ b/test/rest_common.py @@ -1,13 +1,14 @@ import pytest import unittest -import os import shutil import errno from time import sleep from datetime import datetime, timedelta -from roundup.test.tx_Source_detector import init as tx_Source_init from roundup.anypy.cgi_ import cgi +from roundup.anypy.datetime_ import utcnow +from roundup.test.tx_Source_detector import init as tx_Source_init + try: from datetime import timezone @@ -16,14 +17,15 @@ # python 2 from datetime import tzinfo ZERO = timedelta(0) + class UTC(tzinfo): """UTC""" def utcoffset(self, dt): return ZERO - + def tzname(self, dt): return "UTC" - + def dst(self, dt): return ZERO @@ -34,13 +36,12 @@ def dst(self, dt): from roundup.exceptions import * from roundup import password, hyperdb from roundup.rest import RestfulInstance, calculate_etag -from roundup.backends import list_backends from roundup.cgi import client from roundup.anypy.strings import b2s, s2b, us2u import random from roundup.backends.sessions_dbm import OneTimeKeys -from roundup.anypy.dbm_ import anydbm, whichdb +from roundup.anypy.dbm_ import whichdb from .db_test_base import setupTracker @@ -56,10 +57,10 @@ def dst(self, dt): skip_jwt = lambda func, *args, **kwargs: func except ImportError: from .pytest_patcher import mark_class - jwt=None + jwt = None skip_jwt = mark_class(pytest.mark.skip( reason='Skipping JWT tests: jwt library not available')) - + NEEDS_INSTANCE = 1 @@ -109,31 +110,32 @@ def setUp(self): # add set of roles for testing jwt's. self.db.security.addRole(name="User:email", - description="allow email by jwt") + description="allow email by jwt") # allow the jwt to access everybody's email addresses. # this makes it easier to differentiate between User and # User:email roles by accessing the /rest/data/user # endpoint - jwt_perms = self.db.security.addPermission(name='View', - klass='user', - properties=('id', 'realname', 'address', 'username'), - description="Allow jwt access to email", - props_only=False) + jwt_perms = self.db.security.addPermission( + name='View', + klass='user', + properties=('id', 'realname', 'address', 'username'), + description="Allow jwt access to email", + props_only=False) self.db.security.addPermissionToRole("User:email", jwt_perms) self.db.security.addPermissionToRole("User:email", "Rest Access") # add set of roles for testing jwt's. # this is like the user:email role, but it missing access to the rest endpoint. self.db.security.addRole(name="User:emailnorest", - description="allow email by jwt") - jwt_perms = self.db.security.addPermission(name='View', - klass='user', - properties=('id', 'realname', 'address', 'username'), - description="Allow jwt access to email but forget to allow rest", - props_only=False) + description="allow email by jwt") + jwt_perms = self.db.security.addPermission( + name='View', + klass='user', + properties=('id', 'realname', 'address', 'username'), + description="Allow jwt access to email but forget to allow rest", + props_only=False) self.db.security.addPermissionToRole("User:emailnorest", jwt_perms) - if jwt: # must be 32 chars in length minimum (I think this is at least # 256 bits of data) @@ -142,7 +144,7 @@ def setUp(self): self.db.config['WEB_JWT_SECRET'] = secret # generate all timestamps in UTC. - base_datetime = datetime(1970,1,1, tzinfo=myutc) + base_datetime = datetime(1970, 1, 1, tzinfo=myutc) # A UTC timestamp for now. dt = datetime.now(myutc) @@ -158,17 +160,16 @@ def setUp(self): # claims match what cgi/client.py::determine_user # is looking for - claim= { 'sub': self.db.getuid(), + claim = {'sub': self.db.getuid(), 'iss': self.db.config.TRACKER_WEB, 'aud': self.db.config.TRACKER_WEB, - 'roles': [ 'User' ], + 'roles': ['User'], 'iat': now_ts, - 'exp': plus1min_ts, - } + 'exp': plus1min_ts} # in version 2.0.0 and newer jwt.encode returns string # not bytestring. So we have to skip b2s conversion - + if version.parse(jwt.__version__) >= version.parse('2.0.0'): tostr = lambda x: x else: @@ -179,45 +180,53 @@ def setUp(self): # generate invalid claim with expired timestamp self.claim['expired'] = copy(claim) self.claim['expired']['exp'] = expired_ts - self.jwt['expired'] = tostr(jwt.encode(self.claim['expired'], secret, - algorithm='HS256')) - + self.jwt['expired'] = tostr(jwt.encode( + self.claim['expired'], secret, + algorithm='HS256')) + # generate valid claim with user role self.claim['user'] = copy(claim) self.claim['user']['exp'] = plus1min_ts - self.jwt['user'] = tostr(jwt.encode(self.claim['user'], secret, - algorithm='HS256')) + self.jwt['user'] = tostr(jwt.encode( + self.claim['user'], secret, + algorithm='HS256')) # generate invalid claim bad issuer self.claim['badiss'] = copy(claim) self.claim['badiss']['iss'] = "http://someissuer/bugs" - self.jwt['badiss'] = tostr(jwt.encode(self.claim['badiss'], secret, - algorithm='HS256')) + self.jwt['badiss'] = tostr(jwt.encode( + self.claim['badiss'], secret, + algorithm='HS256')) # generate invalid claim bad aud(ience) self.claim['badaud'] = copy(claim) self.claim['badaud']['aud'] = "http://someaudience/bugs" - self.jwt['badaud'] = tostr(jwt.encode(self.claim['badaud'], secret, - algorithm='HS256')) + self.jwt['badaud'] = tostr(jwt.encode( + self.claim['badaud'], secret, + algorithm='HS256')) # generate invalid claim bad sub(ject) self.claim['badsub'] = copy(claim) self.claim['badsub']['sub'] = str("99") - self.jwt['badsub'] = tostr(jwt.encode(self.claim['badsub'], secret, - algorithm='HS256')) + self.jwt['badsub'] = tostr( + jwt.encode(self.claim['badsub'], secret, + algorithm='HS256')) # generate invalid claim bad roles self.claim['badroles'] = copy(claim) - self.claim['badroles']['roles'] = [ "badrole1", "badrole2" ] - self.jwt['badroles'] = tostr(jwt.encode(self.claim['badroles'], secret, - algorithm='HS256')) + self.claim['badroles']['roles'] = ["badrole1", "badrole2"] + self.jwt['badroles'] = tostr(jwt.encode( + self.claim['badroles'], secret, + algorithm='HS256')) # generate valid claim with limited user:email role self.claim['user:email'] = copy(claim) - self.claim['user:email']['roles'] = [ "user:email" ] - self.jwt['user:email'] = tostr(jwt.encode(self.claim['user:email'], secret, - algorithm='HS256')) + self.claim['user:email']['roles'] = ["user:email"] + self.jwt['user:email'] = tostr(jwt.encode( + self.claim['user:email'], secret, + algorithm='HS256')) # generate valid claim with limited user:emailnorest role self.claim['user:emailnorest'] = copy(claim) - self.claim['user:emailnorest']['roles'] = [ "user:emailnorest" ] - self.jwt['user:emailnorest'] = tostr(jwt.encode(self.claim['user:emailnorest'], secret, - algorithm='HS256')) + self.claim['user:emailnorest']['roles'] = ["user:emailnorest"] + self.jwt['user:emailnorest'] = tostr(jwt.encode( + self.claim['user:emailnorest'], secret, + algorithm='HS256')) self.db.tx_Source = 'web' @@ -271,7 +280,7 @@ def tearDown(self): if error.errno not in (errno.ENOENT, errno.ESRCH): raise - def get_header (self, header, not_found=None): + def get_header(self, header, not_found=None): try: return self.headers[header.lower()] except (AttributeError, KeyError, TypeError): @@ -305,13 +314,13 @@ def create_sampledata(self): title='foo1', status=self.db.status.lookup('open'), priority=self.db.priority.lookup('normal'), - nosy = [ "1", "2" ] + nosy=["1", "2"] ) issue_open_norm = self.db.issue.create( title='foo2', status=self.db.status.lookup('open'), priority=self.db.priority.lookup('normal'), - assignedto = "3" + assignedto="3" ) issue_open_crit = self.db.issue.create( title='foo5', @@ -371,31 +380,31 @@ def testGetTransitive(self): sort by status.name (not order) """ base_path = self.db.config['TRACKER_WEB'] + 'rest/data/' - #self.maxDiff=None + # self.maxDiff=None self.create_sampledata() self.db.issue.set('2', status=self.db.status.lookup('closed')) self.db.issue.set('3', status=self.db.status.lookup('chatting')) - expected={'data': - {'@total_size': 2, - 'collection': [ - { 'id': '2', - 'link': base_path + 'issue/2', - 'assignedto.issue': None, - 'status': - { 'id': '10', - 'link': base_path + 'status/10' + expected = {'data': + {'@total_size': 2, + 'collection': [ + {'id': '2', + 'link': base_path + 'issue/2', + 'assignedto.issue': None, + 'status': + {'id': '10', + 'link': base_path + 'status/10' } - }, - { 'id': '1', - 'link': base_path + 'issue/1', - 'assignedto.issue': None, - 'status': - { 'id': '9', - 'link': base_path + 'status/9' + }, + {'id': '1', + 'link': base_path + 'issue/1', + 'assignedto.issue': None, + 'status': + {'id': '9', + 'link': base_path + 'status/9' } - }, - ]} - } + }, + ]} + } form = cgi.FieldStorage() form.list = [ cgi.MiniFieldStorage('status.name', 'o'), @@ -411,7 +420,7 @@ def testGetBadTransitive(self): and a somewhat useful error message. """ base_path = self.db.config['TRACKER_WEB'] + 'rest/data/' - #self.maxDiff=None + # self.maxDiff=None self.create_sampledata() self.db.issue.set('2', status=self.db.status.lookup('closed')) self.db.issue.set('3', status=self.db.status.lookup('chatting')) @@ -471,22 +480,22 @@ def testGetExactMatch(self): """ Retrieve all issues with an exact title """ base_path = self.db.config['TRACKER_WEB'] + 'rest/data/' - #self.maxDiff=None + # self.maxDiff=None self.create_sampledata() self.db.issue.set('2', title='This is an exact match') self.db.issue.set('3', title='This is an exact match') self.db.issue.set('1', title='This is AN exact match') - expected={'data': - {'@total_size': 2, - 'collection': [ - { 'id': '2', - 'link': base_path + 'issue/2', - }, - { 'id': '3', - 'link': base_path + 'issue/3', - }, - ]} - } + expected = {'data': + {'@total_size': 2, + 'collection': [ + {'id': '2', + 'link': base_path + 'issue/2', + }, + {'id': '3', + 'link': base_path + 'issue/3', + }, + ]} + } form = cgi.FieldStorage() form.list = [ cgi.MiniFieldStorage('title:', 'This is an exact match'), @@ -502,7 +511,6 @@ def testOutputFormat(self): self.create_sampledata() base_path = self.db.config['TRACKER_WEB'] + 'rest/data/issue/' - # Check formating for issues status=open; @fields and verbose tests form = cgi.FieldStorage() form.list = [ @@ -511,13 +519,13 @@ def testOutputFormat(self): cgi.MiniFieldStorage('@verbose', '2') ] - expected={'data': - {'@total_size': 3, - 'collection': [ { + expected = {'data': + {'@total_size': 3, + 'collection': [ { 'creator': {'id': '3', 'link': 'http://tracker.example/cgi-bin/roundup.cgi/bugs/rest/data/user/3', 'username': 'joe'}, - 'status': {'id': '9', + 'status': {'id': '9', 'name': 'open', 'link': 'http://tracker.example/cgi-bin/roundup.cgi/bugs/rest/data/status/9'}, 'id': '1', @@ -1094,7 +1102,7 @@ def testRestRateLimit(self): # sqlite or anydbm. So don't need to exercise code. pass - start_time = datetime.utcnow() + start_time = utcnow() # don't set an accept header; json should be the default # use up all our allowed api calls for i in range(calls_per_interval): @@ -1105,7 +1113,7 @@ def testRestRateLimit(self): "/rest/data/user/%s/realname"%self.joeid, self.empty_form) - loop_time = datetime.utcnow() + loop_time = utcnow() self.assertLess((loop_time-start_time).total_seconds(), int(wait_time_str), "Test system is too slow to complete test as configured") @@ -1148,10 +1156,10 @@ def testRestRateLimit(self): wait_time_str) # check as string print("Reset:", self.server.client.additional_headers["X-RateLimit-Reset"]) - print("Now realtime pre-sleep:", datetime.utcnow()) + print("Now realtime pre-sleep:", utcnow()) # sleep as requested so we can do another login sleep(float(wait_time_str) + 0.1) - print("Now realtime post-sleep:", datetime.utcnow()) + print("Now realtime post-sleep:", utcnow()) # this should succeed self.server.client.additional_headers.clear() @@ -1160,7 +1168,7 @@ def testRestRateLimit(self): self.empty_form) print(results) print("Reset:", self.server.client.additional_headers["X-RateLimit-Reset-date"]) - print("Now realtime:", datetime.utcnow()) + print("Now realtime:", utcnow()) print("Now ts header:", self.server.client.additional_headers["Now"]) print("Now date header:", self.server.client.additional_headers["Now-date"]) From aa19d600aef6cb0d3a3cad2e24ed52725adae4dc Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Wed, 26 Jul 2023 11:05:10 -0400 Subject: [PATCH 60/91] fix: restore roundup-admin display output format w/o pragmas Do not indent the display output unless user requested protected fields or headers. This returns the format to what it was prior to 2.3.0. Parsing roundup-admin output programmaticly was never a use case, but don't break the output format without a reason. --- CHANGES.txt | 5 +++++ roundup/admin.py | 12 +++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 0f38e0b0..c324e64f 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -37,6 +37,11 @@ Fixed: - issue2551278 - datetime.datetime.utcnow deprecation. Replace calls with equivalent that produces timezone aware dates rather than naive dates. (John Rouillard) +- when using "roundup-admin display" indent the listing only if + headers or protected fields are requested. This makes the output + look like it did previously to 2.3.0 if the new features aren't + used. Roundup-admin output was never meant to be machine parsed, but + don't break it unless required. (John Rouillard) Features: diff --git a/roundup/admin.py b/roundup/admin.py index 2d551564..ea0d1e02 100644 --- a/roundup/admin.py +++ b/roundup/admin.py @@ -521,6 +521,9 @@ def do_display(self, args): if len(args) < 1: raise UsageError(_('Not enough arguments supplied')) + display_protected = self.settings['display_protected'] + display_header = self.settings['display_header'] + # decode the node designator for designator in args[0].split(','): try: @@ -533,19 +536,22 @@ def do_display(self, args): # display the values normal_props = sorted(cl.properties) - if self.settings['display_protected']: + if display_protected: keys = sorted(cl.getprops()) else: keys = normal_props - if self.settings['display_header']: + if display_header: status = "retired" if cl.is_retired(nodeid) else "active" print('\n[%s (%s)]' % (designator, status)) for key in keys: value = cl.get(nodeid, key) # prepend * for protected properties else just indent # with space. - protected = "*" if key not in normal_props else ' ' + if display_protected or display_header: + protected = "*" if key not in normal_props else ' ' + else: + protected = "" print(_('%(protected)s%(key)s: %(value)s') % locals()) def do_export(self, args, export_files=True): From 566dd9f0cc2e82c0fafe78a673adbd3278f1b1ad Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Wed, 26 Jul 2023 23:59:35 -0400 Subject: [PATCH 61/91] fix: issue2551278 - datetime.datetime.utcnow deprecation. Original fix always failed the import and fell back to the exception case. This method should work and test both code paths in datetime_.py. --- roundup/anypy/datetime_.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/roundup/anypy/datetime_.py b/roundup/anypy/datetime_.py index a5d5e4a3..7dc1f277 100644 --- a/roundup/anypy/datetime_.py +++ b/roundup/anypy/datetime_.py @@ -1,12 +1,13 @@ # https://issues.roundup-tracker.org/issue2551278 # datetime.utcnow deprecated try: - from datetime import now, UTC + from datetime import datetime, UTC def utcnow(): - return now(UTC) + return datetime.now(UTC) + except ImportError: - import datetime + from datetime import datetime def utcnow(): - return datetime.datetime.utcnow() + return datetime.utcnow() From 63156281231ddd2cc09f9d69e54286101eca20f4 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Thu, 27 Jul 2023 00:53:36 -0400 Subject: [PATCH 62/91] fix: issue2551278 - datetime.datetime.utcnow deprecation. We now use the timezone aware utc dates for python 3.11+. But we have to make all the rest of the dates (datetime.min, unix epoch date) timezon aware so we can subtract them. Also need to marshall/unmarshall timezone aware iso formatted date strings. --- roundup/rate_limit.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/roundup/rate_limit.py b/roundup/rate_limit.py index 5ccf3b26..facf1fc2 100644 --- a/roundup/rate_limit.py +++ b/roundup/rate_limit.py @@ -6,6 +6,21 @@ from datetime import timedelta, datetime +try: + # used by python 3.11 and newer use tz aware dates + from datetime import UTC + dt_min = datetime.min.replace(tzinfo=UTC) + # start of unix epoch + dt_epoch = datetime(1970, 1, 1, tzinfo=UTC) + fromisoformat = datetime.fromisoformat +except ImportError: + # python 2.7 and older than 3.11 - use naive dates + dt_min = datetime.min + dt_epoch = datetime(1970, 1, 1) + def fromisoformat(date): + # only for naive dates + return datetime.strptime(date, "%Y-%m-%dT%H:%M:%S.%f") + from roundup.anypy.datetime_ import utcnow @@ -28,7 +43,7 @@ def get_tat(self, key): if key in self.memory: return self.memory[key] else: - return datetime.min + return dt_min def set_tat(self, key, tat): self.memory[key] = tat @@ -40,13 +55,13 @@ def get_tat_as_string(self, key): if key in self.memory: return self.memory[key].isoformat() else: - return datetime.min.isoformat() + return dt_min.isoformat() def set_tat_as_string(self, key, tat): # Take value as string and unmarshall: # YYYY-MM-DDTHH:MM:SS.mmmmmm # to datetime - self.memory[key] = datetime.strptime(tat, "%Y-%m-%dT%H:%M:%S.%f") + self.memory[key] = fromisoformat(tat) def update(self, key, limit, testonly=False): '''Determine if the item associated with the key should be @@ -100,7 +115,7 @@ def status(self, key, limit): seconds_to_tat = (tat - now).total_seconds() ret['X-RateLimit-Reset'] = str(max(seconds_to_tat, 0)) ret['X-RateLimit-Reset-date'] = "%s" % tat - ret['Now'] = str((now - datetime(1970, 1, 1)).total_seconds()) + ret['Now'] = str((now - dt_epoch).total_seconds()) ret['Now-date'] = "%s" % now if self.update(key, limit, testonly=True): From 9dae819cfae6266793ef1c44a8be25c3f1053730 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Thu, 27 Jul 2023 18:47:53 -0400 Subject: [PATCH 63/91] chore: sort imports --- roundup/demo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roundup/demo.py b/roundup/demo.py index 9dfe1479..41449e56 100755 --- a/roundup/demo.py +++ b/roundup/demo.py @@ -5,11 +5,11 @@ from __future__ import print_function import errno +import getopt import os import shutil import socket import sys -import getopt try: import urlparse From 7b60e4d010ba1ce245045a922c29b7253f60333c Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 30 Jul 2023 21:12:11 -0400 Subject: [PATCH 64/91] docs: add required modules for running tests. --- doc/developers.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/developers.txt b/doc/developers.txt index 532d9bad..75312032 100644 --- a/doc/developers.txt +++ b/doc/developers.txt @@ -206,6 +206,16 @@ test suite. Use it by running:: once over the whole test suite. Then subsequent calls will analyze the changed files/functions and run tests that cover those changes. +To run some tests (test_liveserver.py, test_indexer.py, ...) you need +to have some additional modules installed. These include: + + * requests + * mock + +If you are working with a docker container that is set up to execute +Python application and not for development, you will need to install +pytest. + Internationalization Notes -------------------------- From 3b8cee441de686ad1ebe704c7f9f1526c539cb25 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 31 Jul 2023 18:43:56 -0400 Subject: [PATCH 65/91] docs: clarify sentence. --- doc/admin_guide.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/admin_guide.txt b/doc/admin_guide.txt index 8b6a32ef..1c964b57 100644 --- a/doc/admin_guide.txt +++ b/doc/admin_guide.txt @@ -515,7 +515,7 @@ language described at: https://www.sqlite.org/fts5.html#full_text_query_syntax. This supports: -* plain word search (joined with and similar to other search methods) +* plain word search (joined by 'and', similar to other search methods) * phrase search with terms enclosed in quotes (``"``) * proximity search with varying distances using ``NEAR()`` * boolean operations by grouping with parentheses and using ``AND`` From 214446c6a384084b7f75a7406b76481514ac8e59 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 31 Jul 2023 18:50:07 -0400 Subject: [PATCH 66/91] docs: fix sentence. --- doc/admin_guide.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/admin_guide.txt b/doc/admin_guide.txt index 1c964b57..d933bfc1 100644 --- a/doc/admin_guide.txt +++ b/doc/admin_guide.txt @@ -823,7 +823,7 @@ headers can be passed to the ``-I`` option. These could be used in a detector or other tracker extensions, but only one header can be used by the tracker as an authentication header. -To make the tracker honor the new variable changing the tracker +To make the tracker honor the new variable, change the tracker's ``config.ini`` to read:: [web] From 285abbd91f5493b86712522126bddca2d19c7005 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Thu, 3 Aug 2023 17:04:34 -0400 Subject: [PATCH 67/91] refactor(api): early return if REST rate limit is exceeded If the rate limit at the top of dispatch() is exceeded, return rather than running to final return at end of method. This does change a couple of things: 1) output format is always in json 2) json is alays pretty printed Previously, the output format requested by the Accept header or format extension in the URL was used for 1. Similarly value of @pretty was used to control 2. I am trying to reduce the complexities in this routine with the goal of fixing: issue2551289 to return 406 error if the Accept header/format extension is incorrect before executing the request. --- roundup/rest.py | 144 ++++++++++++++---------------------------------- 1 file changed, 41 insertions(+), 103 deletions(-) diff --git a/roundup/rest.py b/roundup/rest.py index 3827e273..b02d9132 100644 --- a/roundup/rest.py +++ b/roundup/rest.py @@ -2042,9 +2042,6 @@ def dispatch(self, method, uri, input): output = None # Before we do anything has the user hit the rate limit. - # This should (but doesn't at the moment) bypass - # all other processing to minimize load of badly - # behaving client. # Get the limit here and not in the init() routine to allow # for a different rate limit per user. @@ -2076,32 +2073,40 @@ def dispatch(self, method, uri, input): if reject: for header, value in limitStatus.items(): self.client.setHeader(header, value) - # User exceeded limits: tell humans how long to wait - # Headers above will do the right thing for api - # aware clients. - try: - retry_after = limitStatus['Retry-After'] - except KeyError: - # handle race condition. If the time between - # the call to grca.update and grca.status - # is sufficient to reload the bucket by 1 - # item, Retry-After will be missing from - # limitStatus. So report a 1 second delay back - # to the client. We treat update as sole - # source of truth for exceeded rate limits. - retry_after = 1 - self.client.setHeader('Retry-After', '1') - - msg = _("Api rate limits exceeded. Please wait: %s seconds.") % retry_after - output = self.error_obj(429, msg, source="ApiRateLimiter") - else: - for header, value in limitStatus.items(): - # Retry-After will be 0 because - # user still has quota available. - # Don't put out the header. - if header in ('Retry-After',): - continue - self.client.setHeader(header, value) + # User exceeded limits: tell humans how long to wait + # Headers above will do the right thing for api + # aware clients. + try: + retry_after = limitStatus['Retry-After'] + except KeyError: + # handle race condition. If the time between + # the call to grca.update and grca.status + # is sufficient to reload the bucket by 1 + # item, Retry-After will be missing from + # limitStatus. So report a 1 second delay back + # to the client. We treat update as sole + # source of truth for exceeded rate limits. + retry_after = 1 + self.client.setHeader('Retry-After', '1') + + msg = _("Api rate limits exceeded. Please wait: %s seconds.") % retry_after + output = self.error_obj(429, msg, source="ApiRateLimiter") + + return self.format_dispatch_output( + self.__default_accept_type, + output, + True # pretty print for this error case as a + # human may read it + ) + + + for header, value in limitStatus.items(): + # Retry-After will be 0 because + # user still has quota available. + # Don't put out the header. + if header in ('Retry-After',): + continue + self.client.setHeader(header, value) # if X-HTTP-Method-Override is set, follow the override method headers = self.client.request.headers @@ -2121,76 +2126,6 @@ def dispatch(self, method, uri, input): 'Ignoring X-HTTP-Method-Override using %s request on %s', method.upper(), uri) - # parse Accept header and get the content type - # Acceptable types ordered with preferred one first - # in list. - try: - accept_header = parse_accept_header(headers.get('Accept')) - except UsageError as e: - output = self.error_obj(406, _("Unable to parse Accept Header. %(error)s. " - "Acceptable types: %(acceptable_types)s") % { - 'error': e.args[0], - 'acceptable_types': " ".join(sorted(self.__accepted_content_type.keys()))}) - accept_header = [] - - if not accept_header: - accept_type = self.__default_accept_type - else: - accept_type = None - for part in accept_header: - if accept_type: - # we accepted the best match, stop searching for - # lower quality matches. - break - if part[0] in self.__accepted_content_type: - accept_type = self.__accepted_content_type[part[0]] - # Version order: - # 1) accept header version=X specifier - # application/vnd.x.y; version=1 - # 2) from type in accept-header type/subtype-vX - # application/vnd.x.y-v1 - # 3) from @apiver in query string to make browser - # use easy - # This code handles 1 and 2. Set api_version to none - # to trigger @apiver parsing below - # Places that need the api_version info should - # use default if version = None - try: - self.api_version = int(part[1]['version']) - except KeyError: - self.api_version = None - except (ValueError, TypeError): - # TypeError if int(None) - msg = ("Unrecognized api version: %s. " - "See /rest without specifying api version " - "for supported versions." % ( - part[1]['version'])) - output = self.error_obj(400, msg) - - # get the request format for response - # priority : extension from uri (/rest/data/issue.json), - # header (Accept: application/json, application/xml) - # default (application/json) - ext_type = os.path.splitext(urlparse(uri).path)[1][1:] - - # Check to see if the length of the extension is less than 6. - # this allows use of .vcard for a future use in downloading - # user info. It also allows passing through larger items like - # JWT that has a final component > 6 items. This method also - # allow detection of mistyped types like jon for json. - if ext_type and (len(ext_type) < 6): - # strip extension so uri make sense - # .../issue.json -> .../issue - uri = uri[:-(len(ext_type) + 1)] - else: - ext_type = None - - # headers.get('Accept') is never empty if called here. - # accept_type will be set to json if there is no Accept header - # accept_type wil be empty only if there is an Accept header - # with invalid values. - data_type = ext_type or accept_type or headers.get('Accept') or "invalid" - if method.upper() == 'OPTIONS': # add access-control-allow-* access-control-max-age to support # CORS preflight @@ -2349,15 +2284,18 @@ def dispatch(self, method, uri, input): output = self.error_obj(405, msg.args[0]) self.client.setHeader("Allow", msg.args[1]) + return self.format_dispatch_output(data_type, output, pretty_output) + + def format_dispatch_output(self, accept_mime_type, output, pretty_print): # Format the content type - if data_type.lower() == "json": + if accept_mime_type.lower() == "json": self.client.setHeader("Content-Type", "application/json") - if pretty_output: + if pretty_print: indent = 4 else: indent = None output = RoundupJSONEncoder(indent=indent).encode(output) - elif data_type.lower() == "xml" and dicttoxml: + elif accept_mime_type.lower() == "xml" and dicttoxml: self.client.setHeader("Content-Type", "application/xml") if 'error' in output: # capture values in error with types unsupported @@ -2390,7 +2328,7 @@ def dispatch(self, method, uri, input): # display acceptable output. self.client.response_code = 406 output = ("Requested content type '%s' is not available.\n" - "Acceptable types: %s" % (data_type, + "Acceptable types: %s" % (accept_mime_type, ", ".join(sorted(self.__accepted_content_type.keys())))) # Make output json end in a newline to From 073c370733bf3eb36e79ada52a1ae81024773a55 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Thu, 3 Aug 2023 18:28:19 -0400 Subject: [PATCH 68/91] fix: replace bad reverted code change; allow js rate headers Last commit included an incorrect undo. I was going to move the Allow header/output format parsing earlier in the dispatch method. But I reverted it incorrectly and removed it instead. It has been added back in the former location. Header that allows javascript access to the rest rate limit header has been moved. The rate limit headers can be accessed by client side javascript regardless of the rate limit being exceeded. --- roundup/rest.py | 87 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/roundup/rest.py b/roundup/rest.py index b02d9132..aac22dc5 100644 --- a/roundup/rest.py +++ b/roundup/rest.py @@ -2092,6 +2092,24 @@ def dispatch(self, method, uri, input): msg = _("Api rate limits exceeded. Please wait: %s seconds.") % retry_after output = self.error_obj(429, msg, source="ApiRateLimiter") + # expose these headers to rest clients. Otherwise they can't + # respond to: + # rate limiting (*RateLimit*, Retry-After) + # obsolete API endpoint (Sunset) + # options request to discover supported methods (Allow) + self.client.setHeader( + "Access-Control-Expose-Headers", + ", ".join([ + "X-RateLimit-Limit", + "X-RateLimit-Remaining", + "X-RateLimit-Reset", + "X-RateLimit-Limit-Period", + "Retry-After", + "Sunset", + "Allow", + ]) + ) + return self.format_dispatch_output( self.__default_accept_type, output, @@ -2126,6 +2144,75 @@ def dispatch(self, method, uri, input): 'Ignoring X-HTTP-Method-Override using %s request on %s', method.upper(), uri) + # parse Accept header and get the content type + # Acceptable types ordered with preferred one first + # in list. + try: + accept_header = parse_accept_header(headers.get('Accept')) + except UsageError as e: + output = self.error_obj(406, _("Unable to parse Accept Header. %(error)s. " + "Acceptable types: %(acceptable_types)s") % { + 'error': e.args[0], + 'acceptable_types': " ".join(sorted(self.__accepted_content_type.keys()))}) + accept_header = [] + + if not accept_header: + accept_type = self.__default_accept_type + else: + accept_type = None + for part in accept_header: + if accept_type: + # we accepted the best match, stop searching for + # lower quality matches. + break + if part[0] in self.__accepted_content_type: + accept_type = self.__accepted_content_type[part[0]] + # Version order: + # 1) accept header version=X specifier + # application/vnd.x.y; version=1 + # 2) from type in accept-header type/subtype-vX + # application/vnd.x.y-v1 + # 3) from @apiver in query string to make browser + # use easy + # This code handles 1 and 2. Set api_version to none + # to trigger @apiver parsing below + # Places that need the api_version info should + # use default if version = None + try: + self.api_version = int(part[1]['version']) + except KeyError: + self.api_version = None + except (ValueError, TypeError): + # TypeError if int(None) + msg = ("Unrecognized api version: %s. " + "See /rest without specifying api version " + "for supported versions." % ( + part[1]['version'])) + output = self.error_obj(400, msg) + + # get the request format for response + # priority : extension from uri (/rest/data/issue.json), + # header (Accept: application/json, application/xml) + # default (application/json) + ext_type = os.path.splitext(urlparse(uri).path)[1][1:] + + # Check to see if the length of the extension is less than 6. + # this allows use of .vcard for a future use in downloading + # user info. It also allows passing through larger items like + # JWT that has a final component > 6 items. This method also + # allow detection of mistyped types like jon for json. + if ext_type and (len(ext_type) < 6): + # strip extension so uri make sense + # .../issue.json -> .../issue + uri = uri[:-(len(ext_type) + 1)] + else: + ext_type = None + + # headers.get('Accept') is never empty if called here. + # accept_type will be set to json if there is no Accept header + # accept_type wil be empty only if there is an Accept header + # with invalid values. + data_type = ext_type or accept_type or headers.get('Accept') or "invalid" if method.upper() == 'OPTIONS': # add access-control-allow-* access-control-max-age to support # CORS preflight From 9abbe9520c257975e244822435b6960df822a8f1 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Thu, 3 Aug 2023 20:19:10 -0400 Subject: [PATCH 69/91] fix: retry_after should be a string. It's only used as the parameter for "%s", so either string or integer would work. But keep the type consistent as a string. Also replace hardcoded string with the variable. --- roundup/rest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roundup/rest.py b/roundup/rest.py index aac22dc5..24f9d268 100644 --- a/roundup/rest.py +++ b/roundup/rest.py @@ -2086,8 +2086,8 @@ def dispatch(self, method, uri, input): # limitStatus. So report a 1 second delay back # to the client. We treat update as sole # source of truth for exceeded rate limits. - retry_after = 1 - self.client.setHeader('Retry-After', '1') + retry_after = '1' + self.client.setHeader('Retry-After', retry_after) msg = _("Api rate limits exceeded. Please wait: %s seconds.") % retry_after output = self.error_obj(429, msg, source="ApiRateLimiter") From 0193db5d8f57898589668768362926cfc920264e Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Tue, 8 Aug 2023 22:47:58 -0400 Subject: [PATCH 70/91] docs: remove COPYING.html from website/www tree. The file should not be published. It gets published as license.html and linked as such from the side menu. --- website/www/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/website/www/conf.py b/website/www/conf.py index 7fccd784..f7bea251 100644 --- a/website/www/conf.py +++ b/website/www/conf.py @@ -89,6 +89,7 @@ 'docs/whatsnew-0.8.txt', 'robots.txt', 'docs/tracker_config.txt', + 'COPYING.txt', '_tmp'] # The reST default role (used for this markup: `text`) to use for all documents. From 7770e03822fad2b132ab52b2841aa169ddddce0f Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Thu, 10 Aug 2023 09:27:11 -0400 Subject: [PATCH 71/91] docs: rewrite index page using GPT based tool. used parapgraph rewriter from ahrefs to freshen up the landing page. --- website/www/index.txt | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/website/www/index.txt b/website/www/index.txt index 6497efa5..515ec4f7 100644 --- a/website/www/index.txt +++ b/website/www/index.txt @@ -69,26 +69,30 @@ Roundup Issue Tracker -Roundup is a simple-to-use and -install issue-tracking system with -command-line, web, REST, XML-RPC and e-mail interfaces. It is based -on the winning design from Ka-Ping Yee in the Software Carpentry -"Track" design competition. +Roundup is an issue-tracking system that boasts a user-friendly +interface and easy installation process. It offers a range of +interfaces, including command-line, web, REST, XML-RPC, and e-mail, +making it a versatile solution for issue tracking. The system is based +on the award-winning design by Ka-Ping Yee, which emerged victorious +in the Software Carpentry “Track” design competition. -It is designed to be customised so you can "track your issues your -way". +Roundup is highly customizable, allowing users to tailor the system to +their specific needs and preferences. -The current stable version of Roundup is 2.3.0. It fixes bugs and -and adds features compared to the 2.2.0 release. +The latest stable version of Roundup is 2.3.0, which includes bug +fixes and additional features compared to the previous 2.2.0 release. -It runs with Python 2.7.12+ or 3.6+. +Roundup is compatible with Python 2.7.12+ or 3.6+. .. admonition:: Python 2 Support - Although the original plan was to support Python 2 until 2025, - CI resources for testing with Python 2 are being phased out by a - number of CI services. Python 3 should be used to deploy new - trackers and older trackers should be `upgraded to use Python 3 - `_. + Despite the initial intention to provide support for Python 2 until + 2025, several Continuous Integration (CI) services are + discontinuing their resources for testing with Python 2. It is + recommended to utilize Python 3 for the deployment of new trackers, + while existing trackers should be upgraded to incorporate Python 3, + as detailed in the following link: + . Release Highlights ================== @@ -162,7 +166,7 @@ other :doc:`documentation `. Roundup has been deployed for: * thing management using the `GTD methodology `_. ...and so on. It's been designed with :doc:`flexibility -` in mind - it's not just another bug +` in mind - it's not merely another bug tracker. From 81e47e9d7854635cb9bef7cb40d046c0c01fbf20 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Thu, 10 Aug 2023 09:50:45 -0400 Subject: [PATCH 72/91] docs: rewrite segments using ahref paragraph rewriter. --- doc/rest.txt | 97 +++++++++++++++++++++++++++------------------------- 1 file changed, 51 insertions(+), 46 deletions(-) diff --git a/doc/rest.txt b/doc/rest.txt index 9faad0a6..c14b7952 100644 --- a/doc/rest.txt +++ b/doc/rest.txt @@ -20,10 +20,10 @@ Introduction After the last 1.6.0 Release, a REST-API developed in 2015 during a Google Summer of Code (GSOC) by Chau Nguyen, supervised by Ezio -Melotti was integrated. The code was updated by Ralf Schlatterbeck -and John Rouillard to fix some shortcomings and provide the necessary -functions for a single page web application, e.g. etag support, -pagination, field embedding among others. +Melotti was integrated. The code was updated by Ralf Schlatterbeck and +John Rouillard to address some limitations and incorporate essential +features for a single page web application, such as etag support, +pagination, and field embedding, among others. Enabling the REST API ===================== @@ -81,46 +81,48 @@ Rate Limiting API Failed Logins ------------------------------- To make brute force password guessing harder, the REST API has an -invalid login rate limiter. It rate limits the number of failed login -attempts with an invalid user or password. Valid login attempts are -managed by the normal API rate limiter. The rate limiter is a GCRA -leaky bucket variant. All APIs (REST/XMLRPC) share the same rate -limiter. The rate limiter for the HTML/web interface is not shared by +invalid login rate limiter. This feature restricts the number of +failed login attempts made with an invalid user or +password. Successful login attempts are limited by the normal API rate +limiter. The rate limiter is a GCRA leaky bucket variant, which is +shared by all API (REST/XMLRPC) endpoints. However it is important to +note that the rate limiter for the HTML/web interface is not shared by the API failed login rate limiter. -It is configured by settings in config.ini. Setting -``api_failed_login_limit`` to a non-zero value enabled the limiter. -Setting it to 0 disables the limiter (not suggested). If a user fails -to log in more than ``api_failed_login_limit`` times in +It is configured through the settings in config.ini. By setting the +value of ``api_failed_login_limit`` to a non-zero value, the limiter +is enabled. Setting it to 0 will disables the limiter (although this +is not recommended). If a user fails to log in more than +``api_failed_login_limit`` times in ``api_failed_login_interval_in_sec`` seconds, a 429 HTTP error will be -returned. The error also tell the user how long they must wait to try -to log in again. - -When a 429 error is returned, the account is locked until enough time -has passed -(``api_failed_login_interval_in_sec/api_failed_login_limit`` seconds) -to make one additional login token available. Any attempt to log in -while it is locked will fail. This is true even if a the correct -password is supplied for a locked account. This means a brute force -attempt can't try more than one password every -``api_failed_login_interval_in_sec/api_failed_login_limit`` seconds on -average. - -The default values allow up to 4 attempts to login before delaying the -user by 2.5 minutes (150 seconds). At this time there is no supported -method to reset the rate limiter. +returned. The error message also tells the user how long to wait +before trying to log in again. + +When a 429 error is returned, the associated account will be +temporarily locked until sufficient time has elapsed to generate an +additional login token. This time period is determined by the values +of the ``api_failed_login_interval_in_sec`` and ``api_failed_login_limit`` +parameters. Any login attempts made during this lockout period will be +unsuccessful, even if the correct password is provided. This +effectively prevents brute force attacks from attempting more than one +password every +``api_failed_login_interval_in_sec/api_failed_login_limit`` seconds on average. + +The system's default settings permit a maximum of four login attempts, +after which the user will experience a delay of 2.5 minutes (150 +seconds). Currently, there is no established procedure for resetting +the rate limiter. Rate Limiting the API --------------------- -This is a work in progress. This version of roundup includes Rate -Limiting for the API (which is different from rate limiting login -attempts on the web interface). +Roundup includes Rate Limiting for the API, which is distinct from +rate limiting login attempts on the web interface. -It is enabled by setting the ``api_calls_per_interval`` and -``api_interval_in_sec`` configuration parameters in the ``[web]`` -section of ``config.ini``. The settings are documented in the -config.ini file. +This feature can be enabled by setting the ``api_calls_per_interval`` +and ``api_interval_in_sec`` configuration parameters in the ``[web]`` +section of the ``config.ini`` file. Details for these settings are +documented in the same file. If ``api_calls_per_interval = 60`` and ``api_interval_in_sec = 60`` the user can make 60 calls in a minute. They can use them all up in the @@ -135,16 +137,19 @@ values that permit one call per second on average: 1/1, 60/60, 3600/3600, but they all have a different maximum burst rates: 1/sec, 60/sec and 3600/sec. -A single page app may make 20 or 30 calls to populate the page (e.g. a -list of open issues). Then wait a few seconds for the user to select -an issue. When displaying the issue, it needs another 20 or calls to -populate status dropdowns, pull the first 10 messages in the issue -etc. Controlling the burst rate as well as the average rate is a -tuning exercise left for the tracker admin. - -Also the rate limit is a little lossy. Under heavy load, it is -possible for it to miscount allowing more than burst count. Errors of -up to 10% have been seen on slower hardware. +In practice, a single page app may require 20 or 30 API calls to +populate the page with data, followed by a few seconds of waiting for +the user to select an issue. When displaying the issue, another 20 or +more calls may be needed to populate status dropdowns, retrieve the +first 10 messages in the issue, and so on. Therefore, controlling both +the burst rate and the average rate is a tuning exercise that is left +to the tracker admin. + +It is worth noting that the rate limit feature may be slightly lossy, +meaning that under heavy load, it may miscount and allow more than the +burst count. On slower hardware, errors of up to 10% have been +observed. Using redis, PostgreSQL, or MySQL for storing ephemeral data +minimizes the loss. Client API ========== From 6746b73f2707fcb1d36d286d18ae04f60c001072 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Thu, 10 Aug 2023 23:17:18 -0400 Subject: [PATCH 73/91] docs: add link to article on using gunicorn with docker. --- doc/installation.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/installation.txt b/doc/installation.txt index fba7c5e5..917eecb0 100644 --- a/doc/installation.txt +++ b/doc/installation.txt @@ -1383,6 +1383,9 @@ If you want you can use a unix domain socket instead. Example: ``--bind unix:///var/run/roundup/tracker.sock`` would be used for the nginx configuration below. +If you are customizing a docker continer to use gunicorn, see +https://pythonspeed.com/articles/gunicorn-in-docker/. + .. index:: pair: web interface; uWSGI single: wsgi; uWSGI From cd4a16185ec43bfee39a5a5b0e207a8a895f07a6 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Fri, 11 Aug 2023 21:37:43 -0400 Subject: [PATCH 74/91] docs: fix grammar --- doc/installation.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/installation.txt b/doc/installation.txt index 917eecb0..bc5d6651 100644 --- a/doc/installation.txt +++ b/doc/installation.txt @@ -1654,7 +1654,7 @@ described on the wiki wiki also documents how to add `multi-factor (MFA)/one time keys/password using TOTP/HOTP `_. If you -have a single sign on system, the `wiki page on using Shibboleth +have a single sign on system see: the `wiki page on using Shibboleth `_ or `LDAP `_ with Roundup. More customisation can be found under the From c60d98fe641ca713010bc46e79b7dfcaeb700921 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Fri, 11 Aug 2023 21:56:08 -0400 Subject: [PATCH 75/91] docs: add possible way to get email into dockerized roundup. --- doc/installation.txt | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/doc/installation.txt b/doc/installation.txt index bc5d6651..460d9a48 100644 --- a/doc/installation.txt +++ b/doc/installation.txt @@ -1697,7 +1697,15 @@ scheduled (cron) job to access email: However running cron in a container is problematic (running busybox crond as root vs. non-root, requiring setgrp privs -etc). Patches for implementing email support are welcome. +etc). + +Using the host's crontab with a command like:: + + docker exec roundup-tracker roundup-mailgw tracker \ + mailbox /var/spool/mail/roundup + +might be a solution if you can mount a mail spool directory or use +pop/imap. Patches for implementing better email support are welcome. If you want to use a MySQL backend, see `Docker-compose Deployment`_ to deploy a Roundup container and a MySQL container From aa7d5aa200a1515a10ea3e2f77f7b4428907ae94 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Tue, 15 Aug 2023 00:29:24 -0400 Subject: [PATCH 76/91] Update dockerfile build: new python base image, reduce disk space use Use newest python:3-alpine. Remove sphinxcontrib libraries and xapian docs that were left around from the build saving 2M. --- scripts/Docker/Dockerfile | 10 +++++----- scripts/Docker/sphinxdeps.txt | 5 +++++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/scripts/Docker/Dockerfile b/scripts/Docker/Dockerfile index f13938d0..b07f42f8 100644 --- a/scripts/Docker/Dockerfile +++ b/scripts/Docker/Dockerfile @@ -27,7 +27,7 @@ ARG source=local ARG pythonversion=3.11 #FROM python:3-alpine -FROM python@sha256:0a56f24afa1fc7f518aa690cb8c7be661225e40b157d9bb8c6ef402164d9faa7 as build +FROM python@sha256:603975e62d85aa07578034d3d10ffa1983b7618a6abb6371cf51941be6b8842c as build # Inherit global values https://github.com/moby/moby/issues/37345 ARG appdir @@ -97,7 +97,8 @@ RUN [ -z "${VERBOSE}" ] || set -xv; \ ./configure --prefix=/usr/local --with-python3 --disable-documentation && \ make && make install && \ pip uninstall --no-cache-dir -y sphinx && \ - pip uninstall --no-cache-dir -y -r $CWD/sphinxdeps.txt + pip uninstall --no-cache-dir -y -r $CWD/sphinxdeps.txt && \ + rm -rf /usr/local/share/doc/xapian-bindings # add requirements for pip here, e.g. Whoosh, gpg, zstd or other # modules not installed in the base library. @@ -157,7 +158,7 @@ RUN if [ -n "$pip_mod" ]; then pip install --no-cache-dir ${pip_mod}; fi # build a new smaller docker image for execution. Build image above # is 1G in size. # FROM python:3-alpine -FROM python@sha256:0a56f24afa1fc7f518aa690cb8c7be661225e40b157d9bb8c6ef402164d9faa7 +FROM python@sha256:603975e62d85aa07578034d3d10ffa1983b7618a6abb6371cf51941be6b8842c # import from global ARG appdir @@ -212,8 +213,7 @@ ARG pythonversion COPY --from=build /usr/local/lib/python${pythonversion}/site-packages /usr/local/lib/python${pythonversion}/site-packages/ COPY --from=build /usr/local/bin/roundup* /usr/local/bin/ COPY --from=build /usr/local/share /usr/local/share/ -COPY scripts/Docker/roundup_start . -COPY scripts/Docker/roundup_healthcheck . +COPY scripts/Docker/roundup_start scripts/Docker/roundup_healthcheck ./ # Do not run roundup as root. This creates roundup user and group. ARG roundup_uid diff --git a/scripts/Docker/sphinxdeps.txt b/scripts/Docker/sphinxdeps.txt index ed6502bc..e5c6d77a 100644 --- a/scripts/Docker/sphinxdeps.txt +++ b/scripts/Docker/sphinxdeps.txt @@ -13,6 +13,11 @@ pyparsing requests six snowballstemmer +sphinxcontrib-applehelp +sphinxcontrib-devhelp +sphinxcontrib-htmlhelp +sphinxcontrib-jsmath +sphinxcontrib-qthelp sphinxcontrib-serializinghtml sphinxcontrib-websupport urllib3 From 4a5fa6b6164df5da48b53598cfcad8190b8592bf Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Wed, 16 Aug 2023 13:35:25 -0400 Subject: [PATCH 77/91] refactor(api): extract api rate limit handling; add default val Add handle_apiRateLimitExceeded to simplify dispatch() method. Add default value for pretty_print=True to format_dispatch_output. --- roundup/rest.py | 154 ++++++++++++++++++++++++++---------------------- 1 file changed, 84 insertions(+), 70 deletions(-) diff --git a/roundup/rest.py b/roundup/rest.py index 24f9d268..06a5d963 100644 --- a/roundup/rest.py +++ b/roundup/rest.py @@ -2037,6 +2037,84 @@ def getRateLimit(self): # disable rate limiting if either parameter is 0 return None + def handle_apiRateLimitExceeded(self, apiRateLimit): + """Determine if the rate limit is exceeded. + + If not exceeded, return False and the rate limit header values. + If exceeded, return error message and None + """ + gcra = Gcra() + # unique key is an "ApiLimit-" prefix and the uid) + apiLimitKey = "ApiLimit-%s" % self.db.getuid() + otk = self.db.Otk + try: + val = otk.getall(apiLimitKey) + gcra.set_tat_as_string(apiLimitKey, val['tat']) + except KeyError: + # ignore if tat not set, it's 1970-1-1 by default. + pass + # see if rate limit exceeded and we need to reject the attempt + reject = gcra.update(apiLimitKey, apiRateLimit) + + # Calculate a timestamp that will make OTK expire the + # unused entry 1 hour in the future + ts = otk.lifetime(3600) + otk.set(apiLimitKey, + tat=gcra.get_tat_as_string(apiLimitKey), + __timestamp=ts) + otk.commit() + + limitStatus = gcra.status(apiLimitKey, apiRateLimit) + if not reject: + return (False, limitStatus) + + for header, value in limitStatus.items(): + self.client.setHeader(header, value) + + # User exceeded limits: tell humans how long to wait + # Headers above will do the right thing for api + # aware clients. + try: + retry_after = limitStatus['Retry-After'] + except KeyError: + # handle race condition. If the time between + # the call to grca.update and grca.status + # is sufficient to reload the bucket by 1 + # item, Retry-After will be missing from + # limitStatus. So report a 1 second delay back + # to the client. We treat update as sole + # source of truth for exceeded rate limits. + retry_after = '1' + self.client.setHeader('Retry-After', retry_after) + + msg = _("Api rate limits exceeded. Please wait: %s seconds.") % retry_after + output = self.error_obj(429, msg, source="ApiRateLimiter") + + # expose these headers to rest clients. Otherwise they can't + # respond to: + # rate limiting (*RateLimit*, Retry-After) + # obsolete API endpoint (Sunset) + # options request to discover supported methods (Allow) + self.client.setHeader( + "Access-Control-Expose-Headers", + ", ".join([ + "X-RateLimit-Limit", + "X-RateLimit-Remaining", + "X-RateLimit-Reset", + "X-RateLimit-Limit-Period", + "Retry-After", + "Sunset", + "Allow", + ]) + ) + + return (self.format_dispatch_output( + self.__default_accept_type, + output, + True # pretty print for this error case as a + # human may read it + ), None) + def dispatch(self, method, uri, input): """format and process the request""" output = None @@ -2048,75 +2126,10 @@ def dispatch(self, method, uri, input): apiRateLimit = self.getRateLimit() if apiRateLimit: # if None, disable rate limiting - gcra = Gcra() - # unique key is an "ApiLimit-" prefix and the uid) - apiLimitKey = "ApiLimit-%s" % self.db.getuid() - otk = self.db.Otk - try: - val = otk.getall(apiLimitKey) - gcra.set_tat_as_string(apiLimitKey, val['tat']) - except KeyError: - # ignore if tat not set, it's 1970-1-1 by default. - pass - # see if rate limit exceeded and we need to reject the attempt - reject = gcra.update(apiLimitKey, apiRateLimit) - - # Calculate a timestamp that will make OTK expire the - # unused entry 1 hour in the future - ts = otk.lifetime(3600) - otk.set(apiLimitKey, - tat=gcra.get_tat_as_string(apiLimitKey), - __timestamp=ts) - otk.commit() - - limitStatus = gcra.status(apiLimitKey, apiRateLimit) - if reject: - for header, value in limitStatus.items(): - self.client.setHeader(header, value) - # User exceeded limits: tell humans how long to wait - # Headers above will do the right thing for api - # aware clients. - try: - retry_after = limitStatus['Retry-After'] - except KeyError: - # handle race condition. If the time between - # the call to grca.update and grca.status - # is sufficient to reload the bucket by 1 - # item, Retry-After will be missing from - # limitStatus. So report a 1 second delay back - # to the client. We treat update as sole - # source of truth for exceeded rate limits. - retry_after = '1' - self.client.setHeader('Retry-After', retry_after) - - msg = _("Api rate limits exceeded. Please wait: %s seconds.") % retry_after - output = self.error_obj(429, msg, source="ApiRateLimiter") - - # expose these headers to rest clients. Otherwise they can't - # respond to: - # rate limiting (*RateLimit*, Retry-After) - # obsolete API endpoint (Sunset) - # options request to discover supported methods (Allow) - self.client.setHeader( - "Access-Control-Expose-Headers", - ", ".join([ - "X-RateLimit-Limit", - "X-RateLimit-Remaining", - "X-RateLimit-Reset", - "X-RateLimit-Limit-Period", - "Retry-After", - "Sunset", - "Allow", - ]) - ) - - return self.format_dispatch_output( - self.__default_accept_type, - output, - True # pretty print for this error case as a - # human may read it - ) - + LimitExceeded, limitStatus = self.handle_apiRateLimitExceeded( + apiRateLimit) + if LimitExceeded: + return LimitExceeded # error message for header, value in limitStatus.items(): # Retry-After will be 0 because @@ -2373,7 +2386,8 @@ def dispatch(self, method, uri, input): return self.format_dispatch_output(data_type, output, pretty_output) - def format_dispatch_output(self, accept_mime_type, output, pretty_print): + def format_dispatch_output(self, accept_mime_type, output, + pretty_print=True): # Format the content type if accept_mime_type.lower() == "json": self.client.setHeader("Content-Type", "application/json") From f3497380d77e96257cfa0b95c9c3ec46f27079d0 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Thu, 17 Aug 2023 11:03:45 -0400 Subject: [PATCH 78/91] fix missing import. It got copied incorrectly from instaltion.txt. --- website/www/index.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/www/index.txt b/website/www/index.txt index 515ec4f7..37bf8d67 100644 --- a/website/www/index.txt +++ b/website/www/index.txt @@ -184,7 +184,7 @@ Follow the source gratification mode with these steps (change the 2. ``tar -xzvf roundup-2.3.0.tar.gz`` * if you don't have a tar command (e.g windows), use: - ``python -c 'import tarfile; tarfile.open(sys.argv[1]).extractall();' roundup-2.3.0.tar.gz`` + ``python -c 'import tarfile, sys; tarfile.open(sys.argv[1]).extractall();' roundup-2.3.0.tar.gz`` 3. ``cd roundup-2.3.0`` From e97d941778825d7fdcfff05c689e809c41361f52 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Thu, 17 Aug 2023 12:58:37 -0400 Subject: [PATCH 79/91] fix: issue2551290? windows install works It looks like windows installs have been broken since at least 2.1.0. Installing on windows triggered an infinite loop. Fix that in two different ways: 1 linux uses lib, windows uses Lib. Use case insenstive match. 2 linux uses / as the root windows uses C:\ (or other drive letter). keep running the loop until the root/head path doesn't change. --- CHANGES.txt | 3 +++ setup.py | 8 +++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index c324e64f..e5ab0e26 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -42,6 +42,9 @@ Fixed: look like it did previously to 2.3.0 if the new features aren't used. Roundup-admin output was never meant to be machine parsed, but don't break it unless required. (John Rouillard) +- issue2551290 - pip install roundup Hangs on Windows 10 + The install under windows goes into an infinite loop using pip or + source install. (John Rouillard) Features: diff --git a/setup.py b/setup.py index d53e69f2..39c92983 100755 --- a/setup.py +++ b/setup.py @@ -87,13 +87,15 @@ def get_prefix(): if prefix: return prefix else: - # get the platform lib path. Must start with / else infinite loop. + # start with the platform library plp = get_path('platlib') # nuke suffix that matches lib/* and return prefix head, tail = os.path.split(plp) - while tail not in ['lib', 'lib64' ] and head != '/': + old_head = None + while tail.lower() not in ['lib', 'lib64' ] and head != old_head: + old_head = head head, tail = os.path.split(head) - if head == '/': + if head == old_head: head = sys.prefix return head From 1eb2d640bb958bbb542a8420316c4eec60f12ecd Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Thu, 17 Aug 2023 13:33:53 -0400 Subject: [PATCH 80/91] docs: add mention of pyreadline3 under windows adds support for CLI editing/history under windows. --- CHANGES.txt | 2 ++ doc/installation.txt | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index e5ab0e26..89e1e945 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -45,6 +45,8 @@ Fixed: - issue2551290 - pip install roundup Hangs on Windows 10 The install under windows goes into an infinite loop using pip or source install. (John Rouillard) +- Document use of pyreadline3 to allow roundup-admin to have CLI editing + on windows. (John Rouillard) Features: diff --git a/doc/installation.txt b/doc/installation.txt index 460d9a48..ded0fe0d 100644 --- a/doc/installation.txt +++ b/doc/installation.txt @@ -284,6 +284,10 @@ Windows Service You can run Roundup as a Windows service if pywin32_ is installed. Otherwise it must be started manually. +pyreadline3 + When running roundup-admin on windows, installing pyreadline3_ will + allow history and editing on the command line. + requests If you are using OAuth authentication with the roundup-mailgw mail gateway you must install the requests_ library. @@ -2103,6 +2107,17 @@ Panel" is directly accessible from "Start". I do not believe this is possible to do in previous versions of Windows. +Use pip to install pyreadline3 for roundup-admin line editing +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you install pyreadline3_ using pip, roundup-admin will support +command line editing and history. + +This will remove the dreaded:: + + Note: command history and editing not available + +warning when starting roundup-admin. Windows Server -------------- @@ -2188,7 +2203,6 @@ If you are using Apache as the webserver you might want to use it with mod_python instead to serve out Roundup. In that case see the mod_python instructions above for details. - Sendmail smrsh -------------- @@ -2296,6 +2310,7 @@ the test. .. _Psycopg2: https://www.psycopg.org/ .. _pyjwt: https://pypi.org/project/PyJWT/ .. _pyopenssl: https://pypi.org/project/pyOpenSSL/ +.. _pyreadline3: https://pypi.org/project/pyreadline3/ .. _pysqlite: https://pysqlite.org/ .. _pytz: https://pypi.org/project/pytz/ .. _pywin32: https://pypi.org/project/pywin32/ From 508f043d986e8396b56a948eb8fc413b407433d3 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Thu, 17 Aug 2023 13:36:43 -0400 Subject: [PATCH 81/91] docs: mention pywin32 in label Most of the labels for optional python packages mention/are the module names except for Windows Server. Mention pywin32 in the label. --- doc/installation.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/installation.txt b/doc/installation.txt index ded0fe0d..604c1a62 100644 --- a/doc/installation.txt +++ b/doc/installation.txt @@ -280,7 +280,7 @@ pyjwt (v1.7.1, v2.0.1 tested). If you don't have it installed, JWT's are not supported. -Windows Service +pywin32 - Windows Service You can run Roundup as a Windows service if pywin32_ is installed. Otherwise it must be started manually. From 782dfb12d7f9557c8de4f569fb0250ed9eafe00e Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Thu, 17 Aug 2023 13:54:01 -0400 Subject: [PATCH 82/91] docs: replace ' with " in windows tarfile examples The examples for using python to unpack a tarfile on windows used ' not ". I thought they were interchangable, but they are not. Only " works like I expect. --- doc/installation.txt | 2 +- website/www/index.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/installation.txt b/doc/installation.txt index 604c1a62..a9962d32 100644 --- a/doc/installation.txt +++ b/doc/installation.txt @@ -76,7 +76,7 @@ program you can run ``python demo.py`` instead.) 1. ``python3 -m pip download roundup`` 2. ``tar -xzvf roundup-2.2.0.tar.gz`` - * if you don't have a tar command, ``python3 -c 'import tarfile, sys; tarfile.open(sys.argv[1]).extractall();' roundup-2.2.0.tar.gz`` can be used. + * if you don't have a tar command, ``python3 -c "import tarfile, sys; tarfile.open(sys.argv[1]).extractall();" roundup-2.2.0.tar.gz`` can be used. 3. ``cd roundup-2.2.0`` 4. ``python3 demo.py`` diff --git a/website/www/index.txt b/website/www/index.txt index 37bf8d67..8623d6d7 100644 --- a/website/www/index.txt +++ b/website/www/index.txt @@ -184,7 +184,7 @@ Follow the source gratification mode with these steps (change the 2. ``tar -xzvf roundup-2.3.0.tar.gz`` * if you don't have a tar command (e.g windows), use: - ``python -c 'import tarfile, sys; tarfile.open(sys.argv[1]).extractall();' roundup-2.3.0.tar.gz`` + ``python -c "import tarfile, sys; tarfile.open(sys.argv[1]).extractall();" roundup-2.3.0.tar.gz`` 3. ``cd roundup-2.3.0`` From 59d8e428c07eed7fdb202d757d55afe63461772e Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sat, 19 Aug 2023 19:14:27 -0400 Subject: [PATCH 83/91] docs: clarify no login rate limit for roundup-xmlrpc-server Login rate limit only available for /xmlrpc endpoint not supplied by roundup-xmlrpc-server endpoint. --- doc/xmlrpc.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/xmlrpc.txt b/doc/xmlrpc.txt index 873155aa..5aaf9c27 100644 --- a/doc/xmlrpc.txt +++ b/doc/xmlrpc.txt @@ -93,7 +93,11 @@ Rate Limiting Failed Logins See the `rest documentation `_ for rate limiting failed -logins on the API. The XML-RPC uses the same method as the REST API. +logins on the API. There is no login rate limiting for the standalone +roundup-xmlrpc-server. Login rate limiting is only for the `/xmlrpc`` +endpoint when the Roundup server is used. + +The XML-RPC uses the same method as the REST API. Rate limiting is shared between the XMLRPC and REST APIs. Client API From 1f556638e3ac7ddc0c4ccbf6f0303685db28b4d4 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 20 Aug 2023 15:49:14 -0400 Subject: [PATCH 84/91] docs: reword to eliminate bare url in prose. --- website/www/index.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/website/www/index.txt b/website/www/index.txt index 8623d6d7..014c60d8 100644 --- a/website/www/index.txt +++ b/website/www/index.txt @@ -90,9 +90,8 @@ Roundup is compatible with Python 2.7.12+ or 3.6+. 2025, several Continuous Integration (CI) services are discontinuing their resources for testing with Python 2. It is recommended to utilize Python 3 for the deployment of new trackers, - while existing trackers should be upgraded to incorporate Python 3, - as detailed in the following link: - . + while existing trackers should be `upgraded to use Python 3. + `_ Release Highlights ================== From c5c6ec909f193cf645f8a65a3024ceb74cd080a0 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Tue, 29 Aug 2023 16:16:21 -0400 Subject: [PATCH 85/91] chore(deps): bump actions/checkout from 3.5.3 to 3.6.0 - https://github.com/roundup-tracker/roundup/pull/46 --- .github/workflows/anchore.yml | 2 +- .github/workflows/ci-test.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/ossf-scorecard.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/anchore.yml b/.github/workflows/anchore.yml index af119314..6eda09af 100644 --- a/.github/workflows/anchore.yml +++ b/.github/workflows/anchore.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout the code - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - name: Build the Docker image run: docker pull python:3-alpine; docker build . --file scripts/Docker/Dockerfile --tag localbuild/testimage:latest - name: List the Docker image diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index f55a0f89..a0a8d64d 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -93,7 +93,7 @@ jobs: # if: {{ false }} # continue running if step fails # continue-on-error: true - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 # Setup version of Python to use - name: Set Up Python ${{ matrix.python-version }} diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 79dee5f3..d850d557 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -49,7 +49,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v2.6.0 + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v2.6.0 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/ossf-scorecard.yml b/.github/workflows/ossf-scorecard.yml index 861f0604..0274b267 100644 --- a/.github/workflows/ossf-scorecard.yml +++ b/.github/workflows/ossf-scorecard.yml @@ -35,7 +35,7 @@ jobs: steps: - name: "Checkout code" - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.1.0 + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.1.0 with: persist-credentials: false From b6c0e58642ad74ab9e13b38aa51a45c5e8b391df Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 3 Sep 2023 14:03:11 -0400 Subject: [PATCH 86/91] refactor: rename the mime type whitelist Whitelist alone is meaningless. Make the purpose of the list more explicit. --- roundup/cgi/client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roundup/cgi/client.py b/roundup/cgi/client.py index 7b3f8558..424a4c12 100644 --- a/roundup/cgi/client.py +++ b/roundup/cgi/client.py @@ -1858,7 +1858,7 @@ def serve_file(self, designator, dre=dre): # mime type detection is performed in cgi.form_parser # everything not here is served as 'application/octet-stream' - whitelist = [ + mime_type_allowlist = [ 'text/plain', 'text/x-csrc', # .c 'text/x-chdr', # .h @@ -1878,7 +1878,7 @@ def serve_file(self, designator, dre=dre): ] if self.instance.config['WEB_ALLOW_HTML_FILE']: - whitelist.append('text/html') + mime_type_allowlist.append('text/html') try: mime_type = klass.get(nodeid, 'type') @@ -1888,7 +1888,7 @@ def serve_file(self, designator, dre=dre): if not mime_type: mime_type = 'text/plain' - if mime_type not in whitelist: + if mime_type not in mime_type_allowlist: mime_type = 'application/octet-stream' # --/ mime-type security From 6cd3f825f40994c2d98b8e5a1e3f3679b5519208 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Sun, 3 Sep 2023 14:03:48 -0400 Subject: [PATCH 87/91] doc: remove whitelist replace with allowing --- doc/admin_guide.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/admin_guide.txt b/doc/admin_guide.txt index d933bfc1..f6ee06e2 100644 --- a/doc/admin_guide.txt +++ b/doc/admin_guide.txt @@ -818,7 +818,7 @@ header to the tracker using:: note that the header is passed exactly as supplied by the upstream server. It is **not** prefixed with ``HTTP_`` like other headers since -you are explicitly whitelisting the header. Multiple comma separated +you are explicitly allowing the header. Multiple comma separated headers can be passed to the ``-I`` option. These could be used in a detector or other tracker extensions, but only one header can be used by the tracker as an authentication header. From 3b17a4ddd48aecf4f456a9bba45726df196bd03c Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Wed, 6 Sep 2023 13:54:48 -0400 Subject: [PATCH 88/91] test: issue2551284 re-enable 3.12 testing See if the new rc 3.12 version is in place and works now. --- .github/workflows/ci-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index a0a8d64d..f9fede67 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -61,9 +61,9 @@ jobs: include: # example: if 3.12 fails the jobs still succeeds - #- python-version: 3.12 - # os: ubuntu-22.04 - # experimental: true + - python-version: 3.12 + os: ubuntu-22.04 + experimental: true # 3.6 not available on new 22.04 runners, so run on 20.04 ubuntu - python-version: 3.6 From 131f045b9779f4699fa964cb3a71eccea67a49c0 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 11 Sep 2023 00:00:57 -0400 Subject: [PATCH 89/91] chore(deps): bump actions/checkout from 3.6.0 to 4.0.0 - https://github.com/roundup-tracker/roundup/pull/47 --- .github/workflows/anchore.yml | 2 +- .github/workflows/ci-test.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/ossf-scorecard.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/anchore.yml b/.github/workflows/anchore.yml index 6eda09af..ab0e43f9 100644 --- a/.github/workflows/anchore.yml +++ b/.github/workflows/anchore.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout the code - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - name: Build the Docker image run: docker pull python:3-alpine; docker build . --file scripts/Docker/Dockerfile --tag localbuild/testimage:latest - name: List the Docker image diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index f9fede67..92de6fed 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -93,7 +93,7 @@ jobs: # if: {{ false }} # continue running if step fails # continue-on-error: true - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 # Setup version of Python to use - name: Set Up Python ${{ matrix.python-version }} diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index d850d557..1caf4cf7 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -49,7 +49,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v2.6.0 + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2.6.0 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/ossf-scorecard.yml b/.github/workflows/ossf-scorecard.yml index 0274b267..fc9b3736 100644 --- a/.github/workflows/ossf-scorecard.yml +++ b/.github/workflows/ossf-scorecard.yml @@ -35,7 +35,7 @@ jobs: steps: - name: "Checkout code" - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.1.0 + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v3.1.0 with: persist-credentials: false From 379c7460ec59aed208d8cdac4d83e9b3549f6338 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 11 Sep 2023 00:01:45 -0400 Subject: [PATCH 90/91] chore(deps): bump coverallsapp/github-action from 2.2.1 to 2.2.3 - https://github.com/roundup-tracker/roundup/pull/48 --- .github/workflows/ci-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 92de6fed..032244c9 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -257,7 +257,7 @@ jobs: - name: Upload coverage to Coveralls # python 2.7 and 3.6 versions of coverage can't produce lcov files. if: matrix.python-version != '2.7' && matrix.python-version != '3.6' - uses: coverallsapp/github-action@95b1a2355bd0e526ad2fd62da9fd386ad4c98474 # master + uses: coverallsapp/github-action@3dfc5567390f6fa9267c0ee9c251e4c8c3f18949 # master with: github-token: ${{ secrets.GITHUB_TOKEN }} path-to-lcov: coverage.lcov @@ -293,7 +293,7 @@ jobs: steps: - name: Coveralls Finished - uses: coverallsapp/github-action@95b1a2355bd0e526ad2fd62da9fd386ad4c98474 # master + uses: coverallsapp/github-action@3dfc5567390f6fa9267c0ee9c251e4c8c3f18949 # master with: github-token: ${{ secrets.github_token }} parallel-finished: true From b835e8801ea8a2e380eec2cb60d50c755567cd84 Mon Sep 17 00:00:00 2001 From: John Rouillard Date: Mon, 11 Sep 2023 00:02:24 -0400 Subject: [PATCH 91/91] chore(deps): bump actions/upload-artifact from 3.1.2 to 3.1.3 - https://github.com/roundup-tracker/roundup/pull/49 --- .github/workflows/ossf-scorecard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ossf-scorecard.yml b/.github/workflows/ossf-scorecard.yml index fc9b3736..69a31cc3 100644 --- a/.github/workflows/ossf-scorecard.yml +++ b/.github/workflows/ossf-scorecard.yml @@ -62,7 +62,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 with: name: SARIF file path: results.sarif