merge default trunk into branch

.github/workflows/anchore.yml

@@ -37,7 +37,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout the code
-      uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+      uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
     - name: Build the Docker image
       run: docker pull python:3-alpine; docker build . --file scripts/Docker/Dockerfile --tag localbuild/testimage:latest
     - name: List the Docker image

.github/workflows/ci-test.yml

@@ -93,11 +93,11 @@ jobs:
         # if: {{ false }}
           # continue running if step fails
         # continue-on-error: true
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
 
       # Setup version of Python to use
       - name: Set Up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
+        uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1
         with:
           python-version: ${{ matrix.python-version }}
           allow-prereleases: true
@@ -184,16 +184,13 @@ jobs:
 
       - name: Install xapian
         run: |
+          set -xv
           sudo apt-get install libxapian-dev
           # Sphinx required to build the xapian python bindings. Use 1.8.5 on
           # older python and newest on newer.
           if [[ $PYTHON_VERSION == "2."* ]]; then pip install sphinx==1.8.5; fi
           if [[ $PYTHON_VERSION == '3.'* ]] ; then pip install sphinx; fi
-          if [[ $PYTHON_VERSION == '3.12'* ]] ; then \
-              XAPIAN_VER=1.4.22; \
-          else
-              XAPIAN_VER=$(dpkg -l libxapian-dev | tail -n 1 | awk '{print $3}' | cut -d '-' -f 1); echo $XAPIAN_VER; \
-          fi
+          XAPIAN_VER=$(dpkg -l libxapian-dev | tail -n 1 | awk '{print $3}' | cut -d '-' -f 1); echo $XAPIAN_VER;
           cd /tmp
           curl -s -O https://oligarchy.co.uk/xapian/$XAPIAN_VER/xapian-bindings-$XAPIAN_VER.tar.xz
           tar -Jxvf xapian-bindings-$XAPIAN_VER.tar.xz
@@ -204,8 +201,19 @@ jobs:
           # 3.11 or newer.
           # Change distutils.sysconfig... to just sysconfig and SO
           # to EXT_SUFFIX to get valid value.
-          if [[ $PYTHON_VERSION == "3."* ]]; then sed -i -e '/PYTHON3_SO=/s/distutils\.//g' -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' configure; ./configure --prefix=$VIRTUAL_ENV --with-python3 --disable-documentation; fi
-          case "$PYTHON_VERSION" in nightly|3.12*) echo skipping xapian build;; *) make && sudo make install; esac
+          if [[ $PYTHON_VERSION == "3."* ]]; then \
+            cp configure configure.FCS; \
+            sed -i \
+              -e '/PYTHON3_SO=/s/distutils\.//g' \
+              -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' \
+              -e '/PYTHON3_CACHE_TAG=/s/imp;print(imp.get_tag())/sys;print(sys.implementation.cache_tag)/' \
+              -e '/PYTHON3_CACHE_OPT1_EXT=/s/imp\.get_tag()/sys.implementation.cache_tag/g' \
+              -e '/PYTHON3_CACHE_OPT1_EXT=/s/imp\b/importlib/g' \
+            configure; \
+            diff -u configure.FCS configure || true; \
+            ./configure --prefix=$VIRTUAL_ENV --with-python3 --disable-documentation; \
+          fi
+          case "$PYTHON_VERSION" in nightly) echo skipping xapian build;; *) make && sudo make install; esac
 
       - name: Install pytest and other packages needed for running tests
         run: pip install flake8 mock pytest pytest-cov requests

.github/workflows/codeql-analysis.yml

@@ -49,7 +49,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2.6.0
+      uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/ossf-scorecard.yml

@@ -35,12 +35,12 @@ jobs:
 
     steps:
       - name: "Checkout code"
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v3.1.0
+        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
         with:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@08b4669551908b1024bb425080c797723083c031 # v2.2.0
+        uses: ossf/scorecard-action@483ef80eb98fb506c348f7d62e28055e49fe2398 # v2.3.0
         with:
           results_file: results.sarif
           results_format: sarif

.grype.yaml

@@ -0,0 +1,3 @@
+ignore:
+  - vulnerability: CVE-2018-20225
+  - vulnerability: CVE-2018-20225-pip

.hgtags

@@ -143,3 +143,4 @@ c90104abe508e3886917243e4acd069c8ef7a1a4 2.2.0
 239d9542b02062c56f88fd1de8b87c4d88d700ad 2.2.0
 51fc06fabcee043db116e2fbdcdcf5e86b67ed3d 2.3.0b2
 913a73b9fab58e9c7e43e1fad379b68cae6ee3ae 2.3.0
+d17e57220a62416fcd192199cf29ca48db3af1a4 2.3.1a0

.travis.yml

@@ -138,7 +138,7 @@ install:
   - if [[ $TRAVIS_PYTHON_VERSION != "3.4"* ]]; then pip install mistune==0.8.4; fi
   - if [[ $TRAVIS_PYTHON_VERSION != "3.4"* && $TRAVIS_PYTHON_VERSION != "2."* ]]; then pip install Markdown; fi
   - pip install 'markdown2<=2.4.8'
-  - pip install brotli
+  - pip install brotli==1.0.9
   # zstd fails to build under python nightly aborting test.
   # allow testing to still happen if the optional package doesn't install.
   - pip install zstd || true
@@ -165,7 +165,7 @@ script:
   - PATH=$VIRTUAL_ENV/bin:$PATH
   - export LD_LIBRARY_PATH=$VIRTUAL_ENV/lib:$LD_LIBRARY_PATH
   - python -c "import sys; print('python version ', sys.version)"
-  - set -xv; if [[ "$TRAVIS_PYTHON_VERSION" != "2."* ]]; then
+  - if [[ "$TRAVIS_PYTHON_VERSION" != "2."* ]]; then
     python -m pytest -r a \
       --durations=20 \
       -W default \
@@ -183,7 +183,7 @@ script:
 
 after_success:
   # from https://docs.codecov.com/docs/codecov-uploader#integrity-checking-the-uploader
-  - curl https://keybase.io/codecovsecurity/pgp_keys.asc |
+  - curl https://keybase.io/codecovsecurity/pgp_keys.asc | \
     gpg --no-default-keyring --keyring trustedkeys.gpg --import # One-time step
   - curl -Os https://uploader.codecov.io/latest/linux/codecov
   - curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM

CHANGES.txt

@@ -47,6 +47,33 @@ Fixed:
   source install. (John Rouillard)
 - Document use of pyreadline3 to allow roundup-admin to have CLI editing
   on windows. (John Rouillard)
+- issue2551293 - remove schema_hook from Tracker instance. Looks like
+  it was an obsolete hook used for testing. Never documented and not
+  accessible from schema.py.
+- Fix roundup-admin security command. Lowercase its optional
+  argument. Roles are indexed by lower case role name. So 'security
+  User' and 'security user' should generate the same output. (John
+  Rouillard from issue on mailing list by Chuck Cunningham)
+- make roundup-server exit more quickly on ^C. This seems to be
+  limited to windows. (John Rouillard)
+- Fix error handling so failure during import of a non-user item
+  doesn't cause a second traceback. (Found by Norbert Schlemmer, fix
+  John Rouillard)
+- Handle out of memory error when importing large trackers in
+  PostgreSQL. (Found by Norbert Schlemmer, extensive testing by
+  Norbert, fix John Rouillard)
+- use unittest.mock rather than mock for
+  test/test_hyperdbvals.py. (found by Ralf Schlatterbeck. Fix John
+  Rouillard)
+- disable proxy with wget in roundup_healthcheck. (Norbert SCHLEMMER
+  Noschvie on github.com)
+- support dicttoxml2.py for Roundup running on 3.7 and
+  newer. dicttoxml uses a type alias: collection.Iterator that is
+  dropped in Python 3.10. (found by Norbert SCHLEMMER, fix John
+  Rouillard)
+- fix repeated password id with user.item.html in all templates except
+  jinja2. (John Rouillard)
+- fix unclosed file when saving index in indexer_dbm.py. (John Rouillard)
 
 Features:
 

RELEASE.txt

@@ -236,7 +236,7 @@ Roundup release checklist:
 
      Also can scan (optionally) using trivy:
 
-        docker run --rm --volume \
+        docker run -it --rm --volume \
 	/var/run/docker.sock:/var/run/docker.sock \
           --name trivy aquasec/trivy:latest image rounduptracker/roundup:2.2.0
 

doc/acknowledgements.txt

@@ -16,6 +16,24 @@ ideas and everything else that helped!
 
 .. _`Announcement with changelog for current release.`: announcement.html
 
+2.4
+---
+
+2.4.0
+~~~~~
+
+Maintainer:  John Rouillard
+
+Release Manager: John Rouillard
+
+Developer activity by changesets::
+
+  TBD
+
+Other contributers
+
+Norbert Schlemmer
+
 2.3
 ---
 

doc/admin_guide.txt

@@ -962,6 +962,16 @@ Migrating Backends
    move the new tracker home into its place.
 9. Restart web and email frontends.
 
+If you are importing into PostgreSQL, it autocommits the data every
+10000 objects/rows by default. This can slow down importing, but it
+prevents an out of memory error caused by using a savepoint for each
+object. You can control the commit frequency by using::
+
+  pragma savepoint_limit=20000
+
+to set a higher or lower number in roundup-admin. In this example a
+commit will be done every 20,000 objects/rows. The pragma can also be
+set on the roundup-admin command line as described below.
 
 Moving a Tracker
 ----------------
@@ -1475,6 +1485,73 @@ Also the tautological::
 Remember the roundup commands that accept multiple designators accept
 them ',' separated so using '-dc' is almost always required.
 
+A Note on Import and Export
+---------------------------
+
+This is a little in the weeds, but I have noticed this and was asked
+about it so I am documenting it for the future.
+
+Running ``roundup-admin`` with ``-V`` to get additional info when
+importing/exporting the tracker generates three types of messages.
+
+For example::
+
+   $ roundup-admin -i tracker -V export ./myExport
+   Exporting priority - 5
+   Exporting Journal for priority
+   Exporting status - 1
+   Exporting Journal for status
+   [...]
+
+
+   $ roundup-admin -i tracker -V import ./myExport
+   Importing priority - 7
+   setting priority 8
+   Importing status - 8
+   setting status 9
+   [...]
+
+Note the numbers for status. Exported ends up at 1, Imported ends up
+at 8 and setting chooses 9. These numbers are derived differently and
+used differently. You can't directly compare them.
+
+``Exporting issue - XXX``:
+
+   ``XXX`` is the id number of the node being exported/processed from
+   the database. The order is determined by sorting by the key of the
+   class (as set by sortkey). If the class key is 'id', then it's a
+   string sort so '9' comes before '1009'. You might notice if the
+   export is slow the numbers jumping around.
+
+   It does not usually end up as the total number of nodes
+   exported. However if it crashes, you know what node it was
+   processing at the time.
+
+   In the example above, the status node with id 1 was the last one
+   when sorted alphabetically by name.
+
+``Importing <class> - XXX``:
+
+    ``XXX`` is the number of the node (not the node id) being
+    imported/currently processed at line XXX+1 in the file. It is an
+    incrementing number starting at 0 and never jumps around. Value 0
+    is consumed when reading the header and not displayed. The final
+    value is the same as the number of objects and one less then the
+    number of lines in the file. If it crashes, you were processing
+    the line at XXX+1.
+
+``setting <class> XXX``:
+
+    ``XXX`` in the setting line should always be one more than the
+    number of imported objects. The setting value is the id for the
+    next created object of that type. So in theory the Importing
+    number should be one less than the setting number.
+
+    However under certain circumstances, Roundup can skip an id
+    number. This can lead to a difference of more than 1 between the
+    Importing and setting numbers. It's not a problem. However setting
+    can (and must) always be higher than the Importing number.
+
 
 .. _`customisation documentation`: customizing.html
 .. _`reference documentation`: reference.html

doc/installation.txt

@@ -1754,10 +1754,13 @@ You can build a docker container in one of 4 modes defined by the
   by some Python users.
 
 ``--build-arg="source=pip_sdist"``
+  Disabled - hopefully it will be available in the future.
   This is meant for maintainer/developer use. It installs using
   pip from a source distribution (sdist) tarball built by
   following the RELEASE.txt. It is meant for testing
-  releases. Normal users/admins should not use it.
+  releases or building a docker image that installs a new pending
+  source distribution release. Normal users/admins should not use it.
+  Use ``local`` or ``pip_local`` instead.
 
 Build a docker container using the code in the current directory,
 with this build command from the top of the source tree::

doc/reference.txt

@@ -144,10 +144,10 @@ Section **main**
   below is used.
 
  dispatcher_email -- ``roundup-admin``
-  The 'dispatcher' is a role that can get notified of new items to the
-  database. It is used by the ERROR_MESSAGES_TO config setting. If the
-  email address doesn't contain an ``@`` part, the MAIL_DOMAIN defined
-  below is used.
+  The 'dispatcher' is a role that can get notified when errors occur
+  while sending email to a user. It is used by the ERROR_MESSAGES_TO config
+  setting. If the email address doesn't contain an ``@`` part, the
+  MAIL_DOMAIN defined below is used.
 
  email_from_tag -- default *blank*
   Additional text to include in the "name" part of the From: address used
@@ -686,7 +686,7 @@ You must never:
 **Remove the user class**
   This class is the only *required* class in Roundup.
 
-**Remove the "username", "address", "password" or "realname" user properties**
+**Remove the "username", "address", "password", "roles" or "realname" user properties**
   Various parts of Roundup require these properties. Don't remove them.
 
 **Change the type of a property**
@@ -1561,8 +1561,17 @@ directories are not on the Python system path when interfaces.py is
 evaluated. You need to add library directories explictly by
 modifying sys.path.
 
+Interfaces.py allows you to interact with any part of Roundup's
+internals. These internals are not as stable as defined interfaces
+(e.g. extensions. detectors, schema). So the code in interfaces.py is
+more likely to need modification when upgrading from version to
+version.  While the developers attempt to keep the examples working,
+it may make more sense to change the internals to make the code
+clearer, add more features etc.
+
 See `Changing How the Core Code Works
-<customizing.html#changing-how-the-core-code-works>`_ for examples.
+<customizing.html#changing-how-the-core-code-works>`_ for examples
+of using interfaces.py.
 
 Database Content
 ================