ci-test.yml

# merged in python-package.yml workflow

# reference docs:
# https://blog.deepjyoti30.dev/tests-github-python
# https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
# https://github.com/pypa/twine/blob/main/.github/workflows/main.yml

name: roundup-ci

on: 
    push:
         # skip if github.ref is 'refs/heads/maint-1.6'
         #   aka github.ref_name of 'maint-1.6'
         # see https://github.com/orgs/community/discussions/26253
         # for mechanism to control matrix based on branch
         branches: [ "*", '!maint-1.6' ]
#    pull_request:
#        branches: [ "master" ]
    schedule:
        # monthly build/check
        - cron: '23 17 1 * *'

# GITHUB_TOKEN only has read repo context.
permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  test:
    name: CI build test

    # run the finalizer for coveralls even if one or more
    # experimental matrix runs fail.
    # continue-on-error: ${{ matrix.experimental }}

    #runs-on: ubuntu-latest
    # use below if running on multiple OS's.
    runs-on: ${{ matrix.os }}

    if: "!contains(github.event.head_commit.message, 'no-github-ci')"

    strategy:
      fail-fast: false
      max-parallel: 4
      matrix:
        # Run in all these versions of Python
        python-version:
          # - "2.7"
          - "3.10"
          # - "3.9"
          - "3.8"
          # - "3.7"
          - "3.11"

        # use for multiple os or ubuntu versions
        #os: [ubuntu-latest, macos-latest, windows-latest]
        # ubuntu latest 22.04 12/2022
        os: [ubuntu-latest, ubuntu-20.04]

        # if the ones above fail. fail the build
        experimental: [false]

        include:
            # example: if 3.12 fails the jobs still succeeds
            - python-version: 3.12
              os: ubuntu-22.04
              experimental: [true]
            # 3.6 not available on new 22.04 runners, so run on 20.04 ubuntu
            - python-version: 3.6
              os: ubuntu-20.04

        exclude:
            # skip all python versions on 20.04 except explicitly included
            - os: ubuntu-20.04

    env:
      # get colorized pytest output even without a controlling tty
      PYTEST_ADDOPTS: "--color=yes"
      #  OS: ${{ matrix.os }}
      PYTHON_VERSION: ${{ matrix.python-version }}

    steps:
      # Checkout the latest code from the repo
      - name: Checkout source
        # example directives:
          # disable step
        # if: {{ false }}
          # continue running if step fails
        # continue-on-error: true
        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3

      # Setup version of Python to use
      - name: Set Up Python ${{ matrix.python-version }}
        uses: actions/setup-python@bd6b4b6205c4dbad673328db7b31b7fab9e241c0 # v4.6.1
        with:
          python-version: ${{ matrix.python-version }}
          allow-prereleases: true
          cache: 'pip'

      - name: Install build tools - setuptools
        run: pip install setuptools

      # Display the Python version being used
      - name: Display Python and key module versions
        run: |
          python -c "import sys; print('python version: ', sys.version)"
          python -c "import sqlite3; print('sqlite version: ', sqlite3.sqlite_version)"
          python -c "import setuptools; print('setuptools version: ', setuptools.__version__);"

      # Install the databases
      - name: Install mysql/mariadb
        run: |
          #set -xv
          # mysql is pre-installed and active but this is the install command
          # sudo apt-get install mysql-server mysql-client
          # set up mysql database
          sudo sed -i -e '/^\[mysqld\]/,/^\[mysql/s/^#* *max_allowed_packet.*/max_allowed_packet = 500M/' /etc/mysql/mysql.conf.d/mysqld.cnf; sleep 3
          #tail -n +0 /etc/mysql/my.cnf /etc/mysql/mysql.conf.d/mysqld.cnf
          #grep max_allowed /etc/mysql/mysql.conf.d/mysqld.cnf
          #ls  /etc/mysql/conf.d/  /etc/mysql/mysql.conf.d/
          #sleep 5
          # try to improve performance speed by disabling some ACID
          # settings and change some layout defaults.
          sudo sed -i -e '$a\innodb_flush_log_at_trx_commit = 2' /etc/mysql/mysql.conf.d/mysqld.cnf
          sudo sed -i -e '$a\innodb_file_per_table = OFF' /etc/mysql/mysql.conf.d/mysqld.cnf
          sudo sed -i -e '$a\innodb_doublewrite=OFF' /etc/mysql/mysql.conf.d/mysqld.cnf
          sudo sed -i -e '$a\innodb_fast_shutdown=2' /etc/mysql/mysql.conf.d/mysqld.cnf
          sudo sed -i -e '$a\innodb_log_file_size=1048576' /etc/mysql/mysql.conf.d/mysqld.cnf
          sudo sed -i -e '$a\innodb_flush_method=O_DIRECT' /etc/mysql/mysql.conf.d/mysqld.cnf
          sudo sed -i -e '$a\innodb_log_buffer_size=3M' /etc/mysql/mysql.conf.d/mysqld.cnf
          sudo sed -i -e '$a\innodb_buffer_pool_size=180M' /etc/mysql/mysql.conf.d/mysqld.cnf
          sleep 3
          sudo service mysql restart
          #sleep 10
          #ps -ef | grep mysqld
          #sudo netstat -anp | grep mysqld
          sudo mysql -u root -proot -e 'CREATE USER "rounduptest"@"localhost" IDENTIFIED WITH mysql_native_password BY "rounduptest"; GRANT ALL on rounduptest.* TO "rounduptest"@"localhost";'

      - name: Install postgres
        run: |
          sudo apt-get update && sudo apt-get install postgresql
          # Disable fsync, full page writes for speed,
          # don't care about data durability when testing
          sudo sed -i -e '$a\fsync = off' /etc/postgresql/*/*/postgresql.conf
          sudo sed -i -e '$a\full_page_writes = off' /etc/postgresql/*/*/postgresql.conf
          sudo sed -i -e '$a\synchronous_commit = off' /etc/postgresql/*/*/postgresql.conf
          sudo service postgresql restart; sleep 10
          # set up postgresql database
          sudo -u postgres psql -c "CREATE ROLE rounduptest WITH CREATEDB LOGIN PASSWORD 'rounduptest';" -U postgres

      - name: install redis
        run: |
          sudo apt-get install redis
          pip install redis

      - name: Update pip
        run: python -m pip install --upgrade pip

      - name: Install python db libraries
        run: pip install psycopg2 mysqlclient

      - name: Install auxiliary packages
        run: |
          sudo apt-get install swig gpgsm libgpgme-dev
          # pygments for markdown2 to highlight code blocks
          pip install 'markdown2<=2.4.8' pygments
          # docutils for ReStructuredText
          pip install beautifulsoup4 brotli docutils gpg jinja2 \
            mistune==0.8.4 pyjwt pytz whoosh

      - name: Install aux packages that need versions differences
        # if zstd fails install, keep going with test, don't abort
        run: |
          set -xv
          pip install zstd || true
          if [[ "$PYTHON_VERSION" != "2."* ]]; then 
              pip install Markdown; fi

      - name: Install xapian
        run: |
          sudo apt-get install libxapian-dev
          # Sphinx required to build the xapian python bindings. Use 1.8.5 on
          # older python and newest on newer.
          if [[ $PYTHON_VERSION == "2."* ]]; then pip install sphinx==1.8.5; fi
          if [[ $PYTHON_VERSION == '3.'* ]] ; then pip install sphinx; fi
          if [[ $PYTHON_VERSION == '3.12'* ]] ; then \
              XAPIAN_VER=1.4.22; \
          else
              XAPIAN_VER=$(dpkg -l libxapian-dev | tail -n 1 | awk '{print $3}' | cut -d '-' -f 1); echo $XAPIAN_VER; \
          fi
          cd /tmp
          curl -s -O https://oligarchy.co.uk/xapian/$XAPIAN_VER/xapian-bindings-$XAPIAN_VER.tar.xz
          tar -Jxvf xapian-bindings-$XAPIAN_VER.tar.xz
          cd xapian-bindings-$XAPIAN_VER/
          if [[ $PYTHON_VERSION == "2."* ]]; then ./configure --prefix=$VIRTUAL_ENV --with-python --disable-documentation; fi
          # edit the configure script.
          # distutils.sysconfig.get_config_vars('SO')  doesn't work for
          # 3.11 or newer.
          # Change distutils.sysconfig... to just sysconfig and SO
          # to EXT_SUFFIX to get valid value.
          if [[ $PYTHON_VERSION == "3."* ]]; then sed -i -e '/PYTHON3_SO=/s/distutils\.//g' -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' configure; ./configure --prefix=$VIRTUAL_ENV --with-python3 --disable-documentation; fi
          case "$PYTHON_VERSION" in nightly|3.12*) echo skipping xapian build;; *) make && sudo make install; esac

      - name: Install pytest and other packages needed for running tests
        run: pip install flake8 mock pytest pytest-cov requests

      - name: Test build roundup and install locale so lang tests work.
        run: |
          sudo apt-get install gettext
          python setup.py build
          (cd locale; make local_install; ls -lR locale/de/LC_MESSAGES)

      - name: run flake8 - abort for syntax error, otherwise warn only
        run: |
            # stop the build for Python syntax errors or undefined names
            # talgettext is a utility function ignore it.
            flake8 roundup --count --select=E9,F63,F7,F82 --show-source --statistics --extend-exclude talgettext.py
            # exit-zero treats all errors as warnings.
            #   The GitHub editor is 127 chars wide
            flake8 roundup --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics

      # Run the tests using pytest with test files in tests directory.
      - name: Run tests
        run: |
          if [[ "$PYTHON_VERSION" != "2."* ]]; then 
            pytest -r a \
              --durations=20 \
              -W default \
              -W "ignore:SelectableGroups:DeprecationWarning" \
              -W "ignore:the imp module:DeprecationWarning:gpg.gpgme:15" \
              -W "ignore:'U' mode::docutils.io"  \
              -W "ignore:unclosed:ResourceWarning:roundup.roundup.demo" \
              -W "ignore:unclosed file:ResourceWarning:enum" \
              -v  test/ --cov=roundup
            if [[ "$PYTHON_VERSION" != "3.6" ]]; then
              # coverage before 3.6 doesn't support lcov output
              coverage lcov
            fi
          else
            # python2 case
            pytest -v -r a --durations=20 test/ --cov=roundup
          fi

      - name: Upload coverage to Codecov
        # see: https://github.com/codecov/codecov-action#usage
        uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # v3.1.4
        with:
          verbose: true
          token: ${{ secrets.CODECOV_TOKEN }}

      - name: Upload coverage to Coveralls
        # python 2.7 and 3.6 versions of coverage can't produce lcov files.
        if: matrix.python-version != '2.7' && matrix.python-version != '3.6'
        uses: coverallsapp/github-action@c7885c00cb7ec0b8f9f5ff3f53cddb980f7a4412 # master
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          path-to-lcov: coverage.lcov
          parallel: run-{{ matrix.python-version }}-{{ matrix.os }}

      - name: test build_doc
        run: |
          python ./setup.py build_doc

      #- name: test docker build current version ubuntu-latest
      #  if: matrix.python-version == '3.10' && matrix.os == 'ubuntu-latest'
      #  run: |
      #     docker build -t roundup-app-dev -f scripts/Docker/Dockerfile .
      #     mkdir tracker; chmod 777 tracker
      #     docker run -d --rm -p 9017:8080 \
      #         -v $PWD/tracker:/usr/src/app/tracker \
      #         roundup-app-dev:latest demo
            # expect 200
      #     curl --fail http://localhost:9017/demo/ > /dev/null

      #- name: test docker build released pip version
      #  run: |
      #     docker build -t roundup-app-rel --build-arg="source=pypi" \
      #         -f scripts/Docker/Dockerfile . 


  # in parallel build coveralls requires a finish step
  finish:
    needs: test
    runs-on: ubuntu-latest

    if: "!contains(github.event.head_commit.message, 'no-github-ci')"
    
    steps:
      - name: Coveralls Finished
        uses: coverallsapp/github-action@c7885c00cb7ec0b8f9f5ff3f53cddb980f7a4412 # master
        with:
          github-token: ${{ secrets.github_token }}
          parallel-finished: true