tox.ini

[tox]
envlist = py{27,36,37,38,39,310}-{linux,macos,windows}-unittests, py{27,36,37,38,39,310}-end2end
skipsdist = true
ignore_basepython_conflict = true
# NOTE: We pass the TERM env to preserve colors
passenv =
    TERM
    XDG_CACHE_HOME
setenv =
  PY_COLORS=1

[gh-actions]
python =
    3.8: py38
    3.9: py39
    3.10: py310
    3.11: py311

[gh-actions:env]
PLATFORM =
    ubuntu: linux
    macos: macos
    windows: windows
KIND =
    end2end: end2end
    unittests: unittests

[testenv]
basepython =
    {py3.6-unit-tests,py3.6-smoke-tests,lint,black,modernize,flake8,pylint,mypy,generate-monitor-docs}: python3.8
    {lint,black,modernize,flake8,pylint,mypy}: {env:LINT_PYTHON_BINARY}
# NOTE: We pass the TERM env to preserve colors
passenv =
    TERM
    XDG_CACHE_HOME
    PYTEST_BENCH_FORCE_UNIT
setenv =
  LINT_FILES_TO_CHECK={env:LINT_FILES_TO_CHECK:*.py scripts/*.py scripts/circleci/*.py benchmarks/scripts/*.py tests/ pylint_plugins/*.py .circleci/*.py scalyr_agent/ scalyr_agent/third_party/tcollector/}
  # Which Python binary to use for various lint targets
  LINT_PYTHON_BINARY={env:LINT_PYTHON_BINARY:python3.8}
  PYTHONPATH={toxinidir}
  PY_COLORS=1
  PYTEST_COMMON_OPTS=-vv -s
  # We make benchmark output less verbose so it's easier to find actual benchmarking data among
  # all the noise
  PYTEST_COMMON_BENCHMARK_OPTS=--benchmark-only --benchmark-name=short --benchmark-columns=min,max,mean,stddev,median,ops,rounds --benchmark-histogram=benchmark_histograms/benchmark
install_command = pip install -U --force-reinstall {opts} {packages}
deps =
    -r dev-requirements-new.txt
    -r benchmarks/micro/requirements-compression-algorithms.txt
allowlist_externals =
    rm
    bash
commands =
    # NOTE 1: We run memory leak tests separately so they run in a separate process and are
    # isolated from other tests
    # NOTE 2: We write junit XML file for nicer visualization on Circle CI
    rm -rf test-results
    py.test tests/unit/ -vv --durations=5 -m "not memory_leak and not json_lib" --junitxml=test-results/junit-1.xml
    py.test tests/unit/test_memory_leaks.py -vv --durations=5 --junitxml=test-results/junit-2.xml
    py.test tests/unit/util/json_util_test.py -vv --durations=5 -m "json_lib" --junitxml=test-results/junit-3.xml

[testenv:py{27,36,37,38,39,310}-{linux,macos,windows}-unittests]
# Instruct virtualenv to upgrade pip to the latest version
download = true
setenv =
  PYTEST_ADDOPTS=-vv --durations=5 --cov-config=.coveragerc --cov=scalyr_agent --cov=tests/
  {[testenv]setenv}
deps =
    pytest-github-actions-annotate-failures
    -r benchmarks/micro/requirements-compression-algorithms.txt
    linux,windows,macos: -r dev-requirements-new.txt
commands =
    # NOTE 1: We run memory leak tests separately so they run in a separate process and are
    # isolated from other tests
    # NOTE 2: We run json util tests in a separate process as they modify global state (cross-test pollution)
    # NOTE 3: We write junit XML file for nicer visualization on Circle CI
    # NOTE 4: We run Windows platform tests separately since they cause issues when running them with other unit tests
    rm -rf test-results .coverage
    py.test tests/unit/ -m "not memory_leak and not json_lib and not windows_platform" --junitxml=test-results/junit-1.xml
    py.test tests/unit/test_memory_leaks.py --cov-append --junitxml=test-results/junit-2.xml
    py.test tests/unit/util/json_util_test.py -m "json_lib" --cov-append --junitxml=test-results/junit-3.xml
    windows: py.test tests/unit/test_platform_windows.py -m "not memory_leak and not json_lib" --cov-append --junitxml=test-results/junit-4.xml

# Target which generates documentation for all the monitors
[testenv:generate-monitor-docs]
deps =
    -r dev-requirements-new.txt
allowlist_externals =
    bash
commands =
    bash -c "scripts/generate-docs-for-all-monitors.sh"


# Lint target which runs all the linting tools such as black, modernize, pylint, flake8, mypy, etc.
# NOTE: We use bash -c since we don't want tox to quote all the arguments, we want globs to
# be expaded
# NOTE: If you update any of the lint targets or the lint target itself, make sure you also update
# corresponding pre commit hook configuration in .pre-commit-config.yaml
[testenv:lint]
basepython = {env:LINT_PYTHON_BINARY}
deps =
    -r dev-requirements-new.txt
    -r lint-requirements.txt
    -r benchmarks/scripts/requirements.txt
    -r benchmarks/micro/requirements-compression-algorithms.txt
    cryptography==2.9.2
commands =
    bash -c "python ./scripts/check-bundled-ca-certs-expirations.py"
    bash -c 'black --check --config pyproject.toml {env:LINT_FILES_TO_CHECK}'
    bash -c 'flake8 --config lint-configs/python/.flake8 {env:LINT_FILES_TO_CHECK}'
    bash -c 'pylint -E --rcfile=./lint-configs/python/.pylintrc {env:LINT_FILES_TO_CHECK}'
    bash -c 'bandit --configfile lint-configs/python/bandit.yaml -lll -r scalyr_agent/'
    bash -c 'mypy --pretty --no-incremental --config-file ./lint-configs/python/mypy.ini {env:LINT_FILES_TO_CHECK}'

[testenv:black]
basepython = {env:LINT_PYTHON_BINARY}
deps =
    -r dev-requirements-new.txt
    -r lint-requirements.txt
    -r benchmarks/scripts/requirements.txt
commands =
    bash -c 'black --config pyproject.toml {env:LINT_FILES_TO_CHECK}'

[testenv:modernize]
basepython = {env:LINT_PYTHON_BINARY}
deps =
    -r dev-requirements-new.txt
    -r lint-requirements.txt
    -r benchmarks/scripts/requirements.txt
commands =
    python .circleci/modernize/modernize.py -j 2

[testenv:flake8]
basepython = {env:LINT_PYTHON_BINARY}
deps =
    -r dev-requirements-new.txt
    -r lint-requirements.txt
    -r benchmarks/scripts/requirements.txt
commands =
    bash -c 'flake8 --config lint-configs/python/.flake8 {env:LINT_FILES_TO_CHECK}'

[testenv:pylint]
basepython = {env:LINT_PYTHON_BINARY}
deps =
    -r dev-requirements-new.txt
    -r lint-requirements.txt
    -r benchmarks/scripts/requirements.txt
    -r tests/ami/requirements.txt
commands =
    bash -c 'pylint -E --rcfile=./lint-configs/python/.pylintrc {env:LINT_FILES_TO_CHECK}'

[testenv:bandit]
basepython = {env:LINT_PYTHON_BINARY}
deps =
    -r dev-requirements-new.txt
    -r lint-requirements.txt
commands =
    bash -c 'bandit --configfile lint-configs/python/bandit.yaml -lll -r scalyr_agent/'

[testenv:mypy]
basepython = {env:LINT_PYTHON_BINARY}
deps =
    -r dev-requirements-new.txt
    -r lint-requirements.txt
    -r benchmarks/scripts/requirements.txt
commands =
    bash -c 'mypy --pretty --no-incremental --config-file ./lint-configs/python/mypy.ini {env:LINT_FILES_TO_CHECK}'

# TODO: Once we make more progress, set coverage % threshold and fail a build if it's not reached
[testenv:coverage]
commands =
    rm -f .coverage
    rm -rf test-results
    py.test tests/unit/ -m "not memory_leak and not json_lib" --cov-config=.coveragerc --cov=scalyr_agent --cov=tests/ --junitxml=test-results/junit-1.xml
    py.test tests/unit/test_memory_leaks.py -vv --durations=5 --cov-config=.coveragerc --cov=scalyr_agent --cov=tests/ --cov-append --junitxml=test-results/junit-2.xml
    py.test tests/unit/util/json_util_test.py -vv --durations=5 -m "json_lib" --cov-config=.coveragerc --cov=scalyr_agent --cov=tests/ --cov-append --junitxml=test-results/junit-3.xml

# Smoke tests related targets
[testenv:py{38,39,310,311}-end2end]
passenv =
    SCALYR_API_KEY
    READ_API_KEY
    SCALYR_SERVER
    TEST_SESSION_SUFFIX
commands =
    rm -rf test-results .coverage
    py.test  tests/end_to_end_tests/test_from_source/test_from_source.py -s -vv --durations=5 --junitxml=test-results/junit-1.xml


# Package smoke tests related targets
[testenv:agent_package_smoke_test_amazonlinux_python2]
basepython = python3.8
passenv =
    TERM
    SCALYR_API_KEY
    READ_API_KEY
    SCALYR_SERVER
    AGENT_HOST_NAME
    DOCKER_CERT_PATH
    DOCKER_HOST
    DOCKER_TLS_VERIFY
commands =
    py.test tests/smoke_tests/package_test.py::test_smoke_package_rpm_python2 -s -vv --durations=5 {posargs}

[testenv:agent_package_smoke_test_amazonlinux_python3]
basepython = python3.8
passenv =
    TERM
    SCALYR_API_KEY
    READ_API_KEY
    SCALYR_SERVER
    AGENT_HOST_NAME
    DOCKER_CERT_PATH
    DOCKER_HOST
    DOCKER_TLS_VERIFY
commands =
    py.test tests/smoke_tests/package_test.py::test_smoke_package_rpm_python3 -s -vv --durations=5 {posargs}

[testenv:agent_package_smoke_test_ubuntu_python2]
basepython = python3.8
passenv =
    TERM
    SCALYR_API_KEY
    READ_API_KEY
    SCALYR_SERVER
    AGENT_HOST_NAME
    DOCKER_CERT_PATH
    DOCKER_HOST
    DOCKER_TLS_VERIFY
commands =
    py.test tests/smoke_tests/package_test.py::test_smoke_package_deb_python2 -s -vv --durations=5 {posargs}

[testenv:agent_package_smoke_test_ubuntu_python3]
basepython = python3.8
passenv =
    TERM
    SCALYR_API_KEY
    READ_API_KEY
    SCALYR_SERVER
    AGENT_HOST_NAME
    DOCKER_CERT_PATH
    DOCKER_HOST
    DOCKER_TLS_VERIFY
commands =
    py.test tests/smoke_tests/package_test.py::test_smoke_package_deb_python3 -s -vv --durations=5 {posargs}

[testenv:agent_distributions_tests_ubuntu1604]
basepython = python3.8
passenv =
    TERM
    SCALYR_API_KEY
    READ_API_KEY
    SCALYR_SERVER
    AGENT_HOST_NAME
    DOCKER_CERT_PATH
    DOCKER_HOST
    DOCKER_TLS_VERIFY
commands =
    # NOTE: We use short traceback formatting since we run py.test inside py.test which results in hard to read output
    py.test tests/distribution/python_version_change_tests/ubuntu1604 -s -vv --tb=short --durations=5 {posargs}

[testenv:agent_distributions_tests_ubuntu2204]
basepython = python3.8
passenv =
    TERM
    SCALYR_API_KEY
    READ_API_KEY
    SCALYR_SERVER
    AGENT_HOST_NAME
    DOCKER_CERT_PATH
    DOCKER_HOST
    DOCKER_TLS_VERIFY
commands =
    # NOTE: We use short traceback formatting since we run py.test inside py.test which results in hard to read output
    py.test tests/distribution/basic_sanity_deb.py tests/distribution/python_version_change_tests/ubuntu2204 -s -vv --tb=short --durations=5 {posargs}

[testenv:agent_distributions_tests_amazonlinux2]
basepython = python3.8
passenv =
    TERM
    SCALYR_API_KEY
    READ_API_KEY
    SCALYR_SERVER
    AGENT_HOST_NAME
    DOCKER_CERT_PATH
    DOCKER_HOST
    DOCKER_TLS_VERIFY
commands =
    # NOTE: We use short traceback formatting since we run py.test inside py.test which results in hard to read output
    py.test tests/distribution/basic_sanity_rpm.py tests/distribution/python_version_change_tests/amazonlinux2_test.py -s -vv --tb=short --durations=5 {posargs}

[testenv:agent_distributions_tests_centos7]
basepython = python3.8
passenv =
    TERM
    SCALYR_API_KEY
    READ_API_KEY
    SCALYR_SERVER
    AGENT_HOST_NAME
    DOCKER_CERT_PATH
    DOCKER_HOST
    DOCKER_TLS_VERIFY
commands =
    # NOTE: We use short traceback formatting since we run py.test inside py.test which results in hard to read output
    py.test tests/distribution/python_version_change_tests/centos7 -s -vv --tb=short --durations=5 {posargs}

[testenv:agent_distributions_tests_centos8]
basepython = python3.8
passenv =
    TERM
    SCALYR_API_KEY
    READ_API_KEY
    SCALYR_SERVER
    AGENT_HOST_NAME
    DOCKER_CERT_PATH
    DOCKER_HOST
    DOCKER_TLS_VERIFY
commands =
    # NOTE: We use short traceback formatting since we run py.test inside py.test which results in hard to read output
    py.test tests/distribution/python_version_change_tests/centos8_test.py -s -vv --tb=short --durations=5 {posargs}

[testenv:agent_deb_package]
basepython = python3.8
passenv =
    TERM
    SCALYR_API_KEY
    READ_API_KEY
    SCALYR_SERVER
    AGENT_HOST_NAME
    DOCKER_CERT_PATH
    DOCKER_HOST
    DOCKER_TLS_VERIFY
commands =
    py.test tests/distribution/deb_package.py -s -vv --tb=short --durations=5 {posargs}

[testenv:agent_rpm_package]
basepython = python3.8
passenv =
    TERM
    SCALYR_API_KEY
    READ_API_KEY
    SCALYR_SERVER
    AGENT_HOST_NAME
    DOCKER_CERT_PATH
    DOCKER_HOST
    DOCKER_TLS_VERIFY
commands =
    py.test tests/distribution/rpm_package.py -s -vv --tb=short --durations=5 {posargs}

[testenv:agent_monitors_ubuntu]
basepython = python3.8
passenv =
    TERM
    SCALYR_API_KEY
    READ_API_KEY
    SCALYR_SERVER
    AGENT_HOST_NAME
    DOCKER_CERT_PATH
    DOCKER_HOST
    DOCKER_TLS_VERIFY
commands =
    py.test tests/utils/tests/ tests/smoke_tests/monitors_test/ -s -vv --tb=short --durations=5 --cov=scalyr_agent --cov=tests/ {posargs}

# Benchmark related targets below
[testenv:micro-benchmarks]
basepython = python
deps =
    -r dev-requirements-new.txt
    -r benchmarks/scripts/requirements.txt
    -r benchmarks/micro/requirements-compression-algorithms.txt
    # need < 9.0.0 for python 2.7
    py-cpuinfo==8.0.0
    pytest-benchmark[histogram]==3.2.3; python_version < '3.11'
    pytest-benchmark[histogram]==4.0.0; python_version == '3.11'
    # Depends on libsnappy-dev / snappy-devel
    # NOTE: We don't include this dependency in requirements.txt to avoid having
    # developer to install it for lint tox target
    python-snappy==0.5.4; python_version < '3.11'
    syslogmp==0.3; python_version >= '3.6'
    faker==4.18.0; python_version > '3.0' and python_version < '3.6'
    faker==14.2.1; python_version >= '3.6'
commands =
    bash -c "rm -rf benchmark_results/*"
    bash -c "rm -rf benchmark_histograms/*"
    bash -c "mkdir -p benchmark_results/"
    py.test {env:PYTEST_COMMON_BENCHMARK_OPTS} --benchmark-json=benchmark_results/1.json benchmarks/micro/test_add_events_request_serialization.py
    py.test {env:PYTEST_COMMON_BENCHMARK_OPTS} --benchmark-json=benchmark_results/2.json --benchmark-group-by=group benchmarks/micro/test_event_serialization.py
    py.test {env:PYTEST_COMMON_BENCHMARK_OPTS} --benchmark-json=benchmark_results/3.json --benchmark-group-by=group,param:log_tuple benchmarks/micro/test_json_serialization.py -k "not test_json_encode_with_custom_options"
    py.test {env:PYTEST_COMMON_BENCHMARK_OPTS} --benchmark-json=benchmark_results/4.json --benchmark-group-by=group,param:keys_count benchmarks/micro/test_json_serialization.py -k "test_json_encode_with_custom_options"
    py.test {env:PYTEST_COMMON_BENCHMARK_OPTS} --benchmark-json=benchmark_results/5.json --benchmark-group-by=group,param:log_tuple benchmarks/micro/test_compression_algorithms.py
    py.test {env:PYTEST_COMMON_BENCHMARK_OPTS} --benchmark-json=benchmark_results/6.json --benchmark-group-by=group,param:with_fraction benchmarks/micro/test_date_parsing.py
    py.test {env:PYTEST_COMMON_BENCHMARK_OPTS} --benchmark-json=benchmark_results/7.json --benchmark-group-by=group,param:message_template benchmarks/micro/test_syslog_parsing.py

[testenv:micro-benchmarks-compression-algorithms]
basepython = python
deps =
    -r dev-requirements-new.txt
    -r benchmarks/scripts/requirements.txt
    -r benchmarks/micro/requirements-compression-algorithms.txt
    pytest-benchmark[histogram]==4.0.0
    # Depends on libsnappy-dev / snappy-devel
    # NOTE: We don't include this dependency in requirements.txt to avoid having
    # developer to install it for lint tox target
    python-snappy==0.5.4; python_version < '3.11'
    prettytable
commands =
    bash -c "rm -rf benchmark_results/*"
    bash -c "rm -rf benchmark_histograms/*"
    bash -c "mkdir -p benchmark_results/"
    py.test {env:PYTEST_COMMON_OPTS} {env:PYTEST_COMMON_BENCHMARK_OPTS} --benchmark-json=benchmark_results/compression_algorithms.json --benchmark-group-by=group,param:log_tuple benchmarks/micro/test_compression_algorithms.py
    python benchmarks/scripts/print_compression_algorithm_results.py benchmark_results/compression_algorithms.json