diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 72d8eba534..a22740197c 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -94,7 +94,11 @@ jobs: - name: Run integration tests run: | cd tests - tmt run -v -eCONTAINER_USED=integration-test-local -eWITH_COVERAGE=1 -eLOG_LEVEL=DEBUG + tmt run -v \ + -eCONTAINER_USED=integration-test-local \ + -eWITH_COVERAGE=1 \ + -eLOG_LEVEL=DEBUG \ + plan --name container - name: Show tmt log output in case of failure if: ${{ failure() }} @@ -106,8 +110,8 @@ jobs: - name: Extract coverage results if: always() run: | - MERGE_FILE="/var/tmp/tmt/run-001/plans/tier0/report/default-0/merged.info" - COVERAGE_DIR="/var/tmp/tmt/run-001/plans/tier0/report/default-0/report/" + MERGE_FILE="/var/tmp/tmt/run-001/plans/container/report/default-0/merged.info" + COVERAGE_DIR="/var/tmp/tmt/run-001/plans/container/report/default-0/report/" if [ -d "$COVERAGE_DIR" ]; then cp -r $COVERAGE_DIR /var/tmp cp $MERGE_FILE /var/tmp diff --git a/.github/workflows/multihost-tests.yml b/.github/workflows/multihost-tests.yml new file mode 100644 index 0000000000..6322a94dd7 --- /dev/null +++ b/.github/workflows/multihost-tests.yml @@ -0,0 +1,62 @@ +name: Integration tests in multi-host mode + +on: + workflow_dispatch: + inputs: + composes: + description: Name of the OS image to use + required: true + type: choice + # based on: + # https://api.dev.testing-farm.io/v0.1/composes + options: + - Fedora-37 + - Fedora-37-aarch64 + - Fedora-38 + - Fedora-38-aarch64 + - Fedora-39 + - Fedora-40 + - Fedora-40-aarch64 + - Fedora-latest + - Fedora-latest-aarch64 + - Fedora-Rawhide + - Fedora-Rawhide-aarch64 + - CentOS-7 + - CentOS-7-aarch64 + - CentOS-Stream-8 + - CentOS-Stream-8-aarch64 + - CentOS-Stream-9 + - CentOS-Stream-9-aarch64 + - CentOS-Stream-10 + +jobs: + + ghrelease: + name: Run Integration Tests on testing farm in multihost mode + runs-on: ubuntu-latest + container: + image: quay.io/bluechi/build-base:latest + + steps: + - name: Checkout sources + uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: 'true' + + - name: Install testing farm CLI + run: | + python3 -m ensurepip --default-pip + python3 -m pip install --upgrade pip + python3 -m pip install tft-cli + + - name: Start Integration Tests on testing farm + run: | + testing-farm version + TESTING_FARM_API_TOKEN=${{ secrets.TESTING_FARM_API_TOKEN }} testing-farm request \ + --path tests \ + --git-ref main \ + --git-url https://github.com/eclipse-bluechi/bluechi.git \ + --compose ${{ inputs.composes }} \ + --plan multihost \ + --pipeline-type tmt-multihost diff --git a/.packit.yml b/.packit.yml index 7c0d682e51..a0a0bc8c33 100644 --- a/.packit.yml +++ b/.packit.yml @@ -45,6 +45,7 @@ jobs: trigger: pull_request identifier: integration-tests fmf_path: tests + tmt_plan: container labels: - standard env: @@ -58,6 +59,7 @@ jobs: trigger: pull_request identifier: integration-tests-valgrind fmf_path: tests + tmt_plan: container labels: - valgrind env: diff --git a/tests/README.md b/tests/README.md index ed1b3e6346..e79360ede3 100644 --- a/tests/README.md +++ b/tests/README.md @@ -71,21 +71,23 @@ systemctl --user start podman.socket Integration tests are executed with [tmt framework](https://github.com/teemtee/tmt). -To run integration tests please execute below command of the top level directory of this project: +To run integration tests please execute below command in the [tests](./) directory: ```shell -tmt run -v +tmt run -v plan --name container ``` This will use latest BlueChi packages from [bluechi-snapshot](https://copr.fedorainfracloud.org/coprs/g/centos-automotive-sig/bluechi-snapshot/) repository. +**Note:** The integration tests can be run in two modes - container and multi-host. For local execution it is advised to select `container` mode (hence the `plan --name container`). + ## Running integration tests with memory leak detection To run integration tests with `valgrind`, set `WITH_VALGRIND` environment variable as follows: ```shell -tmt run -v -eWITH_VALGRIND=1 +tmt run -v -eWITH_VALGRIND=1 plan --name container ``` If `valgrind` detects a memory leak in a test, the test will fail, and the logs will be found in the test `data` directory. @@ -122,7 +124,7 @@ After that step integration tests can be executed using following command: ```shell cd ~/bluechi/tests -tmt run -v -eCONTAINER_USED=integration-test-local +tmt run -v -eCONTAINER_USED=integration-test-local plan --name container ``` ## Creating code coverage report from integration tests execution @@ -139,7 +141,7 @@ createrepo_c ~/bluechi/tests/bluechi-rpms When done, you need to run integration tests with code coverage report enabled: ```shell -tmt run -v -eCONTAINER_USED=integration-test-local -eWITH_COVERAGE=1 +tmt run -v -eCONTAINER_USED=integration-test-local -eWITH_COVERAGE=1 plan --name container ``` After the integration tests finishes, the code coverage html result can be found in `res` subdirectory inside the tmt @@ -173,7 +175,7 @@ More detailed information can be displayed by setting log level to `DEBUG`: ```shell cd ~/bluechi/tests -tmt run -v -eLOG_LEVEL=DEBUG +tmt run -v -eLOG_LEVEL=DEBUG plan --name container ``` ### Using python bindings in tests diff --git a/tests/plans/tier0.fmf b/tests/plans/container.fmf similarity index 94% rename from tests/plans/tier0.fmf rename to tests/plans/container.fmf index 574fc77c4e..c80d771215 100644 --- a/tests/plans/tier0.fmf +++ b/tests/plans/container.fmf @@ -1,4 +1,4 @@ -summary: Tier 0 - Basic functionality tests +summary: Running Integration Tests with containers discover: how: fmf filter: tier:0 diff --git a/tests/plans/multihost.fmf b/tests/plans/multihost.fmf new file mode 100644 index 0000000000..f0eb3145c0 --- /dev/null +++ b/tests/plans/multihost.fmf @@ -0,0 +1,44 @@ +summary: Running Integration Tests with multiple hosts +provision: + - name: test-executor + role: executor + - name: bluechi-controller + role: controller + - name: bluechi-agent-1 + role: agent + - name: bluechi-agent-2 + role: agent +environment: + BLUECHI_CTRL_SVC_PORT: 8420 + + INSTALL_EXECUTOR_DEPS: yes + INSTALL_WORKER_DEPS: yes + + SETUP_SSH: yes + SSH_USER: root + SSH_PASSWORD: root + + LOG_LEVEL: DEBUG + WITH_COVERAGE: 0 + WITH_VALGRIND: 0 +discover: + how: fmf + filter: tag:multihost + where: executor +prepare: + - name: Prepare executor setup + how: shell + script: | + ./scripts/tests-setup.sh setup_multihost_test setup_executor + where: executor + - name: Prepare worker setup + how: shell + script: | + ./scripts/tests-setup.sh setup_multihost_test setup_worker + where: + - controller + - agent +execute: + how: tmt +report: + how: junit