Custom Tests

.github/workflows/qit-custom-test-woocommerce.yml

@@ -0,0 +1,89 @@
+name: QIT Custom Test - WooCommerce
+
+on:
+  # Manually
+  workflow_dispatch:
+
+jobs:
+  playwright-tests:
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        shardIndex: [1, 2, 3, 4]
+        shardTotal: [4]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 18
+      - name: Setup PHP (Cross-platform)
+        uses: shivammathur/setup-php@v2
+        with:
+          php-version: '8.3'
+
+      - name: Enable dev mode
+        run: ./qit dev
+
+      # Delay start of tests based on shardIndex to space out requests and avoid 429.
+      # Example: This step causes shard 1 to start immediately, shard 2 to start after a 10-second delay, shard 3 after a 20-second delay, etc.
+      - name: Wait before running tests
+        run: sleep $(( (${{ matrix.shardIndex }} - 1) * 10 ))
+
+      - name: Connect to Staging QIT
+        run: ./qit backend:add --environment="staging" --qit_secret="${{ secrets.QIT_STAGING_SECRET }}" --manager_url="https://stagingcompatibilitydashboard.wpcomstaging.com"
+
+      - name: Composer install
+        working-directory: src
+        run: composer install
+
+      - name: Start environment
+        working-directory: src
+        run: php qit-cli.php run:e2e https://github.com/woocommerce/woocommerce/releases/download/8.8.1/woocommerce.zip:test:./../_tests/example-tests/woocommerce -v --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+
+      - name: Set the path in an env var
+        if: ${{ !cancelled() }}
+        working-directory: src
+        run: |
+          echo "BLOB_REPORT_PATH=$(php qit-cli.php e2e-report --dir_only)/blob" >> $GITHUB_ENV
+
+      - name: Upload blob report to GitHub Actions Artifacts
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: blob-report-${{ matrix.shardIndex }}
+          path: ${{ env.BLOB_REPORT_PATH }}
+          retention-days: 1
+
+  merge-reports:
+    # Merge reports after playwright-tests, even if some shards have failed
+    if: ${{ !cancelled() }}
+    needs: [playwright-tests]
+
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 18
+
+      - name: Install Playwright
+        run: npm install playwright
+
+      - name: Download blob reports from GitHub Actions Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: all-blob-reports
+          pattern: blob-report-*
+          merge-multiple: true
+
+      - name: Merge into HTML Report
+        run: npx playwright merge-reports --reporter html ./all-blob-reports
+
+      - name: Upload HTML report
+        uses: actions/upload-artifact@v4
+        with:
+          name: html-report--attempt-${{ github.run_attempt }}
+          path: playwright-report
+          retention-days: 14

.github/workflows/qit-environment-dangling-test.yml

@@ -8,7 +8,7 @@ on:
   workflow_dispatch:
 
 jobs:
-  environment_dangling_tests:
+  test_stop_random_container:
     runs-on: ubuntu-latest
     env:
       NO_COLOR: 1
@@ -22,44 +22,62 @@ jobs:
         with:
           php-version: '8.3'
 
-      - name: Composer install (Unix)
+      - name: Run setup script
+        run: bash .github/workflows/tests/environments/setup.sh ${{ secrets.QIT_STAGING_SECRET }} https://stagingcompatibilitydashboard.wpcomstaging.com
+
+      - name: Environment Setup and Checks
+        id: env_setup
+        run: bash .github/workflows/tests/environments/start-and-assert.sh
+
+      - name: Store the environment path in a ENV var
         working-directory: src
-        run: composer install
+        run: echo "TEMPORARY_ENV=$(php qit-cli.php env:list --field=temporary_env)" >> $GITHUB_ENV
 
-      - name: Enable dev mode
+      - name: Stop a random container from that environment.
         working-directory: src
-        run: php qit-cli.php dev
+        run: |
+          CONTAINERS=$(php qit-cli.php env:list --field=docker_images)
+          RANDOM_CONTAINER=$(echo $CONTAINERS | awk '{print $1}')
+          echo "Stopping container: $RANDOM_CONTAINER"
+          docker stop $RANDOM_CONTAINER
 
-      - name: Connect to Staging QIT
+      - name: Run a env:list to trigger the Dangling Environment Cleanup
         working-directory: src
-        run: php qit-cli.php backend:add --environment="staging" --qit_secret="${{ secrets.QIT_STAGING_SECRET }}" --manager_url="https://stagingcompatibilitydashboard.wpcomstaging.com"
+        run: php qit-cli.php env:list
 
-      - name: Add "qit.test" to hosts file (Linux)
-        run: sudo echo "127.0.0.1 qit.test" | sudo tee -a /etc/hosts
+      - name: Environment Cleanup Checks
+        id: env_cleanup
+        run: bash .github/workflows/tests/environments/dangling-cleanup-assert.sh
+
+  test_delete_environment_entry:
+    runs-on: ubuntu-latest
+    env:
+      NO_COLOR: 1
+      QIT_DISABLE_ONBOARDING: yes
+    steps:
+      - name: Checkout code (Cross-platform)
+        uses: actions/checkout@v4
 
-      - name: Start environment
+      - name: Setup PHP (Cross-platform)
+        uses: shivammathur/setup-php@v2
+        with:
+          php-version: '8.3'
+
+      - name: Run setup script
+        run: bash .github/workflows/tests/environments/setup.sh ${{ secrets.QIT_STAGING_SECRET }} https://stagingcompatibilitydashboard.wpcomstaging.com
+
+      - name: Environment Setup and Checks
+        id: env_setup
+        run: bash .github/workflows/tests/environments/start-and-assert.sh
+
+      - name: Find and delete all hidden JSON files in the directory
         working-directory: src
-        run: php qit-cli.php env:up
+        run: php qit-cli.php cache delete environment_monitor
 
-      - name: Test Site Up
-        run: |
-          # Fetch the site URL
-          SITE_URL=$(php qit-cli.php env:list --field=site_url)
-          echo "Site URL: $SITE_URL"
-
-          # Assert home page is 200
-          HTTP_STATUS=$(curl -o /dev/null -s -w "%{http_code}\n" $SITE_URL)
-          if [ "$HTTP_STATUS" -ne 200 ]; then
-            echo "Home page is not up. HTTP status: $HTTP_STATUS"
-            exit 1
-          fi
-          echo "Home page is up. HTTP status: $HTTP_STATUS"
-
-          # Assert the name property in the JSON response
-          JSON_RESPONSE=$(curl -s $SITE_URL/wp-json)
-          NAME_PROPERTY=$(echo $JSON_RESPONSE | jq -r '.name')
-          if [ "$NAME_PROPERTY" != "WooCommerce Core E2E Test Suite" ]; then
-            echo "Name property does not match. Found: $NAME_PROPERTY"
-            exit 1
-          fi
-          echo "Name property matches: $NAME_PROPERTY"
+      - name: Run a env:list to trigger the Dangling Environment Cleanup
+        working-directory: src
+        run: php qit-cli.php env:list
+
+      - name: Environment Cleanup Checks
+        id: env_cleanup
+        run: bash .github/workflows/tests/environments/dangling-cleanup-assert.sh

.github/workflows/qit-environment-test-linux.yml

@@ -1,4 +1,4 @@
-name: QIT Environment Test - Linux and Mac
+name: QIT Environment Test - Linux
 
 on:
   # Every day at 11pm UTC (6pm ET)
@@ -8,19 +8,17 @@ on:
   workflow_dispatch:
 
 jobs:
-  environment_tests:
+  linux_environment_test:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ ubuntu-20.04, macos-13 ]
+        os: [ ubuntu-20.04 ]
         php: [ '7.4' ]
         wordpress: [ 'rc' ]
-        woocommerce: [ 'rc' ]
         include:
           - os: ubuntu-20.04
             php: '8.3'
             wordpress: 'rc'
-            woocommerce: 'rc'
       fail-fast: false
     env:
       NO_COLOR: 1
@@ -34,26 +32,13 @@ jobs:
         with:
           php-version: '8.3'
 
-      - name: Enable dev mode (Cross-platform)
+      - name: Enable dev mode
         run: ./qit dev
 
-      - name: Connect to Staging QIT (Cross-platform)
+      - name: Connect to Staging QIT
         run: ./qit backend:add --environment="staging" --qit_secret="${{ secrets.QIT_STAGING_SECRET }}" --manager_url="https://stagingcompatibilitydashboard.wpcomstaging.com"
 
-      - name: Add "qit.test" to hosts file (Linux)
-        if: matrix.os == 'ubuntu-20.04' || matrix.os == 'macos-13'
-        run: sudo echo "127.0.0.1 qit.test" | sudo tee -a /etc/hosts
-
-      - name: Setup Docker on macOS (Mac)
-        if: matrix.os == 'macos-13'
-        uses: douglascamata/setup-docker-macos-action@v1-alpha
-
-      - name: Check Colima Status (Mac)
-        if: matrix.os == 'macos-13'
-        run: colima status
-
-      - name: Debug Docker and Docker Compose (Unix)
-        if: matrix.os == 'ubuntu-20.04' || matrix.os == 'macos-13'
+      - name: Debug Docker and Docker Compose
         run: |
           echo "Checking Docker version..."
           docker --version || echo "Docker is not installed"
@@ -64,24 +49,19 @@ jobs:
           echo "Checking Docker Compose (standalone) version..."
           docker-compose --version || echo "Docker Compose (standalone) is not installed"
 
-      - name: Benchmark Docker (Unix)
-        if: matrix.os == 'ubuntu-20.04' || matrix.os == 'macos-13'
+      - name: Benchmark Docker
         working-directory: scripts
         run: docker run -v $GITHUB_WORKSPACE/scripts:/scripts -w /scripts alpine sh -c "apk add --no-cache bash wget tar; chmod +x benchmark.sh; ./benchmark.sh"
 
-      - name: Composer install (Unix)
-        if: matrix.os == 'ubuntu-20.04' || matrix.os == 'macos-13'
+      - name: Composer install
         working-directory: src
         run: composer install
 
-      - name: Start environment (Unix)
-        if: matrix.os == 'ubuntu-20.04' || matrix.os == 'macos-13'
+      - name: Start environment
         working-directory: src
-        env:
-          QIT_DOCKER_RUN_TIMEOUT: ${{ (matrix.os == 'ubuntu-20.04') && '300' || '1800' }}
-        run: php qit-cli.php env:up --wordpress_version ${{ matrix.wordpress }} --woocommerce_version ${{ matrix.woocommerce }} --php_version ${{ matrix.php }}
+        run: php qit-cli.php env:up --wordpress_version ${{ matrix.wordpress }} --php_version ${{ matrix.php }}
 
-      - name: Get URL (Cross-platform)
+      - name: Get URL
         working-directory: src
         run: |
           siteUrl=$(php -r "echo exec(PHP_BINARY . ' qit-cli.php env:list --field=site_url');");
@@ -91,7 +71,7 @@ jobs:
         run: |
           php -r "echo 'Site URL: ' . getenv('SITE_URL');"
 
-      - name: Test Site Up (Cross-platform)
+      - name: Test Site Up
         run: |
           php -r "
           \$url = '${{ env.SITE_URL }}';
@@ -105,17 +85,15 @@ jobs:
             exit(1);
           }"
 
-      - name: Query WP JSON (Cross-platform)
+      - name: Query WP JSON
         run: php -r "echo file_get_contents('${{ env.SITE_URL }}/wp-json');"
 
-      - name: Verify PHP Version (Unix)
-        if: matrix.os == 'ubuntu-20.04' || matrix.os == 'macos-13'
+      - name: Verify PHP Version
         working-directory: src
         run: |
           actual_major_minor=$(docker exec $(php qit-cli.php env:list --field=docker_images | grep php) php -r 'echo PHP_MAJOR_VERSION . "." . PHP_MINOR_VERSION;')
           [[ "$actual_major_minor" == "${{ matrix.php }}" ]] || { echo "PHP Major.Minor version mismatch. Expected: ${{ matrix.php }}, Actual: $actual_major_minor"; exit 1; }
 
-      - name: Print WordPress Version (Unix)
-        if: matrix.os == 'ubuntu-20.04' || matrix.os == 'macos-13'
+      - name: Print WordPress Version
         working-directory: src
         run: php qit-cli.php env:exec "wp core version"