Skip to content

Commit

Permalink
Merge branch 'main' into vlt6
Browse files Browse the repository at this point in the history
* main:
  Add support for visual regression tests (#384)
  • Loading branch information
sneridagh committed Oct 21, 2024
2 parents fd62f1c + 27c0a95 commit bea4986
Show file tree
Hide file tree
Showing 175 changed files with 1,253 additions and 67 deletions.
5 changes: 2 additions & 3 deletions .github/workflows/acceptance.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ env:
CYPRESS_RETRIES: 2

jobs:

acceptance:
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name
runs-on: ubuntu-latest
Expand Down Expand Up @@ -178,11 +177,11 @@ jobs:
if: failure()
with:
name: cypress-screenshots-acceptance
path: acceptance/cypress/screenshots
path: cypress/screenshots

# Upload Cypress videos
- uses: actions/upload-artifact@v3
if: failure()
with:
name: cypress-videos-acceptance
path: acceptance/cypress/videos
path: cypress/videos
112 changes: 112 additions & 0 deletions .github/workflows/visual-acceptance.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
name: Visual acceptance tests
on: [push, pull_request]

# # Unkomment if you want a scheduled run
#
# on:
# workflow_dispatch:
# schedule:
# - cron: '0 */12 * * *'

env:
node-version: 20.x

jobs:
visual:
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name
runs-on: ubuntu-latest
env:
CYPRESS_A11Y: true
steps:
- uses: actions/checkout@v4

- name: Use Node.js ${{ env.node-version }}
uses: actions/setup-node@v4
with:
node-version: ${{ env.node-version }}

- name: Enable corepack
run: corepack enable

- name: Get pnpm store directory
shell: bash
run: |
echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
- uses: actions/cache@v4
name: Setup pnpm cache
with:
path: ${{ env.STORE_PATH }}
key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
restore-keys: |
${{ runner.os }}-pnpm-store-
- name: Cache Cypress Binary
id: cache-cypress-binary
uses: actions/cache@v4
with:
path: ~/.cache/Cypress
key: binary-${{ env.node-version }}-${{ hashFiles('pnpm-lock.yaml') }}

- name: Install dependencies
run: make install

- name: Install Cypress if not in cache
if: steps.cache-cypress-binary.outputs.cache-hit != 'true'
working-directory: core/packages/volto
run: make cypress-install

- uses: JarvusInnovations/background-action@v1
name: Start Servers
with:
run: |
make ci-acceptance-server-visual-start &
make ci-acceptance-frontend-visual-start &
# your step-level and job-level environment variables are available to your commands as-is
# npm install will count towards the wait-for timeout
# whenever possible, move unrelated scripts to a different step
# to background multiple processes: add & to the end of the command

wait-on: |
http-get://localhost:8080/Plone
http://localhost:3000
# IMPORTANT: to use environment variables in wait-on, you must use this form: ${{ env.VAR }}
# See wait-on section below for all resource types and prefixes

tail: true # true = stderr,stdout
# This will allow you to monitor the progress live

log-output-resume: stderr
# Eliminates previosuly output stderr log entries from post-run output

wait-for: 10m

log-output: stderr,stdout # same as true

log-output-if: failure
# failure = exit-early or timeout

# working-directory: backend

- run: make ci-acceptance-test-visual

# Upload Cypress screenshots
- uses: actions/upload-artifact@v3
if: failure()
with:
name: cypress-screenshots-visual-acceptance
path: cypress/screenshots

# Upload Cypress videos
- uses: actions/upload-artifact@v3
if: failure()
with:
name: cypress-videos-visual-acceptance
path: cypress/videos

# Upload Cypress visual testing screenshots
- uses: actions/upload-artifact@v3
if: failure()
with:
name: cypress-image-snapshots-visual-acceptance
path: cypress/__image_snapshots__
66 changes: 66 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ GREEN=`tput setaf 2`
RESET=`tput sgr0`
YELLOW=`tput setaf 3`

NODEBIN = ./node_modules/.bin

PLONE_VERSION=6
DOCKER_IMAGE=ghcr.io/kitconcept/voltolighttheme:latest
DOCKER_IMAGE_ACCEPTANCE=plone/server-acceptance:${PLONE_VERSION}
Expand Down Expand Up @@ -149,3 +151,67 @@ acceptance-a11y-test: ## Start a11y Cypress in interactive mode
.PHONY: ci-acceptance-a11y-test
ci-acceptance-a11y-test: ## Run a11y cypress tests in headless mode for CI
CYPRESS_a11y=1 CYPRESS_API_PATH=http://localhost:8080/Plone pnpm exec cypress run --config specPattern=$(CURRENT_DIR)'/cypress/tests/a11y/**/*.{js,jsx,ts,tsx}'

### Visual acceptance tests ###
#
# Useful env vars:
#
# cypress_cumulativeReport: cumulative report file, by default `cumulative.report`
# cypress_cumulativeSummaryReport: cumulative summary report file, by default adding `-summary.report`
# to the end of the cumulative file
# They are placed into `frontend/cypress/config` and
# cannot be outside
# of Cypress due to its virtual filesystem.
#
# Env vars in support of CI runs:
#
# cypress_baseUrl: url of server to run the test against
# e.g. cypress_baseUrl=https://plone-redaktion.de
#
#

.PHONY: acceptance-test-visual
acceptance-test-visual: ## Start visual Cypress Acceptance Tests
NODE_ENV=production CYPRESS_a11y=1 $(NODEBIN)/cypress open --config-file cypress/config/cypress.visual.config.js

# Running the visual tests in headless, cumulative mode will SKIP tests that have run previously, and
# only run the still failing tests HOWEVER the results are not collected in interactive mode due to
# inherent limitations of Cypress. But if you run the tests in cumulative headless mode first,
# then this command can be used to run ONLY the failing tests in interactive mode.
.PHONY: acceptance-test-visual-cumulative
acceptance-test-visual-cumulative: ## Start visual Cypress Acceptance Tests with cumulative filtering
@echo WARNING: cumulative results are NOT collected in interactive mode, use headless mode for collection
NODE_ENV=production CYPRESS_a11y=1 cypress_enableCumulative=true $(NODEBIN)/cypress open --config-file cypress/config/cypress.visual.config.js

.PHONY: ci-acceptance-test-visual
ci-acceptance-test-visual: ## Start visual Cypress Acceptance Tests in headless mode
NODE_ENV=production CYPRESS_a11y=1 $(NODEBIN)/cypress run --browser firefox --config-file cypress/config/cypress.visual.config.js

# Running the visual tests in headless, cumulative mode will SKIP tests that have run previously, and
# only run the still failing tests.
.PHONY: ci-acceptance-test-visual-cumulative
ci-acceptance-test-visual-cumulative: ## Start visual Cypress Acceptance Tests in headless mode
NODE_ENV=production CYPRESS_a11y=1 cypress_enableCumulative=true $(NODEBIN)/cypress run --browser firefox --config-file cypress/config/cypress.visual.config.js

# Automatically update all changed visual snapshots
.PHONY: ci-acceptance-test-visual-update
ci-acceptance-test-visual-update: ## Start visual Cypress Acceptance Tests in headless mode, always pass and update images
NODE_ENV=production CYPRESS_a11y=1 cypress_pluginVisualRegressionUpdateImages=true $(NODEBIN)/cypress run --browser firefox --config-file cypress/config/cypress.visual.config.js

# Automatically update all changed visual snapshots, and
# running the visual tests in headless, cumulative mode will SKIP tests that have run previously, and
# only run the still failing tests
.PHONY: ci-acceptance-test-visual-update-cumulative
ci-acceptance-test-visual-update-cumulative: ## Start visual Cypress Acceptance Tests in headless mode, always pass and update images, with cumulative results
NODE_ENV=production CYPRESS_a11y=1 cypress_enableCumulative=true cypress_pluginVisualRegressionUpdateImages=true $(NODEBIN)/cypress run --browser firefox --config-file cypress/config/cypress.visual.config.js

.PHONY: summarize-cumulative-state
summarize-cumulative-state: ## Summarize cumulative state into ...-summary.report
pnpm summarize-cumulative-state


.PHONY: ci-acceptance-server-visual-start
ci-acceptance-server-visual-start: ci-acceptance-a11y-backend-start

.PHONY: ci-acceptance-frontend-visual-start
ci-acceptance-frontend-visual-start: acceptance-a11y-frontend-prod-start
6 changes: 6 additions & 0 deletions cypress.config.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
const { defineConfig } = require('cypress');
const {
initPlugin: initVisualRegressionPlugin,
} = require('@frsource/cypress-plugin-visual-regression-diff/dist/plugins');

module.exports = defineConfig({
viewportWidth: 1280,
Expand All @@ -9,6 +12,9 @@ module.exports = defineConfig({
runMode: 2,
},
setupNodeEvents(on, config) {
// visual regression testing must be initialized
// when the plugin is installed
initVisualRegressionPlugin(on, config);
on('task', {
table(message) {
console.table(message);
Expand Down
11 changes: 11 additions & 0 deletions cypress/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# Only store the #0.png images, ignore #1.png, #2.png etc...
# Also ignore *.actual.png, *.diff.png
!/__*_snapshots__/**
/__*_snapshots__/*.png
!/__*_snapshots__/*#0.png

# Cumulative report outputs must be in the cypress
# filesystem, but we want to avoid checking them in
/config/cumulative*.report

/screenshots
Loading

0 comments on commit bea4986

Please sign in to comment.