Skip to content
CI

CI #207

Sign in to view logs

CI

CI #207

Workflow file for this run

.github/workflows/ci.yml at 73814cc
name: CI
on:
pull_request: {}
push:
branches:
- main
jobs:
php:
runs-on: ubuntu-22.04
services:
mysql:
image: mysql:8.0.23
ports:
- 13306:13306
env:
MYSQL_ROOT_PASSWORD:
MYSQL_ALLOW_EMPTY_PASSWORD: yes
MYSQL_USER: omegaup
MYSQL_PASSWORD: omegaup
MYSQL_DATABASE: omegaup-test
MYSQL_TCP_PORT: 13306
redis:
image: redis
ports:
- 6379:6379
rabbitmq:
image: rabbitmq:3-management-alpine
ports:
- 5672:5672
- 15672:15672
env:
RABBITMQ_DEFAULT_USER: 'omegaup'
RABBITMQ_DEFAULT_PASS: 'omegaup'
steps:
- uses: actions/checkout@v2
with:
submodules: recursive
fetch-depth: 0
- uses: actions/setup-python@v2
with:
python-version: '3.8'
- name: Add rabbitmq as alias to localhost
run: |
cat << EOF | sudo tee --append /etc/hosts
127.0.0.1 rabbitmq
EOF
- name: Use PHP 8.1
run: |
sudo update-alternatives --install /usr/bin/php php /usr/bin/php8.1 100
# Enable APCu for tests
echo 'apc.enable_cli=1' | sudo tee --append /etc/php/8.1/cli/conf.d/20-apcu.ini >/dev/null
# Enable coverage for XDebug so that codecov can do its magic.
echo 'xdebug.mode=coverage' | sudo tee --append /etc/php/8.1/cli/conf.d/20-xdebug.ini >/dev/null
- name: Create default configuration files
run: |
cat > frontend/tests/test_config.php <<EOF
<?php
define('OMEGAUP_DB_HOST', 'localhost:13306');
define('REDIS_HOST', 'localhost');
define('REDIS_PASS', '');
EOF
- name: Download gitserver
run: |
DOWNLOAD_URL='https://github.com/omegaup/gitserver/releases/download/v1.9.13/omegaup-gitserver.tar.xz'
curl --location "${DOWNLOAD_URL}" | sudo tar -xJv -C /
# omegaup-gitserver depends on libinteractive.
DOWNLOAD_URL='https://github.com/omegaup/libinteractive/releases/download/v2.0.27/libinteractive.jar'
TARGET='/usr/share/java/libinteractive.jar'
sudo curl --location "${DOWNLOAD_URL}" -o "${TARGET}"
- name: Install Python dependencies
run: |
python3 -m pip install --user setuptools
python3 -m pip install --user wheel
python3 -m pip install --user \
-r stuff/requirements.txt
- name: Validate composer.json and composer.lock
run: composer validate
- name: Install composer dependencies
run: composer install --prefer-dist --no-progress
- name: Create database users
run: |
mysql \
-uroot --skip-password --port=13306 --protocol=TCP \
-e 'CREATE USER "omegaup"@"localhost" IDENTIFIED BY "omegaup";'
- name: Validate database migration scripts
run: |
python3 stuff/db-migrate.py --skip-container-check validate
- name: Create test database
run: |
python3 stuff/db-migrate.py \
--skip-container-check \
--username=root --password= --hostname=localhost --port=13306 \
migrate --databases=omegaup-test
- name: Validate database schema
run: |
python3 stuff/policy-tool.py \
--skip-container-check \
--username=root --password= --hostname=localhost --port=13306 \
--database=omegaup-test \
validate
python3 stuff/database_schema.py \
--skip-container-check \
--username=root --password= --hostname=localhost --port=13306 \
--database=omegaup-test \
validate --all < /dev/null
- name: Run tests
timeout-minutes: 20
run: ./stuff/mysql_types.sh
- name: Run Psalm
run: |
# Create optional directories to simplify psalm config.
mkdir -p frontend/www/{phpminiadmin,preguntas}
cat > frontend/server/config.php <<EOF
<?php
define('OMEGAUP_LOG_FILE', 'php://stderr');
EOF
./vendor/bin/psalm \
--output-format=github \
--long-progress \
--show-info=false
- name: Upload code coverage
run: |
curl -Os https://uploader.codecov.io/v0.1.0_5313/linux/codecov
echo 'a229148bfbb9b802a95b4c78501b19630a17784696eb0b5182b40f0e1c0a4b6e codecov' | shasum -a 256 -c
chmod +x codecov
./codecov
- name: Upload artifacts
if: ${{ always() }}
uses: actions/upload-artifact@v2
with:
name: test-logs
path: frontend/tests/runfiles/*/*.log
javascript:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v2
with:
submodules: recursive
- name: Install yarn dependencies
run: |
sudo apt-get update
sudo apt-get install -y ca-certificates curl gnupg
sudo mkdir -p /etc/apt/keyrings
curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | sudo gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
NODE_MAJOR=18
echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | sudo tee /etc/apt/sources.list.d/nodesource.list
sudo apt-get update
sudo apt-get install nodejs -y
sudo apt-get update
sudo apt-get install -y build-essential
sudo apt-get install -y python3
yarn global add node-gyp
yarn install
- name: Run JavaScript tests
run: yarn test:coverage
- name: Upload code coverage
run: |
curl -Os https://uploader.codecov.io/v0.1.0_5313/linux/codecov
echo 'a229148bfbb9b802a95b4c78501b19630a17784696eb0b5182b40f0e1c0a4b6e codecov' | shasum -a 256 -c
chmod +x codecov
./codecov
lint:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v2
with:
submodules: recursive
- name: Use PHP 8.1
run: |
sudo update-alternatives --install /usr/bin/php php /usr/bin/php8.1 100
# Enable APCu for tests
echo 'apc.enable_cli=1' | sudo tee --append /etc/php/8.1/cli/conf.d/20-apcu.ini >/dev/null
# Enable coverage for XDebug so that codecov can do its magic.
echo 'xdebug.mode=coverage' | sudo tee --append /etc/php/8.1/cli/conf.d/20-xdebug.ini >/dev/null
- name: Create default configuration files
run: |
cat > frontend/server/config.php <<EOF
<?php
define('OMEGAUP_LOG_FILE', 'php://stderr');
EOF
- name: Install yarn dependencies
run: |
sudo apt-get update
sudo apt-get install -y ca-certificates curl gnupg
sudo mkdir -p /etc/apt/keyrings
curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | sudo gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
NODE_MAJOR=18
echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | sudo tee /etc/apt/sources.list.d/nodesource.list
sudo apt-get update
sudo apt-get install nodejs -y
sudo apt-get update
sudo apt-get install -y build-essential
sudo apt-get install -y python3
yarn global add node-gyp
yarn install
- name: Install composer dependencies
run: composer install --prefer-dist --no-progress
- name: Install Python dependencies
run: python3 -m pip install --user yapf==0.30.0
- name: Get docker container
run: ./stuff/lint.sh ensure-container
- name: Run linters
run: |
# Create optional directories to simplify psalm config.
mkdir -p frontend/www/{phpminiadmin,preguntas}
cat > frontend/server/config.php <<EOF
<?php
define('OMEGAUP_LOG_FILE', 'php://stderr');
EOF
touch 'frontend/tests/test_config.php'
docker run --rm \
--user "$(id -u)":"$(id -g)" \
--volume "${PWD}:/src" \
--volume "${PWD}:${PWD}" \
--volume "${PWD}:/opt/omegaup" \
--mount 'type=tmpfs,destination=/home/.cache,tmpfs-mode=1777' \
--mount 'type=tmpfs,destination=/home/.local,tmpfs-mode=1777' \
--env 'HOME=/home' \
--env 'MYPYPATH=/opt/omegaup/stuff' \
--env 'VIRTUAL_ENV=/opt/omegaup/stuff/venv' \
--entrypoint='' \
"$(grep CONTAINER_VERSION= stuff/lint.sh | cut -f2 -d=)" \
bash -c "\
python3 -m venv /opt/omegaup/stuff/venv && \
. /opt/omegaup/stuff/venv/bin/activate && \
python3 -m pip install wheel && \
python3 -m pip install -r /opt/omegaup/stuff/requirements.txt"
./stuff/lint.sh --diagnostics-output=github validate --all < /dev/null
- name: Run Psalm
run: |
find frontend/ \
-name *.php \
-and -not -wholename 'frontend/server/libs/third_party/*' \
-and -not -wholename 'frontend/tests/badges/*' \
-and -not -wholename 'frontend/tests/controllers/*' \
-and -not -wholename 'frontend/tests/runfiles/*' \
-and -not -wholename 'frontend/www/preguntas/*' \
| xargs ./vendor/bin/psalm \
--output-format=github \
--long-progress \
--show-info=false
- name: Find unused translation strings
run: ./stuff/unused_translation_strings.py
- name: Validate generated Python API syntax
run: php frontend/server/cmd/APITool.php --file api.py | yapf > /dev/null
cypress:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v2
with:
submodules: recursive
- name: Install wait-for-it
run: |
sudo apt-get update -y && sudo apt-get install -y wait-for-it
- name: Download the Docker containers
run: |
docker-compose pull
- name: Start the Docker containers
run: |
docker-compose up --no-build --detach
# Add the container names as aliases to localhost
cat << EOF | sudo tee --append /etc/hosts
127.0.0.1 mysql grader runner gitserver broadcaster redis
EOF
# Wait for some of the containers
wait-for-it -t 30 mysql:13306
wait-for-it -t 30 gitserver:33861
wait-for-it -t 30 redis:6379
for i in $(seq 10); do
if [[ "$(mysql \
-uomegaup -pomegaup --host=mysql --port=13306 --skip-column-names --batch \
-e 'SELECT 1;' || echo 0)" == "1" ]]; then
break
fi
sleep 1
done
- name: Create default configuration files
run: |
cat > frontend/server/config.php <<EOF
<?php
define('OMEGAUP_ALLOW_PRIVILEGE_SELF_ASSIGNMENT', true);
define('OMEGAUP_CSP_LOG_FILE', '/tmp/csp.log');
define('OMEGAUP_DB_HOST', 'mysql:13306');
define('OMEGAUP_DB_NAME', 'omegaup');
define('OMEGAUP_DB_PASS', 'omegaup');
define('OMEGAUP_DB_USER', 'omegaup');
define('OMEGAUP_ENVIRONMENT', 'development');
define('OMEGAUP_LOG_FILE', '/tmp/omegaup.log');
define('OMEGAUP_ENABLE_SOCIAL_MEDIA_RESOURCES', false);
define('OMEGAUP_GITSERVER_URL', 'http://gitserver:33861');
define('OMEGAUP_GRADER_URL', 'https://grader:21680');
define('OMEGAUP_GITSERVER_SECRET_TOKEN', 'secret');
define('REDIS_HOST', 'redis');
define('REDIS_PASS', '');
define('TEMPLATE_CACHE_DIR', '/tmp');
define('OMEGAUP_CSRF_HOSTS', ['frontend', '127.0.0.1']);
EOF
cat > ~/.my.cnf << EOF
[client]
port=13306
host=mysql
protocol=tcp
user=root
password=omegaup
EOF
- name: Install Python dependencies
run: |
python3 -m pip install --user setuptools
python3 -m pip install --user wheel
python3 -m pip install --user \
-r stuff/requirements.txt
- name: Install composer dependencies
run: composer install --prefer-dist --no-progress
- name: Initialize database
run: |
mysql \
-e 'CREATE USER "omegaup"@"localhost" IDENTIFIED BY "omegaup";'
# Create directory to prevent --purge from failing.
sudo mkdir -p /var/lib/omegaup
sudo chown "$(id -u)":"$(id -g)" /var/lib/omegaup
docker-compose exec --user "$(id -u)":"$(id -g)" -T frontend bash -c "\
python3 -m venv /opt/omegaup/stuff/venv && \
python3 -m pip install wheel && \
python3 -m pip install -r /opt/omegaup/stuff/requirements.txt"
docker-compose exec -T frontend python3 stuff/bootstrap-environment.py --purge --verbose
- name: Install yarn dependencies
run: |
sudo apt-get update
sudo apt-get install -y ca-certificates curl gnupg
sudo mkdir -p /etc/apt/keyrings
curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | sudo gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
NODE_MAJOR=18
echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | sudo tee /etc/apt/sources.list.d/nodesource.list
sudo apt-get update
sudo apt-get install nodejs -y
sudo apt-get update
sudo apt-get install -y build-essential
sudo apt-get install -y python3
yarn global add node-gyp
yarn install
- name: Build webpack resources
run: yarn build
- name: Wait for dependencies
run: |
wait-for-it -t 30 grader:21680
- name: Run Cypress tests
run: |
./node_modules/.bin/cypress run --browser chrome
- name: Collect container logs
if: ${{ always() }}
run: |
mkdir -p frontend/tests/runfiles/containers
for name in $(docker ps --format '{{.Names}}'); do
docker logs --timestamps --since 1970-01-01T00:00:00 "${name}" > \
"frontend/tests/runfiles/containers/${name}.log" 2>&1
done
- name: Upload cypress screenshot artifacts
if: ${{ always() }}
uses: actions/upload-artifact@v2
with:
name: test-logs
path: cypress/screenshots
- name: Upload cypress videos artifacts
if: ${{ always() }}
uses: actions/upload-artifact@v2
with:
name: test-logs
path: cypress/videos
python:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v2
with:
submodules: recursive
fetch-depth: 0
- name: Install wait-for-it
run: |
sudo apt-get update -y && sudo apt-get install -y wait-for-it
- name: Install retry
run: |
sudo apt-get update -y && sudo apt-get install -y retry
- name: Download the Docker containers
run: |
docker-compose pull
- name: Start the Docker containers
run: |
docker-compose up --no-build --detach
# Add the container names as aliases to localhost
cat << EOF | sudo tee --append /etc/hosts
127.0.0.1 rabbitmq mysql
EOF
# Wait for some of the containers
wait-for-it -t 30 mysql:13306
retry -t 5 -d 5 -- docker-compose exec -T rabbitmq rabbitmqctl await_startup
for i in $(seq 10); do
if [[ "$(mysql \
-uomegaup -pomegaup --host=mysql --port=13306 --skip-column-names --batch \
-e 'SELECT 1;' || echo 0)" == "1" ]]; then
break
fi
sleep 1
done
- name: Create default configuration files
run: |
cat > frontend/server/config.php <<EOF
<?php
define('OMEGAUP_ALLOW_PRIVILEGE_SELF_ASSIGNMENT', true);
define('OMEGAUP_CSP_LOG_FILE', '/tmp/csp.log');
define('OMEGAUP_DB_HOST', 'mysql:13306');
define('OMEGAUP_DB_NAME', 'omegaup');
define('OMEGAUP_DB_PASS', 'omegaup');
define('OMEGAUP_DB_USER', 'omegaup');
define('OMEGAUP_ENVIRONMENT', 'development');
define('OMEGAUP_LOG_FILE', '/tmp/omegaup.log');
define('OMEGAUP_ENABLE_SOCIAL_MEDIA_RESOURCES', false);
define('OMEGAUP_GITSERVER_URL', 'http://gitserver:33861');
define('OMEGAUP_GRADER_URL', 'https://grader:21680');
define('OMEGAUP_GITSERVER_SECRET_TOKEN', 'secret');
define('REDIS_HOST', 'redis');
define('REDIS_PASS', '');
define('TEMPLATE_CACHE_DIR', '/tmp');
EOF
cat > ~/.my.cnf << EOF
[client]
port=13306
host=mysql
protocol=tcp
user=root
password=omegaup
EOF
- name: Install composer dependencies
run: composer install --prefer-dist --no-progress
- name: Create database users
run: |
mysql -e 'CREATE USER "omegaup"@"localhost" IDENTIFIED BY "omegaup";'
mysql -e 'CREATE DATABASE IF NOT EXISTS `omegaup`;'
mysql -e 'GRANT ALL ON `omegaup`.* TO "omegaup"@"localhost";'
- name: Install Python dependencies
run: |
docker-compose exec --user "$(id -u)":"$(id -g)" -T frontend bash -c "\
python3 -m venv /opt/omegaup/stuff/venv && \
python3 -m pip install wheel && \
python3 -m pip install -r /opt/omegaup/stuff/requirements.txt"
- name: Bootstrap environment
run: |
# Create directory to prevent --purge from failing.
sudo mkdir -p /var/lib/omegaup
sudo chown "$(id -u)":"$(id -g)" /var/lib/omegaup
docker-compose exec -T frontend python3 stuff/bootstrap-environment.py --purge --verbose
- name: Run Python tests
run: |
docker-compose exec -T frontend python3 -m pytest --verbose --timeout=20 ./stuff/
- name: Collect container logs
if: ${{ always() }}
run: |
mkdir -p /tmp/container-logs
for name in $(docker ps --format '{{.Names}}'); do
docker logs --timestamps --since 1970-01-01T00:00:00 "${name}" > \
"/tmp/container-logs/${name}.log" 2>&1
done
- name: Upload artifacts
if: ${{ always() }}
uses: actions/upload-artifact@v2
with:
name: python-test-logs
path: /tmp/container-logs/*.log