From 8712cd92ba1a2949f43b0fe7c16e225521eb582e Mon Sep 17 00:00:00 2001 From: Romain Le Cellier Date: Fri, 3 Nov 2023 15:15:16 +0100 Subject: [PATCH 1/2] =?UTF-8?q?=F0=9F=91=B7(global)=20circleci=20condition?= =?UTF-8?q?al=20testing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use passculture circleci configuration example to skip job unless files have been modified in the right folder. See: https://github.com/pass-culture/pass-culture-main/blob/9b8c259d683fa4fafa0a0621e83092b09f606927/.circleci/config.yml#L123-L163 --- .circleci/config.yml | 77 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 674e72c84d..6195dfabdc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,6 +18,49 @@ parameters: type: boolean default: false +commands: + skip_unchanged: + parameters: + except_branches: + default: master + description: Comma-separated list of git branches where we should never skip tests. + type: string + git_base_branch: + default: origin/master + description: The branch to compare against when looking for changing. + type: string + paths: + description: The paths to look for changes in. + type: string + description: Skip if no changes to "<< parameters.paths >>" + steps: + - run: + name: Skip job when not modifying this folder + command: | + echo "Skip debugging" + echo "--------------" + GIT_BASE_BRANCH=<> + PATHS=<> + EXCEPT_BRANCHES=<> + echo "GIT_BASE_BRANCH=$GIT_BASE_BRANCH" + echo "PATHS=$PATHS" + echo "EXCEPT_BRANCHES=$EXCEPT_BRANCHES" + + BRANCH_NAME="$(git rev-parse --abbrev-ref HEAD)" + echo "BRANCH_NAME=$BRANCH_NAME" + + IFS=' ' read -ra paths_arr \<<< "$PATHS" + echo "paths_arr=" "${paths_arr[@]}" + + CHANGED_FILES="$(git diff "$GIT_BASE_BRANCH"..."$(git symbolic-ref --short -q HEAD)" --name-only --diff-filter=ACMRT "${paths_arr[@]}")"; + echo "CHANGED_FILES=$CHANGED_FILES" + + if [[ "$EXCEPT_BRANCHES" == *"${BRANCH_NAME}"* ]] || [ -n "${CHANGED_FILES}" ]; + then + exit 0; + fi + circleci step halt; + aliases: - &checkout_fun checkout: @@ -189,6 +232,8 @@ jobs: working_directory: ~/fun steps: - checkout + - skip_unchanged: + paths: ~/fun/src/richie - restore_cache: keys: - v2-back-dependencies-{{ .Revision }} @@ -211,6 +256,8 @@ jobs: working_directory: ~/fun/src/richie steps: - *checkout_fun + - skip_unchanged: + paths: ~/fun/src/richie - restore_cache: keys: - v2-back-dependencies-{{ .Revision }} @@ -237,6 +284,8 @@ jobs: working_directory: ~/fun steps: - checkout + - skip_unchanged: + paths: ~/fun/src/richie - restore_cache: keys: - v2-back-dependencies-{{ .Revision }} @@ -267,6 +316,8 @@ jobs: working_directory: ~/fun steps: - checkout + - skip_unchanged: + paths: ~/fun/src/richie - attach_workspace: at: ~/fun - run: @@ -329,6 +380,8 @@ jobs: working_directory: ~/fun steps: - checkout + - skip_unchanged: + paths: ~/fun/src/richie - restore_cache: keys: - v2-back-dependencies-{{ .Revision }} @@ -422,6 +475,8 @@ jobs: working_directory: ~/fun steps: - checkout + - skip_unchanged: + paths: ~/fun/src/richie - restore_cache: keys: - v2-back-dependencies-{{ .Revision }} @@ -509,6 +564,8 @@ jobs: working_directory: ~/fun steps: - checkout + - skip_unchanged: + paths: ~/fun/src/richie - restore_cache: keys: - v2-back-dependencies-{{ .Revision }} @@ -590,6 +647,8 @@ jobs: working_directory: ~/fun steps: - checkout + - skip_unchanged: + paths: ~/fun/src/richie - restore_cache: keys: - v2-back-dependencies-{{ .Revision }} @@ -631,6 +690,8 @@ jobs: working_directory: ~/fun steps: - checkout + - skip_unchanged: + paths: ~/fun/src/richie # Ensure we restore frontend production builds in Richie's static # directory - attach_workspace: @@ -660,6 +721,8 @@ jobs: working_directory: ~/fun steps: - checkout + - skip_unchanged: + paths: ~/fun/src/richie # Restore built python packages - attach_workspace: at: ~/fun @@ -683,6 +746,8 @@ jobs: working_directory: ~/fun/src/frontend steps: - *checkout_fun + - skip_unchanged: + paths: ~/fun/src/frontend - *restore_node_modules # If the yarn.lock file is not up-to-date with the package.json file, # using the --frozen-lockfile should fail. @@ -716,6 +781,8 @@ jobs: working_directory: ~/fun/src/frontend steps: - *checkout_fun + - skip_unchanged: + paths: ~/fun/src/frontend - *restore_node_modules - run: name: Build front-end application in production mode @@ -745,6 +812,8 @@ jobs: working_directory: ~/fun-site-factory/frontend steps: - *checkout_fun + - skip_unchanged: + paths: ~/fun-site-factory/frontend - *restore_node_modules - run: name: Initialize a frontend project that uses richie-education module @@ -855,6 +924,8 @@ jobs: working_directory: ~/fun/src/frontend steps: - *checkout_fun + - skip_unchanged: + paths: ~/fun/src/frontend - *restore_node_modules - run: name: Lint code with eslint @@ -874,6 +945,8 @@ jobs: working_directory: ~/fun/src/frontend steps: - *checkout_fun + - skip_unchanged: + paths: ~/fun/src/frontend - *restore_node_modules - run: name: Run tests @@ -888,6 +961,8 @@ jobs: working_directory: ~/wrk steps: - *checkout_fun + - skip_unchanged: + paths: ~/wrk - run: name: Check SCSS library command: | @@ -911,6 +986,8 @@ jobs: working_directory: ~/fun steps: - *checkout_fun + - skip_unchanged: + paths: ~/fun/src/frontend - run: name: Authenticate with registry command: echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > ~/fun/.npmrc From 4fba713130dec1fa1f515d705b4dbbb927dada1b Mon Sep 17 00:00:00 2001 From: jbpenrath Date: Tue, 12 Dec 2023 20:41:28 +0100 Subject: [PATCH 2/2] !wip --- .circleci/config.yml | 141 ++++++++++++++++++++----------------------- 1 file changed, 64 insertions(+), 77 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6195dfabdc..f87deb35bd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,7 +17,24 @@ parameters: nightly-round: type: boolean default: false +aliases: + - &checkout_fun + checkout: + path: ~/fun + + - &restore_node_modules + restore_cache: + name: Restore node_modules cache + keys: + - v8-front-dependencies-{{ checksum "~/fun/src/frontend/yarn.lock" }} + - v8-front-dependencies- + - &install-libcairo2-dev + run: + name: Install cairo package (required by easy_thumbnails[svg]) + command: | + sudo apt-get update + sudo apt-get install -y libcairo2-dev commands: skip_unchanged: parameters: @@ -34,11 +51,12 @@ commands: type: string description: Skip if no changes to "<< parameters.paths >>" steps: + - *checkout_fun - run: - name: Skip job when not modifying this folder + name: Check if job should be skipped + # Skip job when there is no diff in the paths + # except when the branch is the base one or the job is run on a tag command: | - echo "Skip debugging" - echo "--------------" GIT_BASE_BRANCH=<> PATHS=<> EXCEPT_BRANCHES=<> @@ -46,39 +64,17 @@ commands: echo "PATHS=$PATHS" echo "EXCEPT_BRANCHES=$EXCEPT_BRANCHES" - BRANCH_NAME="$(git rev-parse --abbrev-ref HEAD)" - echo "BRANCH_NAME=$BRANCH_NAME" - IFS=' ' read -ra paths_arr \<<< "$PATHS" echo "paths_arr=" "${paths_arr[@]}" - CHANGED_FILES="$(git diff "$GIT_BASE_BRANCH"..."$(git symbolic-ref --short -q HEAD)" --name-only --diff-filter=ACMRT "${paths_arr[@]}")"; + CHANGED_FILES="$(git diff "$GIT_BASE_BRANCH"..."$CIRCLE_BRANCH" --name-only --diff-filter=ACMRT "${paths_arr[@]}")"; echo "CHANGED_FILES=$CHANGED_FILES" - if [[ "$EXCEPT_BRANCHES" == *"${BRANCH_NAME}"* ]] || [ -n "${CHANGED_FILES}" ]; + if [[ -n "${CIRCLE_TAG}" ]] || [[ "$EXCEPT_BRANCHES" == *"${CIRCLE_BRANCH}"* ]] || [ -n "${CHANGED_FILES}" ]; then exit 0; fi circleci step halt; - -aliases: - - &checkout_fun - checkout: - path: ~/fun - - - &restore_node_modules - restore_cache: - name: Restore node_modules cache - keys: - - v8-front-dependencies-{{ checksum "~/fun/src/frontend/yarn.lock" }} - - v8-front-dependencies- - - - &install-libcairo2-dev - run: - name: Install cairo package (required by easy_thumbnails[svg]) - command: | - sudo apt-get update - sudo apt-get install -y libcairo2-dev jobs: # Git jobs # Check that the git history is clean and complies with our expectations @@ -231,9 +227,6 @@ jobs: password: $DOCKER_PASS working_directory: ~/fun steps: - - checkout - - skip_unchanged: - paths: ~/fun/src/richie - restore_cache: keys: - v2-back-dependencies-{{ .Revision }} @@ -255,9 +248,6 @@ jobs: password: $DOCKER_PASS working_directory: ~/fun/src/richie steps: - - *checkout_fun - - skip_unchanged: - paths: ~/fun/src/richie - restore_cache: keys: - v2-back-dependencies-{{ .Revision }} @@ -283,9 +273,6 @@ jobs: password: $DOCKER_PASS working_directory: ~/fun steps: - - checkout - - skip_unchanged: - paths: ~/fun/src/richie - restore_cache: keys: - v2-back-dependencies-{{ .Revision }} @@ -316,8 +303,6 @@ jobs: working_directory: ~/fun steps: - checkout - - skip_unchanged: - paths: ~/fun/src/richie - attach_workspace: at: ~/fun - run: @@ -379,9 +364,6 @@ jobs: REDIS_MASTER_HOST: redis-primary working_directory: ~/fun steps: - - checkout - - skip_unchanged: - paths: ~/fun/src/richie - restore_cache: keys: - v2-back-dependencies-{{ .Revision }} @@ -474,9 +456,6 @@ jobs: REDIS_MASTER_HOST: redis-primary working_directory: ~/fun steps: - - checkout - - skip_unchanged: - paths: ~/fun/src/richie - restore_cache: keys: - v2-back-dependencies-{{ .Revision }} @@ -563,9 +542,6 @@ jobs: REDIS_MASTER_HOST: redis-primary working_directory: ~/fun steps: - - checkout - - skip_unchanged: - paths: ~/fun/src/richie - restore_cache: keys: - v2-back-dependencies-{{ .Revision }} @@ -646,9 +622,6 @@ jobs: REDIS_MASTER_HOST: redis-primary working_directory: ~/fun steps: - - checkout - - skip_unchanged: - paths: ~/fun/src/richie - restore_cache: keys: - v2-back-dependencies-{{ .Revision }} @@ -690,8 +663,6 @@ jobs: working_directory: ~/fun steps: - checkout - - skip_unchanged: - paths: ~/fun/src/richie # Ensure we restore frontend production builds in Richie's static # directory - attach_workspace: @@ -721,8 +692,6 @@ jobs: working_directory: ~/fun steps: - checkout - - skip_unchanged: - paths: ~/fun/src/richie # Restore built python packages - attach_workspace: at: ~/fun @@ -745,9 +714,6 @@ jobs: password: $DOCKER_PASS working_directory: ~/fun/src/frontend steps: - - *checkout_fun - - skip_unchanged: - paths: ~/fun/src/frontend - *restore_node_modules # If the yarn.lock file is not up-to-date with the package.json file, # using the --frozen-lockfile should fail. @@ -780,9 +746,6 @@ jobs: password: $DOCKER_PASS working_directory: ~/fun/src/frontend steps: - - *checkout_fun - - skip_unchanged: - paths: ~/fun/src/frontend - *restore_node_modules - run: name: Build front-end application in production mode @@ -809,11 +772,8 @@ jobs: auth: username: $DOCKER_USER password: $DOCKER_PASS - working_directory: ~/fun-site-factory/frontend + working_directory: ~/fun/frontend steps: - - *checkout_fun - - skip_unchanged: - paths: ~/fun-site-factory/frontend - *restore_node_modules - run: name: Initialize a frontend project that uses richie-education module @@ -923,9 +883,6 @@ jobs: password: $DOCKER_PASS working_directory: ~/fun/src/frontend steps: - - *checkout_fun - - skip_unchanged: - paths: ~/fun/src/frontend - *restore_node_modules - run: name: Lint code with eslint @@ -944,9 +901,6 @@ jobs: resource_class: large working_directory: ~/fun/src/frontend steps: - - *checkout_fun - - skip_unchanged: - paths: ~/fun/src/frontend - *restore_node_modules - run: name: Run tests @@ -958,11 +912,8 @@ jobs: auth: username: $DOCKER_USER password: $DOCKER_PASS - working_directory: ~/wrk + working_directory: ~/fun steps: - - *checkout_fun - - skip_unchanged: - paths: ~/wrk - run: name: Check SCSS library command: | @@ -985,9 +936,6 @@ jobs: password: $DOCKER_PASS working_directory: ~/fun steps: - - *checkout_fun - - skip_unchanged: - paths: ~/fun/src/frontend - run: name: Authenticate with registry command: echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > ~/fun/.npmrc @@ -1045,34 +993,52 @@ workflows: # # Build, lint and test the front-end apps - build-front: + pre-steps: + - skip_unchanged: + paths: ~/fun/src/frontend filters: tags: only: /.*/ - build-front-production: + pre-steps: + - skip_unchanged: + paths: ~/fun/src/frontend requires: - build-front filters: tags: only: /.*/ - build-dependent-front: + pre-steps: + - skip_unchanged: + paths: ~/fun/src/frontend requires: - build-front filters: tags: only: /.*/ - lint-front: + pre-steps: + - skip_unchanged: + paths: ~/fun/src/frontend requires: - build-front filters: tags: only: /.*/ - test-front: + pre-steps: + - skip_unchanged: + paths: ~/fun/src/frontend requires: - lint-front filters: tags: only: /.*/ - test-front-package: + pre-steps: + - skip_unchanged: + paths: ~/fun/src/frontend requires: - lint-front filters: @@ -1124,16 +1090,25 @@ workflows: # Build, lint and test production and development Docker images # (debian-based) - build-back: + pre-steps: + - skip_unchanged: + paths: ~/fun/src/richie filters: tags: only: /.*/ - lint-back: + pre-steps: + - skip_unchanged: + paths: ~/fun/src/richie requires: - build-back filters: tags: only: /.*/ - test-back-mysql-5: + pre-steps: + - skip_unchanged: + paths: ~/fun/src/richie requires: - build-back - build-front-production @@ -1141,6 +1116,9 @@ workflows: tags: only: /.*/ - test-back-mysql-8: + pre-steps: + - skip_unchanged: + paths: ~/fun/src/richie requires: - build-back - build-front-production @@ -1148,6 +1126,9 @@ workflows: tags: only: /.*/ - test-back-postgresql: + pre-steps: + - skip_unchanged: + paths: ~/fun/src/richie requires: - build-back - build-front-production @@ -1155,6 +1136,9 @@ workflows: tags: only: /.*/ - test-back-postgresql-es6: + pre-steps: + - skip_unchanged: + paths: ~/fun/src/richie requires: - build-back - build-front-production @@ -1166,6 +1150,9 @@ workflows: # # Extract strings and upload them to our translation management SaaS - build-back-i18n: + pre-steps: + - skip_unchanged: + paths: ~/fun/src/richie requires: - build-back filters: