config.yml

version: 2.1

orbs:
  slack: circleci/slack@3.4.2

parameters:
  start_install:
    type: boolean
    default: false
  wait_for_install:
    type: boolean
    default: false
  finish_install:
    type: boolean
    default: false
  test_and_deploy:
    type: boolean
    default: false
  test_and_deploy_qa:
    type: boolean
    default: false
  switch_production:
    type: boolean
    default: false
  server_env:
    default: 'qa'
    description: The target server environment for installation
    type: enum
    enum: ['qa', 'reserve']

workflows:
  start_install:
    when: << pipeline.parameters.start_install >>
    jobs:
      - start_install

  start_install_qa_automatic:
    jobs:
      - launch_qa_install

    triggers:
        # Every third day at 1AM UTC, meaning 03:00 (winter) or 04:00 (summer) Helsinki time
      - schedule:
          cron: "0 1 */3 * *"
          filters:
            branches:
              only:
                - master

  start_install_reserve_automatic:
    jobs:
      - launch_reserve_install

    triggers:
        # Every Friday at 19:00 (winter) or 20:00 (summer) Helsinki time depending on DST
        # The estimated installation time is ~65 hours, which consists of:
        #  20h all software installation + planet osm data import
        #  40h for tile cache warming (12 styles, ~3.5h per style)
        #  5h for visual diff testing setp
        #
        # Install should be ready for production push on Monday at mid day
        # Syntax: https://pubs.opengroup.org/onlinepubs/7908799/xcu/crontab.html
      - schedule:
          cron: "0 17 * * 5"
          filters:
            branches:
              only:
                - master

  # Remote server will start this step after installation is ready to proceed
  # This would also work on it's own as a polling job, but it spends so much computation
  # credits in Circle CI it makes no sense
  wait_for_install:
    when: << pipeline.parameters.wait_for_install >>
    jobs:
      - wait_for_install

  finish_install:
    when: << pipeline.parameters.finish_install >>
    jobs:
      - finish_install

  switch_production:
    when: << pipeline.parameters.switch_production >>
    jobs:
      - switch_production

  test_and_deploy_qa:
    when: << pipeline.parameters.test_and_deploy_qa >>
    jobs:
      - restart_services
      - test_server:
          requires:
            - restart_services

  # This is a separate task so that we are able to independently install the server and
  # test installation if needed. Not sure if this separation will be useful but at least
  # in dev phase it helps
  test_and_deploy:
    when: << pipeline.parameters.test_and_deploy >>
    jobs:
      - restart_services
      - test_server:
          requires:
            - restart_services
      - slack/approval-notification:
          message: "Production deployment is pending approval. Visual diffs are available in artifacts."
          requires:
            - test_server
      - approval_to_deploy:
          type: approval
          requires:
            - test_server
      - deploy_to_production:
          requires:
            - approval_to_deploy
      - update_snapshots:
          requires:
            - deploy_to_production

jobs:
  launch_qa_install:
    docker:
      - image: circleci/python:3.8
    steps:
      - checkout

      - run:
          name: Launch QA installation
          command: |
            bash .circleci/launch-start-install.sh qa

      - slack/status:
          fail_only: true
          only_for_branches: master

  launch_reserve_install:
    docker:
      - image: circleci/python:3.8
    steps:
      - slack/notify:
          message: "Starting reserve production server installation"

      - checkout

      - run:
          name: Launch Reserve installation
          command: |
            bash .circleci/launch-start-install.sh reserve

      - slack/status:
          failure_message: ':red_circle: A $CIRCLE_JOB job for << pipeline.parameters.server_env >> environment has failed!'
          fail_only: true
          only_for_branches: master

  start_install:
    docker:
      - image: circleci/python:3.8
    steps:
      - checkout

      - run:
          name: Install dependencies
          command: |
            pip install -r requirements.txt

      - run:
          name: Start server installation
          command: |
            export ALVAR_SERVER_ENV="<< pipeline.parameters.server_env >>"
            python manage-servers.py start_install $ALVAR_SERVER_ENV

      - slack/status:
          failure_message: ':red_circle: A $CIRCLE_JOB job for << pipeline.parameters.server_env >> environment has failed!'
          fail_only: true
          only_for_branches: master

  wait_for_install:
    docker:
      - image: circleci/python:3.8
    steps:
      - checkout

      - run:
          name: Install dependencies
          command: |
            pip install -r requirements.txt

      - run:
          name: Wait for install to finish
          command: |
            export ALVAR_SERVER_ENV="<< pipeline.parameters.server_env >>"
            bash .circleci/wait-loop.sh $ALVAR_SERVER_ENV

      - slack/status:
          failure_message: ':red_circle: A $CIRCLE_JOB job for << pipeline.parameters.server_env >> environment has failed!'
          fail_only: true
          only_for_branches: master

  finish_install:
    docker:
      - image: circleci/python:3.8
    steps:
      - checkout

      - run:
          name: Install dependencies
          command: |
            pip install -r requirements.txt

      - run:
          name: Wait for install to finish
          command: |
            export ALVAR_SERVER_ENV="<< pipeline.parameters.server_env >>"
            mkdir -p ~/project/artifacts
            python manage-servers.py finish_install $ALVAR_SERVER_ENV
            python manage-servers.py download_file $ALVAR_SERVER_ENV '/home/alvar/screenlog.0' ~/project/artifacts/screenlog.0

      - run:
          name: Launch test and deploy
          command: |
            export ALVAR_SERVER_ENV="<< pipeline.parameters.server_env >>"
            bash .circleci/launch-test-and-deploy.sh $ALVAR_SERVER_ENV

      - store_artifacts:
          path: ~/project/artifacts

      - slack/status:
          failure_message: ':red_circle: A $CIRCLE_JOB job for << pipeline.parameters.server_env >> environment has failed!'
          fail_only: true
          only_for_branches: master

  test_server:
    docker:
      - image: circleci/node:10
    resource_class: large
    steps:
      - checkout

      - run:
          name: Install dependencies
          command: |
            # When taking SSH into the box, it had python 3.5.3, but no pip
            sudo apt-get install python3-pip
            pip3 install -r requirements.txt

      - deploy:
          name: Run tests for << pipeline.parameters.server_env >> server
          command: |
            export ALVAR_SERVER_ENV="<< pipeline.parameters.server_env >>"
            IP=$(python3 manage-servers.py get_ip $ALVAR_SERVER_ENV)

            git clone "https://alvarcarto-integration:$GITHUB_INTEGRATION_USER_TOKEN@github.com/alvarcarto/alvarcarto-render-snapshot.git"
            cd alvarcarto-render-snapshot
            npm i

            # RENDER_API_KEY and PLACEMENT_API_KEY in circle ci env variables
            export RENDER_API_URL="http://$IP/render"
            export PLACEMENT_API_URL="http://$IP/placement"
            export AWS_S3_BUCKET_NAME=alvarcarto-render-snapshots
            export AWS_ACCESS_KEY_ID="$SNAPSHOT_TOOL_AWS_ACCESS_KEY_ID"
            export AWS_SECRET_ACCESS_KEY="$SNAPSHOT_TOOL_AWS_SECRET_ACCESS_KEY"

            # Test only one render via Caddy server. This makes sure caddy has been properly installed
            NODE_OPTIONS=--max_old_space_size=6000 npm run compare -- --main-location-id hki_c --services tile --only 'mapStyle:bw'

            # Test more extensively via direct node APIs
            export RENDER_API_URL="http://$IP:8001"
            export PLACEMENT_API_URL="http://$IP:8003"
            if [ "$ALVAR_SERVER_ENV" == "reserve" ]; then
              NODE_OPTIONS=--max_old_space_size=6000 npm run compare -- --concurrency 3
            else
              NODE_OPTIONS=--max_old_space_size=6000 npm run compare -- --concurrency 3 --main-location-id hki_c --only 'locationId:hki_c' --only 'orientation:portrait' --only 'size:+(null|A6|30x40cm)' --only 'mapStyle:bw' --only 'zoomLevel:+(11|14)'
            fi

      - store_artifacts:
          path: ~/project/alvarcarto-render-snapshot/images

      - slack/notify:
          message: "<< pipeline.parameters.server_env >> testing has finished! Visual diffs are now available."
          title: 'See report'
          title_link: 'https://alvarcarto-render-snapshots.s3-eu-west-1.amazonaws.com/temp/ci-build-${CIRCLE_BUILD_NUM}/report.html'

      - slack/status:
          failure_message: ':red_circle: A $CIRCLE_JOB job for << pipeline.parameters.server_env >> environment has failed!'
          fail_only: true
          only_for_branches: master

  restart_services:
    docker:
      - image: circleci/python:3.8
    steps:
      - checkout

      - run:
          name: Install dependencies
          command: |
            pip install -r requirements.txt

      - deploy:
          name: Restart all services (cleans pg-data/base/pgsql_tmp)
          command: |
            export ALVAR_SERVER_ENV="<< pipeline.parameters.server_env >>"
            python manage-servers.py restart_all_services $ALVAR_SERVER_ENV
            echo "Waiting until services are back up .."
            sleep 120

      - slack/status:
          failure_message: ':red_circle: A $CIRCLE_JOB job for << pipeline.parameters.server_env >> environment has failed!'
          fail_only: true
          only_for_branches: master

  deploy_to_production:
    docker:
      - image: circleci/python:3.8
    steps:
      - checkout

      - run:
          name: Install dependencies
          command: |
            pip install -r requirements.txt

      - deploy:
          name: Promote release candidate server to production via DNS
          command: |
            python manage-servers.py promote_reserve_to_production

      - run:
          name: Clear previously cached tiles and renders
          command: |
            python manage-servers.py purge_cloudflare_cache

      - slack/status:
          failure_message: ':red_circle: A $CIRCLE_JOB job for << pipeline.parameters.server_env >> environment has failed!'
          fail_only: true
          only_for_branches: master

  switch_production:
    docker:
      - image: circleci/python:3.8
    steps:
      - checkout

      - run:
          name: Install dependencies
          command: |
            pip install -r requirements.txt

      - deploy:
          name: Switch production and reserve DNS entries
          command: |
            python manage-servers.py promote_reserve_to_production

      - slack/status:
          failure_message: ':red_circle: A $CIRCLE_JOB job has failed!'
          fail_only: true
          only_for_branches: master

  update_snapshots:
    docker:
      - image: circleci/node:10
    resource_class: large
    steps:
      - checkout

      - run:
          name: Install dependencies
          command: |
            # When taking SSH into the box, it had python 3.5.3, but no pip
            sudo apt-get install python3-pip
            pip3 install -r requirements.txt

      - deploy:
          name: Update poster snapshots to S3
          command: |
            IP=$(python3 manage-servers.py get_ip production)

            git clone "https://alvarcarto-integration:$GITHUB_INTEGRATION_USER_TOKEN@github.com/alvarcarto/alvarcarto-render-snapshot.git"
            cd alvarcarto-render-snapshot
            npm install

            # RENDER_API_KEY and PLACEMENT_API_KEY in circle ci env variables
            export RENDER_API_URL="http://$IP:8001"
            export PLACEMENT_API_URL="http://$IP:8003"
            export AWS_S3_BUCKET_NAME=alvarcarto-render-snapshots
            export AWS_ACCESS_KEY_ID="$SNAPSHOT_TOOL_AWS_ACCESS_KEY_ID"
            export AWS_SECRET_ACCESS_KEY="$SNAPSHOT_TOOL_AWS_SECRET_ACCESS_KEY"
            echo "Add snapshots for production testing"
            NODE_OPTIONS=--max_old_space_size=6000 npm run snapshot -- --concurrency 2
            echo "Add snapshots for QA testing (only uses helsinki location as the QA server doesn't have whole planet data)"
            NODE_OPTIONS=--max_old_space_size=6000 npm run snapshot -- --concurrency 2 --main-location-id hki_c --only 'locationId:hki_c' --only 'orientation:portrait'

      - slack/status:
          failure_message: ':red_circle: A $CIRCLE_JOB job for << pipeline.parameters.server_env >> environment has failed!'
          fail_only: true
          only_for_branches: master