Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: move all served files to $web; add CDN, custom domain #8

Merged
merged 11 commits into from
Sep 4, 2024
2 changes: 1 addition & 1 deletion .env-example
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
DATA_REGISTRATION=ckan-registry
DATA_REGISTRY_BASE_URL=https://iatiregistry.org/api/3/action/package_search

BLOB_STORAGE_BASE_PUBLIC_URL=http://127.0.0.1:10000/devstoreaccount1
WEB_BASE_URL=http://127.0.0.1:10000/devstoreaccount1

NUMBER_DOWNLOADER_THREADS=1 # makes for easier testing locally

Expand Down
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ __pycache__

/.env

/azure-provision/default-github-config.env

/azure-deployment/azure-resource-manager-deployment-manifest.yml
/azure-deployment/manual-azure-deploy-secrets.env
/azure-deployment/manual-azure-deploy-variables.env
Expand Down
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,8 @@ This automated test environment is configured via the following files:

You can use the Mockoon GUI application to edit the mockoon server configuration file (`mockoon-registration-and-data-server-config.json`).

The automated tests are safe to run alongside the `docker compose` setup for development.

## Provisioning and Deployment

### Initial Provisioning
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ properties: # Properties of container group
value: "#DATA_REGISTRATION#"
- name: DATA_REGISTRY_BASE_URL
value: "#DATA_REGISTRY_BASE_URL#"
- name: BLOB_STORAGE_BASE_PUBLIC_URL
value: "https://sabulkdataservice#TARGET_ENVIRONMENT#.blob.core.windows.net"
- name: WEB_BASE_URL
value: "#WEB_BASE_URL#"
- name: NUMBER_DOWNLOADER_THREADS
value: "#NUMBER_DOWNLOADER_THREADS#"
- name: FORCE_REDOWNLOAD_AFTER_HOURS
Expand Down
1 change: 1 addition & 0 deletions azure-deployment/generate-manifest-from-template.sh
Original file line number Diff line number Diff line change
Expand Up @@ -49,3 +49,4 @@ sed -i ''s^#REMOVE_LAST_GOOD_DOWNLOAD_AFTER_FAILING_HOURS#^$REMOVE_LAST_GOOD_DOW
sed -i ''s^#ZIP_WORKING_DIR#^$ZIP_WORKING_DIR^g'' ./azure-deployment/azure-resource-manager-deployment-manifest.yml
sed -i ''s^#AZURE_STORAGE_BLOB_CONTAINER_NAME_IATI_XML#^$AZURE_STORAGE_BLOB_CONTAINER_NAME_IATI_XML^g'' ./azure-deployment/azure-resource-manager-deployment-manifest.yml
sed -i ''s^#AZURE_STORAGE_BLOB_CONTAINER_NAME_IATI_ZIP#^$AZURE_STORAGE_BLOB_CONTAINER_NAME_IATI_ZIP^g'' ./azure-deployment/azure-resource-manager-deployment-manifest.yml
sed -i ''s^#WEB_BASE_URL#^$WEB_BASE_URL^g'' ./azure-deployment/azure-resource-manager-deployment-manifest.yml
32 changes: 21 additions & 11 deletions azure-deployment/manual-azure-deploy-from-local.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,44 +13,54 @@ if [ ! -d ".git" ]; then
exit 1
fi

git remote -v | grep "IATI/bulk-data-service.git" > /dev/null
if [ ! -f "./azure-deployment/manual-azure-deploy-secrets.env" ]; then
echo "$0: there must be a file 'manual-azure-deploy-secrets.env' in"
echo "'azure-deployment' containing the secrets. See the examples in manual-azure-deploy-secrets-example.env'"
exit 1
fi

if [ "$?" != 0 ]; then
echo "$0: script must be run from the root of the bulk-data-service repository"
if [ ! -f "./azure-deployment/manual-azure-deploy-variables.env" ]; then
echo "$0: there must be a file 'manual-azure-deploy-variables.env' in"
echo "'azure-deployment' containing the config variables. See example: manual-azure-deploy-variables-example.env'"
exit 1
fi

. ./manual-azure-deploy-secrets.env

(git remote -v 2> /dev/null | grep "IATI/bulk-data-service.git" > /dev/null) || (echo "$0: script must be run from the root of the bulk-data-service repository"; exit 1)

. ./azure-deployment/manual-azure-deploy-secrets.env

TARGET_ENVIRONMENT=$1

APP_NAME=bulk-data-service

RESOURCE_GROUP_NAME=rg-${APP_NAME}-${TARGET_ENVIRONMENT}
RESOURCE_GROUP_NAME="rg-${APP_NAME}-${TARGET_ENVIRONMENT}"

CONTAINER_GROUP_INSTANCE_NAME="aci-${APP_NAME}-${TARGET_ENVIRONMENT}"

CONTAINER_GROUP_INSTANCE_NAME=aci-${APP_NAME}-${TARGET_ENVIRONMENT}
DOCKER_IMAGE_TAG=$(git log -n1 --format=format:"%H")

LOCAL_DEPLOY=true

echo "Generating Azure ARM deployment manifest from template"
. ./azure-deployment/generate-manifest-from-template.sh

# build the docker image for the Bulk Data Service
docker build . -t criati.azurecr.io/bulk-data-service-$TARGET_ENVIRONMENT
docker build . -t "criati.azurecr.io/bulk-data-service-$TARGET_ENVIRONMENT:$DOCKER_IMAGE_TAG"

# push Bulk Data Service image to Azure
docker push criati.azurecr.io/bulk-data-service-$TARGET_ENVIRONMENT
docker push "criati.azurecr.io/bulk-data-service-$TARGET_ENVIRONMENT:$DOCKER_IMAGE_TAG"

# now configure, build and push the docker image for the nginx reverse proxy

# create password file
htpasswd -c -b ./azure-deployment/nginx-reverse-proxy/htpasswd prom $PROM_NGINX_REVERSE_PROXY_PASSWORD
htpasswd -c -b ./azure-deployment/nginx-reverse-proxy/htpasswd prom "$PROM_NGINX_REVERSE_PROXY_PASSWORD"

# make the image for the nginx reverse proxy (for putting HTTP basic auth on the
# prom client)
docker build ./azure-deployment/nginx-reverse-proxy -t criati.azurecr.io/bds-prom-nginx-reverse-proxy-$TARGET_ENVIRONMENT
docker build ./azure-deployment/nginx-reverse-proxy -t "criati.azurecr.io/bds-prom-nginx-reverse-proxy-$TARGET_ENVIRONMENT:$DOCKER_IMAGE_TAG"

docker push criati.azurecr.io/bds-prom-nginx-reverse-proxy-$TARGET_ENVIRONMENT
docker push "criati.azurecr.io/bds-prom-nginx-reverse-proxy-$TARGET_ENVIRONMENT:$DOCKER_IMAGE_TAG"


echo az container delete \
Expand Down
3 changes: 1 addition & 2 deletions azure-deployment/manual-azure-deploy-variables-example.env
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,7 @@
AZURE_STORAGE_BLOB_CONTAINER_NAME_IATI_XML=iati-xml
AZURE_STORAGE_BLOB_CONTAINER_NAME_IATI_ZIP=iati-zip

# Value of BLOB_STORAGE_BASE_PUBLIC_URL generated automatically by deploy scripts
# BLOB_STORAGE_BASE_PUBLIC_URL=
WEB_BASE_URL=

DATA_REGISTRATION=ckan-registry
DATA_REGISTRY_BASE_URL=https://iatiregistry.org/api/3/action/package_search
Expand Down
36 changes: 36 additions & 0 deletions azure-provision/add-default-config-to-github-variables.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
#!/usr/bin/bash

set -o errexit # abort on nonzero exitstatus
set -o nounset # abort on unbound variable
set -o pipefail # don't hide errors within pipes

if [[ ! -v "1" ]]; then
echo "usage: $0 TARGET_ENVIRONMENT"
echo " TARGET_ENVIRONMENT should likely be 'test', 'dev', or 'prod'"
exit 1
fi

if [[ ! -d ".git" ]]; then
echo "$0: script must be run from the root of the bulk-data-service repository"
exit 1
fi

(git remote -v 2> /dev/null | grep "IATI/bulk-data-service.git" > /dev/null) || (echo "$0: script must be run from the root of the bulk-data-service repository"; exit 1)

if [[ "$1" == "" ]]; then
echo "TARGET_ENVIRONMENT cannot be empty"
exit 2
fi

if [[ $(which gh > /dev/null) ]]; then
echo "This script requires the Github command line client to be installed"
exit 3
fi

TARGET_ENVIRONMENT="$1"

cp -f azure-provision/default-github-config-template.env azure-provision/default-github-config.env

sed -i "s/^/${TARGET_ENVIRONMENT^^}/g" azure-provision/default-github-config.env

gh variable set --env-file ./azure-provision/default-github-config.env
Loading