Remove development azurite keys (#235)

* remove development azurite keys * add AZURITE_DEFAULT_KEY to workflows * rename azurite default key * fix validate * lint fix * add defult key before validate * test * revert test change * remove dash from docker-compose * revert change * replace docker-compose with docker compose * add AZURITE_ACCOUNT_KEY as env var * update azurite account key --------- Co-authored-by: elay <[email protected]>
microsoft · Aug 13, 2024 · 822f8f5 · 822f8f5
1 parent d2b46c9
commit 822f8f5
Show file tree

Hide file tree

Showing 19 changed files with 72 additions and 47 deletions.
diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml
@@ -24,6 +24,11 @@ jobs:
           tenant-id: ${{ fromJSON(secrets.SECURE_AZURE_CREDENTIALS).tenantId }}
           subscription-id: ${{ fromJSON(secrets.SECURE_AZURE_CREDENTIALS).subscriptionId }}
 
+      - name: Set Azurite Default Key
+        run: |
+          echo "AZURITE_ACCOUNT_KEY=${{ secrets.AZURITE_ACCOUNT_KEY }}" >> $GITHUB_ENV
+          echo "Using Azurite default key: $AZURITE_ACCOUNT_KEY"
+
       - name: Authenticate
         run: ./scripts/ciauthenticate
 

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -11,6 +11,11 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - name: Set Azurite Default Key
+        run: |
+          echo "AZURITE_ACCOUNT_KEY=${{ secrets.AZURITE_ACCOUNT_KEY }}" >> $GITHUB_ENV
+          echo "Using Azurite default key: $AZURITE_ACCOUNT_KEY"
+
       - name: Run cibuild
         run: ./scripts/cibuild
 
@@ -22,5 +27,11 @@ jobs:
         with:
           python-version: "3.10" # stac-api-validator requires >= 3.10
           cache: "pip"
+
+      - name: Set Azurite Default Key
+        run: |
+          echo "AZURITE_ACCOUNT_KEY=${{ secrets.AZURITE_ACCOUNT_KEY }}" >> $GITHUB_ENV
+          echo "Using Azurite default key: $AZURITE_ACCOUNT_KEY"
+
       - name: Validate
         run: ./scripts/validate
diff --git a/README.md b/README.md
@@ -61,6 +61,14 @@ This project uses a variation on [scripts to rule them all](https://github.com/g
 
 #### Environment setup and building images
 
+Before setting up the local environment, ensure that you have set the AZURITE_ACCOUNT_KEY environment variable. 
+The account key can be found in the [Azurite GitHub repository](https://github.com/Azure/Azurite?tab=readme-ov-file#usage-with-azure-storage-sdks-or-tools)
+
+For example, you can set the environment variable in your terminal with:
+```console
+> export AZURITE_ACCOUNT_KEY=<azurite_account_key>
+```
+
 To set up a local environment, use
 
 ```console

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
@@ -18,19 +18,20 @@ services:
       - TILER_HREF=http://localhost:8080/data/
 
       # Azure Storage
+      - AZURITE_ACCOUNT_KEY=${AZURITE_ACCOUNT_KEY}
       - PCAPIS_COLLECTION_CONFIG__ACCOUNT_URL=http://azurite:10002/devstoreaccount1
       - PCAPIS_COLLECTION_CONFIG__ACCOUNT_NAME=devstoreaccount1
-      - PCAPIS_COLLECTION_CONFIG__ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+      - PCAPIS_COLLECTION_CONFIG__ACCOUNT_KEY=${AZURITE_ACCOUNT_KEY}
       - PCAPIS_COLLECTION_CONFIG__TABLE_NAME=collectionconfig
 
       - PCAPIS_CONTAINER_CONFIG__ACCOUNT_URL=http://azurite:10002/devstoreaccount1
       - PCAPIS_CONTAINER_CONFIG__ACCOUNT_NAME=devstoreaccount1
-      - PCAPIS_CONTAINER_CONFIG__ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+      - PCAPIS_CONTAINER_CONFIG__ACCOUNT_KEY=${AZURITE_ACCOUNT_KEY}
       - PCAPIS_CONTAINER_CONFIG__TABLE_NAME=containerconfig
 
       - PCAPIS_IP_EXCEPTION_CONFIG__ACCOUNT_URL=http://azurite:10002/devstoreaccount1
       - PCAPIS_IP_EXCEPTION_CONFIG__ACCOUNT_NAME=devstoreaccount1
-      - PCAPIS_IP_EXCEPTION_CONFIG__ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+      - PCAPIS_IP_EXCEPTION_CONFIG__ACCOUNT_KEY=${AZURITE_ACCOUNT_KEY}
       - PCAPIS_IP_EXCEPTION_CONFIG__TABLE_NAME=ipexceptionlist
 
       # Redis
@@ -92,19 +93,20 @@ services:
       - WORKERS_PER_CORE=1
 
       # Azure Storage
+      - AZURITE_ACCOUNT_KEY=${AZURITE_ACCOUNT_KEY}
       - PCAPIS_COLLECTION_CONFIG__ACCOUNT_URL=http://azurite:10002/devstoreaccount1
       - PCAPIS_COLLECTION_CONFIG__ACCOUNT_NAME=devstoreaccount1
-      - PCAPIS_COLLECTION_CONFIG__ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+      - PCAPIS_COLLECTION_CONFIG__ACCOUNT_KEY=${AZURITE_ACCOUNT_KEY}
       - PCAPIS_COLLECTION_CONFIG__TABLE_NAME=collectionconfig
 
       - PCAPIS_CONTAINER_CONFIG__ACCOUNT_URL=http://azurite:10002/devstoreaccount1
       - PCAPIS_CONTAINER_CONFIG__ACCOUNT_NAME=devstoreaccount1
-      - PCAPIS_CONTAINER_CONFIG__ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+      - PCAPIS_CONTAINER_CONFIG__ACCOUNT_KEY=${AZURITE_ACCOUNT_KEY}
       - PCAPIS_CONTAINER_CONFIG__TABLE_NAME=containerconfig
 
       - PCAPIS_IP_EXCEPTION_CONFIG__ACCOUNT_URL=http://azurite:10002/devstoreaccount1
       - PCAPIS_IP_EXCEPTION_CONFIG__ACCOUNT_NAME=devstoreaccount1
-      - PCAPIS_IP_EXCEPTION_CONFIG__ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+      - PCAPIS_IP_EXCEPTION_CONFIG__ACCOUNT_KEY=${AZURITE_ACCOUNT_KEY}
       - PCAPIS_IP_EXCEPTION_CONFIG__TABLE_NAME=ipexceptionlist
 
       # Redis

diff --git a/pc-funcs.dev.env b/pc-funcs.dev.env
@@ -1,16 +1,15 @@
-AzureWebJobsStorage=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1;TableEndpoint=http://azurite:10002/devstoreaccount1;
 WEBSITE_HOSTNAME=funcs:8083
-
 ANIMATION_OUTPUT_STORAGE_URL="http://azurite:10000/devstoreaccount1/output/animations"
-ANIMATION_OUTPUT_ACCOUNT_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+
 ANIMATION_API_ROOT_URL="https://planetarycomputer-staging.microsoft.com/api/data/v1"
 ANIMATION_TILE_REQUEST_CONCURRENCY=2
 
 IMAGE_OUTPUT_STORAGE_URL="http://azurite:10000/devstoreaccount1/output/images"
-IMAGE_OUTPUT_ACCOUNT_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+
 IMAGE_API_ROOT_URL="https://planetarycomputer-staging.microsoft.com/api/data/v1"
 IMAGE_TILE_REQUEST_CONCURRENCY=2
-
 STORAGE_ACCOUNT_URL=https://pctapisstagingsa.table.core.windows.net/
 BANNED_IP_TABLE=blobstoragebannedip
-LOG_ANALYTICS_WORKSPACE_ID=78d48390-b6bb-49a9-b7fd-a86f6522e9c4
+LOG_ANALYTICS_WORKSPACE_ID=78d48390-b6bb-49a9-b7fd-a86f6522e9c4
+
+AZURITE_ACCOUNT_KEY=${AZURITE_ACCOUNT_KEY}
diff --git a/pc-stac.dev.env b/pc-stac.dev.env
@@ -13,6 +13,7 @@ WEB_CONCURRENCY=1
 USE_API_HYDRATE=TRUE
 
 # Azure Storage
+AZURITE_ACCOUNT_KEY=${AZURITE_ACCOUNT_KEY}
 PCAPIS_COLLECTION_CONFIG__ACCOUNT_URL=http://azurite:10002/devstoreaccount1
 PCAPIS_COLLECTION_CONFIG__ACCOUNT_NAME=devstoreaccount1
 PCAPIS_COLLECTION_CONFIG__TABLE_NAME=collectionconfig

diff --git a/pccommon/pccommon/constants.py b/pccommon/pccommon/constants.py
@@ -1,3 +1,5 @@
+import os
+
 from opencensus.trace.attributes_helper import COMMON_ATTRIBUTES
 
 DEFAULT_COLLECTION_CONFIG_TABLE_NAME = "collectionconfig"
@@ -34,7 +36,4 @@
 # This is the Azurite storage account key.
 # This is not a key for a real Storage Account and is publicly accessible
 # on Azurite's GitHub repo. This is used only in development.
-AZURITE_ACCOUNT_KEY = (
-    "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUz"
-    "FT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
-)
+AZURITE_ACCOUNT_KEY: str = os.environ.get("AZURITE_ACCOUNT_KEY", "")
diff --git a/pcfuncs/tests/ipban/test_ipban.py b/pcfuncs/tests/ipban/test_ipban.py
@@ -13,6 +13,8 @@
 from ipban.models import UpdateBannedIPTask
 from pytest_mock import MockerFixture
 
+from pccommon.constants import AZURITE_ACCOUNT_KEY
+
 MOCK_LOGS_QUERY_RESULT = [("192.168.1.1", 8000), ("192.168.1.4", 12000)]
 TEST_ID = str(uuid.uuid4()).replace("-", "")  # dash is not allowed in table name
 TEST_BANNED_IP_TABLE = f"testblobstoragebannedip{TEST_ID}"
@@ -58,8 +60,7 @@ def mock_clients(
     logs_query_client.query_workspace.return_value = mock_response
     CONNECTION_STRING: str = (
         "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;"
-        "AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsu"
-        "Fq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;"
+        f"AccountKey={AZURITE_ACCOUNT_KEY};"
         "TableEndpoint=http://azurite:10002/devstoreaccount1;"
     )
 

diff --git a/scripts/bin/setup_azurite.py b/scripts/bin/setup_azurite.py
@@ -14,6 +14,7 @@
 from pccommon.config.collections import CollectionConfig, CollectionConfigTable
 from pccommon.config.containers import ContainerConfig, ContainerConfigTable
 from pccommon.constants import (
+    AZURITE_ACCOUNT_KEY,
     DEFAULT_COLLECTION_CONFIG_TABLE_NAME,
     DEFAULT_CONTAINER_CONFIG_TABLE_NAME,
     DEFAULT_IP_EXCEPTION_CONFIG_TABLE_NAME,
@@ -27,9 +28,7 @@
 
 AZURITE_CONNECT_STRING = (
     "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;"
-    "AccountKey="
-    "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq"
-    "/K1SZFPTOtr/KBHBeksoGMGw==;"
+    f"AccountKey={AZURITE_ACCOUNT_KEY};"
     "BlobEndpoint=http://azurite:10000/devstoreaccount1;"
     "QueueEndpoint=http://azurite:10001/devstoreaccount1;"
     "TableEndpoint=http://azurite:10002/devstoreaccount1;"

diff --git a/scripts/cideploy b/scripts/cideploy
@@ -47,7 +47,7 @@ while [[ "$#" -gt 0 ]]; do case $1 in
         ;;
     esac done
 
-DOCKER_COMPOSE="docker-compose -f deployment/docker-compose.yml"
+DOCKER_COMPOSE="docker compose -f deployment/docker-compose.yml"
 
 # Set terraform to deploy to dev or staging
 if [ "${DEV_DEPLOY}" = "1" ]; then

diff --git a/scripts/cipublish-func b/scripts/cipublish-func
@@ -30,7 +30,7 @@ while [[ "$#" -gt 0 ]]; do case $1 in
         ;;
     esac done
 
-DOCKER_COMPOSE="docker-compose -f deployment/docker-compose.yml"
+DOCKER_COMPOSE="docker compose -f deployment/docker-compose.yml"
 
 # Set terraform to deploy to dev or staging
 if [[ "${CI}" != "true" ]]; then

diff --git a/scripts/console b/scripts/console
@@ -57,15 +57,15 @@ while [[ "$#" > 0 ]]; do case $1 in
 if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
 
     if [[ "${DB_CONSOLE}" ]]; then
-        docker-compose \
+        docker compose \
             -f docker-compose.yml \
             exec database psql postgres://username:password@database:5432/postgis
 
         exit 0
     fi
 
     if [[ "${DEV_TILER_CONSOLE}" ]]; then
-        docker-compose \
+        docker compose \
             -f docker-compose.yml \
             -f docker-compose.dev.yml \
             run --rm \
@@ -75,7 +75,7 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
     fi
 
     if [[ "${FUNCS_CONSOLE}" ]]; then
-        docker-compose \
+        docker compose \
             -f docker-compose.yml \
             run --rm \
             funcs /bin/bash
@@ -84,7 +84,7 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
     fi
 
     if [[ "${DEPLOY_CONSOLE}" ]]; then
-        GIT_COMMIT="$(git rev-parse --short HEAD)" docker-compose \
+        GIT_COMMIT="$(git rev-parse --short HEAD)" docker compose \
             -f deployment/docker-compose.yml \
             run --rm \
             deploy /bin/bash
@@ -93,7 +93,7 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
     fi
 
     if [[ "${DEV_STAC_CONSOLE}" ]]; then
-        docker-compose \
+        docker compose \
             -f docker-compose.yml \
             -f docker-compose.dev.yml \
             run --rm \

diff --git a/scripts/format b/scripts/format
@@ -16,31 +16,31 @@ Runs formatting for the project.
 
 if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
 
-    docker-compose  \
+    docker compose  \
         -f docker-compose.yml \
         -f docker-compose.dev.yml \
         run --rm \
         stac-dev scripts/bin/format-common;
 
-    docker-compose  \
+    docker compose  \
         -f docker-compose.yml \
         -f docker-compose.dev.yml \
         run --rm \
         tiler-dev scripts/bin/format-tiler;
 
-    docker-compose  \
+    docker compose  \
         -f docker-compose.yml \
         -f docker-compose.dev.yml \
         run --rm \
         stac-dev scripts/bin/format-stac;
 
-    docker-compose  \
+    docker compose  \
         -f docker-compose.yml \
         -f docker-compose.dev.yml \
         run --rm \
         funcs /bin/bash -c "cd /opt/src && scripts/bin/format-funcs";
 
-    docker-compose  \
+    docker compose  \
         -f docker-compose.yml \
         -f docker-compose.dev.yml \
         run --rm \

diff --git a/scripts/generate-requirements b/scripts/generate-requirements
@@ -5,28 +5,28 @@ set -e
 echo "== Generating requirements.txt ..."
 pip_compile_options="--emit-index-url --annotate --verbose"
 
-docker-compose \
+docker compose \
     -f docker-compose.yml \
     -f docker-compose.dev.yml \
     run --rm \
     stac-dev \
     pip-compile ./pcstac/setup.py --extra server -o pcstac/requirements-server.txt $pip_compile_options
 
-docker-compose \
+docker compose \
     -f docker-compose.yml \
     -f docker-compose.dev.yml \
     run --rm \
     tiler-dev \
     pip-compile ./pctiler/setup.py --extra server -o pctiler/requirements-server.txt $pip_compile_options
 
-docker-compose \
+docker compose \
     -f docker-compose.yml \
     -f docker-compose.dev.yml \
     run --rm \
     tiler-dev \
     pip-compile ./pctiler/setup.py --extra dev -o pctiler/requirements-dev.txt $pip_compile_options
 
-docker-compose \
+docker compose \
     -f docker-compose.yml \
     -f docker-compose.dev.yml \
     run --rm \

diff --git a/scripts/migrate b/scripts/migrate
@@ -16,7 +16,7 @@ Run migrations against the development database.
 if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
 
     # Run database migrations
-    docker-compose \
+    docker compose \
         -f docker-compose.yml \
         run --rm stac \
         bash -c "pypgstac pgready && pypgstac migrate"

diff --git a/scripts/server b/scripts/server
@@ -26,7 +26,7 @@ while [[ "$#" > 0 ]]; do case $1 in
 esac; done
 
 if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
-    docker-compose \
+    docker compose \
         -f docker-compose.yml \
         up ${DETACHED_ARG} $@;
 fi
diff --git a/scripts/setup b/scripts/setup
@@ -27,15 +27,15 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
 
     echo "Ingesting development data..."
 
-    docker-compose \
+    docker compose \
         -f docker-compose.yml \
         run --rm \
         stac \
         python3 /opt/src/pcstac/tests/loadtestdata.py
 
     echo "Setting up azurite..."
 
-    docker-compose \
+    docker compose \
         -f docker-compose.yml \
         -f docker-compose.dev.yml \
         run --rm \

diff --git a/scripts/test b/scripts/test
@@ -65,15 +65,15 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
 
     if [ -z "${TILER_ONLY}${STAC_ONLY}${FUNCS_ONLY}" ]; then
 
-        docker-compose \
+        docker compose \
             -f docker-compose.yml \
             -f docker-compose.dev.yml \
             run --rm \
             stac-dev scripts/bin/test-common
     fi
 
     if [ -z "${STAC_ONLY}${COMMON_ONLY}${FUNCS_ONLY}" ]; then
-        docker-compose \
+        docker compose \
             -f docker-compose.yml \
             -f docker-compose.dev.yml \
             run --rm \
@@ -82,7 +82,7 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
 
     if [ -z "${TILER_ONLY}${COMMON_ONLY}${FUNCS_ONLY}" ]; then
 
-        docker-compose \
+        docker compose \
             -f docker-compose.yml \
             -f docker-compose.dev.yml \
             run --rm \
@@ -91,7 +91,7 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
 
     if [ -z "${STAC_ONLY}${TILER_ONLY}${COMMON_ONLY}" ]; then
 
-        docker-compose \
+        docker compose \
             -f docker-compose.yml \
             run --rm \
             funcs /bin/bash -c "cd /opt/src && scripts/bin/test-funcs ${NO_INTEGRATION}"