From b11dc2cb3c93385722526b9a703e595d00339fb4 Mon Sep 17 00:00:00 2001 From: rv2931 Date: Tue, 27 Feb 2024 23:25:44 +0100 Subject: [PATCH] Makefile (#1) * try: makefile * fix: docker compose * feat: generate PATH_ENV only if != None * Makefile launch-amp-data * fix: BLOOM_DEV_DOCKER use ${POSTGRES_PORT} + clean * merged * update config.__dict__ * fix: scripts/update_env.sh add exec mod * fix: remove .env loading in entreypoint.sh. No need as --env-file is now used * fix: back to app port instead of POSTGRES_PORT * fix: docker compose instead of docker-compose (plugin instead of command) * fix: correct name of generate env /tmp/.docker.${APP_ENV} * feat add EXPOSED_POSTGRES_PORT * fix: Makefile ++ * fix: Makefile production add --host POSTGRES_HOSTNAME * fix: Makefile working + add POSTGRES_IP for production server --------- Co-authored-by: RV --- .env.dev | 2 + .env.prod | 1 + .env.template | 1 + .env.test | 1 + Makefile | 59 ++++++++++++-------- bloom/config.py | 93 ++++++++++++++++++------------- docker-env/docker-compose-db.yaml | 16 +++--- docker-env/entrypoint.sh | 4 -- scripts/update_env.sh | 5 ++ 9 files changed, 108 insertions(+), 74 deletions(-) create mode 100755 scripts/update_env.sh diff --git a/.env.dev b/.env.dev index e69de29b..21c588fd 100644 --- a/.env.dev +++ b/.env.dev @@ -0,0 +1,2 @@ +EXPOSED_POSTGRES_PORT=25432 +POSTGRES_USER=bloomdev diff --git a/.env.prod b/.env.prod index e69de29b..50e0433a 100644 --- a/.env.prod +++ b/.env.prod @@ -0,0 +1 @@ +POSTGRES_USER=bloom-prod diff --git a/.env.template b/.env.template index 01a50525..b7fef70e 100644 --- a/.env.template +++ b/.env.template @@ -1,6 +1,7 @@ # these values are used in the local docker env. You can use "localhost" hostname if you run the application without docker APP_ENV=dev POSTGRES_HOSTNAME=postgres_bloom +POSTGRES_IP=172.18.0.100 POSTGRES_USER=bloom_user POSTGRES_PASSWORD=bloom POSTGRES_DB=bloom_db diff --git a/.env.test b/.env.test index e69de29b..eeecb238 100644 --- a/.env.test +++ b/.env.test @@ -0,0 +1 @@ +POSTGRES_PORT=15432 \ No newline at end of file diff --git a/Makefile b/Makefile index ba6e511d..d83dd98d 100644 --- a/Makefile +++ b/Makefile @@ -1,59 +1,74 @@ VERSION ?= 1.0.0 -BLOOM_DEV_DOCKER = @docker run --name bloom-test --mount type=bind,source="$(shell pwd)",target=/source_code --env-file ./.env.test --network=bloom_net -p 8501:8501 -BLOOM_PRODUCTION_DOCKER = @docker run --mount type=bind,source="$(shell pwd)",target=/source_code --env-file ./.env --log-driver json-file --log-opt max-size=10M --log-opt max-file=3 --entrypoint /entrypoint.sh +BLOOM_DEV_DOCKER = docker run --name bloom-dev --mount type=bind,source="$(shell pwd)",target=/source_code --env-file /tmp/.env.docker.dev --network=bloom_net -e POSTGRES_PORT=5432 -p 8501:8501 +BLOOM_PRODUCTION_DOCKER = docker run --add-host ${POSTGRES_HOSTNAME}:$${POSTGRES_IP} --mount type=bind,source="$(shell pwd)",target=/source_code --env-file /tmp/.env.docker.prod --log-driver json-file --log-opt max-size=10M --log-opt max-file=3 --entrypoint /entrypoint.sh -UPDATE_SETTINGS = @python -c "from bloom.config import settings" +EXPORT_ENV_DEV = @export $(shell cat /tmp/.env.docker.dev | grep -v "#" | xargs -d '\r') +EXPORT_ENV_TEST = @export $(shell cat /tmp/.env.docker.test | grep -v "#" | xargs -d '\r') +EXPORT_ENV_PROD = @export $(shell cat /tmp/.env.docker.prod | grep -v "#" | xargs -d '\r') + +UPDATE_ENV_DEV = @$(shell pwd)/scripts/update_env.sh dev /tmp/.env.docker.dev +UPDATE_ENV_TEST = @$(shell pwd)/scripts/update_env.sh test /tmp/.env.docker.test +UPDATE_ENV_PROD = @$(shell pwd)/scripts/update_env.sh prod /tmp/.env.docker.prod -# Regenerate .env file according to .env.template, .env.local, APP_ENV value, .env.${APP_ENV}, .env.${APP_ENV}.local -update_env: - $(UPDATE_SETTINGS) - build: @docker build -t d4g/bloom:${VERSION} --platform linux/amd64 -f docker-env/Dockerfile . @docker tag d4g/bloom:${VERSION} d4g/bloom:latest +test-config: + $(UPDATE_ENV_DEV) + $(EXPORT_ENV_DEV) && env + launch-dev-db: - @docker compose -f docker-env/docker-compose-db.yaml up -d + $(UPDATE_ENV_DEV) + $(EXPORT_ENV_DEV) && docker compose -f docker-env/docker-compose-db.yaml up -d @sleep 20 - $(BLOOM_DEV_DOCKER) --rm d4g/bloom:${VERSION} alembic upgrade head - $(BLOOM_DEV_DOCKER) --rm d4g/bloom:${VERSION} /venv/bin/python3 alembic/init_script/load_vessels_data.py + $(EXPORT_ENV_DEV) && $(BLOOM_DEV_DOCKER) --rm d4g/bloom:${VERSION} alembic upgrade head + $(EXPORT_ENV_DEV) && $(BLOOM_DEV_DOCKER) --rm d4g/bloom:${VERSION} /venv/bin/python3 alembic/init_script/load_vessels_data.py load-amp-data: - $(BLOOM_DEV_DOCKER) --rm d4g/bloom:${VERSION} /venv/bin/python3 alembic/init_script/load_amp_data.py + $(UPDATE_ENV_DEV) + $(EXPORT_ENV_DEV) && $(BLOOM_DEV_DOCKER) --rm d4g/bloom:${VERSION} /venv/bin/python3 alembic/init_script/load_amp_data.py load-test-positions-data: - $(BLOOM_DEV_DOCKER) --rm d4g/bloom:${VERSION} /venv/bin/python3 alembic/init_script/load_positions_data.py + $(UPDATE_ENV_DEV) + $(EXPORT_ENV_DEV) && $(BLOOM_DEV_DOCKER) --rm d4g/bloom:${VERSION} /venv/bin/python3 alembic/init_script/load_positions_data.py launch-dev-container: - $(BLOOM_DEV_DOCKER) -dti d4g/bloom:${VERSION} /bin/bash + $(UPDATE_ENV_DEV) + $(EXPORT_ENV_DEV) && $(BLOOM_DEV_DOCKER) -dti d4g/bloom:${VERSION} /bin/bash launch-dev-app: - $(BLOOM_DEV_DOCKER) --rm d4g/bloom:${VERSION} /venv/bin/python3 app.py + $(UPDATE_ENV_DEV) + $(EXPORT_ENV_DEV) && $(BLOOM_DEV_DOCKER) --rm d4g/bloom:${VERSION} /venv/bin/python3 app.py launch-test: - $(BLOOM_DEV_DOCKER) --rm d4g/bloom:${VERSION} tox -vv + $(UPDATE_ENV_DEV) + $(EXPORT_DEV_DEV) && $(BLOOM_DEV_DOCKER) --rm d4g/bloom:${VERSION} tox -vv rm-dev-db: - @docker-compose -f docker-env/docker-compose-db.yaml stop - @docker-compose -f docker-env/docker-compose-db.yaml rm + @docker compose -f docker-env/docker-compose-db.yaml stop + @docker compose -f docker-env/docker-compose-db.yaml rm rm-dev-env: @docker stop bloom-test @docker rm bloom-test init-production: - $(BLOOM_PRODUCTION_DOCKER) --name bloom-production-db-init --rm d4g/bloom:${VERSION} alembic upgrade head - $(BLOOM_PRODUCTION_DOCKER) --name bloom-production-db-init --rm d4g/bloom:${VERSION} /venv/bin/python3 alembic/init_script/load_vessels_data.py + $(UPDATE_ENV_PROD) + $(EXPORT_ENV_PROD) && $(BLOOM_PRODUCTION_DOCKER) --name bloom-production-db-init --rm d4g/bloom:${VERSION} alembic upgrade head + $(EXPORT_ENV_PROD) && $(BLOOM_PRODUCTION_DOCKER) --name bloom-production-db-init --rm d4g/bloom:${VERSION} /venv/bin/python3 alembic/init_script/load_vessels_data.py launch-production: - $(BLOOM_PRODUCTION_DOCKER) --name bloom-production -d d4g/bloom:${VERSION} cron -f -L 2 + $(UPDATE_ENV_PROD) + $(EXPORT_ENV_PROD) && $(BLOOM_PRODUCTION_DOCKER) --name bloom-production -d d4g/bloom:${VERSION} cron -f -L 2 launch-production-app: - $(BLOOM_PRODUCTION_DOCKER) --name bloom-production-app --rm d4g/bloom:${VERSION} /venv/bin/python3 app.py + $(UPDATE_ENV_PROD) + $(EXPORT_ENV_PROD) && $(BLOOM_PRODUCTION_DOCKER) --name bloom-production-app --rm d4g/bloom:${VERSION} /venv/bin/python3 app.py dump-dev-db: $(BLOOM_DEV_DOCKER) --rm postgres:latest sh -c 'export PGPASSWORD=$$POSTGRES_PASSWORD && pg_dump -Fc $$POSTGRES_DB -h $$POSTGRES_HOSTNAME -p $$POSTGRES_PORT -U $$POSTGRES_USER> /source_code/bloom_$(shell date +%Y%m%d_%H%M).dump' dump-db: - @docker run --mount type=bind,source="$(shell pwd)",target=/source_code --env-file ./.env.test --network=bloom_net --rm postgres:latest sh -c 'export PGPASSWORD=$$POSTGRES_PASSWORD && pg_dump -Fc $$POSTGRES_DB -h $$POSTGRES_HOSTNAME -p $$POSTGRES_PORT -U $$POSTGRES_USER> /source_code/bloom_$(shell date +%Y%m%d_%H%M).dump' \ No newline at end of file + @docker run --mount type=bind,source="$(shell pwd)",target=/source_code --env-file ./.env.test --network=bloom_net --rm postgres:latest sh -c 'export PGPASSWORD=$$POSTGRES_PASSWORD && pg_dump -Fc $$POSTGRES_DB -h $$POSTGRES_HOSTNAME -p $$POSTGRES_PORT -U $$POSTGRES_USER> /source_code/bloom_$(shell date +%Y%m%d_%H%M).dump' diff --git a/bloom/config.py b/bloom/config.py index ad3d2bdd..e18b4bce 100644 --- a/bloom/config.py +++ b/bloom/config.py @@ -3,7 +3,20 @@ from pydantic import BaseSettings -def extract_values_from_file(filename:str,config:dict,allow_extend:bool=False): +def extract_values_from_env(config:dict,allow_extend:bool=False): + """ function that extrat key=value pairs from a file + Parameters: + - config: dict to extend/update with new key/value pairs found in environment + - allow_extend: allows to extend extracted keys with new keys that are not in actuel config if True, + restrict to already existing keys in config of False + Returns a dict contains key/value + """ + for k,v in os.environ.items(): + if k in config.keys() or allow_extend == True: + config[k]=v + return config + +def extract_values_from_file(filename:str,config:dict,allow_extend:bool=False,env_priority=True): """ function that extrat key=value pairs from a file Parameters: - filename: filename/filepath from which to extract key/value pairs found in .env.* file @@ -22,80 +35,80 @@ def extract_values_from_file(filename:str,config:dict,allow_extend:bool=False): # Then adding/updating key/value if split[0] in config.keys() or allow_extend == True: config[split[0]]=split[1] + if env_priority: extract_values_from_env(config,allow_extend=False) return config - -def extract_values_from_env(config:dict,allow_extend:bool=False): - """ function that extrat key=value pairs from a file - Parameters: - - config: dict to extend/update with new key/value pairs found in environment - - allow_extend: allows to extend extracted keys with new keys that are not in actuel config if True, - restrict to already existing keys in config of False - Returns a dict contains key/value - """ - for k,v in os.environ.items(): - if k in config.keys() or allow_extend == True: - config[k]=v - return config - + class Settings(BaseSettings): APP_ENV:str=None db_url:str=None def __init__(self,*arg, **args): super().__init__(self,*arg, **args) - # Default app_env is 'dev' + # Default APP_ENV is 'dev' self.APP_ENV='dev' + # Destination file of "env" merged config + # Usefull to set it to docker.${APP_ENV} when generated for docker + PATH_ENV=os.getenv('PATH_ENV',Path(os.path.dirname(__file__)).joinpath(f"../.env")) + # dict to store temporary/overrided config parameters config={} - if os.environ.get('BLOOM_CONFIG') != None: - file_to_process=os.environ.get('BLOOM_CONFIG') - if os.path.isfile(file_to_process): extract_values_from_file(file_to_process,config,allow_extend=True) - # Extract .env.template as default values # The keys present in .env.template now will restrict keys that are extracted from following files # So all parameters MUST HAVE a default value declared in .env.template to be loaded file_to_process=Path(os.path.dirname(__file__)).joinpath(f"../.env.template") - if os.path.isfile(file_to_process): extract_values_from_file(file_to_process,config,allow_extend=True) + if os.path.isfile(file_to_process): extract_values_from_file(file_to_process,config,allow_extend=True,env_priority=True) + + # Extract from file pointed by BLOOM_CONFIG en var + if os.environ.get('BLOOM_CONFIG') != None: + file_to_process=os.environ.get('BLOOM_CONFIG') + if os.path.isfile(file_to_process): extract_values_from_file(file_to_process,config,allow_extend=True,env_priority=True) # Extract .env.local and override existing values # We restrict extracted keys to the keys already existing in .env.template file_to_process=Path(os.path.dirname(__file__)).joinpath(f"../.env.local") - if os.path.isfile(file_to_process): extract_values_from_file(file_to_process,config,allow_extend=False) + if os.path.isfile(file_to_process): extract_values_from_file(file_to_process,config,allow_extend=False,env_priority=True) - # Extract .env.${app_env} and override existing values + # Extract .env.${APP_ENV} and override existing values # We restrict extracted keys to the keys already existing in .env.template - if 'app_env' in config: - file_to_process=Path(os.path.dirname(__file__)).joinpath(f"../.env.{config['app_env']}") - if os.path.isfile(file_to_process): extract_values_from_file(file_to_process,config,allow_extend=False) + if 'APP_ENV' in config: + file_to_process=Path(os.path.dirname(__file__)).joinpath(f"../.env.{config['APP_ENV']}") + if os.path.isfile(file_to_process): extract_values_from_file(file_to_process,config,allow_extend=False,env_priority=True) - # Extract .env.${app_env}.local and override existing values + # Extract .env.${APP_ENV}.local and override existing values # We restrict extracted keys to the keys already existing in .env.template - if 'app_env' in config: - file_to_process=Path(os.path.dirname(__file__)).joinpath(f"../.env.{config['app_env']}.local") - if os.path.isfile(file_to_process): extract_values_from_file(file_to_process,config,allow_extend=False) + if 'APP_ENV' in config: + file_to_process=Path(os.path.dirname(__file__)).joinpath(f"../.env.{config['APP_ENV']}.local") + if os.path.isfile(file_to_process): extract_values_from_file(file_to_process,config,allow_extend=False,env_priority=True) # Extract and oerride values from environment variables # We restrict extracted keys to the keys already existing in .env.template extract_values_from_env(config,allow_extend=False) - # Now all .env.* files has been merged, we write the cumulated result to .env - # .env is for compliance with docker/docker-compose standard - file_to_process=Path(os.path.dirname(__file__)).joinpath(f"../.env") - f = open(file_to_process, "w") - f.truncate(0) - f.write("# This file was generated automaticaly by bloom.config\n# Don't modify values directly here\n# Use .env.* files instead then restart application\n") - for k,v in config.items(): - f.write(f"{k}={v}\n") - f.close() # Now we extract key/value pairs from new .env and add them to current class as attributes - if os.path.isfile(file_to_process): extract_values_from_file(file_to_process,self.__dict__,allow_extend=True) + self.__dict__.update(config) # Set the db_url attribute containing connection string to the database self.db_url = ( f"postgresql://" f"{self.POSTGRES_USER}:{self.POSTGRES_PASSWORD}" f"@{self.POSTGRES_HOSTNAME}:{self.POSTGRES_PORT}/{self.POSTGRES_DB}") + + + # Destination file of "env" merged config + # Usefull to set it to docker.${APP_ENV} when generated for docker + # If not defined, is None and no merged file is generated + PATH_ENV=os.getenv('PATH_ENV') + if PATH_ENV != None: + # Now all .env.* files has been merged, we write the cumulated result to .env + # .env is for compliance with docker/docker-compose standard + print(f"writing {PATH_ENV}") + f = open(PATH_ENV, "w") + f.truncate(0) + f.write("# This file was generated automaticaly by bloom.config\n# Don't modify values directly here\n# Use .env.* files instead then restart application\n") + for k,v in config.items(): + f.write(f"{k}={v}\n") + f.close() srid: int = 4326 diff --git a/docker-env/docker-compose-db.yaml b/docker-env/docker-compose-db.yaml index 8defa88a..5a9032a1 100644 --- a/docker-env/docker-compose-db.yaml +++ b/docker-env/docker-compose-db.yaml @@ -3,13 +3,13 @@ services: postgres: container_name: postgres_bloom environment: - HOSTNAME: postgres_bloom - POSTGRES_DB: bloom_db - POSTGRES_PASSWORD: bloom - POSTGRES_USER: bloom_user + HOSTNAME: ${POSTGRES_HOSTNAME} + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_USER: ${POSTGRES_USER} image: postgis/postgis:14-3.3-alpine ports: - - 5480:5432 + - ${EXPOSED_POSTGRES_PORT}:5432 networks: - bloom_net restart: unless-stopped @@ -18,8 +18,8 @@ services: container_name: pgadmin_bloom image: dpage/pgadmin4 environment: - PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL:-test@test.com} - PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD:-test} + PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL:-dev@dev.com} + PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD:-dev} volumes: - ./pgpassfile:/pgadmin4/pgpassfile - ./pgadmin-servers.json:/pgadmin4/servers.json @@ -31,4 +31,4 @@ services: networks: bloom_net: - name: bloom_net \ No newline at end of file + name: bloom_net diff --git a/docker-env/entrypoint.sh b/docker-env/entrypoint.sh index 5914e212..25a787f2 100644 --- a/docker-env/entrypoint.sh +++ b/docker-env/entrypoint.sh @@ -1,7 +1,5 @@ #!/bin/bash -cat /source_code/.env >> /etc/environment - # Store all APP_BLOOM ENV variables to local file accessible to current user to share them with cron task # Due to the fact that cron process doesn't access to declared ENV vars and doesn't load user profiles # The entrypoint.sh script stores ENV vars at runtime in the ~/.env file as key=value pairs @@ -13,8 +11,6 @@ env | egrep '^(POSTGRES_.*|SPIRE_TOKEN.*)' > ~/.env ln -sf /dev/stdout /var/log/syslog -source /source_code/.env - # execute CMD echo "$@" exec "$@" diff --git a/scripts/update_env.sh b/scripts/update_env.sh new file mode 100755 index 00000000..65ac52c1 --- /dev/null +++ b/scripts/update_env.sh @@ -0,0 +1,5 @@ +#!/bin/bash +export APP_ENV=$1 +export PATH_ENV=$2 +/venv/bin/python3 -c "from bloom.config import settings" +export $(cat $2 | grep -v "#" | xargs -d '\r'); \ No newline at end of file