Makefile (#1)

* try: makefile

* fix: docker compose

* feat: generate PATH_ENV only if != None

* Makefile launch-amp-data

* fix: BLOOM_DEV_DOCKER use ${POSTGRES_PORT} + clean

* merged

* update config.__dict__

* fix: scripts/update_env.sh add exec mod

* fix: remove .env loading in entreypoint.sh. No need as --env-file is now used

* fix: back to app port instead of POSTGRES_PORT

* fix: docker compose instead of docker-compose (plugin instead of command)

* fix: correct name of generate env /tmp/.docker.${APP_ENV}

* feat add EXPOSED_POSTGRES_PORT

* fix: Makefile ++

* fix: Makefile production add --host POSTGRES_HOSTNAME

* fix: Makefile working + add POSTGRES_IP for production server

---------

Co-authored-by: RV <RV>

.env.dev

@@ -0,0 +1,2 @@
+EXPOSED_POSTGRES_PORT=25432
+POSTGRES_USER=bloomdev

.env.prod

@@ -0,0 +1 @@
+POSTGRES_USER=bloom-prod

.env.template

@@ -1,6 +1,7 @@
 # these values are used in the local docker env. You can use "localhost" hostname if you run the application without docker 
 APP_ENV=dev
 POSTGRES_HOSTNAME=postgres_bloom
+POSTGRES_IP=172.18.0.100
 POSTGRES_USER=bloom_user
 POSTGRES_PASSWORD=bloom
 POSTGRES_DB=bloom_db

.env.test

@@ -0,0 +1 @@
+POSTGRES_PORT=15432

Makefile

@@ -1,59 +1,74 @@
 VERSION ?= 1.0.0
 
-BLOOM_DEV_DOCKER = @docker run --name bloom-test  --mount type=bind,source="$(shell pwd)",target=/source_code --env-file ./.env.test --network=bloom_net -p 8501:8501
-BLOOM_PRODUCTION_DOCKER = @docker run --mount type=bind,source="$(shell pwd)",target=/source_code --env-file ./.env --log-driver json-file --log-opt max-size=10M --log-opt max-file=3 --entrypoint /entrypoint.sh
+BLOOM_DEV_DOCKER = docker run --name bloom-dev  --mount type=bind,source="$(shell pwd)",target=/source_code --env-file /tmp/.env.docker.dev --network=bloom_net -e POSTGRES_PORT=5432 -p 8501:8501
+BLOOM_PRODUCTION_DOCKER = docker run --add-host ${POSTGRES_HOSTNAME}:$${POSTGRES_IP} --mount type=bind,source="$(shell pwd)",target=/source_code --env-file /tmp/.env.docker.prod --log-driver json-file --log-opt max-size=10M --log-opt max-file=3 --entrypoint /entrypoint.sh
 
-UPDATE_SETTINGS = @python -c "from bloom.config import settings"
+EXPORT_ENV_DEV = @export $(shell cat /tmp/.env.docker.dev | grep -v "#" | xargs -d '\r')
+EXPORT_ENV_TEST = @export $(shell cat /tmp/.env.docker.test | grep -v "#" | xargs -d '\r')
+EXPORT_ENV_PROD = @export $(shell cat /tmp/.env.docker.prod | grep -v "#" | xargs -d '\r')
+
+UPDATE_ENV_DEV = @$(shell pwd)/scripts/update_env.sh dev /tmp/.env.docker.dev
+UPDATE_ENV_TEST = @$(shell pwd)/scripts/update_env.sh test /tmp/.env.docker.test
+UPDATE_ENV_PROD = @$(shell pwd)/scripts/update_env.sh prod /tmp/.env.docker.prod
 
-# Regenerate .env file according to .env.template, .env.local, APP_ENV value, .env.${APP_ENV},  .env.${APP_ENV}.local
-update_env:
-	$(UPDATE_SETTINGS)
-
 build:
 	@docker build -t d4g/bloom:${VERSION} --platform linux/amd64  -f docker-env/Dockerfile .
 	@docker tag d4g/bloom:${VERSION} d4g/bloom:latest
 
+test-config:
+	$(UPDATE_ENV_DEV)
+	$(EXPORT_ENV_DEV) && env 
+
 launch-dev-db:
-	@docker compose -f docker-env/docker-compose-db.yaml up -d
+	$(UPDATE_ENV_DEV)
+	$(EXPORT_ENV_DEV) && docker compose -f docker-env/docker-compose-db.yaml up -d
 	@sleep 20
-	$(BLOOM_DEV_DOCKER) --rm d4g/bloom:${VERSION} alembic upgrade head
-	$(BLOOM_DEV_DOCKER) --rm d4g/bloom:${VERSION} /venv/bin/python3 alembic/init_script/load_vessels_data.py
+	$(EXPORT_ENV_DEV) && $(BLOOM_DEV_DOCKER) --rm d4g/bloom:${VERSION} alembic upgrade head
+	$(EXPORT_ENV_DEV) && $(BLOOM_DEV_DOCKER) --rm d4g/bloom:${VERSION} /venv/bin/python3 alembic/init_script/load_vessels_data.py
 
 load-amp-data:
-	$(BLOOM_DEV_DOCKER) --rm d4g/bloom:${VERSION} /venv/bin/python3 alembic/init_script/load_amp_data.py
+	$(UPDATE_ENV_DEV)
+	$(EXPORT_ENV_DEV) && $(BLOOM_DEV_DOCKER) --rm d4g/bloom:${VERSION} /venv/bin/python3 alembic/init_script/load_amp_data.py
 
 load-test-positions-data:
-	$(BLOOM_DEV_DOCKER) --rm d4g/bloom:${VERSION} /venv/bin/python3 alembic/init_script/load_positions_data.py
+	$(UPDATE_ENV_DEV)
+	$(EXPORT_ENV_DEV) && $(BLOOM_DEV_DOCKER) --rm d4g/bloom:${VERSION} /venv/bin/python3 alembic/init_script/load_positions_data.py
 
 launch-dev-container:
-	$(BLOOM_DEV_DOCKER) -dti  d4g/bloom:${VERSION} /bin/bash
+	$(UPDATE_ENV_DEV)
+	$(EXPORT_ENV_DEV) && $(BLOOM_DEV_DOCKER) -dti  d4g/bloom:${VERSION} /bin/bash
 
 launch-dev-app:
-	$(BLOOM_DEV_DOCKER) --rm d4g/bloom:${VERSION} /venv/bin/python3 app.py
+	$(UPDATE_ENV_DEV)
+	$(EXPORT_ENV_DEV) && $(BLOOM_DEV_DOCKER) --rm d4g/bloom:${VERSION} /venv/bin/python3 app.py
 
 launch-test:
-	$(BLOOM_DEV_DOCKER) --rm d4g/bloom:${VERSION} tox -vv
+	$(UPDATE_ENV_DEV)
+	$(EXPORT_DEV_DEV) && $(BLOOM_DEV_DOCKER) --rm d4g/bloom:${VERSION} tox -vv
 
 rm-dev-db:
-	@docker-compose -f docker-env/docker-compose-db.yaml stop
-	@docker-compose -f docker-env/docker-compose-db.yaml rm
+	@docker compose -f docker-env/docker-compose-db.yaml stop
+	@docker compose -f docker-env/docker-compose-db.yaml rm
 
 rm-dev-env:
 	@docker stop bloom-test
 	@docker rm bloom-test
 
 init-production:
-	$(BLOOM_PRODUCTION_DOCKER) --name bloom-production-db-init --rm d4g/bloom:${VERSION} alembic upgrade head
-	$(BLOOM_PRODUCTION_DOCKER) --name bloom-production-db-init --rm d4g/bloom:${VERSION} /venv/bin/python3 alembic/init_script/load_vessels_data.py
+	$(UPDATE_ENV_PROD)
+	$(EXPORT_ENV_PROD) && $(BLOOM_PRODUCTION_DOCKER) --name bloom-production-db-init --rm d4g/bloom:${VERSION} alembic upgrade head
+	$(EXPORT_ENV_PROD) && $(BLOOM_PRODUCTION_DOCKER) --name bloom-production-db-init --rm d4g/bloom:${VERSION} /venv/bin/python3 alembic/init_script/load_vessels_data.py
 
 launch-production:
-	$(BLOOM_PRODUCTION_DOCKER) --name bloom-production -d d4g/bloom:${VERSION} cron -f -L 2
+	$(UPDATE_ENV_PROD)
+	$(EXPORT_ENV_PROD) && $(BLOOM_PRODUCTION_DOCKER) --name bloom-production -d d4g/bloom:${VERSION} cron -f -L 2
 
 launch-production-app:
-	 $(BLOOM_PRODUCTION_DOCKER) --name bloom-production-app --rm d4g/bloom:${VERSION} /venv/bin/python3 app.py
+	$(UPDATE_ENV_PROD)
+	$(EXPORT_ENV_PROD) && $(BLOOM_PRODUCTION_DOCKER) --name bloom-production-app --rm d4g/bloom:${VERSION} /venv/bin/python3 app.py
 
 dump-dev-db:
 	$(BLOOM_DEV_DOCKER) --rm postgres:latest sh -c 'export PGPASSWORD=$$POSTGRES_PASSWORD && pg_dump -Fc $$POSTGRES_DB -h $$POSTGRES_HOSTNAME -p $$POSTGRES_PORT -U $$POSTGRES_USER> /source_code/bloom_$(shell date +%Y%m%d_%H%M).dump'
 
 dump-db:
-	@docker run --mount type=bind,source="$(shell pwd)",target=/source_code --env-file ./.env.test --network=bloom_net --rm postgres:latest sh -c 'export PGPASSWORD=$$POSTGRES_PASSWORD && pg_dump -Fc $$POSTGRES_DB -h $$POSTGRES_HOSTNAME -p $$POSTGRES_PORT -U $$POSTGRES_USER> /source_code/bloom_$(shell date +%Y%m%d_%H%M).dump'
+	@docker run --mount type=bind,source="$(shell pwd)",target=/source_code --env-file ./.env.test --network=bloom_net --rm postgres:latest sh -c 'export PGPASSWORD=$$POSTGRES_PASSWORD && pg_dump -Fc $$POSTGRES_DB -h $$POSTGRES_HOSTNAME -p $$POSTGRES_PORT -U $$POSTGRES_USER> /source_code/bloom_$(shell date +%Y%m%d_%H%M).dump'

bloom/config.py

@@ -3,7 +3,20 @@
 
 from pydantic import BaseSettings
 
-def extract_values_from_file(filename:str,config:dict,allow_extend:bool=False):
+def extract_values_from_env(config:dict,allow_extend:bool=False):
+    """ function that extrat key=value pairs from a file
+    Parameters:
+    - config: dict to extend/update with new key/value pairs found in environment
+    - allow_extend: allows to extend extracted keys with new keys that are not in actuel config if True,
+                    restrict to already existing keys in config of False
+    Returns a dict contains key/value
+    """
+    for k,v in os.environ.items():
+        if k in config.keys() or allow_extend == True:
+            config[k]=v
+    return config
+
+def extract_values_from_file(filename:str,config:dict,allow_extend:bool=False,env_priority=True):
     """ function that extrat key=value pairs from a file
     Parameters:
     - filename: filename/filepath from which to extract key/value pairs found in .env.* file
@@ -22,80 +35,80 @@ def extract_values_from_file(filename:str,config:dict,allow_extend:bool=False):
             # Then adding/updating key/value
             if split[0] in config.keys() or allow_extend == True:
                 config[split[0]]=split[1]
+        if env_priority: extract_values_from_env(config,allow_extend=False)
     return config
-
-def extract_values_from_env(config:dict,allow_extend:bool=False):
-    """ function that extrat key=value pairs from a file
-    Parameters:
-    - config: dict to extend/update with new key/value pairs found in environment
-    - allow_extend: allows to extend extracted keys with new keys that are not in actuel config if True,
-                    restrict to already existing keys in config of False
-    Returns a dict contains key/value
-    """
-    for k,v in os.environ.items():
-        if k in config.keys() or allow_extend == True:
-            config[k]=v
-    return config
-
+
 class Settings(BaseSettings):
     APP_ENV:str=None
     db_url:str=None
     def __init__(self,*arg, **args):
         super().__init__(self,*arg, **args)
-        # Default app_env is 'dev'
+        # Default APP_ENV is 'dev'
         self.APP_ENV='dev'
 
+        # Destination file of "env" merged config
+        # Usefull to set it to docker.${APP_ENV} when generated for docker
+        PATH_ENV=os.getenv('PATH_ENV',Path(os.path.dirname(__file__)).joinpath(f"../.env"))
+
         # dict to store temporary/overrided config parameters
         config={}
 
-        if os.environ.get('BLOOM_CONFIG') != None:
-            file_to_process=os.environ.get('BLOOM_CONFIG')
-            if os.path.isfile(file_to_process): extract_values_from_file(file_to_process,config,allow_extend=True)
-
         # Extract .env.template as default values
         # The keys present in .env.template now will restrict keys that are extracted from following files
         # So all parameters MUST HAVE a default value declared in .env.template to be loaded
         file_to_process=Path(os.path.dirname(__file__)).joinpath(f"../.env.template")
-        if os.path.isfile(file_to_process): extract_values_from_file(file_to_process,config,allow_extend=True)
+        if os.path.isfile(file_to_process): extract_values_from_file(file_to_process,config,allow_extend=True,env_priority=True)
+
+        # Extract from file pointed by BLOOM_CONFIG en var
+        if os.environ.get('BLOOM_CONFIG') != None:
+            file_to_process=os.environ.get('BLOOM_CONFIG')
+            if os.path.isfile(file_to_process): extract_values_from_file(file_to_process,config,allow_extend=True,env_priority=True)
 
         # Extract .env.local and override existing values
         # We restrict extracted keys to the keys already existing in .env.template
         file_to_process=Path(os.path.dirname(__file__)).joinpath(f"../.env.local")
-        if os.path.isfile(file_to_process): extract_values_from_file(file_to_process,config,allow_extend=False)
+        if os.path.isfile(file_to_process): extract_values_from_file(file_to_process,config,allow_extend=False,env_priority=True)
 
-        # Extract .env.${app_env} and override existing values
+        # Extract .env.${APP_ENV} and override existing values
         # We restrict extracted keys to the keys already existing in .env.template
-        if 'app_env' in config:
-            file_to_process=Path(os.path.dirname(__file__)).joinpath(f"../.env.{config['app_env']}")
-            if os.path.isfile(file_to_process): extract_values_from_file(file_to_process,config,allow_extend=False)
+        if 'APP_ENV' in config:
+            file_to_process=Path(os.path.dirname(__file__)).joinpath(f"../.env.{config['APP_ENV']}")
+            if os.path.isfile(file_to_process): extract_values_from_file(file_to_process,config,allow_extend=False,env_priority=True)
 
-        # Extract .env.${app_env}.local and override existing values
+        # Extract .env.${APP_ENV}.local and override existing values
         # We restrict extracted keys to the keys already existing in .env.template
-        if 'app_env' in config:
-            file_to_process=Path(os.path.dirname(__file__)).joinpath(f"../.env.{config['app_env']}.local")
-            if os.path.isfile(file_to_process): extract_values_from_file(file_to_process,config,allow_extend=False)
+        if 'APP_ENV' in config:
+            file_to_process=Path(os.path.dirname(__file__)).joinpath(f"../.env.{config['APP_ENV']}.local")
+            if os.path.isfile(file_to_process): extract_values_from_file(file_to_process,config,allow_extend=False,env_priority=True)
 
         # Extract and oerride values from environment variables
         # We restrict extracted keys to the keys already existing in .env.template
         extract_values_from_env(config,allow_extend=False)
 
-        # Now all .env.* files has been merged, we write the cumulated result to .env
-        # .env is for compliance with docker/docker-compose standard
-        file_to_process=Path(os.path.dirname(__file__)).joinpath(f"../.env")
-        f = open(file_to_process, "w")
-        f.truncate(0)
-        f.write("# This file was generated automaticaly by bloom.config\n# Don't modify values directly here\n# Use .env.* files instead then restart application\n")
-        for k,v in config.items():
-            f.write(f"{k}={v}\n")
-        f.close()
         # Now we extract key/value pairs from new .env and add them to current class as attributes
-        if os.path.isfile(file_to_process): extract_values_from_file(file_to_process,self.__dict__,allow_extend=True)
+        self.__dict__.update(config)
 
 
         # Set the db_url attribute containing connection string to the database
         self.db_url = ( f"postgresql://"
                         f"{self.POSTGRES_USER}:{self.POSTGRES_PASSWORD}"
                         f"@{self.POSTGRES_HOSTNAME}:{self.POSTGRES_PORT}/{self.POSTGRES_DB}")
+
+
+        # Destination file of "env" merged config
+        # Usefull to set it to docker.${APP_ENV} when generated for docker
+        # If not defined, is None and no merged file is generated
+        PATH_ENV=os.getenv('PATH_ENV')
+        if PATH_ENV != None:
+            # Now all .env.* files has been merged, we write the cumulated result to .env
+            # .env is for compliance with docker/docker-compose standard
+            print(f"writing {PATH_ENV}")
+            f = open(PATH_ENV, "w")
+            f.truncate(0)
+            f.write("# This file was generated automaticaly by bloom.config\n# Don't modify values directly here\n# Use .env.* files instead then restart application\n")
+            for k,v in config.items():
+                f.write(f"{k}={v}\n")
+            f.close()
 
 
     srid: int = 4326

docker-env/docker-compose-db.yaml

@@ -3,13 +3,13 @@ services:
   postgres:
     container_name: postgres_bloom
     environment:
-      HOSTNAME: postgres_bloom
-      POSTGRES_DB: bloom_db
-      POSTGRES_PASSWORD: bloom
-      POSTGRES_USER: bloom_user
+      HOSTNAME: ${POSTGRES_HOSTNAME}
+      POSTGRES_DB: ${POSTGRES_DB}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      POSTGRES_USER: ${POSTGRES_USER}
     image: postgis/postgis:14-3.3-alpine
     ports:
-      - 5480:5432
+      - ${EXPOSED_POSTGRES_PORT}:5432
     networks:
         - bloom_net
     restart: unless-stopped
@@ -18,8 +18,8 @@ services:
       container_name: pgadmin_bloom
       image: dpage/pgadmin4
       environment:
-        PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL:-test@test.com}
-        PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD:-test}
+        PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL:-dev@dev.com}
+        PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD:-dev}
       volumes:
         - ./pgpassfile:/pgadmin4/pgpassfile
         - ./pgadmin-servers.json:/pgadmin4/servers.json
@@ -31,4 +31,4 @@ services:
 
 networks:
   bloom_net:
-    name: bloom_net
+    name: bloom_net

docker-env/entrypoint.sh

@@ -1,7 +1,5 @@
 #!/bin/bash
 
-cat /source_code/.env >> /etc/environment
-
 # Store all APP_BLOOM ENV variables to local file accessible to current user to share them with cron task
 # Due to the fact that cron process doesn't access to declared ENV vars and doesn't load user profiles
 # The entrypoint.sh script stores ENV vars at runtime in the ~/.env file as key=value pairs
@@ -13,8 +11,6 @@ env | egrep '^(POSTGRES_.*|SPIRE_TOKEN.*)' > ~/.env
 
 ln -sf /dev/stdout /var/log/syslog
 
-source /source_code/.env
-
 # execute CMD
 echo "$@"
 exec "$@"

scripts/update_env.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+export APP_ENV=$1
+export PATH_ENV=$2
+/venv/bin/python3 -c "from bloom.config import settings"
+export $(cat $2 | grep -v "#" | xargs -d '\r');