All package update

.gitignore

@@ -2,6 +2,7 @@
 
 /data/
 !/data/README.md
+/secrets/
 
 
 # From https://github.com/github/gitignore/blob/main/Python.gitignore

docker-compose.yaml

@@ -25,7 +25,6 @@ services:
     #stdin_open: true
     volumes:
       - ./:/project/
-      - ./data:/project/data
     environment:
       <<: *common-env
       POSTGRES_PORT: 5432

pyproject.toml

@@ -9,37 +9,38 @@ description = "Bloom scrapping application"
 authors = ["Your Name <[email protected]>"]
 
 [tool.poetry.dependencies]
-alembic = "^1.10.2"
-autopep8 = "1.5.4"
-dependency-injector = "^4.41.0"
-dependency_injection = "1.2.0"
-folium = "^0.14.0"
-geoalchemy2 = "^0.13.1"
-geopandas = "0.14.3"
-geopy = "^2.4.0"
-gql = "^3.4.0"
-matplotlib = "^3.8.0"
-openpyxl = "^3.1.0"
-pandas = "1.4.4"
-poetry = "^1.8"
-psycopg2-binary = "^2.9.6"
-pydantic = "1.10.14"
-python = ">=3.9,<3.9.7 || >3.9.7,<3.12"
-python-dotenv = "^1.0.0"
-pyyaml = "^6.0"
-requests = "^2.31"
-requests-toolbelt = "^1.0"
-shapely = "2.0.1"
-slack-sdk = "^3.21.3"
-sqlalchemy = "^2.0"
-streamlit = "^1.27.2"
-streamlit-folium = "^0.15.0"
+alembic = "~1.13"
+autopep8 = "~2.0"
+dependency-injector = "~4.41"
+dependency_injection = "~1.2"
+folium = "~0.16"
+GeoAlchemy2 = "~0.14"
+geopandas = "~0.14"
+geopy = "~2.4"
+gql = "~3.5"
+matplotlib = "~3.8"
+openpyxl = "~3.1"
+pandas = "~2.2"
+poetry = "~1.8"
+psycopg2-binary = "~2.9"
+pydantic = "~2.6"
+pydantic-settings = "~2.2"
+python = ">=3.9,<3.9.7 || >3.9.7,<4.0"
+python-dotenv = "~1.0"
+PyYAML = "~6.0"
+requests = "~2.31"
+requests-toolbelt = "~1.0"
+shapely = "~2.0"
+slack-sdk = "~3.27"
+SQLAlchemy = "~2.0"
+streamlit = "~1.32"
+streamlit-folium = "~0.18"
 
 
 [tool.poetry.group.dev.dependencies]
-pre-commit = "^3.0.3"
-ruff = "^0.0.254"
-tox = "^4.4.8"
+pre-commit = "~3.6"
+ruff = "~0.3"
+tox = "~4.14"
 
 [tool.ruff]
 select = [

src/bloom/config.py

@@ -1,116 +1,66 @@
 import os
 from pathlib import Path
 
-from pydantic import BaseSettings
-
-def extract_values_from_env(config:dict,allow_extend:bool=False) -> dict:
-    """ function that extrat key=value pairs from a file
-    Parameters:
-    - config: dict to extend/update with new key/value pairs found in environment
-    - allow_extend: allows to extend extracted keys with new keys that are not in
-                    actuel config if True, restrict to already existing keys in config of False
-    Returns a dict contains key/value
-    """
-    for k,v in os.environ.items():
-        # Processing of indirect affectation via [ATTR]_FILE=VALUE_PATH => ATTR=VALUE
-        if k.lower() in [f"{k}_FILE".lower() for k in config.keys()]\
-            and ( k.removesuffix('_FILE').lower() in config.keys() or  allow_extend == True)\
-            and Path(v).is_file():
-                with Path.open(v, mode='r') as file:
-                    config[k.removesuffix('_FILE').lower()]=file.readline().strip()
-        # Processing of direct affectation via ATTR=VALUE
-        # if extracted key already exist in config OR if allowed to add new keys to config
-        # Then adding/updating key/value
-        if k.lower() in [k.lower() for k in config.keys()] or allow_extend == True:
-            config[k.lower()]=v
-    return config
-
-def extract_values_from_file(filename:str,config:dict,
-                             allow_extend:bool=False,
-                             env_priority:bool=True
-                             )-> dict:
-    """ function that extrat key=value pairs from a file
-    Parameters:
-    - filename: filename/filepath from which to extract key/value pairs found in .env.* file
-    - config: dict to extend/update with new key/value pairs
-    - allow_extend: allows to extend extracted keys with new keys that are not in actuel
-                    config if True, restrict to already existing keys in config of False
-    Returns a dict contains key/value
-    """ 
-    filepath=Path(Path(__file__).parent).joinpath(filename)
-    with Path.open(filepath) as file:
-        for line in file:
-            # Split line at first occurence of '='.
-            # This allows to have values containing '=' character
-            split=line.strip().split('=',1)
-            # if extraction contains 2 items and strictly 2 items
-            split_succeed=2
-            if(len(split)==split_succeed):
-                k=split[0]
-                v=split[1]
-                # Processing of indirect affectation via [ATTR]_FILE=VALUE_PATH => ATTR=VALUE
-                if k.lower() in [f"{k}_FILE".lower() for k in config.keys()]\
-                    and ( k.removesuffix('_FILE').lower() in config.keys() or  allow_extend == True)\
-                    and Path(v).is_file():
-                        with Path(v).open( mode='r') as file_value:
-                            config[k.removesuffix('_FILE').lower()]=file_value.readline().strip()
-                # if extracted key already exist in config OR if allowed to add new keys to
-                # config then adding/updating key/value
-                if k.lower() in [k.lower() for k in config.keys()] or allow_extend == True:
-                    config[k.lower()]=v
-            # If env priority True, then overriding all values with ENV values before ending
-            if env_priority:
-                extract_values_from_env(config,allow_extend=False)
-    return config
+from pydantic_settings import BaseSettings, SettingsConfigDict
+from typing import Any
 
+from pydantic import (
+    AliasChoices,
+    AmqpDsn,
+    BaseModel,
+    Field,
+    ImportString,
+    PostgresDsn,
+    RedisDsn,
+    field_validator,
+    model_validator
+)
 
 class Settings(BaseSettings):
-    # Déclaration des attributs/paramètres disponibles au sein de la class settings
-    postgres_user:str = None
-    postgres_password:str = None
-    postgres_hostname:str = None
-    postgres_port:str = None
-    postgres_db:str = None
-    srid: int = 4326
-    db_url:str = None
-    spire_token:str = None
-    data_folder:str=None
-    logging_level:str="INFO"
+    model_config = SettingsConfigDict(
+        # validate_assignment=True allows to update db_url value as soon as one of
+        # postgres_user, postgres_password, postgres_hostname, postgres_port, postgres_db
+        # is modified then db_url_update is called
+        validate_assignment=True,
+        # env_ignore_empty=False take env as it is and if declared but empty then empty
+        # the setting value
+        env_ignore_empty=True,
+        env_nested_delimiter='__',
+        env_file='.env',
+        env_file_encoding = 'utf-8',
+        extra='ignore'
+        )
 
-    def __init__(self):
-        super().__init__(self)
-        # Default values
-        self.srid = 4326
-        self.data_folder = Path(__file__).parent.parent.parent.joinpath('./data')
+    # Déclaration des attributs/paramètres disponibles au sein de la class settings
+    postgres_user:str = Field(default='')
+    postgres_password:str = Field(default='')
+    postgres_hostname:str = Field(min_length=1,
+                                  default='localhost')
+    postgres_port:int = Field(gt=1024,
+                                  default=5432)
 
-        # Si le fichier de configuration à charger est précisé par la variable d'environnement
-        # BLOOM_CONFIG alors on charge ce fichier, sinon par défaut c'est <project>/.env
-        bloom_config=os.getenv('BLOOM_CONFIG',Path(__file__).parent.parent.parent
-                                                                            .joinpath(".env"))
-
-        # Ici on charge les paramètres à partir du fichier BLOOM_CONFIG
-        # et on mets à jour directement les valeurs des paramètres en tant qu'attribut de la
-        # la classe courante Settings en attanquant le self.__dict__
-        # Ces variables sont systmétiquement convertie en lower case
-        # 
-        # allow_extend=False précise que seuls les attributs déjà existants dans la config
-        # passée en paramètres (ici self.__dict__) sont mis à jour. Pas de nouveau paramètres
-        # Cela singifie que pour rendre accessible un nouveau paramètre il faut le déclaré
-        # dans la liste des attributs de la classe Settings
-        #
-        # env_priority=true signifie que si un paramètres est présent dans la classe Settings,
-        # mas aussi dans le fichier BLOOM_CONFIG ainsi qu'en tant que variable d'environnement
-        # alors c'est la valeur de la variable d'environnement qui sera chargée au final
-        # La priorité est donnée aux valeur de l'environnement selon le standard Docker
-        if Path(bloom_config).exists():
-            extract_values_from_file(bloom_config,self.__dict__,allow_extend=False,
-                                                                            env_priority=True)
-        else:
-            extract_values_from_env(self.__dict__,allow_extend=False)
+    postgres_db:str = Field(min_length=1,max_length=32,pattern=r'^(?:[a-zA-Z]|_)[\w\d_]*$')
+    srid: int = Field(default=4326)
+    spire_token:str = Field(default='')
+    data_folder:str=Field(default=str(Path(__file__).parent.parent.parent.joinpath('./data')))
+    db_url:str=Field(default='')
 
-        self.db_url = ( f"postgresql://{self.postgres_user}:"
-                  f"{self.postgres_password}@{self.postgres_hostname}:"
-                  f"{self.postgres_port}/{self.postgres_db}")
+    logging_level:str=Field(
+                                default="INFO",
+                                pattern=r'NOTSET|DEBUG|INFO|WARNING|ERROR|CRITICAL'
+                            )
+
+    @model_validator(mode='after')
+    def update_db_url(self)->dict:
+        new_url= f"postgresql://{self.postgres_user}:"\
+                          f"{self.postgres_password}@{self.postgres_hostname}:"\
+                          f"{self.postgres_port}/{self.postgres_db}"
+        if self.db_url != new_url:
+           self.db_url = new_url
+        return self
 
 
-settings = Settings()
+settings = Settings(_env_file=os.getenv('BLOOM_CONFIG',
+                                    Path(__file__).parent.parent.parent.joinpath('.env')),
+                    _secrets_dir=os.getenv('BLOOM_SECRETS_DIR',
+                                    Path(__file__).parent.parent.parent.joinpath('./secrets')))

src/bloom/infra/repositories/repository_vessel.py

@@ -215,7 +215,7 @@ def convert_wkb_to_point(x: Any) -> Union[ Point, None ]:
                                         "course",
                                         "heading",
                                         "rot"])
-            df = df.astype({"timestamp": 'datetime64'})
+            df = df.astype({"timestamp": 'datetime64[ms]'})
         else:
             df = (
                 pd.DataFrame([p.__dict__ for p in positions])