diff --git a/.gitignore b/.gitignore index 55af72a6..5ea2266f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ /data/ !/data/README.md +/secrets/ # From https://github.com/github/gitignore/blob/main/Python.gitignore diff --git a/poetry.lock b/poetry.lock index 4fd5ca09..c2b9a26d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2526,6 +2526,25 @@ files = [ [package.dependencies] typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" +[[package]] +name = "pydantic-settings" +version = "2.2.1" +description = "Settings management using Pydantic" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic_settings-2.2.1-py3-none-any.whl", hash = "sha256:0235391d26db4d2190cb9b31051c4b46882d28a51533f97440867f012d4da091"}, + {file = "pydantic_settings-2.2.1.tar.gz", hash = "sha256:00b9f6a5e95553590434c0fa01ead0b216c3e10bc54ae02e37f359948643c5ed"}, +] + +[package.dependencies] +pydantic = ">=2.3.0" +python-dotenv = ">=0.21.0" + +[package.extras] +toml = ["tomli (>=2.0.1)"] +yaml = ["pyyaml (>=6.0.1)"] + [[package]] name = "pydeck" version = "0.8.0" @@ -3295,13 +3314,13 @@ sqlcipher = ["sqlcipher3_binary"] [[package]] name = "streamlit" -version = "1.32.1" +version = "1.32.2" description = "A faster way to build and share data apps" optional = false python-versions = ">=3.8, !=3.9.7" files = [ - {file = "streamlit-1.32.1-py2.py3-none-any.whl", hash = "sha256:fe30ce26f08a5b50b3cb2b349c49c0ad9d3bba6c5ed2f19aac05e39026a30fcc"}, - {file = "streamlit-1.32.1.tar.gz", hash = "sha256:ec6400496f678852143cbc23c4c43889f78b6c93c2e2756fd8e060cccde4b8fd"}, + {file = "streamlit-1.32.2-py2.py3-none-any.whl", hash = "sha256:a0b8044e76fec364b07be145f8b40dbd8d083e20ebbb189ceb1fa9423f3dedea"}, + {file = "streamlit-1.32.2.tar.gz", hash = "sha256:1258b9cbc3ff957bf7d09b1bfc85cedc308f1065b30748545295a9af8d5577ab"}, ] [package.dependencies] @@ -3766,4 +3785,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.9.7 || >3.9.7,<4.0" -content-hash = "abcbcdc93161c793637b9c43b74910df822b9fd6ba773fbf7e488da814ebf1a2" +content-hash = "3b6bc49f105811c0635fce909835e5105928f022c1a00f8eadcd3dcdc99003af" diff --git a/pyproject.toml b/pyproject.toml index b1b6121f..ac0de9fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ pandas = ">=2.2" poetry = ">=1.8" psycopg2-binary = ">=2.9.6" pydantic = ">=1.10.14" +pydantic-settings = ">0" python = ">=3.9,<3.9.7 || >3.9.7,<4.0" python-dotenv = ">=1.0.0" pyyaml = ">=6.0" diff --git a/secrets/.keep b/secrets/.keep new file mode 100644 index 00000000..e69de29b diff --git a/src/bloom/config.py b/src/bloom/config.py index 2e26195c..254eedc2 100644 --- a/src/bloom/config.py +++ b/src/bloom/config.py @@ -1,116 +1,66 @@ import os from pathlib import Path -from pydantic import BaseSettings - -def extract_values_from_env(config:dict,allow_extend:bool=False) -> dict: - """ function that extrat key=value pairs from a file - Parameters: - - config: dict to extend/update with new key/value pairs found in environment - - allow_extend: allows to extend extracted keys with new keys that are not in - actuel config if True, restrict to already existing keys in config of False - Returns a dict contains key/value - """ - for k,v in os.environ.items(): - # Processing of indirect affectation via [ATTR]_FILE=VALUE_PATH => ATTR=VALUE - if k.lower() in [f"{k}_FILE".lower() for k in config.keys()]\ - and ( k.removesuffix('_FILE').lower() in config.keys() or allow_extend == True)\ - and Path(v).is_file(): - with Path.open(v, mode='r') as file: - config[k.removesuffix('_FILE').lower()]=file.readline().strip() - # Processing of direct affectation via ATTR=VALUE - # if extracted key already exist in config OR if allowed to add new keys to config - # Then adding/updating key/value - if k.lower() in [k.lower() for k in config.keys()] or allow_extend == True: - config[k.lower()]=v - return config - -def extract_values_from_file(filename:str,config:dict, - allow_extend:bool=False, - env_priority:bool=True - )-> dict: - """ function that extrat key=value pairs from a file - Parameters: - - filename: filename/filepath from which to extract key/value pairs found in .env.* file - - config: dict to extend/update with new key/value pairs - - allow_extend: allows to extend extracted keys with new keys that are not in actuel - config if True, restrict to already existing keys in config of False - Returns a dict contains key/value - """ - filepath=Path(Path(__file__).parent).joinpath(filename) - with Path.open(filepath) as file: - for line in file: - # Split line at first occurence of '='. - # This allows to have values containing '=' character - split=line.strip().split('=',1) - # if extraction contains 2 items and strictly 2 items - split_succeed=2 - if(len(split)==split_succeed): - k=split[0] - v=split[1] - # Processing of indirect affectation via [ATTR]_FILE=VALUE_PATH => ATTR=VALUE - if k.lower() in [f"{k}_FILE".lower() for k in config.keys()]\ - and ( k.removesuffix('_FILE').lower() in config.keys() or allow_extend == True)\ - and Path(v).is_file(): - with Path(v).open( mode='r') as file_value: - config[k.removesuffix('_FILE').lower()]=file_value.readline().strip() - # if extracted key already exist in config OR if allowed to add new keys to - # config then adding/updating key/value - if k.lower() in [k.lower() for k in config.keys()] or allow_extend == True: - config[k.lower()]=v - # If env priority True, then overriding all values with ENV values before ending - if env_priority: - extract_values_from_env(config,allow_extend=False) - return config +from pydantic_settings import BaseSettings, SettingsConfigDict +from typing import Any +from pydantic import ( + AliasChoices, + AmqpDsn, + BaseModel, + Field, + ImportString, + PostgresDsn, + RedisDsn, + field_validator, + model_validator +) class Settings(BaseSettings): - # Déclaration des attributs/paramètres disponibles au sein de la class settings - postgres_user:str = None - postgres_password:str = None - postgres_hostname:str = None - postgres_port:str = None - postgres_db:str = None - srid: int = 4326 - db_url:str = None - spire_token:str = None - data_folder:str=None - logging_level:str="INFO" + model_config = SettingsConfigDict( + # validate_assignment=True allows to update db_url value as soon as one of + # postgres_user, postgres_password, postgres_hostname, postgres_port, postgres_db + # is modified then db_url_update is called + validate_assignment=True, + # env_ignore_empty=False take env as it is and if declared but empty then empty + # the setting value + env_ignore_empty=True, + env_nested_delimiter='__', + env_file='.env', + env_file_encoding = 'utf-8', + extra='ignore' + ) - def __init__(self): - super().__init__(self) - # Default values - self.srid = 4326 - self.data_folder = Path(__file__).parent.parent.parent.joinpath('./data') + # Déclaration des attributs/paramètres disponibles au sein de la class settings + postgres_user:str = Field(default='') + postgres_password:str = Field(default='') + postgres_hostname:str = Field(min_length=1, + default='localhost') + postgres_port:int = Field(gt=1024, + default=5432) - # Si le fichier de configuration à charger est précisé par la variable d'environnement - # BLOOM_CONFIG alors on charge ce fichier, sinon par défaut c'est /.env - bloom_config=os.getenv('BLOOM_CONFIG',Path(__file__).parent.parent.parent - .joinpath(".env")) - - # Ici on charge les paramètres à partir du fichier BLOOM_CONFIG - # et on mets à jour directement les valeurs des paramètres en tant qu'attribut de la - # la classe courante Settings en attanquant le self.__dict__ - # Ces variables sont systmétiquement convertie en lower case - # - # allow_extend=False précise que seuls les attributs déjà existants dans la config - # passée en paramètres (ici self.__dict__) sont mis à jour. Pas de nouveau paramètres - # Cela singifie que pour rendre accessible un nouveau paramètre il faut le déclaré - # dans la liste des attributs de la classe Settings - # - # env_priority=true signifie que si un paramètres est présent dans la classe Settings, - # mas aussi dans le fichier BLOOM_CONFIG ainsi qu'en tant que variable d'environnement - # alors c'est la valeur de la variable d'environnement qui sera chargée au final - # La priorité est donnée aux valeur de l'environnement selon le standard Docker - if Path(bloom_config).exists(): - extract_values_from_file(bloom_config,self.__dict__,allow_extend=False, - env_priority=True) - else: - extract_values_from_env(self.__dict__,allow_extend=False) + postgres_db:str = Field(min_length=1,max_length=32,pattern=r'^(?:[a-zA-Z]|_)[\w\d_]*$') + srid: int = Field(default=4326) + spire_token:str = Field(default='') + data_folder:str=Field(default=str(Path(__file__).parent.parent.parent.joinpath('./data'))) + db_url:str=Field(default='') - self.db_url = ( f"postgresql://{self.postgres_user}:" - f"{self.postgres_password}@{self.postgres_hostname}:" - f"{self.postgres_port}/{self.postgres_db}") + logging_level:str=Field( + default="INFO", + pattern=r'NOTSET|DEBUG|INFO|WARNING|ERROR|CRITICAL' + ) + + @model_validator(mode='after') + def update_db_url(self)->dict: + new_url= f"postgresql://{self.postgres_user}:"\ + f"{self.postgres_password}@{self.postgres_hostname}:"\ + f"{self.postgres_port}/{self.postgres_db}" + if self.db_url != new_url: + self.db_url = new_url + return self -settings = Settings() +settings = Settings(_env_file=os.getenv('BLOOM_CONFIG', + Path(__file__).parent.parent.parent.joinpath('.env')), + _secrets_dir=os.getenv('BLOOM_SECRETS_DIR', + Path(__file__).parent.parent.parent.joinpath('./secrets')))