Merge pull request #46 from 6G-SANDBOX/dev

6G-SANDBOX · Oct 30, 2024 · 586160e · 586160e
2 parents f645e5d + 865238a
commit 586160e
Show file tree

Hide file tree

Showing 22 changed files with 161 additions and 176 deletions.
diff --git a/.gitignore b/.gitignore
@@ -121,6 +121,7 @@ celerybeat.pid
 
 # Environments
 .env.dev
+.env.prod
 .env
 .venv
 env/

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,13 +1,22 @@
 # Changelog
 
+## [v0.4.1] - 2024-10-30
+
+### Fixed
+
+- Generalization of the functions defined in the file `file_handler.py` in the `utils` directory.
+- Purge trial network when state is **validated**.
+- Email validation updated to skip MX record check for `MAIL_USERNAME`. Now allows sending emails from domains without an MX record by setting `check_deliverability=False` in the `validate_email` function.
+- Remove `users` collection create by default. The correct name of collection is `user`.
+
 ## [v0.4.0] - 2024-10-24
 
 ### Added
 
 - **Concurrency** support in the Flask application by integrating **Gunicorn** to handle multiple simultaneous requests, improving responsiveness and performance.
 - **Trial network isolation** to run several networks in parallel. The number of parallel trial networks is determined by the value of the environment variable `GUNICORN_WORKERS` minus 1. By default, `GUNICORN_WORKERS` is set to 3, allowing two trial networks to be executed concurrently. To execute sequentially, the number of `GUNICORN_WORKERS` has to be 2.
 - **TNLCM folder is created in jenkins.** For each trial network its own pipeline is created to guarantee the isolation and to be able to execute several at the same time.
-- **TRIAL_NETWORKS readme file** added in the `tn_template_lib` directory to explain defined trial networks.
+- **TRIAL_NETWORKS readme file** added in the `docs` directory to explain defined trial networks.
 - **Initial CLI handler** to execute commands.
 - **Git switch method** for streamlined repository management.
 - **Callback collection** in database for answers generated by jenkins on entity deployment.
@@ -166,6 +175,7 @@
 
 - Frontend implementation.
 
+[v0.4.1]: https://github.com/6G-SANDBOX/TNLCM/compare/v0.4.0...v0.4.1
 [v0.4.0]: https://github.com/6G-SANDBOX/TNLCM/compare/v0.3.1...v0.4.0
 [v0.3.1]: https://github.com/6G-SANDBOX/TNLCM/compare/v0.3.0...v0.3.1
 [v0.3.0]: https://github.com/6G-SANDBOX/TNLCM/compare/v0.2.1...v0.3.0

diff --git a/conf/__init__.py b/conf/__init__.py
@@ -3,7 +3,7 @@
 from dotenv import load_dotenv
 
 dotenv_path = os.path.join(os.getcwd(), ".env")
-# dotenv_path_dev = os.path.join(os.getcwd(), ".env.dev")
+# dotenv_path = os.path.join(os.getcwd(), ".env.dev")
 load_dotenv(dotenv_path=dotenv_path)
 
 from core.logs.log_handler import log_handler

diff --git a/conf/mail.py b/conf/mail.py
@@ -41,10 +41,11 @@ class MailSettings:
     if missing_variables:
         raise UndefinedEnvVariableError(missing_variables)
     try:
-        valid = validate_email(MAIL_USERNAME)
+        valid = validate_email(MAIL_USERNAME, check_deliverability=False)
         email = valid.email
     except EmailNotValidError:
         raise UndefinedEnvVariableError("Invalid 'MAIL_USERNAME' entered in .env file", 500)
+
     MAIL_PORT = int(MAIL_PORT)
     MAIL_USE_TLS = str_to_bool(MAIL_USE_TLS)
     MAIL_USE_SSL = str_to_bool(MAIL_USE_SSL)

diff --git a/core/database/tnlcm-structure.js b/core/database/tnlcm-structure.js
@@ -38,8 +38,8 @@ const digest = "sha256";
 const hash = crypto.pbkdf2Sync(tnlcmAdminPassword, salt, iterations, keyLength, digest).toString("hex");
 const hashedPassword = `pbkdf2:sha256:${iterations}$${salt}$${hash}`;
 
-// Insert administrator user in the users collection
-db.users.insertOne({
+// Insert administrator user in the user collection
+db.user.insertOne({
     username: tnlcmAdminUser,
     password: hashedPassword,
     email: tnlcmAdminEmail,

diff --git a/core/exceptions/exceptions_handler.py b/core/exceptions/exceptions_handler.py
@@ -13,67 +13,67 @@ def __init__(self, message: str, error_code: int):
         self.error_code = error_code
         log_handler.error(f"Error {self.error_code}: {message}")
 
-###############################
-#### Environment exception ####
-###############################
+###################################
+###### Environment exception ######
+###################################
 class UndefinedEnvVariableError(CustomException):
     """Error thrown when the variables are undefined in the .env file"""
     def __init__(self, missing_variables):
         message = f"Set the value of the variables {', '.join(missing_variables)} in the .env file"
         super().__init__(message, 500)
 
-###############################
-###### MongoDB exception ######
-###############################
+###################################
+######## MongoDB exception ########
+###################################
 class CustomMongoDBException(CustomException):
     """Base class for MongoDB related exceptions"""
     pass
 
-##############################
-##### Callback exception #####
-##############################
+###################################
+####### Callback exception ########
+###################################
 class CustomCallbackException(CustomException):
     """Base class for callback related exceptions"""
     pass
 
-##############################
-#### 6G-Library exception ####
-##############################
+###################################
+###### 6G-Library exception #######
+###################################
 class CustomSixGLibraryException(CustomException):
     """Base class for 6G-Library related exceptions"""
     pass
 
-##############################
-# 6G-Sandbox-Sites exception #
-##############################
+###################################
+### 6G-Sandbox-Sites exception ####
+###################################
 class CustomSixGSandboxSitesException(CustomException):
     """Base class for 6G-Sandbox-Sites related exceptions"""
     pass
 
-##############################
-###### GitHub exception ######
-##############################
+###################################
+######## GitHub exception #########
+###################################
 class CustomGitException(CustomException):
     """Base class for GitHub related exceptions"""
     pass
 
-##############################
-## Trial Networks exception ##
-##############################
+###################################
+##### Trial Network exception #####
+###################################
 class CustomTrialNetworkException(CustomException):
     """Base class for trial network related errors"""
     pass
 
-###########################
-#### Jenkins exception ####
-###########################
+###################################
+######## Jenkins exception ########
+###################################
 class CustomJenkinsException(CustomException):
     """Base class for Jenkins related exceptions"""
     pass
 
-####################################
-#### Resource manager exception ####
-####################################
+###################################
+### Resource manager exception ####
+###################################
 class CustomResourceManagerException(CustomException):
     """Base class for resource manager related exceptions"""
     pass
diff --git a/core/jenkins/jenkins_handler.py b/core/jenkins/jenkins_handler.py
@@ -9,7 +9,7 @@
 from core.logs.log_handler import log_handler
 from core.models import CallbackModel, TrialNetworkModel
 from core.sixg_library.sixg_library_handler import SixGLibraryHandler
-from core.utils.file_handler import save_yaml
+from core.utils.file_handler import load_file, save_yaml
 from core.exceptions.exceptions_handler import CustomJenkinsException
 
 class JenkinsHandler:
@@ -129,34 +129,34 @@ def trial_network_deployment(self) -> None:
             entity_name_input_file_path = os.path.join(self.trial_network.directory_path, f"{self.trial_network.tn_id}-{entity_name}.yaml")
             save_yaml(data=entity_data["input"], file_path=entity_name_input_file_path)
             log_handler.info(f"[{self.trial_network.tn_id}] - Created input file for entity '{entity_name}' to send to Jenkins pipeline")
-            with open(entity_name_input_file_path, "rb") as entity_name_input_file:
-                file = {"FILE": (entity_name_input_file_path, entity_name_input_file)}
-                log_handler.info(f"[{self.trial_network.tn_id}] - Add Jenkins parameters to the pipeline of the '{entity_name}' entity")
-                jenkins_build_job_url = self.jenkins_client.build_job_url(name=self.trial_network.jenkins_deploy_pipeline, parameters=self._jenkins_deployment_parameters(component_type, custom_name, debug))
-                response = post(jenkins_build_job_url, auth=(self.jenkins_username, self.jenkins_token), files=file)
-                log_handler.info(f"[{self.trial_network.tn_id}] - Deployment request code of the '{entity_name}' entity '{response.status_code}'")
-                if response.status_code != 201:
-                    raise CustomJenkinsException(f"Error in the response received by Jenkins when trying to deploy the '{entity_name}' entity", response.status_code)
-                next_build_number = self.jenkins_client.get_job_info(name=self.trial_network.jenkins_deploy_pipeline)["nextBuildNumber"]
-                while not self.jenkins_client.get_job_info(name=self.trial_network.jenkins_deploy_pipeline)["lastCompletedBuild"]:
-                    log_handler.info(f"[{self.trial_network.tn_id}] - Deploying '{entity_name}'")
-                    sleep(10)
-                while next_build_number != self.jenkins_client.get_job_info(name=self.trial_network.jenkins_deploy_pipeline)["lastCompletedBuild"]["number"]:
-                    log_handler.info(f"[{self.trial_network.tn_id}] - Deploying '{entity_name}'")
-                    sleep(10)
-                if self.jenkins_client.get_job_info(name=self.trial_network.jenkins_deploy_pipeline)["lastSuccessfulBuild"]["number"] != next_build_number:
-                    raise CustomJenkinsException(f"Pipeline for the entity '{entity_name}' has failed", 500)
-                log_handler.info(f"[{self.trial_network.tn_id}] - Entity '{entity_name}' successfully deployed")
-                sleep(3)
-                callback = CallbackModel.objects(tn_id=self.trial_network.tn_id, entity_name=entity_name).first()
-                if not callback:
-                    raise CustomJenkinsException(f"Callback with the results of the entity '{entity_name}' not found", 404)
-                del deployed_descriptor[entity_name]
-                self.trial_network.set_deployed_descriptor(deployed_descriptor)
-                self.trial_network.save()
-                log_handler.info(f"[{self.trial_network.tn_id}] - End of deployment of entity '{entity_name}'")
-            if not os.path.join(f"{self.trial_network.directory_path}", f"{self.trial_network.tn_id}.md"):
-                raise CustomJenkinsException(f"File with the report of the trial network '{self.trial_network.tn_id}' not found", 404)
+            entity_name_input_file_content = load_file(entity_name_input_file_path, mode="rb", encoding=None)
+            file = {"FILE": (entity_name_input_file_path, entity_name_input_file_content)}
+            log_handler.info(f"[{self.trial_network.tn_id}] - Add Jenkins parameters to the pipeline of the '{entity_name}' entity")
+            jenkins_build_job_url = self.jenkins_client.build_job_url(name=self.trial_network.jenkins_deploy_pipeline, parameters=self._jenkins_deployment_parameters(component_type, custom_name, debug))
+            response = post(jenkins_build_job_url, auth=(self.jenkins_username, self.jenkins_token), files=file)
+            log_handler.info(f"[{self.trial_network.tn_id}] - Deployment request code of the '{entity_name}' entity '{response.status_code}'")
+            if response.status_code != 201:
+                raise CustomJenkinsException(f"Error in the response received by Jenkins when trying to deploy the '{entity_name}' entity", response.status_code)
+            next_build_number = self.jenkins_client.get_job_info(name=self.trial_network.jenkins_deploy_pipeline)["nextBuildNumber"]
+            while not self.jenkins_client.get_job_info(name=self.trial_network.jenkins_deploy_pipeline)["lastCompletedBuild"]:
+                log_handler.info(f"[{self.trial_network.tn_id}] - Deploying '{entity_name}'")
+                sleep(10)
+            while next_build_number != self.jenkins_client.get_job_info(name=self.trial_network.jenkins_deploy_pipeline)["lastCompletedBuild"]["number"]:
+                log_handler.info(f"[{self.trial_network.tn_id}] - Deploying '{entity_name}'")
+                sleep(10)
+            if self.jenkins_client.get_job_info(name=self.trial_network.jenkins_deploy_pipeline)["lastSuccessfulBuild"]["number"] != next_build_number:
+                raise CustomJenkinsException(f"Pipeline for the entity '{entity_name}' has failed", 500)
+            log_handler.info(f"[{self.trial_network.tn_id}] - Entity '{entity_name}' successfully deployed")
+            sleep(3)
+            callback = CallbackModel.objects(tn_id=self.trial_network.tn_id, entity_name=entity_name).first()
+            if not callback:
+                raise CustomJenkinsException(f"Callback with the results of the entity '{entity_name}' not found", 404)
+            del deployed_descriptor[entity_name]
+            self.trial_network.set_deployed_descriptor(deployed_descriptor)
+            self.trial_network.save()
+            log_handler.info(f"[{self.trial_network.tn_id}] - End of deployment of entity '{entity_name}'")
+        if not os.path.join(f"{self.trial_network.directory_path}", f"{self.trial_network.tn_id}.md"):
+            raise CustomJenkinsException(f"File with the report of the trial network '{self.trial_network.tn_id}' not found", 404)
 
     def generate_jenkins_destroy_pipeline(self, jenkins_destroy_pipeline: str) -> str:
         """
@@ -193,7 +193,7 @@ def _jenkins_destroy_parameters(self) -> dict:
 
         :return: dictionary containing mandatory and optional parameters for the Jenkins destroy pipeline, ``dict``
         """
-        tn_components_types = self.trial_network.get_tn_components_types()
+        tn_components_types = self.trial_network.get_components_types()
         metadata_part = self.sixg_library_handler.get_tn_components_parts(parts=["metadata"], tn_components_types=tn_components_types)["metadata"]
         sorted_descriptor = self.trial_network.sorted_descriptor["trial_network"]
         entities_with_destroy_script = []

diff --git a/core/models/callback.py b/core/models/callback.py
@@ -3,7 +3,7 @@
 from mongoengine import Document, StringField, DictField
 
 from conf import SixGLibrarySettings, TnlcmSettings
-from core.utils.file_handler import load_yaml, save_json, append_markdown
+from core.utils.file_handler import load_yaml, save_json, save_file
 from core.utils.parser_handler import decode_base64
 from core.exceptions.exceptions_handler import CustomCallbackException
 
@@ -62,7 +62,7 @@ def save_data_file(self, data: dict) -> None:
         :param data: dictionary with decode values, ``dict``
         """
         save_json(data=data, file_path=os.path.join(TnlcmSettings.TRIAL_NETWORKS_DIRECTORY, self.tn_id, f"{self.entity_name}.json"))
-        append_markdown(data=self.markdown, file_path=os.path.join(TnlcmSettings.TRIAL_NETWORKS_DIRECTORY, self.tn_id, f"{self.tn_id}.md"))
+        save_file(data=self.markdown, file_path=os.path.join(TnlcmSettings.TRIAL_NETWORKS_DIRECTORY, self.tn_id, f"{self.tn_id}.md"), mode="a", encoding="utf-8")
 
     def matches_expected_output(self) -> bool:
         """

diff --git a/core/models/trial_network.py b/core/models/trial_network.py
@@ -1,15 +1,15 @@
 import os
 import re
 
-from yaml import safe_load, YAMLError
+from yaml import safe_load
 from datetime import datetime, timezone
 from string import ascii_lowercase, digits
 from random import choice
 from werkzeug.utils import secure_filename
 from werkzeug.datastructures import FileStorage
 from mongoengine import Document, StringField, DictField, DateTimeField
 
-from core.utils.file_handler import load_markdown
+from core.utils.file_handler import load_file
 from core.exceptions.exceptions_handler import CustomTrialNetworkException
 
 STATE_MACHINE = {"validated", "suspended", "activated", "failed", "destroyed"}
@@ -79,21 +79,17 @@ def set_state(self, state: str) -> None:
             raise CustomTrialNetworkException(f"Trial network '{state}' state not found", 404)
         self.state = state
 
-    def set_raw_descriptor(self, tn_descriptor_file: FileStorage) -> None:
+    def set_raw_descriptor(self, file: FileStorage) -> None:
         """
-        Set the trial network raw descriptor from a file.
+        Set the trial network raw descriptor from a file
         
-        :param tn_descriptor_file: descriptor file containing YAML data, ``FileStorage``
+        :param file: descriptor file containing YAML data, ``FileStorage``
         :raises CustomTrialNetworkException:
         """
-        try:
-            filename = secure_filename(tn_descriptor_file.filename)
-            if '.' in filename and filename.split('.')[-1].lower() in ["yml", "yaml"]:
-                self.raw_descriptor = safe_load(tn_descriptor_file.stream)
-            else:
-                raise CustomTrialNetworkException("Invalid descriptor format. Only 'yml' or 'yaml' files will be further processed", 422)
-        except YAMLError:
-            raise CustomTrialNetworkException("Trial network descriptor is not parsed correctly", 422)
+        filename = secure_filename(file.filename)
+        if not "." in filename or not filename.split(".")[-1].lower() in ["yml", "yaml"]:
+            raise CustomTrialNetworkException("Invalid descriptor format. Only 'yml' or 'yaml' files will be further processed", 422)
+        self.raw_descriptor = safe_load(file.stream)
 
     def set_sorted_descriptor(self) -> None:
         """
@@ -126,7 +122,7 @@ def set_report(self, file_path: str) -> None:
 
         :param file_path: path to the markdown report file, ``str``
         """
-        self.report = load_markdown(file_path=file_path)
+        self.report = load_file(file_path=file_path, mode="rt", encoding="utf-8")
 
     def set_directory_path(self, directory_path: str) -> None:
         """
@@ -252,7 +248,7 @@ def _validate_list_of_networks(
             if not self._logical_expression(tn_components_types, bool_expresion, tn_descriptor, component_name):
                 raise CustomTrialNetworkException(f"Component '{component_name}' in the list does not match the type '{bool_expresion}'", 422)
 
-    def validate_tn_descriptor(self, tn_components_types: set, tn_component_inputs: dict) -> None:
+    def validate_descriptor(self, tn_components_types: set, tn_component_inputs: dict) -> None:
         """
         If the descriptor follows the correct scheme
         It starts with trial_network and there is only that key
@@ -350,9 +346,9 @@ def validate_tn_descriptor(self, tn_components_types: set, tn_component_inputs:
                                 if not input_descriptor_component[input_sixg_library_key] in sixg_library_choices:
                                     raise CustomTrialNetworkException(f"Component '{component_type}'. Value of the '{input_sixg_library_key}' field has to be one of then: '{sixg_library_choices}'", 422)
 
-    def get_tn_components_types(self) -> set:
+    def get_components_types(self) -> set:
         """
-        Function to get a set with components types that are in the descriptor
+        Function to get the components types that are in the descriptor
 
         :return: set with components that compose trial network descriptor, ``set``
         """

diff --git a/core/models/user.py b/core/models/user.py
@@ -22,9 +22,9 @@ def set_password(self, secret: str) -> None:
         """
         self.password = generate_password_hash(secret, method="pbkdf2")
 
-    def check_password(self, secret: str) -> bool:
+    def verify_password(self, secret: str) -> bool:
         """
-        Check the hash associated with the password
+        Verify the hash associated with the password
         
         :param secret: password value, ``str``
         :return: True if the provided password matches the stored hash. Otherwise False, ``bool``

diff --git a/core/models/verification_token.py b/core/models/verification_token.py
@@ -12,12 +12,12 @@ class VerificationTokenModel(Document):
         "collection": "verification_token"
     }
 
-    def to_dict(self):
+    def to_dict(self) -> dict:
         return {
             "new_account_email": self.new_account_email,
             "verification_token": self.verification_token,
             "creation_date": self.creation_date.isoformat() if self.creation_date else None
         }
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         return "<VerificationToken #%s: %s>" % (self.new_account_email, self.verification_token)