Merge pull request #42 from 6G-SANDBOX/dev

New release 0.4.0

.dockerignore

@@ -0,0 +1,9 @@
+.github/
+.vscode/
+docs/
+scripts/
+tn_template_lib/
+venv/
+**/__pycache__/
+core/logs/executions/
+core/trial_networks/

.env.template

@@ -1,18 +1,18 @@
-# FLASK_ENV
-#   1. development
-#   2. production
-#   3. testing
-FLASK_ENV="production"
+# GUNICORN
+GUNICORN_WORKERS=3
+GUNICORN_TIMEOUT=7200
+GUNICORN_LOG_LEVEL="INFO"
 
 # TNLCM
+TNLCM_LOG_LEVEL="INFO"
 TNLCM_ADMIN_USER=""
 TNLCM_ADMIN_PASSWORD=""
-TNLCM_ADMIN_EMAIL=""
+TNLCM_ADMIN_EMAIL="[email protected]"
 TNLCM_HOST=""
 TNLCM_PORT=5000
 TNLCM_CALLBACK="http://${TNLCM_HOST}:${TNLCM_PORT}/tnlcm/callback"
 
-# Jenkins connection
+# JENKINS
 JENKINS_HOST=""
 JENKINS_PORT=8080
 JENKINS_URL="http://${JENKINS_HOST}:${JENKINS_PORT}"
@@ -22,37 +22,35 @@ JENKINS_TOKEN=""
 JENKINS_DEPLOY_PIPELINE="TN_DEPLOY"
 JENKINS_DESTROY_PIPELINE="TN_DESTROY"
 
-# MongoDB database
-MONGO_INITDB_ROOT_USERNAME="tnlcm-database-user"
-MONGO_INITDB_ROOT_PASSWORD="tnlcm-database-password"
+# MONGODB
 MONGO_HOST="127.0.0.1"
 MONGO_PORT=27017
 MONGO_DATABASE="tnlcm-database"
-# MongoDB connect database with frontend
+
+# MONGO-EXPRESS
 ME_CONFIG_MONGODB_ADMINUSERNAME="tnlcm-database-user"
 ME_CONFIG_MONGODB_ADMINPASSWORD="tnlcm-database-password"
-ME_CONFIG_MONGODB_URL="mongodb://${ME_CONFIG_MONGODB_ADMINUSERNAME}:${ME_CONFIG_MONGODB_ADMINPASSWORD}@${MONGO_DATABASE}:${MONGO_PORT}/?authSource=admin"
-# MongoDB database frontend dashboard
+ME_CONFIG_MONGODB_ENABLE_ADMIN="false"
+ME_CONFIG_MONGODB_URL="mongodb://${ME_CONFIG_MONGODB_ADMINUSERNAME}:${ME_CONFIG_MONGODB_ADMINPASSWORD}@${MONGO_HOST}:${MONGO_PORT}/${MONGO_DATABASE}"
+ME_CONFIG_BASICAUTH="true"
+ME_CONFIG_SITE_SESSIONSECRET="secret"
+VCAP_APP_HOST="0.0.0.0"
 ME_CONFIG_BASICAUTH_USERNAME="tnlcm"
 ME_CONFIG_BASICAUTH_PASSWORD="tnlcm"
 
-# Connect TNLCM with MongoDB
-MONGO_URI="mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@${MONGO_HOST}:${MONGO_PORT}/${MONGO_DATABASE}?authSource=admin"
-
-# 6G-Library connection
+# 6G-LIBRARY
 GITHUB_6G_LIBRARY_HTTPS_URL="https://github.com/6G-SANDBOX/6G-Library.git"
 GITHUB_6G_LIBRARY_BRANCH="main"
 GITHUB_6G_LIBRARY_REPOSITORY_NAME="6G-Library"
 
-# 6G-Sandbox-Sites connection
+# 6G-SANDBOX-SITES
 GITHUB_6G_SANDBOX_SITES_HTTPS_URL="https://github.com/6G-SANDBOX/6G-Sandbox-Sites.git"
 GITHUB_6G_SANDBOX_SITES_BRANCH="main"
 GITHUB_6G_SANDBOX_SITES_REPOSITORY_NAME="6G-Sandbox-Sites"
-# Ansible password to decrypt file 6G-Sandbox-Sites
-#   Not use " or ' or `
-ANSIBLE_VAULT=''
+# Ansible password to decrypt file 6G-Sandbox-Sites (not use \" or \' or \`)
+SITES_TOKEN=''
 
-# Mail constants
+# MAIL
 MAIL_SERVER="smtp.gmail.com"
 MAIL_PORT=465
 MAIL_USE_TLS=False

.github/CODEOWNERS

@@ -0,0 +1,5 @@
+# This is a comment.
+# Each line is a file pattern followed by one or more owners.
+
+# These owners will be the default owners for everything in the repo. Unless a later match takes precedence, @CarlosAndreo will be requested for review when someone opens a pull request.
+* @CarlosAndreo

.gitignore

@@ -164,8 +164,6 @@ cython_debug/
 .vscode/
 
 # TNLCM
-core/callback/reports/
 core/temp/files/
 core/logs/executions/
-core/sixg_library/6G-Library/
-core/sixg_sandbox_sites/6G-Sandbox-Sites/
+core/trial_networks/

CHANGELOG.md

@@ -1,5 +1,50 @@
 # Changelog
 
+## [v0.4.0] - 2024-10-24
+
+### Added
+
+- **Concurrency** support in the Flask application by integrating **Gunicorn** to handle multiple simultaneous requests, improving responsiveness and performance.
+- **Trial network isolation** to run several networks in parallel. The number of parallel trial networks is determined by the value of the environment variable `GUNICORN_WORKERS` minus 1. By default, `GUNICORN_WORKERS` is set to 3, allowing two trial networks to be executed concurrently. To execute sequentially, the number of `GUNICORN_WORKERS` has to be 2.
+- **TNLCM folder is created in jenkins.** For each trial network its own pipeline is created to guarantee the isolation and to be able to execute several at the same time.
+- **TRIAL_NETWORKS readme file** added in the `tn_template_lib` directory to explain defined trial networks.
+- **Initial CLI handler** to execute commands.
+- **Git switch method** for streamlined repository management.
+- **Callback collection** in database for answers generated by jenkins on entity deployment.
+- **Utils directory** with required functions.
+- **Scripts directory** with the following deployment scripts:
+  - `deploy_vm.sh`: deploy TNLCM and MongoDB in a virtual machine.
+  - `deploy_docker.sh`: deploy TNLCM and MongoDB using Docker.
+
+### Changed
+
+- Updated Python to version **3.13.0**.
+- Updated MongoDB to version **8.0**.
+- **Improved README documentation** for better clarity and usage guidelines.
+- Migrated from the `ansible-vault` library to `ansible-core` since only this module is of interest.
+- Renamed `ANSIBLE_VAULT` variable in `.env.template` to `SITES_TOKEN` for clarity.
+- Moved the implementation from the `resource_manager` directory to the resource manager within the `models` directory.
+- Rename `trial_networks`, `users` and `verification_tokens` collections to `trial_network`, `user` and `verification_token`.
+- Rename columns `tn_state`, `tn_date_created_utc`, `tn_raw_descriptor`, `tn_sorted_descriptor`, `tn_deployed_descriptor`, `tn_report` and `tn_directory_path` to `state`, `date_created_utc`, `raw_descriptor`, `sorted_descriptor`, `deployed_descriptor`, `report` and `directory_path` in `trial_network` collection.
+
+### Fixed
+
+- Resolved a bug when creating a trial network, where components were defined in the 6G-Sandbox-Sites repository but not in the 6G-Library.
+- Trial network can be destroyed if it is in **failed** state.
+- Corrected function documentation to reflect accurate information.
+- Message display when checking if a user can access a trial network.
+- Enhanced logging for TNLCM processes.
+- Issues with JWT and MongoDB error handling.
+- Simplified class exception handling for improved readability and maintainability.
+- Descriptors are converted from being stored in json to dictionaries in the `trial_networks` collection of the database.
+
+### Removed
+
+- **Deprecated API namespaces**:
+  - Trial network templates.
+  - 6G-Library.
+  - 6G-Sandbox-Sites.
+
 ## [v0.3.1] - 2024-10-04
 
 ### Added
@@ -121,6 +166,7 @@
 
 - Frontend implementation.
 
+[v0.4.0]: https://github.com/6G-SANDBOX/TNLCM/compare/v0.3.1...v0.4.0
 [v0.3.1]: https://github.com/6G-SANDBOX/TNLCM/compare/v0.3.0...v0.3.1
 [v0.3.0]: https://github.com/6G-SANDBOX/TNLCM/compare/v0.2.1...v0.3.0
 [v0.2.1]: https://github.com/6G-SANDBOX/TNLCM/compare/v0.2.0...v0.2.1

app.py

@@ -1,28 +1,19 @@
-import os
-
 from flask import Flask
 from flask_restx import Api
 from flask_jwt_extended import JWTManager
 from flask_cors import CORS
-from waitress import serve
 
-from conf import TnlcmSettings, ProductionConfig, DevelopmentConfig, TestingConfig
+from conf import TnlcmSettings, FlaskConf
 from core.logs.log_handler import log_handler
 from core.mail.mail import init_mail
 from core.database.database import init_db
-from core.routes import callback_namespace, debug_namespace, sixg_library_namespace, sixg_sandbox_sites_namespace, trial_network_namespace, user_namespace, verification_token_namespace
+from core.routes import callback_namespace, debug_namespace, trial_network_namespace, user_namespace, verification_token_namespace
 
 app = Flask(__name__)
 CORS(app)
 JWTManager(app)
 
-flask_env = os.getenv("FLASK_ENV").upper()
-if flask_env == "PRODUCTION":
-    app.config.from_object(ProductionConfig)
-elif flask_env == "DEVELOPMENT":
-    app.config.from_object(DevelopmentConfig)
-else:
-    app.config.from_object(TestingConfig)
+app.config.from_object(FlaskConf)
 
 init_db(app)
 init_mail(app)
@@ -37,16 +28,8 @@
 
 api.add_namespace(callback_namespace, path="/tnlcm/callback")
 api.add_namespace(debug_namespace, path="/tnlcm/debug")
-api.add_namespace(sixg_library_namespace, path="/tnlcm/6G-Library")
-api.add_namespace(sixg_sandbox_sites_namespace, path="/tnlcm/6G-Sandbox-Sites")
 api.add_namespace(trial_network_namespace, path="/tnlcm/trial-network")
 api.add_namespace(user_namespace, path="/tnlcm/user")
 api.add_namespace(verification_token_namespace, path="/tnlcm/verification-token")
 
-log_handler.info(f"Start Server Trial Network Life Cycle Manager (TNLCM) on http://0.0.0.0:{TnlcmSettings.TNLCM_PORT}")
-
-if __name__ == "__main__":
-    if flask_env == "PRODUCTION":
-        serve(app, host="0.0.0.0", port=TnlcmSettings.TNLCM_PORT)
-    else:
-        app.run(host="0.0.0.0", port=TnlcmSettings.TNLCM_PORT)
+log_handler.info(f"Start Trial Network Lifecycle Manager (TNLCM) v{TnlcmSettings.VERSION} on http://0.0.0.0:{TnlcmSettings.TNLCM_PORT}")

conf/__init__.py

@@ -2,17 +2,19 @@
 
 from dotenv import load_dotenv
 
-from core.logs.log_handler import log_handler
-
 dotenv_path = os.path.join(os.getcwd(), ".env")
-log_handler.info(f"Loading the '{dotenv_path}' file of the project")
+# dotenv_path_dev = os.path.join(os.getcwd(), ".env.dev")
 load_dotenv(dotenv_path=dotenv_path)
 
+from core.logs.log_handler import log_handler
+
+log_handler.debug(f"Loading the '{dotenv_path}' file of TNLCM")
+
 from .mail import MailSettings
 from .mongodb import MongoDBSettings
 from .jenkins import JenkinsSettings
 from .repository import RepositorySettings
 from .sixg_library import SixGLibrarySettings
 from .sixg_sandbox_sites import SixGSandboxSitesSettings
 from .tnlcm import TnlcmSettings
-from .config import ProductionConfig, DevelopmentConfig, TestingConfig
+from .flask_conf import FlaskConf

conf/flask_conf.py

@@ -0,0 +1,39 @@
+import os
+
+from core.logs.log_handler import log_handler
+from conf import MailSettings, MongoDBSettings
+
+class FlaskConf(object):
+    """
+    Flask Settings
+    """
+
+    DEBUG = False
+    TESTING = False
+    SECRET_KEY = os.getenv("SECRET_KEY") or "clave"
+    ERROR_404_HELP = False
+
+    ME_CONFIG_MONGODB_URL = MongoDBSettings.ME_CONFIG_MONGODB_URL
+
+    MAIL_SERVER = MailSettings.MAIL_SERVER
+    MAIL_PORT = MailSettings.MAIL_PORT
+    MAIL_USE_TLS = MailSettings.MAIL_USE_TLS
+    MAIL_USE_SSL = MailSettings.MAIL_USE_SSL
+    MAIL_USERNAME = MailSettings.MAIL_USERNAME
+    MAIL_PASSWORD = MailSettings.MAIL_PASSWORD
+
+    config_dict = {
+        "DEBUG": DEBUG,
+        "TESTING": TESTING,
+        "SECRET_KEY": SECRET_KEY,
+        "ERROR_404_HELP": ERROR_404_HELP,
+        "ME_CONFIG_MONGODB_URL": ME_CONFIG_MONGODB_URL,
+        "MAIL_SERVER": MAIL_SERVER,
+        "MAIL_PORT": MAIL_PORT,
+        "MAIL_USE_TLS": MAIL_USE_TLS,
+        "MAIL_USE_SSL": MAIL_USE_SSL,
+        "MAIL_USERNAME": MAIL_USERNAME,
+        "MAIL_PASSWORD": MAIL_PASSWORD,
+    }
+
+    log_handler.info(f"Load Flask configuration: {config_dict}")

conf/gunicorn_conf.py

@@ -0,0 +1,43 @@
+import os
+
+from conf import TnlcmSettings
+from core.logs.log_handler import log_handler
+from core.exceptions.exceptions_handler import UndefinedEnvVariableError
+
+# Number of worker processes to handle requests
+workers = os.getenv("GUNICORN_WORKERS")
+
+# Log level for output verbosity
+loglevel = os.getenv("GUNICORN_LOG_LEVEL")
+
+# Request timeout in seconds (35 minutes). The time of the component that takes the longest time to deploy
+timeout = os.getenv("GUNICORN_TIMEOUT")
+
+# Address and port Gunicorn will bind to
+bind = f"0.0.0.0:{TnlcmSettings.TNLCM_PORT}"
+
+# WSGI entry point for the application
+wsgi_app = "app:app"
+
+# Maximum number of pending connections
+backlog = 1024
+
+missing_variables = []
+if not workers:
+    missing_variables.append("GUNICORN_WORKERS")
+if not workers:
+    missing_variables.append("GUNICORN_LOG_LEVEL")
+if not workers:
+    missing_variables.append("GUNICORN_TIMEOUT")
+if missing_variables:
+    raise UndefinedEnvVariableError(missing_variables)
+
+config_dict = {
+    "WORKERS": workers,
+    "BIND": bind,
+    "TIMEOUT": timeout,
+    "LOGLEVEL": loglevel,
+    "BACKLOG": backlog,
+}
+
+log_handler.info(f"Load gunicorn configuration: {config_dict}")

conf/jenkins.py

@@ -7,8 +7,6 @@ class JenkinsSettings:
     """
     Jenkins Settings
     """
-
-    log_handler.info("Load Jenkins configuration")
 
     JENKINS_HOST = os.getenv("JENKINS_HOST")
     JENKINS_PORT = os.getenv("JENKINS_PORT")
@@ -36,4 +34,17 @@ class JenkinsSettings:
     if not JENKINS_DESTROY_PIPELINE:
         missing_variables.append("JENKINS_DESTROY_PIPELINE")
     if missing_variables:
-        raise UndefinedEnvVariableError(missing_variables)
+        raise UndefinedEnvVariableError(missing_variables)
+
+    config_dict = {
+        "JENKINS_HOST": JENKINS_HOST,
+        "JENKINS_PORT": JENKINS_PORT,
+        "JENKINS_URL": JENKINS_URL,
+        "JENKINS_USERNAME": JENKINS_USERNAME,
+        "JENKINS_PASSWORD": JENKINS_PASSWORD,
+        "JENKINS_TOKEN": JENKINS_TOKEN,
+        "JENKINS_DEPLOY_PIPELINE": JENKINS_DEPLOY_PIPELINE,
+        "JENKINS_DESTROY_PIPELINE": JENKINS_DESTROY_PIPELINE,
+    }
+
+    log_handler.info(f"Load Jenkins configuration: {config_dict}")