self review

README.md

@@ -91,7 +91,9 @@ A locally-focused workflow (local development, local execution) with the CLI may
 - [`lean create-project`](#lean-create-project)
 - [`lean data download`](#lean-data-download)
 - [`lean data generate`](#lean-data-generate)
+- [`lean decrypt`](#lean-decrypt)
 - [`lean delete-project`](#lean-delete-project)
+- [`lean encrypt`](#lean-encrypt)
 - [`lean init`](#lean-init)
 - [`lean library add`](#lean-library-add)
 - [`lean library remove`](#lean-library-remove)
@@ -145,7 +147,8 @@ Options:
                                   Update the Lean configuration file to retrieve data from the given provider
   --terminal-link-connection-type [DAPI|SAPI]
                                   Terminal Link Connection Type [DAPI, SAPI]
-  --terminal-link-auth-id TEXT    The Auth ID of the TerminalLink server
+  --terminal-link-server-auth-id TEXT
+                                  The Auth ID of the TerminalLink server
   --terminal-link-environment [Production|Beta]
                                   The environment to run in
   --terminal-link-server-host TEXT
@@ -321,7 +324,8 @@ Options:
   --samco-trading-segment [equity|commodity]
                                   EQUITY if you are trading equities on NSE or BSE, COMMODITY if you are trading
                                   commodities on MCX
-  --terminal-link-auth-id TEXT    The Auth ID of the TerminalLink server
+  --terminal-link-server-auth-id TEXT
+                                  The Auth ID of the TerminalLink server
   --terminal-link-environment [Production|Beta]
                                   The environment to run in
   --terminal-link-server-host TEXT
@@ -545,6 +549,9 @@ Usage: lean cloud pull [OPTIONS]
 Options:
   --project TEXT   Name or id of the project to pull (all cloud projects if not specified)
   --pull-bootcamp  Pull Boot Camp projects (disabled by default)
+  --encrypt        Pull your cloud files and encrypt them before saving on your local drive
+  --decrypt        Pull your cloud files and decrypt them before saving on your local drive
+  --key FILE       Path to the encryption key to use
   --verbose        Enable debug logging
   --help           Show this message and exit.
 ```
@@ -566,6 +573,9 @@ Usage: lean cloud push [OPTIONS]
 
 Options:
   --project DIRECTORY  Path to the local project to push (all local projects if not specified)
+  --encrypt            Push your local files and encrypt them before saving on the cloud
+  --decrypt            Push your local files and decrypt them before saving on the cloud
+  --key FILE           Path to the encryption key to use
   --verbose            Enable debug logging
   --help               Show this message and exit.
 ```
@@ -795,6 +805,23 @@ Options:
 
 _See code: [lean/commands/data/generate.py](lean/commands/data/generate.py)_
 
+### `lean decrypt`
+
+Decrypt your local project using the specified decryption key.
+
+```
+Usage: lean decrypt [OPTIONS] PROJECT
+
+  Decrypt your local project using the specified decryption key.
+
+Options:
+  --key FILE  Path to the decryption key to use
+  --verbose   Enable debug logging
+  --help      Show this message and exit.
+```
+
+_See code: [lean/commands/decrypt.py](lean/commands/decrypt.py)_
+
 ### `lean delete-project`
 
 Alias for 'project-delete'
@@ -813,6 +840,23 @@ Options:
 
 _See code: [lean/commands/delete_project.py](lean/commands/delete_project.py)_
 
+### `lean encrypt`
+
+Encrypt your local project using the specified encryption key.
+
+```
+Usage: lean encrypt [OPTIONS] PROJECT
+
+  Encrypt your local project using the specified encryption key.
+
+Options:
+  --key FILE  Path to the encryption key to use
+  --verbose   Enable debug logging
+  --help      Show this message and exit.
+```
+
+_See code: [lean/commands/encrypt.py](lean/commands/encrypt.py)_
+
 ### `lean init`
 
 Scaffold a Lean configuration file and data directory.
@@ -1060,7 +1104,8 @@ Options:
                                   commodities on MCX
   --terminal-link-connection-type [DAPI|SAPI]
                                   Terminal Link Connection Type [DAPI, SAPI]
-  --terminal-link-auth-id TEXT    The Auth ID of the TerminalLink server
+  --terminal-link-server-auth-id TEXT
+                                  The Auth ID of the TerminalLink server
   --terminal-link-environment [Production|Beta]
                                   The environment to run in
   --terminal-link-server-host TEXT
@@ -1515,7 +1560,8 @@ Options:
                                   Update the Lean configuration file to retrieve data from the given provider
   --terminal-link-connection-type [DAPI|SAPI]
                                   Terminal Link Connection Type [DAPI, SAPI]
-  --terminal-link-auth-id TEXT    The Auth ID of the TerminalLink server
+  --terminal-link-server-auth-id TEXT
+                                  The Auth ID of the TerminalLink server
   --terminal-link-environment [Production|Beta]
                                   The environment to run in
   --terminal-link-server-host TEXT

lean/commands/cloud/pull.py

@@ -24,10 +24,10 @@
 @option("--pull-bootcamp", is_flag=True, default=False, help="Pull Boot Camp projects (disabled by default)")
 @option("--encrypt",
         is_flag=True, default=False,
-        help="Encrypt your cloud files with a key")
+        help="Pull your cloud files and encrypt them before saving on your local drive")
 @option("--decrypt",
         is_flag=True, default=False,
-        help="Decrypt your cloud files with a key")
+        help="Pull your cloud files and decrypt them before saving on your local drive")
 @option("--key",
               type=PathParameter(exists=True, file_okay=True, dir_okay=False),
               help="Path to the encryption key to use")
@@ -41,11 +41,9 @@ def pull(project: Optional[str], pull_bootcamp: bool, encrypt: Optional[bool], d
 
     encryption_action = None
 
-    if encrypt and decrypt:
-        raise RuntimeError(f"Cannot encrypt and decrypt at the same time.")
-    if key is None and (encrypt or decrypt):
-        raise RuntimeError(f"Encryption key is required when encrypting or decrypting.")
-
+    from lean.components.util.encryption_helper import validate_user_inputs_for_cloud_push_pull_commands
+    validate_user_inputs_for_cloud_push_pull_commands(encrypt, decrypt, key)
+
     if encrypt:
         encryption_action = ActionType.ENCRYPT
     if decrypt:
@@ -77,8 +75,11 @@ def pull(project: Optional[str], pull_bootcamp: bool, encrypt: Optional[bool], d
     if project is None and not pull_bootcamp:
         projects_to_pull = [p for p in projects_to_pull if not p.name.startswith("Boot Camp/")]
 
-    if  len(projects_to_pull) > 1 and key is not None:
+    if key is not None and len(projects_to_pull) > 1:
         raise RuntimeError(f"Cannot encrypt or decrypt more than one project at a time.")
 
+    # the encryption key info is available when reading the project individually from API
+    projects_to_pull = [api_client.projects.get(project.projectId, project.organizationId) if project.encrypted == True else project for project in projects_to_pull]
+
     pull_manager = container.pull_manager
     pull_manager.pull_projects(projects_to_pull, all_projects, encryption_action, key)

lean/commands/cloud/push.py

@@ -27,10 +27,10 @@
         help="Path to the local project to push (all local projects if not specified)")
 @option("--encrypt",
         is_flag=True, default=False,
-        help="Encrypt your cloud files with a key")
+        help="Push your local files and encrypt them before saving on the cloud")
 @option("--decrypt",
         is_flag=True, default=False,
-        help="Decrypt your cloud files with a key")
+        help="Push your local files and decrypt them before saving on the cloud")
 @option("--key",
               type=PathParameter(exists=True, file_okay=True, dir_okay=False),
               help="Path to the encryption key to use")
@@ -42,13 +42,10 @@ def push(project: Optional[Path], encrypt: Optional[bool], decrypt: Optional[boo
     This command will delete cloud files which don't have a local counterpart.
     """
     push_manager = container.push_manager
-    encryption_key_id = None
     encryption_action = None
 
-    if encrypt and decrypt:
-        raise RuntimeError(f"Cannot encrypt and decrypt at the same time.")
-    if key is None and (encrypt or decrypt):
-        raise RuntimeError(f"Encryption key is required when encrypting or decrypting.")
+    from lean.components.util.encryption_helper import validate_user_inputs_for_cloud_push_pull_commands
+    validate_user_inputs_for_cloud_push_pull_commands(encrypt, decrypt, key)
 
     if encrypt:
         encryption_action = ActionType.ENCRYPT
@@ -63,14 +60,9 @@ def push(project: Optional[Path], encrypt: Optional[bool], decrypt: Optional[boo
             raise RuntimeError(f"'{project}' is not a Lean project")
 
         if encrypt and key is not None:
-            from lean.components.util.encryption_helper import get_project_key_hash
-            # lets check if the given key is registered with the cloud
-            organization_id = container.organization_manager.try_get_working_organization_id()
-            available_encryption_keys = container.api_client.encryption_keys.list(organization_id)['keys']
-            encryption_key_id = get_project_key_hash(key)
-            if (not any(found_key for found_key in available_encryption_keys if found_key['hash'] == encryption_key_id)):
-                raise RuntimeError(f"Given encryption key is not registered with the cloud.")
-
+            from lean.components.util.encryption_helper import validate_encryption_key_registered_with_cloud
+            validate_encryption_key_registered_with_cloud(key, container.organization_manager, container.api_client) 
+
         push_manager.push_project(project, encryption_action, key)
     else:
         if key is not None:

lean/commands/decrypt.py

@@ -26,11 +26,8 @@
               help="Path to the decryption key to use")
 def decrypt(project: Path,
             key: Optional[Path]) -> None:
-    """Decrypt the project using the specified decryption key.
-
-    :param project: The project to decrypt
-    :param key: The path to the decryption key to use
-    """
+    """Decrypt your local project using the specified decryption key."""
+
     logger = container.logger
     project_manager = container.project_manager
     project_config_manager = container.project_config_manager

lean/commands/encrypt.py

@@ -26,11 +26,7 @@
               help="Path to the encryption key to use")
 def encrypt(project: Path,
             key: Optional[Path]) -> None:
-    """Encrypt the project using the specified encryption key.
-
-    :param project: The project to encrypt
-    :param key: The path to the encryption key to use
-    """
+    """Encrypt your local project using the specified encryption key."""
 
     logger = container.logger
     project_manager = container.project_manager

lean/components/cloud/pull_manager.py

@@ -178,10 +178,12 @@ def _pull_project(self, project: QCProject, encryption_action: Optional[ActionTy
 
         project_config = self._project_config_manager.get_project_config(local_project_path)
         local_encryption_state = project_config.get("encrypted", False)
-
+        local_encryption_key = project_config.get("encryption-key-path", None)
+        if local_encryption_key is not None:
+            local_encryption_key = Path(local_encryption_key)
         # Handle mismatch cases
         from lean.components.util.encryption_helper import validate_key_and_encryption_state_for_cloud_project
-        validate_key_and_encryption_state_for_cloud_project(project, local_encryption_state, encryption_key)
+        validate_key_and_encryption_state_for_cloud_project(project, local_encryption_state, encryption_key, local_encryption_key, self._logger)
 
         # Pull the cloud files to the local drive
         self._pull_files(project, local_project_path, encryption_action, encryption_key)

lean/components/cloud/push_manager.py

@@ -109,6 +109,9 @@ def _push_project(self, project_path: Path, organization_id: str, encryption_act
         project_config = self._project_config_manager.get_project_config(project_path)
         cloud_id = project_config.get("cloud-id")
         local_encryption_state = project_config.get("encrypted", False)
+        local_encryption_key = project_config.get("encryption-key-path", None)
+        if local_encryption_key is not None:
+            local_encryption_key = Path(local_encryption_key)
 
         # check if project name is valid or if rename is required
         if cloud_id is not None:
@@ -149,8 +152,10 @@ def _push_project(self, project_path: Path, organization_id: str, encryption_act
 
         # Handle mismatch cases
         from lean.components.util.encryption_helper import validate_key_and_encryption_state_for_cloud_project
-        validate_key_and_encryption_state_for_cloud_project(cloud_project, local_encryption_state, encryption_key)
-
+        validate_key_and_encryption_state_for_cloud_project(cloud_project, local_encryption_state, encryption_key, local_encryption_key, self._logger)
+        if local_encryption_state == True and encryption_key is None and local_encryption_key is not None:
+            encryption_key = local_encryption_key
+            encryption_action = ActionType.ENCRYPT if local_encryption_state else ActionType.DECRYPT
         # Finalize pushing by updating locally modified metadata, files and libraries
         self._push_metadata(project_path, cloud_project, encryption_action, encryption_key)
 
@@ -233,14 +238,16 @@ def _push_metadata(self, project: Path, cloud_project: QCProject, encryption_act
         update_args["files"] = self._get_files(project, encryption_action, encryption_key)
         update_args["libraries"] = self._get_local_libraries_cloud_ids(project)
 
+        # default value
+        update_args["encryption_key"] = ''
         if (encryption_action is not None and encryption_key is not None):
+            # lets check if the given key is registered with the cloud
+            from lean.components.util.encryption_helper import validate_encryption_key_registered_with_cloud, get_project_key_hash
+            validate_encryption_key_registered_with_cloud(encryption_key, self._organization_manager, self._api_client) 
+
             if encryption_action == ActionType.ENCRYPT:
-                from lean.components.util.encryption_helper import get_project_key_hash
                 encryption_key_id = get_project_key_hash(encryption_key)
                 update_args["encryption_key"] = encryption_key_id
-            else:
-                # decryption case: Lets reset the value in the Cloud.
-                update_args["encryption_key"] = ''
 
         if update_args != {}:
             self._api_client.projects.update(cloud_project.projectId, **update_args)

lean/components/util/encryption_helper.py

@@ -13,6 +13,7 @@
 
 from typing import List
 from pathlib import Path
+from lean.components.util.logger import Logger
 from base64 import b64decode, b64encode
 from cryptography.hazmat.primitives import padding
 from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
@@ -22,6 +23,8 @@
 from lean.components.config.project_config_manager import ProjectConfigManager
 from lean.models.api import QCProject, QCFullFile
 from lean.components.config.storage import Storage
+from lean.components.api.api_client import APIClient
+from lean.components.util.organization_manager import OrganizationManager
 
 def calculate_md5(input_string: str):
     """Calculate the md5 hash of a string
@@ -137,11 +140,30 @@ def get_and_validate_user_input_encryption_key(user_input_key: Path, project_con
         raise RuntimeError(f"Provided encryption key ({user_input_key}) does not match the encryption key in the project ({project_config_encryption_key})")
     return project_config_encryption_key
 
-def validate_key_and_encryption_state_for_cloud_project(project: QCProject, local_project_encryption_state: bool, encryption_key: Path) -> None:
+def validate_user_inputs_for_cloud_push_pull_commands(encrypt: bool, decrypt: bool, key: Path):
+    if encrypt and decrypt:
+        raise RuntimeError(f"Cannot encrypt and decrypt at the same time.")
+    if key is None and (encrypt or decrypt):
+        raise RuntimeError(f"Encryption key is required when encrypting or decrypting.")
+    if key is not None and not encrypt and not decrypt:
+            raise RuntimeError(f"Encryption key can only be specified when encrypting or decrypting.")
+
+def validate_encryption_key_registered_with_cloud(user_key: Path, organization_manager: OrganizationManager, api_client: APIClient):
+    # lets check if the given key is registered with the cloud
+    organization_id = organization_manager.try_get_working_organization_id()
+    available_encryption_keys = api_client.encryption_keys.list(organization_id)['keys']
+    encryption_key_id = get_project_key_hash(user_key)
+    if (not any(found_key for found_key in available_encryption_keys if found_key['hash'] == encryption_key_id)):
+        raise RuntimeError(f"Given encryption key is not registered with the cloud.")
+
+def validate_key_and_encryption_state_for_cloud_project(project: QCProject, local_project_encryption_state: bool, encryption_key: Path, local_encryption_key: Path, logger:Logger) -> None:
+    if not encryption_key and project.encryptionKey and local_encryption_key and local_encryption_key.exists() and get_project_key_hash(local_encryption_key) != project.encryptionKey.id:
+        raise RuntimeError(f"Encryption Key mismatch. Local Project Key: {local_encryption_key}. Cloud Project Key: {project.encryptionKey.name}. Please provide correct encryption key for project '{project.name}' to proceed.")
     if not encryption_key and bool(project.encrypted) != bool(local_project_encryption_state):
-        raise RuntimeError(f"Project encryption state mismatch. Local Project Encrypted: {bool(local_project_encryption_state)}. Cloud Project Encrypted {bool(project.encrypted)}. Please provide encryption key to pull project '{project.name}'")
-    if project.encryptionKey and encryption_key and get_project_key_hash(encryption_key) != project.encryptionKey.id:
-        raise RuntimeError(f"Encryption Key mismatch. Local Project Key hash: {get_project_key_hash(encryption_key)}. Cloud Project Key Hash {project.encryptionKey.id}. Please provide correct encryption key to pull project '{project.name}'")
+        logger.warn(f"Force Overwrite: Project encryption state mismatch. Local Project Encrypted: {bool(local_project_encryption_state)}. Cloud Project Encrypted: {bool(project.encrypted)}.")
+        return
+    if encryption_key and project.encryptionKey and get_project_key_hash(encryption_key) != project.encryptionKey.id:
+        raise RuntimeError(f"Encryption Key mismatch. Local Project Key hash: {get_project_key_hash(encryption_key)}. Cloud Project Key Hash: {project.encryptionKey.id}. Please provide correct encryption key for project '{project.name}' to proceed.")
 
 def get_appropriate_files_from_cloud_project(project: QCProject, cloud_files: List[QCFullFile], encryption_key: Path, organization_id: str, encryption_action: ActionType) -> List[QCFullFile]:
     if encryption_action == ActionType.DECRYPT: