Implement Support For Project Encryption (#373)

* implement support for project encryption

* add remaning tests

* fix tests

* address review

* fix line endings

* remove duplicate tests

* remove duplicate conversion

* add test

* self review

* fix tests

* use debug log instead

README.md

@@ -91,7 +91,9 @@ A locally-focused workflow (local development, local execution) with the CLI may
 - [`lean create-project`](#lean-create-project)
 - [`lean data download`](#lean-data-download)
 - [`lean data generate`](#lean-data-generate)
+- [`lean decrypt`](#lean-decrypt)
 - [`lean delete-project`](#lean-delete-project)
+- [`lean encrypt`](#lean-encrypt)
 - [`lean init`](#lean-init)
 - [`lean library add`](#lean-library-add)
 - [`lean library remove`](#lean-library-remove)
@@ -547,6 +549,9 @@ Usage: lean cloud pull [OPTIONS]
 Options:
   --project TEXT   Name or id of the project to pull (all cloud projects if not specified)
   --pull-bootcamp  Pull Boot Camp projects (disabled by default)
+  --encrypt        Pull your cloud files and encrypt them before saving on your local drive
+  --decrypt        Pull your cloud files and decrypt them before saving on your local drive
+  --key FILE       Path to the encryption key to use
   --verbose        Enable debug logging
   --help           Show this message and exit.
 ```
@@ -568,6 +573,9 @@ Usage: lean cloud push [OPTIONS]
 
 Options:
   --project DIRECTORY  Path to the local project to push (all local projects if not specified)
+  --encrypt            Push your local files and encrypt them before saving on the cloud
+  --decrypt            Push your local files and decrypt them before saving on the cloud
+  --key FILE           Path to the encryption key to use
   --verbose            Enable debug logging
   --help               Show this message and exit.
 ```
@@ -797,6 +805,23 @@ Options:
 
 _See code: [lean/commands/data/generate.py](lean/commands/data/generate.py)_
 
+### `lean decrypt`
+
+Decrypt your local project using the specified decryption key.
+
+```
+Usage: lean decrypt [OPTIONS] PROJECT
+
+  Decrypt your local project using the specified decryption key.
+
+Options:
+  --key FILE  Path to the decryption key to use
+  --verbose   Enable debug logging
+  --help      Show this message and exit.
+```
+
+_See code: [lean/commands/decrypt.py](lean/commands/decrypt.py)_
+
 ### `lean delete-project`
 
 Alias for 'project-delete'
@@ -815,6 +840,23 @@ Options:
 
 _See code: [lean/commands/delete_project.py](lean/commands/delete_project.py)_
 
+### `lean encrypt`
+
+Encrypt your local project using the specified encryption key.
+
+```
+Usage: lean encrypt [OPTIONS] PROJECT
+
+  Encrypt your local project using the specified encryption key.
+
+Options:
+  --key FILE  Path to the encryption key to use
+  --verbose   Enable debug logging
+  --help      Show this message and exit.
+```
+
+_See code: [lean/commands/encrypt.py](lean/commands/encrypt.py)_
+
 ### `lean init`
 
 Scaffold a Lean configuration file and data directory.

lean/commands/__init__.py

@@ -19,6 +19,8 @@
 from lean.commands.create_project import create_project
 from lean.commands.delete_project import delete_project
 from lean.commands.data import data
+from lean.commands.decrypt import decrypt
+from lean.commands.encrypt import encrypt
 from lean.commands.init import init
 from lean.commands.library import library
 from lean.commands.live.live import live
@@ -35,6 +37,8 @@
 lean.add_command(config)
 lean.add_command(cloud)
 lean.add_command(data)
+lean.add_command(decrypt)
+lean.add_command(encrypt)
 lean.add_command(library)
 lean.add_command(live)
 lean.add_command(login)

lean/commands/cloud/__init__.py

@@ -35,4 +35,4 @@ def cloud() -> None:
 cloud.add_command(optimize)
 cloud.add_command(live)
 cloud.add_command(status)
-cloud.add_command(object_store)
+cloud.add_command(object_store)

lean/commands/cloud/pull.py

@@ -12,23 +12,43 @@
 # limitations under the License.
 
 from typing import Optional
-
+from pathlib import Path
 from click import command, option
 
-from lean.click import LeanCommand
+from lean.click import LeanCommand, PathParameter
 from lean.container import container
-
+from lean.models.encryption import ActionType
 
 @command(cls=LeanCommand)
 @option("--project", type=str, help="Name or id of the project to pull (all cloud projects if not specified)")
 @option("--pull-bootcamp", is_flag=True, default=False, help="Pull Boot Camp projects (disabled by default)")
-def pull(project: Optional[str], pull_bootcamp: bool) -> None:
+@option("--encrypt",
+        is_flag=True, default=False,
+        help="Pull your cloud files and encrypt them before saving on your local drive")
+@option("--decrypt",
+        is_flag=True, default=False,
+        help="Pull your cloud files and decrypt them before saving on your local drive")
+@option("--key",
+              type=PathParameter(exists=True, file_okay=True, dir_okay=False),
+              help="Path to the encryption key to use")
+def pull(project: Optional[str], pull_bootcamp: bool, encrypt: Optional[bool], decrypt: Optional[bool], key: Optional[Path]) -> None:
     """Pull projects from QuantConnect to the local drive.
 
     This command overrides the content of local files with the content of their respective counterparts in the cloud.
 
     This command will not delete local files for which there is no counterpart in the cloud.
     """
+
+    encryption_action = None
+
+    from lean.components.util.encryption_helper import validate_user_inputs_for_cloud_push_pull_commands
+    validate_user_inputs_for_cloud_push_pull_commands(encrypt, decrypt, key)
+
+    if encrypt:
+        encryption_action = ActionType.ENCRYPT
+    if decrypt:
+        encryption_action = ActionType.DECRYPT
+
     # Parse which projects need to be pulled
     project_id = None
     project_name = None
@@ -55,5 +75,11 @@ def pull(project: Optional[str], pull_bootcamp: bool) -> None:
     if project is None and not pull_bootcamp:
         projects_to_pull = [p for p in projects_to_pull if not p.name.startswith("Boot Camp/")]
 
+    if key is not None and len(projects_to_pull) > 1:
+        raise RuntimeError(f"Cannot encrypt or decrypt more than one project at a time.")
+
+    # the encryption key info is available when reading the project individually from API
+    projects_to_pull = [api_client.projects.get(project.projectId, project.organizationId) if project.encrypted == True else project for project in projects_to_pull]
+
     pull_manager = container.pull_manager
-    pull_manager.pull_projects(projects_to_pull, all_projects)
+    pull_manager.pull_projects(projects_to_pull, all_projects, encryption_action, key)

lean/commands/cloud/push.py

@@ -19,20 +19,38 @@
 from lean.click import LeanCommand, PathParameter
 from lean.constants import PROJECT_CONFIG_FILE_NAME
 from lean.container import container
-
+from lean.models.encryption import ActionType
 
 @command(cls=LeanCommand)
 @option("--project",
         type=PathParameter(exists=True, file_okay=False, dir_okay=True),
         help="Path to the local project to push (all local projects if not specified)")
-def push(project: Optional[Path]) -> None:
+@option("--encrypt",
+        is_flag=True, default=False,
+        help="Push your local files and encrypt them before saving on the cloud")
+@option("--decrypt",
+        is_flag=True, default=False,
+        help="Push your local files and decrypt them before saving on the cloud")
+@option("--key",
+              type=PathParameter(exists=True, file_okay=True, dir_okay=False),
+              help="Path to the encryption key to use")
+def push(project: Optional[Path], encrypt: Optional[bool], decrypt: Optional[bool], key: Optional[Path]) -> None:
     """Push local projects to QuantConnect.
 
     This command overrides the content of cloud files with the content of their respective local counterparts.
 
     This command will delete cloud files which don't have a local counterpart.
     """
     push_manager = container.push_manager
+    encryption_action = None
+
+    from lean.components.util.encryption_helper import validate_user_inputs_for_cloud_push_pull_commands
+    validate_user_inputs_for_cloud_push_pull_commands(encrypt, decrypt, key)
+
+    if encrypt:
+        encryption_action = ActionType.ENCRYPT
+    if decrypt:
+        encryption_action = ActionType.DECRYPT
 
     # Parse which projects need to be pushed
     if project is not None:
@@ -41,7 +59,13 @@ def push(project: Optional[Path]) -> None:
         if not project_config.file.exists():
             raise RuntimeError(f"'{project}' is not a Lean project")
 
-        push_manager.push_project(project)
+        if encrypt and key is not None:
+            from lean.components.util.encryption_helper import validate_encryption_key_registered_with_cloud
+            validate_encryption_key_registered_with_cloud(key, container.organization_manager, container.api_client) 
+
+        push_manager.push_project(project, encryption_action, key)
     else:
+        if key is not None:
+            raise RuntimeError(f"Encryption key can only be specified when pushing a single project.")
         projects_to_push = [p.parent for p in Path.cwd().rglob(PROJECT_CONFIG_FILE_NAME)]
-        push_manager.push_projects(projects_to_push)
+        push_manager.push_projects(projects_to_push, encryption_action, key)

lean/commands/decrypt.py

@@ -0,0 +1,62 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pathlib import Path
+from typing import Optional
+from click import command, option, argument
+
+from lean.click import LeanCommand, PathParameter
+from lean.container import container
+
+
+@command(cls=LeanCommand)
+@argument("project", type=PathParameter(exists=True, file_okay=False, dir_okay=True))
+@option("--key",
+              type=PathParameter(exists=True, file_okay=True, dir_okay=False),
+              help="Path to the decryption key to use")
+def decrypt(project: Path,
+            key: Optional[Path]) -> None:
+    """Decrypt your local project using the specified decryption key."""
+
+    logger = container.logger
+    project_manager = container.project_manager
+    project_config_manager = container.project_config_manager
+    project_config = project_config_manager.get_project_config(project)
+
+    # Check if the project is already decrypted
+    if not project_config.get("encrypted", False):
+        logger.info(f"Successfully decrypted project {project}")
+        return
+
+    decryption_key: Path = project_config.get('encryption-key-path', None)
+    from lean.components.util.encryption_helper import get_and_validate_user_input_encryption_key
+    decryption_key = get_and_validate_user_input_encryption_key(key, decryption_key)
+
+    organization_id = container.organization_manager.try_get_working_organization_id()
+
+    source_files = project_manager.get_source_files(project)
+    try:
+        from lean.components.util.encryption_helper import get_decrypted_file_content_for_local_project
+        decrypted_data = get_decrypted_file_content_for_local_project(project, 
+                                source_files, decryption_key, project_config_manager, organization_id)
+    except Exception as e:
+        raise RuntimeError(f"Could not decrypt project {project}: {e}")
+
+    for file, decrypted in zip(source_files, decrypted_data):
+        with open(file, 'w') as f:
+            f.write(decrypted)
+
+    # Mark the project as decrypted
+    project_config.set('encrypted', False)
+    project_config.delete('encryption-key-path')
+    logger.info(f"Successfully decrypted project {project}")

lean/commands/encrypt.py

@@ -0,0 +1,64 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pathlib import Path
+from typing import Optional
+from click import command, option, argument
+
+from lean.click import LeanCommand, PathParameter
+from lean.container import container
+
+
+@command(cls=LeanCommand)
+@argument("project", type=PathParameter(exists=True, file_okay=False, dir_okay=True))
+@option("--key",
+              type=PathParameter(exists=True, file_okay=True, dir_okay=False),
+              help="Path to the encryption key to use")
+def encrypt(project: Path,
+            key: Optional[Path]) -> None:
+    """Encrypt your local project using the specified encryption key."""
+
+    logger = container.logger
+    project_manager = container.project_manager
+    project_config_manager = container.project_config_manager
+    project_config = project_config_manager.get_project_config(project)
+
+    # Check if the project is already encrypted
+    if project_config.get('encrypted', False):
+        logger.info(f"Local files encrypted successfully.")
+        return
+
+    encryption_key: Path = project_config.get('encryption-key-path', None)
+    from lean.components.util.encryption_helper import get_and_validate_user_input_encryption_key
+    encryption_key = get_and_validate_user_input_encryption_key(key, encryption_key)
+
+    organization_id = container.organization_manager.try_get_working_organization_id()
+
+    source_files = project_manager.get_source_files(project)
+    try:
+        from lean.components.util.encryption_helper import get_encrypted_file_content_for_local_project
+        encrypted_data = get_encrypted_file_content_for_local_project(project, 
+                            source_files, encryption_key, project_config_manager, organization_id)
+    except Exception as e:
+        raise RuntimeError(f"Could not encrypt project {project}: {e}")
+    for file, encrypted in zip(source_files, encrypted_data):
+        with open(file, 'w') as f:
+            f.write(encrypted)
+
+    # Mark the project as encrypted
+    project_config.set('encrypted', True)
+    project_config.set('encryption-key-path', str(encryption_key))
+    logger.info(f"Local files encrypted successfully with key {encryption_key}")
+
+
+

lean/components/api/api_client.py

@@ -17,6 +17,7 @@
 from lean.components.api.backtest_client import BacktestClient
 from lean.components.api.compile_client import CompileClient
 from lean.components.api.data_client import DataClient
+from lean.components.api.encryption_keys_client import EncryptionKeysClient
 from lean.components.api.file_client import FileClient
 from lean.components.api.lean_client import LeanClient
 from lean.components.api.live_client import LiveClient
@@ -55,6 +56,7 @@ def __init__(self, logger: Logger, http_client: HTTPClient, user_id: str, api_to
         self.backtests = BacktestClient(self)
         self.compiles = CompileClient(self)
         self.data = DataClient(self, http_client)
+        self.encryption_keys = EncryptionKeysClient(self)
         self.files = FileClient(self)
         self.live = LiveClient(self)
         self.market = MarketClient(self)