New ebs volume (#277)

* creating new directory, creating new EBS volume, creating mounting point

* changing dir in engine util

* adjusting new volume size

* changing conflicting ebs names

* fixing block device path shell script

* trying different path name

* trying different directory name

* chaning file path name

* changing dir name back

* adding error handling to engine shell script

* formatting tf file

* adding some logging and cleaning shell script

* add volume path

* running ruf

* adding logging to available space

* adding logging to engine.py

* change log type

* adding another log case

* changing path for checking disk space

* more debugging

* adding more logging

* fixing clean up dir

* updating cloned_repo volume size

* removing comment

backend/aws_env.py

@@ -62,7 +62,8 @@ def parse_args():
 print("PRIORITY_TASK_QUEUE=%s" % priority_task_queue)
 print("ECR=%s" % ecr)
 # HOST_WORKING_DIR is mapped into the engine/plugin docker containers for holding the cloned repo
-print("HOST_WORKING_DIR=/data/artemis/work")
+# print("HOST_WORKING_DIR=/data/artemis/work")
+print("HOST_WORKING_DIR=/cloned_repos")
 print("INSTANCE_ID=%s" % ident["instanceId"])
 print("DEFAULT_DEPTH=500")
 print("CALLBACK_QUEUE=%s" % callback_queue)

backend/docker-compose.aws.yml

@@ -5,7 +5,7 @@ services:
     scale: 3
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
-      - /data/artemis/work:/work
+      - /cloned_repos:/work
     env_file:
       - .env
     logging:

backend/engine/engine.py

@@ -79,7 +79,7 @@ def process(msg, manager=None):  # pylint: disable=too-many-statements
 
     if action.lower() == "scan":
         if not check_disk_space(details.get("repo_size", 0)):
-            log.info("Scan failed because not enough " "disk space (repo size: %d KB)", details["repo_size"])
+            log.error("Scan failed because not enough " "disk space (repo size: %d KB)", details["repo_size"])
             engine_processor.update_scan_status(
                 "error", end_time=get_utc_datetime(), errors=["Repo too large (%d KB)" % details["repo_size"]]
             )
@@ -132,6 +132,7 @@ def process(msg, manager=None):  # pylint: disable=too-many-statements
 
 def cleanup(working_dir, scan_id):
     try:
+        log.info(f"cleaning up cloned repo at {working_dir, scan_id}")
         shutil.rmtree(os.path.join(working_dir, scan_id))
     except FileNotFoundError:
         # If path doesn't exist our work is done

backend/engine/utils/engine.py

@@ -17,7 +17,7 @@ def check_disk_space(repo_size: int, available_space=None) -> bool:
     Compares the repo size * 2 to the available disk space.
     :param available_space: For testing: provides the available space in KB
     :param repo_size: size of the repository in KB
-    :return: True if the available space is greater than the repo size * 2, otherwise False
+    :return: True if the available space is greater than the repo size, otherwise False
     """
     if available_space is None:
         s = os.statvfs("/work")
@@ -28,8 +28,7 @@ def check_disk_space(repo_size: int, available_space=None) -> bool:
         extra={"repo_size": repo_size, "available_disk_space": available_space},
     )
 
-    # The worst repo encountered (so far) is twice the size on disk
-    if (repo_size * 2) >= available_space:
+    if repo_size >= available_space:
         return False
     return True
 

backend/terraform/modules/analyzer/modules/engine_cluster/main.tf

@@ -38,6 +38,7 @@ locals {
       aws_region                   = var.aws_region
       docker_compose_ver           = var.docker_compose_ver
       engine_block_device          = var.engine_block_device
+      repos_block_device           = var.repos_block_device
       application                  = var.app
       region                       = var.aws_region
       domain_name                  = var.domain_name
@@ -83,6 +84,15 @@ resource "aws_launch_template" "engine-template" {
     }
   }
 
+  block_device_mappings {
+    device_name = var.repos_block_device
+
+    ebs {
+      volume_size = var.repos_volume_size
+      volume_type = "gp3"
+    }
+  }
+
   metadata_options {
     http_endpoint               = "enabled"
     http_tokens                 = "required"

backend/terraform/modules/analyzer/modules/engine_cluster/templates/user_data/engine.sh

@@ -9,6 +9,7 @@ export "aws_region"
 export "datadog_enabled"
 export "docker_compose_ver"
 export "engine_block_device"
+export "repos_block_device"
 export "github_app_id"
 export "plugin_java_heap_size"
 export "private_docker_repos_key"
@@ -23,10 +24,25 @@ export "ghas_enabled"
 export "revproxy_domain_substring"
 
 # Format and mount data volume
-mkfs -t ext4 "${engine_block_device}"
-mkdir -p /data
-mount "${engine_block_device}" /data
-chmod 755 /data
+if mount | grep -q "${engine_block_device}"; then
+  echo "${engine_block_device} is already mounted. Skipping format and mount." >> /var/log/my-script-log.log
+else
+  mkfs -t ext4 "${engine_block_device}"
+  mkdir -p /data
+  mount "${engine_block_device}" /data
+  chmod 755 /data
+fi
+
+# Format and mount repo volume
+if mount | grep -q "${repos_block_device}"; then
+  echo "${repos_block_device} is already mounted. Skipping format and mount." >> /var/log/my-script-log.log
+else
+  echo "Formatting ${repos_block_device}..." >> /var/log/my-script-log.log
+  mkfs -t ext4 "${repos_block_device}"
+  mkdir -p /cloned_repos
+  mount "${repos_block_device}" /cloned_repos
+  chmod 755 /cloned_repos
+fi
 
 # Make sure packages are up-to-date
 yum -y update

backend/terraform/modules/analyzer/modules/engine_cluster/variables.tf

@@ -96,14 +96,24 @@ variable "engine_scale_down_threshold" {
 
 variable "engine_volume_size" {
   description = "Engine instance volume size"
-  default     = 45
+  default     = 35
+}
+
+variable "repos_volume_size" {
+  description = "Cloned repos volume size"
+  default     = 25
 }
 
 variable "engine_block_device" {
   description = "Engine instance volume block device"
   default     = "/dev/sdf"
 }
 
+variable "repos_block_device" {
+  description = "Engine instance volume block device for cloned repos"
+  default     = "/dev/sdg"
+}
+
 variable "engine_log_retention" {
   description = "Engine log retention time (in days)"
   default     = 30