ec2: add cleanup for VPC

ocw/lib/EC2.py

@@ -6,6 +6,8 @@
 from botocore.exceptions import ClientError
 import re
 from datetime import date, datetime, timedelta
+from ocw.lib.emailnotify import send_mail
+import traceback
 import time
 
 
@@ -216,6 +218,121 @@ def cleanup_all(self):
         self.cleanup_images()
         self.cleanup_snapshots()
         self.cleanup_volumes()
+        self.cleanup_uploader_vpcs()
+
+    def delete_vpc(self, region, vpcId):
+        if PCWConfig.getBoolean('cleanup/vpc-notify-only', self._namespace):
+            return False
+        try:
+            vpc = self.ec2_resource(region).Vpc(vpcId)
+            for subnet in vpc.subnets.all():
+                if len(list(subnet.instances.all())):
+                    self.log_warn('{} has associated instance(s) so can not be deleted', vpcId)
+                    return False
+            self.log_info('{} has no associated instances. Initializing cleanup of it', vpcId)
+
+            self.delete_internet_gw(vpc)
+            self.delete_routing_tables(vpc)
+            self.delete_vpc_endpoints(region, vpcId)
+            self.delete_security_groups(vpc)
+            self.delete_vpc_peering_connections(region, vpcId)
+            self.delete_network_acls(vpc)
+            self.delete_vpc_subnets(vpc)
+            if self.dry_run:
+                self.log_info('Deletion of VPC skipped due to dry_run mode')
+            else:
+                # finally, delete the vpc
+                self.ec2_resource(region).meta.client.delete_vpc(VpcId=vpcId)
+        except Exception as e:
+            self.log_err("{} on VPC deletion. {}", type(e).__name__, traceback.format_exc())
+            send_mail('{} on VPC deletion in [{}]'.format(type(e).__name__, self._namespace), traceback.format_exc())
+            return False
+        return True
+
+    def delete_vpc_subnets(self, vpc):
+        for subnet in vpc.subnets.all():
+            for interface in subnet.network_interfaces.all():
+                if self.dry_run:
+                    self.log_info('Deletion of {} skipped due to dry_run mode', interface)
+                else:
+                    self.log_info('Deleting {}', interface)
+                    interface.delete()
+            if self.dry_run:
+                self.log_info('Deletion of {} skipped due to dry_run mode', subnet)
+            else:
+                self.log_info('Deleting {}', subnet)
+                subnet.delete()
+
+    def delete_network_acls(self, vpc):
+        for netacl in vpc.network_acls.all():
+            if not netacl.is_default:
+                if self.dry_run:
+                    self.log_info('Deletion of {} skipped due to dry_run mode', netacl)
+                else:
+                    self.log_info('Deleting {}', netacl)
+                    netacl.delete()
+
+    def delete_vpc_peering_connections(self, region, vpcId):
+        response = self.ec2_client(region).describe_vpc_peering_connections(
+            Filters=[{'Name': 'requester-vpc-info.vpc-id', 'Values': [vpcId]}])
+        for vpcpeer in response['VpcPeeringConnections']:
+            vpcpeer_connection = self.ec2_resource(region).VpcPeeringConnection(vpcpeer['VpcPeeringConnectionId'])
+            if self.dry_run:
+                self.log_info('Deletion of {} skipped due to dry_run mode', vpcpeer_connection)
+            else:
+                self.log_info('Deleting {}', vpcpeer_connection)
+                vpcpeer_connection.delete()
+
+    def delete_security_groups(self, vpc):
+        for sg in vpc.security_groups.all():
+            if sg.group_name != 'default':
+                if self.dry_run:
+                    self.log_info('Deletion of {} skipped due to dry_run mode', sg)
+                else:
+                    self.log_info('Deleting {}', sg)
+                    sg.delete()
+
+    def delete_vpc_endpoints(self, region, vpcId):
+        response = self.ec2_client(region).describe_vpc_endpoints(Filters=[{'Name': 'vpc-id', 'Values': [vpcId]}])
+        for ep in response['VpcEndpoints']:
+            if self.dry_run:
+                self.log_info('Deletion of {} skipped due to dry_run mode', ep)
+            else:
+                self.log_info('Deleting {}', ep)
+                self.ec2_client(region).delete_vpc_endpoints(VpcEndpointIds=[ep['VpcEndpointId']])
+
+    def delete_routing_tables(self, vpc):
+        for rt in vpc.route_tables.all():
+            # we can not delete main RouteTable's , not main one don't have associations_attributes
+            if len(rt.associations_attribute) == 0:
+                if self.dry_run:
+                    self.log_info('{} will be not deleted due to dry_run mode', rt)
+                else:
+                    self.log_info('Deleting {}', rt)
+                    rt.delete()
+
+    def delete_internet_gw(self, vpc):
+        for gw in vpc.internet_gateways.all():
+            if self.dry_run:
+                self.log_info('{} will be not deleted due to dry_run mode', gw)
+            else:
+                self.log_info('Deleting {}', gw)
+                vpc.detach_internet_gateway(InternetGatewayId=gw.id)
+                gw.delete()
+
+    def cleanup_uploader_vpcs(self):
+        if PCWConfig.getBoolean('cleanup/vpc_cleanup', self._namespace):
+            for region in self.all_regions:
+                response = self.ec2_client(region).describe_vpcs(
+                    Filters=[{'Name': 'isDefault', 'Values': ['false']},
+                             {'Name': 'tag:Name', 'Values': ['uploader-*']}])
+                for vpc in response['Vpcs']:
+                    self.log_info('{} in {} looks like uploader leftover. (OwnerId={}).', vpc['VpcId'], region,
+                                  vpc['OwnerId'])
+                    if not self.delete_vpc(region, vpc['VpcId']) and not self.dry_run:
+                        send_mail('VPC deletion locked by running VMs',
+                                  'Uploader leftover {} (OwnerId={}) in {} is locked'.format(vpc['VpcId'],
+                                                                                             vpc['OwnerId'], region))
 
     def cleanup_images(self):
         for region in self.all_regions:

ocw/lib/emailnotify.py

@@ -7,6 +7,7 @@
 import json
 import smtplib
 import logging
+from email.mime.text import MIMEText
 
 logger = logging.getLogger(__name__)
 
@@ -33,7 +34,7 @@ def draw_instance_table(objects):
 
 
 def send_leftover_notification():
-    if PCWConfig().has('notify'):
+    if PCWConfig.has('notify'):
         o = Instance.objects
         o = o.filter(active=True, csp_info__icontains='openqa_created_by',
                      age__gt=timedelta(hours=PCWConfig.get_feature_property('notify', 'age-hours')))
@@ -59,20 +60,17 @@ def send_cluster_notification(namespace, clusters):
 
 
 def send_mail(subject, message, receiver_email=None):
-    if PCWConfig().has('notify'):
-        smtp_server = PCWConfig().get_feature_property('notify', 'smtp')
-        port = PCWConfig().get_feature_property('notify', 'smtp-port')
-        sender_email = PCWConfig().get_feature_property('notify', 'from')
+    if PCWConfig.has('notify'):
+        smtp_server = PCWConfig.get_feature_property('notify', 'smtp')
+        port = PCWConfig.get_feature_property('notify', 'smtp-port')
+        sender_email = PCWConfig.get_feature_property('notify', 'from')
         if receiver_email is None:
-            receiver_email = PCWConfig().get_feature_property('notify', 'to')
-        email = '''\
-    Subject: [Openqa-Cloud-Watch] {subject}
-    From: {_from}
-    To: {_to}
-
-    {message}
-    '''.format(subject=subject, _from=sender_email, _to=receiver_email, message=message)
+            receiver_email = PCWConfig.get_feature_property('notify', 'to')
+        mimetext = MIMEText(message)
+        mimetext['Subject'] = '[Openqa-Cloud-Watch] {}'.format(subject)
+        mimetext['From'] = sender_email
+        mimetext['To'] = receiver_email
         logger.info("Send Email To:'%s' Subject:'[Openqa-Cloud-Watch] %s'", receiver_email, subject)
         server = smtplib.SMTP(smtp_server, port)
         server.ehlo()
-        server.sendmail(sender_email, receiver_email.split(','), email)
+        server.sendmail(sender_email, receiver_email.split(','), mimetext.as_string())

ocw/lib/provider.py

@@ -11,7 +11,7 @@ class Provider:
 
     def __init__(self, namespace: str):
         self._namespace = namespace
-        self.dry_run = PCWConfig().getBoolean('default/dry_run')
+        self.dry_run = PCWConfig.getBoolean('default/dry_run')
         self.logger = logging.getLogger(self.__module__)
 
     def older_than_min_age(self, age):

ocw/lib/vault.py

@@ -18,11 +18,11 @@ class Vault:
     extra_time = 600
 
     def __init__(self, vault_namespace):
-        self.url = PCWConfig().get_feature_property('vault', 'url')
-        self.user = PCWConfig().get_feature_property('vault', 'user')
+        self.url = PCWConfig.get_feature_property('vault', 'url')
+        self.user = PCWConfig.get_feature_property('vault', 'user')
         self.namespace = vault_namespace
-        self.password = PCWConfig().get_feature_property('vault', 'password')
-        self.certificate_dir = PCWConfig().get_feature_property('vault', 'cert_dir')
+        self.password = PCWConfig.get_feature_property('vault', 'password')
+        self.certificate_dir = PCWConfig.get_feature_property('vault', 'cert_dir')
         if PCWConfig.getBoolean('vault/use-file-cache') and self._getAuthCacheFile().exists():
             logger.info('Loading cached credentials')
             self.auth_json = self.loadAuthCache()
@@ -102,7 +102,7 @@ def getCredentials(self):
         raise NotImplementedError
 
     def getData(self, name=None):
-        use_file_cache = PCWConfig().getBoolean('vault/use-file-cache')
+        use_file_cache = PCWConfig.getBoolean('vault/use-file-cache')
         if self.auth_json is None and use_file_cache:
             self.auth_json = self.loadAuthCache()
         if self.isExpired():
@@ -129,7 +129,7 @@ def isExpired(self):
         return expire < datetime.today() + timedelta(seconds=self.extra_time)
 
     def renew(self):
-        if PCWConfig().getBoolean('vault/use-file-cache') and self._getAuthCacheFile().exists():
+        if PCWConfig.getBoolean('vault/use-file-cache') and self._getAuthCacheFile().exists():
             self._getAuthCacheFile().unlink()
         self.revoke()
         self.getData()

tests/generators.py

@@ -7,6 +7,7 @@
 max_image_age_hours = 20
 azure_storage_resourcegroup = 'openqa'
 ec2_max_snapshot_age_days = 1
+ec2_max_volumes_age_days = 5
 
 
 class MockImage:
@@ -15,7 +16,7 @@ def __init__(self, name, last_modified=None):
         self.last_modified = last_modified
 
 
-def mock_get_feature_property(feature: str, property: str, namespace: str):
+def mock_get_feature_property(feature: str, property: str, namespace: str = None):
     if property == 'min-image-age-hours':
         return min_image_age_hours
     elif property == 'max-images-per-flavor':
@@ -26,6 +27,8 @@ def mock_get_feature_property(feature: str, property: str, namespace: str):
         return azure_storage_resourcegroup
     elif property == 'ec2-max-snapshot-age-days':
         return ec2_max_snapshot_age_days
+    elif property == 'ec2-max-volumes-age-days':
+        return ec2_max_volumes_age_days
 
 
 class ec2_meta_mock: