ec2: add cleanup for VPC

ocw/lib/EC2.py

@@ -6,6 +6,8 @@
 from botocore.exceptions import ClientError
 import re
 from datetime import date, datetime, timedelta
+from ocw.lib.emailnotify import send_mail
+import traceback
 import time
 
 
@@ -189,6 +191,88 @@ def parse_image_name(self, img_name):
     def cleanup_all(self):
         self.cleanup_images()
         self.cleanup_snapshots()
+        self.cleanup_uploader_vpcs()
+
+    def delete_vpc(self, region, vpcId):
+        if PCWConfig.getBoolean('cleanup/vpc_only_notify', self._namespace):
+            return False
+        try:
+            vpc = self.ec2_resource(region).Vpc(vpcId)
+            for subnet in vpc.subnets.all():
+                if len(list(subnet.instances.all())):
+                    self.log_warn('{} has associated instance(s) so can not be deleted', vpcId)
+                    return False
+            self.log_info('{} has no associated instances. Initializing cleanup of it', vpcId)
+            for gw in vpc.internet_gateways.all():
+                self.log_info('Deleting {}', gw)
+                vpc.detach_internet_gateway(InternetGatewayId=gw.id)
+                gw.delete()
+            # delete all route table associations
+            for rt in vpc.route_tables.all():
+                for route in rt.routes:
+                    self.log_info('Deleting route - {}', route)
+                    self.ec2_resource(region).meta.client.delete_route(
+                        DestinationCidrBlock=route.destination_cidr_block,
+                        RouteTableId=rt.id)
+                for rta in rt.associations:
+                    self.log_info('Found rta - {}', rta)
+                    if not rta.main:
+                        self.log_info('Deleting {}', rta)
+                        rta.delete()
+
+            # delete our endpoints
+            response = self.ec2_client(region).describe_vpc_endpoints(Filters=[{'Name': 'vpc-id', 'Values': [vpcId]}])
+            for ep in response['VpcEndpoints']:
+                self.log_info('Deleting {}', ep)
+                self.ec2_client(region).delete_vpc_endpoints(VpcEndpointIds=[ep['VpcEndpointId']])
+            # delete our security groups
+            for sg in vpc.security_groups.all():
+                if sg.group_name != 'default':
+                    self.log_info('Deleting {}', sg)
+                    sg.delete()
+            # delete any vpc peering connections
+            response = self.ec2_client(region).describe_vpc_peering_connections(
+                Filters=[{'Name': 'requester-vpc-info.vpc-id', 'Values': [vpcId]}])
+            for vpcpeer in response['VpcPeeringConnections']:
+                vpcpeer_connection = self.ec2_resource(region).VpcPeeringConnection(vpcpeer['VpcPeeringConnectionId'])
+                self.log_info('Deleting {}', vpcpeer_connection)
+                vpcpeer_connection.delete()
+            # delete non-default network acls
+            for netacl in vpc.network_acls.all():
+                if not netacl.is_default:
+                    self.log_info('Deleting {}', netacl)
+                    netacl.delete()
+            # delete network interfaces
+            for subnet in vpc.subnets.all():
+                for interface in subnet.network_interfaces.all():
+                    self.log_info('Deleting {}', interface)
+                    interface.delete()
+                self.log_info('Deleting {}', subnet)
+                subnet.delete()
+            # finally, delete the vpc
+            self.ec2_resource(region).meta.client.delete_vpc(VpcId=vpcId)
+        except Exception as e:
+            self.log_err("{} on VPC {} deletion. {}", type(e).__name__, vpc, traceback.format_exc())
+            send_mail('{} on VPC deletion in [{}]'.format(type(e).__name__, self._namespace), traceback.format_exc())
+        return True
+
+    def cleanup_uploader_vpcs(self):
+        if PCWConfig().getBoolean('cleanup/vpc_cleanup', self._namespace):
+            for region in self.all_regions:
+                response = self.ec2_client(region).describe_vpcs(
+                    Filters=[{'Name': 'isDefault', 'Values': ['false']},
+                             {'Name': 'tag:Name', 'Values': ['uploader-*']}])
+                for vpc in response['Vpcs']:
+                    self.log_info('{} in {} looks like uploader leftover. (OwnerId={}).', vpc['VpcId'], region,
+                                  vpc['OwnerId'])
+                    if self.dry_run:
+                        self.log_info("VPC deletion {} skipped due to dry run mode", vpc['VpcId'])
+                    else:
+                        if not self.delete_vpc(region, vpc['VpcId']):
+                            send_mail('VPC deletion locked by running VMs',
+                                      'Uploader leftover {} (OwnerId={}) in {} is locked'.format(vpc['VpcId'],
+                                                                                                 vpc['OwnerId'],
+                                                                                                 region))
 
     def cleanup_images(self):
         for region in self.all_regions:

ocw/lib/emailnotify.py

@@ -7,6 +7,7 @@
 import json
 import smtplib
 import logging
+from email.mime.text import MIMEText
 
 logger = logging.getLogger(__name__)
 
@@ -65,14 +66,11 @@ def send_mail(subject, message, receiver_email=None):
         sender_email = PCWConfig().get_feature_property('notify', 'from')
         if receiver_email is None:
             receiver_email = PCWConfig().get_feature_property('notify', 'to')
-        email = '''\
-    Subject: [Openqa-Cloud-Watch] {subject}
-    From: {_from}
-    To: {_to}
-
-    {message}
-    '''.format(subject=subject, _from=sender_email, _to=receiver_email, message=message)
+        mimetext = MIMEText(message)
+        mimetext['Subject'] = '[Openqa-Cloud-Watch] {}'.format(subject)
+        mimetext['From'] = sender_email
+        mimetext['To'] = receiver_email
         logger.info("Send Email To:'%s' Subject:'[Openqa-Cloud-Watch] %s'", receiver_email, subject)
         server = smtplib.SMTP(smtp_server, port)
         server.ehlo()
-        server.sendmail(sender_email, receiver_email.split(','), email)
+        server.sendmail(sender_email, receiver_email.split(','), mimetext.as_string())

tests/test_pcwconfig.py

@@ -99,3 +99,34 @@ def test_get_providers_for_existed_feature(pcw_file):
     """)
     providers = PCWConfig.get_providers_for('providerfeature', 'fake')
     assert not {'azure'} ^ set(providers)
+
+def test_getBoolean_notdefined(pcw_file):
+    assert not PCWConfig.getBoolean('feature/bool_property')
+
+def test_getBoolean_notdefined_namespace(pcw_file):
+    assert not PCWConfig.getBoolean('feature/bool_property','random_namespace')
+
+def test_getBoolean_defined(pcw_file):
+    set_pcw_ini(pcw_file, """
+    [feature]
+    bool_property = True
+    """)
+    assert PCWConfig.getBoolean('feature/bool_property')
+
+def test_getBoolean_defined_namespace(pcw_file):
+    set_pcw_ini(pcw_file, """
+    [feature]
+    bool_property = False
+    [feature.namespace.random_namespace]
+    bool_property = True
+    """)
+    assert PCWConfig.getBoolean('feature/bool_property','random_namespace')
+
+def test_getBoolean_namespace_but_not_defined(pcw_file):
+    set_pcw_ini(pcw_file, """
+    [feature]
+    bool_property = True
+    [feature.namespace.random_namespace]
+    providers = azure
+    """)
+    assert PCWConfig.getBoolean('feature/bool_property','random_namespace')

webui/settings.py

@@ -258,8 +258,16 @@ def has(setting: str) -> bool:
             return False
 
     @staticmethod
-    def getBoolean(config_path: str, default=False) -> bool:
-        value = ConfigFile().get(config_path, default)
+    def getBoolean(config_path: str, namespace: str = None, default=False) -> bool:
+        if namespace:
+            (feature, property) = config_path.split('/')
+            setting = '{}.namespace.{}/{}'.format(feature, namespace, property)
+            if PCWConfig.has(setting):
+                value = ConfigFile().get(setting)
+            else:
+                value = ConfigFile().get(config_path, default)
+        else:
+            value = ConfigFile().get(config_path, default)
         if isinstance(value, bool):
             return value
         return bool(re.match("^(true|on|1|yes)$", str(value), flags=re.IGNORECASE))