ec2: add cleanup for VPC

ocw/lib/EC2.py

@@ -6,6 +6,8 @@
 from botocore.exceptions import ClientError
 import re
 from datetime import date, datetime, timedelta
+from ocw.lib.emailnotify import send_mail
+import traceback
 import time
 
 
@@ -216,6 +218,105 @@ def cleanup_all(self):
         self.cleanup_images()
         self.cleanup_snapshots()
         self.cleanup_volumes()
+        self.cleanup_uploader_vpcs()
+
+    def delete_vpc(self, region, vpcId):
+        if PCWConfig.getBoolean('cleanup/vpc_only_notify', self._namespace):
+            return False
+        try:
+            vpc = self.ec2_resource(region).Vpc(vpcId)
+            for subnet in vpc.subnets.all():
+                if len(list(subnet.instances.all())):
+                    self.log_warn('{} has associated instance(s) so can not be deleted', vpcId)
+                    return False
+            self.log_info('{} has no associated instances. Initializing cleanup of it', vpcId)
+            for gw in vpc.internet_gateways.all():
+                if self.dry_run:
+                    self.log_info('{} will be not deleted due to dry_run mode', gw)
+                else:
+                    self.log_info('Deleting {}', gw)
+                    vpc.detach_internet_gateway(InternetGatewayId=gw.id)
+                    gw.delete()
+            # delete all route table's
+            for rt in vpc.route_tables.all():
+                # we can not delete main RouteTable's , not main one don't have associations_attributes
+                if len(rt.associations_attribute) == 0:
+                    if self.dry_run:
+                        self.log_info('{} will be not deleted due to dry_run mode', rt)
+                    else:
+                        self.log_info('Deleting {}', rt)
+                        rt.delete()
+
+            # delete our endpoints
+            response = self.ec2_client(region).describe_vpc_endpoints(Filters=[{'Name': 'vpc-id', 'Values': [vpcId]}])
+            for ep in response['VpcEndpoints']:
+                if self.dry_run:
+                    self.log_info('Deletion of {} skipped due to dry_run mode', ep)
+                else:
+                    self.log_info('Deleting {}', ep)
+                    self.ec2_client(region).delete_vpc_endpoints(VpcEndpointIds=[ep['VpcEndpointId']])
+            # delete our security groups
+            for sg in vpc.security_groups.all():
+                if sg.group_name != 'default':
+                    if self.dry_run:
+                        self.log_info('Deletion of {} skipped due to dry_run mode', sg)
+                    else:
+                        self.log_info('Deleting {}', sg)
+                        sg.delete()
+            # delete any vpc peering connections
+            response = self.ec2_client(region).describe_vpc_peering_connections(
+                Filters=[{'Name': 'requester-vpc-info.vpc-id', 'Values': [vpcId]}])
+            for vpcpeer in response['VpcPeeringConnections']:
+                vpcpeer_connection = self.ec2_resource(region).VpcPeeringConnection(vpcpeer['VpcPeeringConnectionId'])
+                if self.dry_run:
+                    self.log_info('Deletion of {} skipped due to dry_run mode', vpcpeer_connection)
+                else:
+                    self.log_info('Deleting {}', vpcpeer_connection)
+                    vpcpeer_connection.delete()
+            # delete non-default network acls
+            for netacl in vpc.network_acls.all():
+                if not netacl.is_default:
+                    if self.dry_run:
+                        self.log_info('Deletion of {} skipped due to dry_run mode', netacl)
+                    else:
+                        self.log_info('Deleting {}', netacl)
+                        netacl.delete()
+            # delete network interfaces
+            for subnet in vpc.subnets.all():
+                for interface in subnet.network_interfaces.all():
+                    if self.dry_run:
+                        self.log_info('Deletion of {} skipped due to dry_run mode', interface)
+                    else:
+                        self.log_info('Deleting {}', interface)
+                        interface.delete()
+                if self.dry_run:
+                    self.log_info('Deletion of {} skipped due to dry_run mode', subnet)
+                else:
+                    self.log_info('Deleting {}', subnet)
+                    subnet.delete()
+            if self.dry_run:
+                self.log_info('Deletion of VPC skipped due to dry_run mode')
+            else:
+                # finally, delete the vpc
+                self.ec2_resource(region).meta.client.delete_vpc(VpcId=vpcId)
+        except Exception as e:
+            self.log_err("{} on VPC {} deletion. {}", type(e).__name__, vpc, traceback.format_exc())
+            send_mail('{} on VPC deletion in [{}]'.format(type(e).__name__, self._namespace), traceback.format_exc())
+        return True
+
+    def cleanup_uploader_vpcs(self):
+        if PCWConfig.getBoolean('cleanup/vpc_cleanup', self._namespace):
+            for region in self.all_regions:
+                response = self.ec2_client(region).describe_vpcs(
+                    Filters=[{'Name': 'isDefault', 'Values': ['false']},
+                             {'Name': 'tag:Name', 'Values': ['uploader-*']}])
+                for vpc in response['Vpcs']:
+                    self.log_info('{} in {} looks like uploader leftover. (OwnerId={}).', vpc['VpcId'], region,
+                                  vpc['OwnerId'])
+                    if not self.delete_vpc(region, vpc['VpcId']) and not self.dry_run:
+                        send_mail('VPC deletion locked by running VMs',
+                                  'Uploader leftover {} (OwnerId={}) in {} is locked'.format(vpc['VpcId'],
+                                                                                             vpc['OwnerId'], region))
 
     def cleanup_images(self):
         for region in self.all_regions:

ocw/lib/emailnotify.py

@@ -7,6 +7,7 @@
 import json
 import smtplib
 import logging
+from email.mime.text import MIMEText
 
 logger = logging.getLogger(__name__)
 
@@ -33,7 +34,7 @@ def draw_instance_table(objects):
 
 
 def send_leftover_notification():
-    if PCWConfig().has('notify'):
+    if PCWConfig.has('notify'):
         o = Instance.objects
         o = o.filter(active=True, csp_info__icontains='openqa_created_by',
                      age__gt=timedelta(hours=PCWConfig.get_feature_property('notify', 'age-hours')))
@@ -59,20 +60,17 @@ def send_cluster_notification(namespace, clusters):
 
 
 def send_mail(subject, message, receiver_email=None):
-    if PCWConfig().has('notify'):
-        smtp_server = PCWConfig().get_feature_property('notify', 'smtp')
-        port = PCWConfig().get_feature_property('notify', 'smtp-port')
-        sender_email = PCWConfig().get_feature_property('notify', 'from')
+    if PCWConfig.has('notify'):
+        smtp_server = PCWConfig.get_feature_property('notify', 'smtp')
+        port = PCWConfig.get_feature_property('notify', 'smtp-port')
+        sender_email = PCWConfig.get_feature_property('notify', 'from')
         if receiver_email is None:
-            receiver_email = PCWConfig().get_feature_property('notify', 'to')
-        email = '''\
-    Subject: [Openqa-Cloud-Watch] {subject}
-    From: {_from}
-    To: {_to}
-
-    {message}
-    '''.format(subject=subject, _from=sender_email, _to=receiver_email, message=message)
+            receiver_email = PCWConfig.get_feature_property('notify', 'to')
+        mimetext = MIMEText(message)
+        mimetext['Subject'] = '[Openqa-Cloud-Watch] {}'.format(subject)
+        mimetext['From'] = sender_email
+        mimetext['To'] = receiver_email
         logger.info("Send Email To:'%s' Subject:'[Openqa-Cloud-Watch] %s'", receiver_email, subject)
         server = smtplib.SMTP(smtp_server, port)
         server.ehlo()
-        server.sendmail(sender_email, receiver_email.split(','), email)
+        server.sendmail(sender_email, receiver_email.split(','), mimetext.as_string())

ocw/lib/provider.py

@@ -11,7 +11,7 @@ class Provider:
 
     def __init__(self, namespace: str):
         self._namespace = namespace
-        self.dry_run = PCWConfig().getBoolean('default/dry_run')
+        self.dry_run = PCWConfig.getBoolean('default/dry_run')
         self.logger = logging.getLogger(self.__module__)
 
     def older_than_min_age(self, age):

ocw/lib/vault.py

@@ -18,11 +18,11 @@ class Vault:
     extra_time = 600
 
     def __init__(self, vault_namespace):
-        self.url = PCWConfig().get_feature_property('vault', 'url')
-        self.user = PCWConfig().get_feature_property('vault', 'user')
+        self.url = PCWConfig.get_feature_property('vault', 'url')
+        self.user = PCWConfig.get_feature_property('vault', 'user')
         self.namespace = vault_namespace
-        self.password = PCWConfig().get_feature_property('vault', 'password')
-        self.certificate_dir = PCWConfig().get_feature_property('vault', 'cert_dir')
+        self.password = PCWConfig.get_feature_property('vault', 'password')
+        self.certificate_dir = PCWConfig.get_feature_property('vault', 'cert_dir')
         if PCWConfig.getBoolean('vault/use-file-cache') and self._getAuthCacheFile().exists():
             logger.info('Loading cached credentials')
             self.auth_json = self.loadAuthCache()
@@ -102,7 +102,7 @@ def getCredentials(self):
         raise NotImplementedError
 
     def getData(self, name=None):
-        use_file_cache = PCWConfig().getBoolean('vault/use-file-cache')
+        use_file_cache = PCWConfig.getBoolean('vault/use-file-cache')
         if self.auth_json is None and use_file_cache:
             self.auth_json = self.loadAuthCache()
         if self.isExpired():
@@ -129,7 +129,7 @@ def isExpired(self):
         return expire < datetime.today() + timedelta(seconds=self.extra_time)
 
     def renew(self):
-        if PCWConfig().getBoolean('vault/use-file-cache') and self._getAuthCacheFile().exists():
+        if PCWConfig.getBoolean('vault/use-file-cache') and self._getAuthCacheFile().exists():
             self._getAuthCacheFile().unlink()
         self.revoke()
         self.getData()

tests/test_ec2.py

@@ -4,6 +4,7 @@
 from tests.generators import min_image_age_hours, max_image_age_hours
 from datetime import datetime, timezone, timedelta
 from botocore.exceptions import ClientError
+import smtplib
 
 
 def test_parse_image_name(monkeypatch):
@@ -191,3 +192,39 @@ def delete_volume(VolumeId):
 
     assert len(deleted_volumes) == 1
     assert deleted_volumes[0] == volumeid_to_delete
+
+def test_cleanup_uploader_vpc(monkeypatch):
+    def mocked_ec2_client():
+        pass
+
+    def mocked_get_boolean(config_path, field=None):
+        if config_path == 'default/dry_run':
+            return False
+        else:
+            return True
+
+    cnt = 0
+
+    def mocked_send_email(subject, body):
+        global cnt
+        cnt += 1
+
+    response = {
+        'Vpcs': [
+            {'VpcId': 'someId', 'OwnerId': 'someId'}
+        ]
+    }
+    monkeypatch.setattr(EC2, 'check_credentials', lambda *args, **kwargs: True)
+    monkeypatch.setattr(EC2, 'ec2_client', lambda self, region: mocked_ec2_client)
+    monkeypatch.setattr(EC2, 'get_all_regions', lambda self: ['eu-central'])
+    monkeypatch.setattr(EC2, 'delete_vpc', lambda self, region, vpcId: False)
+    monkeypatch.setattr(PCWConfig, 'getBoolean', mocked_get_boolean)
+    monkeypatch.setattr(PCWConfig, 'get_feature_property', lambda section, field: -1)
+    monkeypatch.setattr(smtplib.SMTP, 'sendmail', mocked_send_email)
+
+    mocked_ec2_client.describe_vpcs = lambda Filters: response
+
+    ec2 = EC2('fake')
+    ec2.cleanup_uploader_vpcs()
+
+    assert cnt == 1

tests/test_pcwconfig.py

@@ -99,3 +99,34 @@ def test_get_providers_for_existed_feature(pcw_file):
     """)
     providers = PCWConfig.get_providers_for('providerfeature', 'fake')
     assert not {'azure'} ^ set(providers)
+
+def test_getBoolean_notdefined(pcw_file):
+    assert not PCWConfig.getBoolean('feature/bool_property')
+
+def test_getBoolean_notdefined_namespace(pcw_file):
+    assert not PCWConfig.getBoolean('feature/bool_property','random_namespace')
+
+def test_getBoolean_defined(pcw_file):
+    set_pcw_ini(pcw_file, """
+    [feature]
+    bool_property = True
+    """)
+    assert PCWConfig.getBoolean('feature/bool_property')
+
+def test_getBoolean_defined_namespace(pcw_file):
+    set_pcw_ini(pcw_file, """
+    [feature]
+    bool_property = False
+    [feature.namespace.random_namespace]
+    bool_property = True
+    """)
+    assert PCWConfig.getBoolean('feature/bool_property','random_namespace')
+
+def test_getBoolean_namespace_but_not_defined(pcw_file):
+    set_pcw_ini(pcw_file, """
+    [feature]
+    bool_property = True
+    [feature.namespace.random_namespace]
+    providers = azure
+    """)
+    assert PCWConfig.getBoolean('feature/bool_property','random_namespace')

webui/settings.py

@@ -259,8 +259,16 @@ def has(setting: str) -> bool:
             return False
 
     @staticmethod
-    def getBoolean(config_path: str, default=False) -> bool:
-        value = ConfigFile().get(config_path, default)
+    def getBoolean(config_path: str, namespace: str = None, default=False) -> bool:
+        if namespace:
+            (feature, property) = config_path.split('/')
+            setting = '{}.namespace.{}/{}'.format(feature, namespace, property)
+            if PCWConfig.has(setting):
+                value = ConfigFile().get(setting)
+            else:
+                value = ConfigFile().get(config_path, default)
+        else:
+            value = ConfigFile().get(config_path, default)
         if isinstance(value, bool):
             return value
         return bool(re.match("^(true|on|1|yes)$", str(value), flags=re.IGNORECASE))