From 7acec00faceef3805e2c636c229a5243bc231496 Mon Sep 17 00:00:00 2001 From: Anton Smorodskyi Date: Sun, 8 Nov 2020 15:36:51 +0100 Subject: [PATCH] ec2: add cleanup for VPC --- ocw/lib/EC2.py | 101 ++++++++++++++++++++++++++++++++++++++++ ocw/lib/emailnotify.py | 26 +++++------ ocw/lib/provider.py | 2 +- ocw/lib/vault.py | 12 ++--- tests/test_ec2.py | 37 +++++++++++++++ tests/test_pcwconfig.py | 31 ++++++++++++ webui/settings.py | 12 ++++- 7 files changed, 198 insertions(+), 23 deletions(-) diff --git a/ocw/lib/EC2.py b/ocw/lib/EC2.py index 820df6ff..0939c687 100644 --- a/ocw/lib/EC2.py +++ b/ocw/lib/EC2.py @@ -6,6 +6,8 @@ from botocore.exceptions import ClientError import re from datetime import date, datetime, timedelta +from ocw.lib.emailnotify import send_mail +import traceback import time @@ -216,6 +218,105 @@ def cleanup_all(self): self.cleanup_images() self.cleanup_snapshots() self.cleanup_volumes() + self.cleanup_uploader_vpcs() + + def delete_vpc(self, region, vpcId): + if PCWConfig.getBoolean('cleanup/vpc_only_notify', self._namespace): + return False + try: + vpc = self.ec2_resource(region).Vpc(vpcId) + for subnet in vpc.subnets.all(): + if len(list(subnet.instances.all())): + self.log_warn('{} has associated instance(s) so can not be deleted', vpcId) + return False + self.log_info('{} has no associated instances. Initializing cleanup of it', vpcId) + for gw in vpc.internet_gateways.all(): + if self.dry_run: + self.log_info('{} will be not deleted due to dry_run mode', gw) + else: + self.log_info('Deleting {}', gw) + vpc.detach_internet_gateway(InternetGatewayId=gw.id) + gw.delete() + # delete all route table's + for rt in vpc.route_tables.all(): + # we can not delete main RouteTable's , not main one don't have associations_attributes + if len(rt.associations_attribute) == 0: + if self.dry_run: + self.log_info('{} will be not deleted due to dry_run mode', rt) + else: + self.log_info('Deleting {}', rt) + rt.delete() + + # delete our endpoints + response = self.ec2_client(region).describe_vpc_endpoints(Filters=[{'Name': 'vpc-id', 'Values': [vpcId]}]) + for ep in response['VpcEndpoints']: + if self.dry_run: + self.log_info('Deletion of {} skipped due to dry_run mode', ep) + else: + self.log_info('Deleting {}', ep) + self.ec2_client(region).delete_vpc_endpoints(VpcEndpointIds=[ep['VpcEndpointId']]) + # delete our security groups + for sg in vpc.security_groups.all(): + if sg.group_name != 'default': + if self.dry_run: + self.log_info('Deletion of {} skipped due to dry_run mode', sg) + else: + self.log_info('Deleting {}', sg) + sg.delete() + # delete any vpc peering connections + response = self.ec2_client(region).describe_vpc_peering_connections( + Filters=[{'Name': 'requester-vpc-info.vpc-id', 'Values': [vpcId]}]) + for vpcpeer in response['VpcPeeringConnections']: + vpcpeer_connection = self.ec2_resource(region).VpcPeeringConnection(vpcpeer['VpcPeeringConnectionId']) + if self.dry_run: + self.log_info('Deletion of {} skipped due to dry_run mode', vpcpeer_connection) + else: + self.log_info('Deleting {}', vpcpeer_connection) + vpcpeer_connection.delete() + # delete non-default network acls + for netacl in vpc.network_acls.all(): + if not netacl.is_default: + if self.dry_run: + self.log_info('Deletion of {} skipped due to dry_run mode', netacl) + else: + self.log_info('Deleting {}', netacl) + netacl.delete() + # delete network interfaces + for subnet in vpc.subnets.all(): + for interface in subnet.network_interfaces.all(): + if self.dry_run: + self.log_info('Deletion of {} skipped due to dry_run mode', interface) + else: + self.log_info('Deleting {}', interface) + interface.delete() + if self.dry_run: + self.log_info('Deletion of {} skipped due to dry_run mode', subnet) + else: + self.log_info('Deleting {}', subnet) + subnet.delete() + if self.dry_run: + self.log_info('Deletion of VPC skipped due to dry_run mode') + else: + # finally, delete the vpc + self.ec2_resource(region).meta.client.delete_vpc(VpcId=vpcId) + except Exception as e: + self.log_err("{} on VPC {} deletion. {}", type(e).__name__, vpc, traceback.format_exc()) + send_mail('{} on VPC deletion in [{}]'.format(type(e).__name__, self._namespace), traceback.format_exc()) + return True + + def cleanup_uploader_vpcs(self): + if PCWConfig.getBoolean('cleanup/vpc_cleanup', self._namespace): + for region in self.all_regions: + response = self.ec2_client(region).describe_vpcs( + Filters=[{'Name': 'isDefault', 'Values': ['false']}, + {'Name': 'tag:Name', 'Values': ['uploader-*']}]) + for vpc in response['Vpcs']: + self.log_info('{} in {} looks like uploader leftover. (OwnerId={}).', vpc['VpcId'], region, + vpc['OwnerId']) + if not self.delete_vpc(region, vpc['VpcId']) and not self.dry_run: + send_mail('VPC deletion locked by running VMs', + 'Uploader leftover {} (OwnerId={}) in {} is locked'.format(vpc['VpcId'], + vpc['OwnerId'], region)) def cleanup_images(self): for region in self.all_regions: diff --git a/ocw/lib/emailnotify.py b/ocw/lib/emailnotify.py index 9e922241..bfca5bcc 100644 --- a/ocw/lib/emailnotify.py +++ b/ocw/lib/emailnotify.py @@ -7,6 +7,7 @@ import json import smtplib import logging +from email.mime.text import MIMEText logger = logging.getLogger(__name__) @@ -33,7 +34,7 @@ def draw_instance_table(objects): def send_leftover_notification(): - if PCWConfig().has('notify'): + if PCWConfig.has('notify'): o = Instance.objects o = o.filter(active=True, csp_info__icontains='openqa_created_by', age__gt=timedelta(hours=PCWConfig.get_feature_property('notify', 'age-hours'))) @@ -59,20 +60,17 @@ def send_cluster_notification(namespace, clusters): def send_mail(subject, message, receiver_email=None): - if PCWConfig().has('notify'): - smtp_server = PCWConfig().get_feature_property('notify', 'smtp') - port = PCWConfig().get_feature_property('notify', 'smtp-port') - sender_email = PCWConfig().get_feature_property('notify', 'from') + if PCWConfig.has('notify'): + smtp_server = PCWConfig.get_feature_property('notify', 'smtp') + port = PCWConfig.get_feature_property('notify', 'smtp-port') + sender_email = PCWConfig.get_feature_property('notify', 'from') if receiver_email is None: - receiver_email = PCWConfig().get_feature_property('notify', 'to') - email = '''\ - Subject: [Openqa-Cloud-Watch] {subject} - From: {_from} - To: {_to} - - {message} - '''.format(subject=subject, _from=sender_email, _to=receiver_email, message=message) + receiver_email = PCWConfig.get_feature_property('notify', 'to') + mimetext = MIMEText(message) + mimetext['Subject'] = '[Openqa-Cloud-Watch] {}'.format(subject) + mimetext['From'] = sender_email + mimetext['To'] = receiver_email logger.info("Send Email To:'%s' Subject:'[Openqa-Cloud-Watch] %s'", receiver_email, subject) server = smtplib.SMTP(smtp_server, port) server.ehlo() - server.sendmail(sender_email, receiver_email.split(','), email) + server.sendmail(sender_email, receiver_email.split(','), mimetext.as_string()) diff --git a/ocw/lib/provider.py b/ocw/lib/provider.py index af004867..907ecd20 100644 --- a/ocw/lib/provider.py +++ b/ocw/lib/provider.py @@ -11,7 +11,7 @@ class Provider: def __init__(self, namespace: str): self._namespace = namespace - self.dry_run = PCWConfig().getBoolean('default/dry_run') + self.dry_run = PCWConfig.getBoolean('default/dry_run') self.logger = logging.getLogger(self.__module__) def older_than_min_age(self, age): diff --git a/ocw/lib/vault.py b/ocw/lib/vault.py index e5b89446..a2910bc9 100644 --- a/ocw/lib/vault.py +++ b/ocw/lib/vault.py @@ -18,11 +18,11 @@ class Vault: extra_time = 600 def __init__(self, vault_namespace): - self.url = PCWConfig().get_feature_property('vault', 'url') - self.user = PCWConfig().get_feature_property('vault', 'user') + self.url = PCWConfig.get_feature_property('vault', 'url') + self.user = PCWConfig.get_feature_property('vault', 'user') self.namespace = vault_namespace - self.password = PCWConfig().get_feature_property('vault', 'password') - self.certificate_dir = PCWConfig().get_feature_property('vault', 'cert_dir') + self.password = PCWConfig.get_feature_property('vault', 'password') + self.certificate_dir = PCWConfig.get_feature_property('vault', 'cert_dir') if PCWConfig.getBoolean('vault/use-file-cache') and self._getAuthCacheFile().exists(): logger.info('Loading cached credentials') self.auth_json = self.loadAuthCache() @@ -102,7 +102,7 @@ def getCredentials(self): raise NotImplementedError def getData(self, name=None): - use_file_cache = PCWConfig().getBoolean('vault/use-file-cache') + use_file_cache = PCWConfig.getBoolean('vault/use-file-cache') if self.auth_json is None and use_file_cache: self.auth_json = self.loadAuthCache() if self.isExpired(): @@ -129,7 +129,7 @@ def isExpired(self): return expire < datetime.today() + timedelta(seconds=self.extra_time) def renew(self): - if PCWConfig().getBoolean('vault/use-file-cache') and self._getAuthCacheFile().exists(): + if PCWConfig.getBoolean('vault/use-file-cache') and self._getAuthCacheFile().exists(): self._getAuthCacheFile().unlink() self.revoke() self.getData() diff --git a/tests/test_ec2.py b/tests/test_ec2.py index 087f3bfa..e061fff2 100644 --- a/tests/test_ec2.py +++ b/tests/test_ec2.py @@ -4,6 +4,7 @@ from tests.generators import min_image_age_hours, max_image_age_hours from datetime import datetime, timezone, timedelta from botocore.exceptions import ClientError +import smtplib def test_parse_image_name(monkeypatch): @@ -191,3 +192,39 @@ def delete_volume(VolumeId): assert len(deleted_volumes) == 1 assert deleted_volumes[0] == volumeid_to_delete + +def test_cleanup_uploader_vpc(monkeypatch): + def mocked_ec2_client(): + pass + + def mocked_get_boolean(config_path, field=None): + if config_path == 'default/dry_run': + return False + else: + return True + + cnt = 0 + + def mocked_send_email(subject, body): + global cnt + cnt += 1 + + response = { + 'Vpcs': [ + {'VpcId': 'someId', 'OwnerId': 'someId'} + ] + } + monkeypatch.setattr(EC2, 'check_credentials', lambda *args, **kwargs: True) + monkeypatch.setattr(EC2, 'ec2_client', lambda self, region: mocked_ec2_client) + monkeypatch.setattr(EC2, 'get_all_regions', lambda self: ['eu-central']) + monkeypatch.setattr(EC2, 'delete_vpc', lambda self, region, vpcId: False) + monkeypatch.setattr(PCWConfig, 'getBoolean', mocked_get_boolean) + monkeypatch.setattr(PCWConfig, 'get_feature_property', lambda section, field: -1) + monkeypatch.setattr(smtplib.SMTP, 'sendmail', mocked_send_email) + + mocked_ec2_client.describe_vpcs = lambda Filters: response + + ec2 = EC2('fake') + ec2.cleanup_uploader_vpcs() + + assert cnt == 1 diff --git a/tests/test_pcwconfig.py b/tests/test_pcwconfig.py index 4a7f9344..c152f445 100644 --- a/tests/test_pcwconfig.py +++ b/tests/test_pcwconfig.py @@ -99,3 +99,34 @@ def test_get_providers_for_existed_feature(pcw_file): """) providers = PCWConfig.get_providers_for('providerfeature', 'fake') assert not {'azure'} ^ set(providers) + +def test_getBoolean_notdefined(pcw_file): + assert not PCWConfig.getBoolean('feature/bool_property') + +def test_getBoolean_notdefined_namespace(pcw_file): + assert not PCWConfig.getBoolean('feature/bool_property','random_namespace') + +def test_getBoolean_defined(pcw_file): + set_pcw_ini(pcw_file, """ + [feature] + bool_property = True + """) + assert PCWConfig.getBoolean('feature/bool_property') + +def test_getBoolean_defined_namespace(pcw_file): + set_pcw_ini(pcw_file, """ + [feature] + bool_property = False + [feature.namespace.random_namespace] + bool_property = True + """) + assert PCWConfig.getBoolean('feature/bool_property','random_namespace') + +def test_getBoolean_namespace_but_not_defined(pcw_file): + set_pcw_ini(pcw_file, """ + [feature] + bool_property = True + [feature.namespace.random_namespace] + providers = azure + """) + assert PCWConfig.getBoolean('feature/bool_property','random_namespace') diff --git a/webui/settings.py b/webui/settings.py index 2368d4ea..1969898f 100644 --- a/webui/settings.py +++ b/webui/settings.py @@ -259,8 +259,16 @@ def has(setting: str) -> bool: return False @staticmethod - def getBoolean(config_path: str, default=False) -> bool: - value = ConfigFile().get(config_path, default) + def getBoolean(config_path: str, namespace: str = None, default=False) -> bool: + if namespace: + (feature, property) = config_path.split('/') + setting = '{}.namespace.{}/{}'.format(feature, namespace, property) + if PCWConfig.has(setting): + value = ConfigFile().get(setting) + else: + value = ConfigFile().get(config_path, default) + else: + value = ConfigFile().get(config_path, default) if isinstance(value, bool): return value return bool(re.match("^(true|on|1|yes)$", str(value), flags=re.IGNORECASE))