diff --git a/ocw/lib/EC2.py b/ocw/lib/EC2.py index 820df6ff..631ca90c 100644 --- a/ocw/lib/EC2.py +++ b/ocw/lib/EC2.py @@ -6,6 +6,8 @@ from botocore.exceptions import ClientError import re from datetime import date, datetime, timedelta +from ocw.lib.emailnotify import send_mail +import traceback import time @@ -216,6 +218,121 @@ def cleanup_all(self): self.cleanup_images() self.cleanup_snapshots() self.cleanup_volumes() + self.cleanup_uploader_vpcs() + + def delete_vpc(self, region, vpcId): + if PCWConfig.getBoolean('cleanup/vpc-notify-only', self._namespace): + return False + try: + vpc = self.ec2_resource(region).Vpc(vpcId) + for subnet in vpc.subnets.all(): + if len(list(subnet.instances.all())): + self.log_warn('{} has associated instance(s) so can not be deleted', vpcId) + return False + self.log_info('{} has no associated instances. Initializing cleanup of it', vpcId) + + self.delete_internet_gw(vpc) + self.delete_routing_tables(vpc) + self.delete_vpc_endpoints(region, vpcId) + self.delete_security_groups(vpc) + self.delete_vpc_peering_connections(region, vpcId) + self.delete_network_acls(vpc) + self.delete_vpc_subnets(vpc) + if self.dry_run: + self.log_info('Deletion of VPC skipped due to dry_run mode') + else: + # finally, delete the vpc + self.ec2_resource(region).meta.client.delete_vpc(VpcId=vpcId) + except Exception as e: + self.log_err("{} on VPC deletion. {}", type(e).__name__, traceback.format_exc()) + send_mail('{} on VPC deletion in [{}]'.format(type(e).__name__, self._namespace), traceback.format_exc()) + return False + return True + + def delete_vpc_subnets(self, vpc): + for subnet in vpc.subnets.all(): + for interface in subnet.network_interfaces.all(): + if self.dry_run: + self.log_info('Deletion of {} skipped due to dry_run mode', interface) + else: + self.log_info('Deleting {}', interface) + interface.delete() + if self.dry_run: + self.log_info('Deletion of {} skipped due to dry_run mode', subnet) + else: + self.log_info('Deleting {}', subnet) + subnet.delete() + + def delete_network_acls(self, vpc): + for netacl in vpc.network_acls.all(): + if not netacl.is_default: + if self.dry_run: + self.log_info('Deletion of {} skipped due to dry_run mode', netacl) + else: + self.log_info('Deleting {}', netacl) + netacl.delete() + + def delete_vpc_peering_connections(self, region, vpcId): + response = self.ec2_client(region).describe_vpc_peering_connections( + Filters=[{'Name': 'requester-vpc-info.vpc-id', 'Values': [vpcId]}]) + for vpcpeer in response['VpcPeeringConnections']: + vpcpeer_connection = self.ec2_resource(region).VpcPeeringConnection(vpcpeer['VpcPeeringConnectionId']) + if self.dry_run: + self.log_info('Deletion of {} skipped due to dry_run mode', vpcpeer_connection) + else: + self.log_info('Deleting {}', vpcpeer_connection) + vpcpeer_connection.delete() + + def delete_security_groups(self, vpc): + for sg in vpc.security_groups.all(): + if sg.group_name != 'default': + if self.dry_run: + self.log_info('Deletion of {} skipped due to dry_run mode', sg) + else: + self.log_info('Deleting {}', sg) + sg.delete() + + def delete_vpc_endpoints(self, region, vpcId): + response = self.ec2_client(region).describe_vpc_endpoints(Filters=[{'Name': 'vpc-id', 'Values': [vpcId]}]) + for ep in response['VpcEndpoints']: + if self.dry_run: + self.log_info('Deletion of {} skipped due to dry_run mode', ep) + else: + self.log_info('Deleting {}', ep) + self.ec2_client(region).delete_vpc_endpoints(VpcEndpointIds=[ep['VpcEndpointId']]) + + def delete_routing_tables(self, vpc): + for rt in vpc.route_tables.all(): + # we can not delete main RouteTable's , not main one don't have associations_attributes + if len(rt.associations_attribute) == 0: + if self.dry_run: + self.log_info('{} will be not deleted due to dry_run mode', rt) + else: + self.log_info('Deleting {}', rt) + rt.delete() + + def delete_internet_gw(self, vpc): + for gw in vpc.internet_gateways.all(): + if self.dry_run: + self.log_info('{} will be not deleted due to dry_run mode', gw) + else: + self.log_info('Deleting {}', gw) + vpc.detach_internet_gateway(InternetGatewayId=gw.id) + gw.delete() + + def cleanup_uploader_vpcs(self): + if PCWConfig.getBoolean('cleanup/vpc_cleanup', self._namespace): + for region in self.all_regions: + response = self.ec2_client(region).describe_vpcs( + Filters=[{'Name': 'isDefault', 'Values': ['false']}, + {'Name': 'tag:Name', 'Values': ['uploader-*']}]) + for vpc in response['Vpcs']: + self.log_info('{} in {} looks like uploader leftover. (OwnerId={}).', vpc['VpcId'], region, + vpc['OwnerId']) + if not self.delete_vpc(region, vpc['VpcId']) and not self.dry_run: + send_mail('VPC deletion locked by running VMs', + 'Uploader leftover {} (OwnerId={}) in {} is locked'.format(vpc['VpcId'], + vpc['OwnerId'], region)) def cleanup_images(self): for region in self.all_regions: diff --git a/ocw/lib/emailnotify.py b/ocw/lib/emailnotify.py index 9e922241..bfca5bcc 100644 --- a/ocw/lib/emailnotify.py +++ b/ocw/lib/emailnotify.py @@ -7,6 +7,7 @@ import json import smtplib import logging +from email.mime.text import MIMEText logger = logging.getLogger(__name__) @@ -33,7 +34,7 @@ def draw_instance_table(objects): def send_leftover_notification(): - if PCWConfig().has('notify'): + if PCWConfig.has('notify'): o = Instance.objects o = o.filter(active=True, csp_info__icontains='openqa_created_by', age__gt=timedelta(hours=PCWConfig.get_feature_property('notify', 'age-hours'))) @@ -59,20 +60,17 @@ def send_cluster_notification(namespace, clusters): def send_mail(subject, message, receiver_email=None): - if PCWConfig().has('notify'): - smtp_server = PCWConfig().get_feature_property('notify', 'smtp') - port = PCWConfig().get_feature_property('notify', 'smtp-port') - sender_email = PCWConfig().get_feature_property('notify', 'from') + if PCWConfig.has('notify'): + smtp_server = PCWConfig.get_feature_property('notify', 'smtp') + port = PCWConfig.get_feature_property('notify', 'smtp-port') + sender_email = PCWConfig.get_feature_property('notify', 'from') if receiver_email is None: - receiver_email = PCWConfig().get_feature_property('notify', 'to') - email = '''\ - Subject: [Openqa-Cloud-Watch] {subject} - From: {_from} - To: {_to} - - {message} - '''.format(subject=subject, _from=sender_email, _to=receiver_email, message=message) + receiver_email = PCWConfig.get_feature_property('notify', 'to') + mimetext = MIMEText(message) + mimetext['Subject'] = '[Openqa-Cloud-Watch] {}'.format(subject) + mimetext['From'] = sender_email + mimetext['To'] = receiver_email logger.info("Send Email To:'%s' Subject:'[Openqa-Cloud-Watch] %s'", receiver_email, subject) server = smtplib.SMTP(smtp_server, port) server.ehlo() - server.sendmail(sender_email, receiver_email.split(','), email) + server.sendmail(sender_email, receiver_email.split(','), mimetext.as_string()) diff --git a/ocw/lib/provider.py b/ocw/lib/provider.py index af004867..907ecd20 100644 --- a/ocw/lib/provider.py +++ b/ocw/lib/provider.py @@ -11,7 +11,7 @@ class Provider: def __init__(self, namespace: str): self._namespace = namespace - self.dry_run = PCWConfig().getBoolean('default/dry_run') + self.dry_run = PCWConfig.getBoolean('default/dry_run') self.logger = logging.getLogger(self.__module__) def older_than_min_age(self, age): diff --git a/ocw/lib/vault.py b/ocw/lib/vault.py index e5b89446..a2910bc9 100644 --- a/ocw/lib/vault.py +++ b/ocw/lib/vault.py @@ -18,11 +18,11 @@ class Vault: extra_time = 600 def __init__(self, vault_namespace): - self.url = PCWConfig().get_feature_property('vault', 'url') - self.user = PCWConfig().get_feature_property('vault', 'user') + self.url = PCWConfig.get_feature_property('vault', 'url') + self.user = PCWConfig.get_feature_property('vault', 'user') self.namespace = vault_namespace - self.password = PCWConfig().get_feature_property('vault', 'password') - self.certificate_dir = PCWConfig().get_feature_property('vault', 'cert_dir') + self.password = PCWConfig.get_feature_property('vault', 'password') + self.certificate_dir = PCWConfig.get_feature_property('vault', 'cert_dir') if PCWConfig.getBoolean('vault/use-file-cache') and self._getAuthCacheFile().exists(): logger.info('Loading cached credentials') self.auth_json = self.loadAuthCache() @@ -102,7 +102,7 @@ def getCredentials(self): raise NotImplementedError def getData(self, name=None): - use_file_cache = PCWConfig().getBoolean('vault/use-file-cache') + use_file_cache = PCWConfig.getBoolean('vault/use-file-cache') if self.auth_json is None and use_file_cache: self.auth_json = self.loadAuthCache() if self.isExpired(): @@ -129,7 +129,7 @@ def isExpired(self): return expire < datetime.today() + timedelta(seconds=self.extra_time) def renew(self): - if PCWConfig().getBoolean('vault/use-file-cache') and self._getAuthCacheFile().exists(): + if PCWConfig.getBoolean('vault/use-file-cache') and self._getAuthCacheFile().exists(): self._getAuthCacheFile().unlink() self.revoke() self.getData() diff --git a/tests/generators.py b/tests/generators.py index ad1687b7..da1c66a7 100644 --- a/tests/generators.py +++ b/tests/generators.py @@ -7,6 +7,7 @@ max_image_age_hours = 20 azure_storage_resourcegroup = 'openqa' ec2_max_snapshot_age_days = 1 +ec2_max_volumes_age_days = 5 class MockImage: @@ -15,7 +16,7 @@ def __init__(self, name, last_modified=None): self.last_modified = last_modified -def mock_get_feature_property(feature: str, property: str, namespace: str): +def mock_get_feature_property(feature: str, property: str, namespace: str = None): if property == 'min-image-age-hours': return min_image_age_hours elif property == 'max-images-per-flavor': @@ -26,6 +27,8 @@ def mock_get_feature_property(feature: str, property: str, namespace: str): return azure_storage_resourcegroup elif property == 'ec2-max-snapshot-age-days': return ec2_max_snapshot_age_days + elif property == 'ec2-max-volumes-age-days': + return ec2_max_volumes_age_days class ec2_meta_mock: diff --git a/tests/test_ec2.py b/tests/test_ec2.py index 087f3bfa..91d0e675 100644 --- a/tests/test_ec2.py +++ b/tests/test_ec2.py @@ -1,86 +1,269 @@ from ocw.lib.EC2 import EC2 from webui.settings import PCWConfig from tests.generators import mock_get_feature_property -from tests.generators import min_image_age_hours, max_image_age_hours +from tests.generators import min_image_age_hours, max_image_age_hours, ec2_max_volumes_age_days from datetime import datetime, timezone, timedelta from botocore.exceptions import ClientError +import pytest +older_then_min_age = (datetime.now(timezone.utc) - timedelta(hours=min_image_age_hours + 1)).isoformat() +# used by test_delete_vpc_deleting_everything test. Needs to be global due to use in ec2_patch fixture +delete_vpc_calls_stack = [] + + +# fixture setting up a commonly used mocks +@pytest.fixture +def ec2_patch(monkeypatch): + def mocked_ec2_resource(): + pass + + def mocked_vpc(vpcId): + return MockedVpc(vpcId) + + # used only in test_delete_vpc_deleting_everything. needs to be here because it is single place where we + # mocking mocking ec2_resource + def mocked_boto3_delete_vpc(VpcId): + delete_vpc_calls_stack.append('boto3_delete_vpc') + + # only reason we need it is to mock long call chain + def mocked_meta(): + pass + + # only reason we need it is to mock long call chain + def mocked_client(): + pass -def test_parse_image_name(monkeypatch): monkeypatch.setattr(EC2, 'check_credentials', lambda *args, **kwargs: True) - monkeypatch.setattr(EC2, 'get_all_regions', lambda self:['region1','region2']) - monkeypatch.setattr(PCWConfig, 'get_feature_property', lambda *args, **kwargs: "FOOF") - ec2 = EC2('fake') - - assert ec2.parse_image_name('openqa-SLES12-SP5-EC2.x86_64-0.9.1-BYOS-Build1.55.raw.xz') == { - 'key': '12-SP5-EC2-BYOS-x86_64', - 'build': '0.9.1-1.55' - } - assert ec2.parse_image_name('openqa-SLES15-SP2.x86_64-0.9.3-EC2-HVM-Build1.10.raw.xz') == { - 'key': '15-SP2-EC2-HVM-x86_64', - 'build': '0.9.3-1.10' - } - assert ec2.parse_image_name('openqa-SLES15-SP2-BYOS.x86_64-0.9.3-EC2-HVM-Build1.10.raw.xz') == { - 'key': '15-SP2-EC2-HVM-BYOS-x86_64', - 'build': '0.9.3-1.10' - } - - assert ec2.parse_image_name('openqa-SLES15-SP2.aarch64-0.9.3-EC2-HVM-Build1.49.raw.xz') == { - 'key': '15-SP2-EC2-HVM-aarch64', - 'build': '0.9.3-1.49' - } - - assert ec2.parse_image_name('openqa-SLES12-SP4-EC2-HVM-BYOS.x86_64-0.9.2-Build2.56.raw.xz') == { - 'key': '12-SP4-EC2-HVM-BYOS-x86_64', - 'build': '0.9.2-2.56' - } - - assert ec2.parse_image_name('openqa-SLES15-SP2-CHOST-BYOS.x86_64-0.9.3-EC2-Build1.11.raw.xz') == { - 'key': '15-SP2-EC2-CHOST-BYOS-x86_64', - 'build': '0.9.3-1.11' - } - - assert ec2.parse_image_name('do not match') is None - - -def test_cleanup_images(monkeypatch): - newer_then_min_age = datetime.now(timezone.utc).isoformat() - older_then_min_age = (datetime.now(timezone.utc) - timedelta(hours=min_image_age_hours+1)).isoformat() - older_then_max_age = (datetime.now(timezone.utc) - timedelta(hours=max_image_age_hours+1)).isoformat() - - response = { - 'Images': [ - {'Name': 'SomeThingElse', - 'CreationDate': older_then_max_age, 'ImageId': 0}, - {'Name': 'openqa-SLES15-SP2-BYOS.x86_64-0.9.3-EC2-HVM-Build1.10.raw.xz', - 'CreationDate': newer_then_min_age, 'ImageId': 1}, - {'Name': 'openqa-SLES15-SP2-BYOS.x86_64-0.9.3-EC2-HVM-Build1.11.raw.xz', - 'CreationDate': older_then_min_age, 'ImageId': 2}, - {'Name': 'openqa-SLES15-SP2-BYOS.x86_64-0.9.3-EC2-HVM-Build1.12.raw.xz', - 'CreationDate': older_then_min_age, 'ImageId': 3}, - {'Name': 'openqa-SLES15-SP2-BYOS.x86_64-0.9.3-EC2-HVM-Build1.13.raw.xz', - 'CreationDate': older_then_max_age, 'ImageId': 4}, - ] - } + monkeypatch.setattr(EC2, 'get_all_regions', lambda self: ['region1']) + monkeypatch.setattr(PCWConfig, 'get_feature_property', mock_get_feature_property) + monkeypatch.setattr(EC2, 'ec2_client', lambda self, region: MockedEC2Client()) + monkeypatch.setattr(EC2, 'needs_to_delete_snapshot', lambda *args, **kwargs: True) + monkeypatch.setattr(EC2, 'ec2_resource', lambda self, region: mocked_ec2_resource) + + mocked_ec2_resource.Vpc = mocked_vpc + mocked_ec2_resource.meta = mocked_meta + mocked_ec2_resource.VpcPeeringConnection = lambda id: MockedVpcPeeringConnection() + mocked_meta.client = mocked_client + # don't mix up this with EC2.delete_vpc . this one is boto3 side of the call + mocked_client.delete_vpc = mocked_boto3_delete_vpc + return EC2('fake') + + +@pytest.fixture +def ec2_patch_for_vpc(ec2_patch, monkeypatch): + def mocked_get_boolean(config_path, field=None): + # all places where this fixture is called needs to have dry_run=False + # most of tests except test_delete_vpc_exception_swallow does not care about vpc-notify-only + # because delete_vpc ( only place where it currently used ) is mocked + return config_path not in ['default/dry_run', 'cleanup/vpc-notify-only'] + + # needs within emailnotify.send_mail + def mocked_has(config_path): + return config_path == 'notify' + + def mock_local_get_feature_property(feature: str, property: str, namespace: str = None): + # within emailnotify.send_mail we needs sane strings + if feature == 'notify' and property in ('to', 'from'): + return 'email' + else: + return -1 + + MockedEC2Client.response = { + 'Vpcs': [ + {'VpcId': 'someId', 'OwnerId': 'someId'} + ] + } + monkeypatch.setattr(PCWConfig, 'getBoolean', mocked_get_boolean) + monkeypatch.setattr(PCWConfig, 'has', mocked_has) + monkeypatch.setattr(PCWConfig, 'get_feature_property', mock_local_get_feature_property) + monkeypatch.setattr('smtplib.SMTP', lambda arg1, arg2: MockedSMTP()) + return EC2('fake') + + +class MockedEC2Client(): + response = {} deleted_images = list() + deleted_volumes = list() + snapshotid_to_delete = 'delete_me' + volumeid_to_delete = 'delete_me' + snapshotid_i_have_ami = 'you_can_not_delete_me' + delete_snapshot_raise_error = False + delete_vpc_endpoints_called = False + + ec2_snapshots = {snapshotid_to_delete: 'snapshot', snapshotid_i_have_ami: 'snapshot'} + + def describe_images(self, *args, **kwargs): + return MockedEC2Client.response + + def describe_snapshots(self, *args, **kwargs): + return MockedEC2Client.response + + def deregister_image(self, *args, **kwargs): + MockedEC2Client.deleted_images.append(kwargs['ImageId']) + + def delete_snapshot(self, SnapshotId): + if MockedEC2Client.delete_snapshot_raise_error: + error_response = {'Error': {'Code': 'InvalidSnapshot.InUse', 'Message': 'Message'}} + raise ClientError(error_response=error_response, operation_name='delete_snapshot') + else: + MockedEC2Client.ec2_snapshots.pop(SnapshotId, None) + + def delete_volume(self, VolumeId): + MockedEC2Client.deleted_volumes.append(VolumeId) + + def describe_volumes(self, *args, **kwargs): + return MockedEC2Client.response + + def describe_vpcs(self, Filters): + return MockedEC2Client.response + + def describe_vpc_endpoints(self, Filters): + return MockedEC2Client.response + + def delete_vpc_endpoints(self, VpcEndpointIds): + MockedEC2Client.delete_vpc_endpoints_called = True + + def describe_vpc_peering_connections(self, Filters): + return MockedEC2Client.response - def mocked_ec2_client(): + +class MockedSMTP: + smtp_calls = 0 + + def ehlo(self): pass - mocked_ec2_client.describe_images = lambda *args, **kwargs: response - mocked_ec2_client.deregister_image = lambda *args, **kwargs: deleted_images.append(kwargs['ImageId']) - monkeypatch.setattr(PCWConfig, 'get_feature_property', mock_get_feature_property) - monkeypatch.setattr(EC2, 'check_credentials', lambda *args, **kwargs: True) - monkeypatch.setattr(EC2, 'get_all_regions', lambda self:['region1']) - monkeypatch.setattr(EC2, 'ec2_client', lambda self, region: mocked_ec2_client) + def sendmail(self, sender_email, receiver_email, mimetext): + MockedSMTP.smtp_calls += 1 - ec2 = EC2('fake') - ec2.cleanup_images() - assert deleted_images == [2, 3, 4] +class MockedInstances: + is_empty = False - deleted_images = list() - response = { + def all(self): + if MockedInstances.is_empty: + return [] + else: + return ['subnet'] + + +class MockedInterface: + delete_called = False + + def delete(self): + MockedInterface.delete_called = True + + +class MockedNetworkInterfaces: + def all(self): + return [MockedInterface()] + + +# this mock is used to replace several totally different classes +# but they all have common ground -> delete method +class MockedCollectionItem: + delete_called = 0 + + def __init__(self): + # for mocking subnets while checking for running instances + self.instances = MockedInstances() + # for mocking subnets while deleting vpc_endpoints + self.network_interfaces = MockedNetworkInterfaces() + # for mocking internet gateways + self.id = 'id' + # for mocking routing tables + self.associations_attribute = [] + # for mocking security_groups and also name should not be equal 'default' + self.group_name = 'NOT_default' + # for mocking network_acls should be False + self.is_default = False + + def delete(self): + MockedCollectionItem.delete_called += 1 + + +class MockedCollectionWithAllMethod: + + def all(self): + return [MockedCollectionItem()] + + +class MockedVpc: + cnt_calls = 0 + detach_internet_gateway_called = 0 + + def __init__(self, vpcId): + MockedVpc.cnt_calls += 1 + self.subnets = MockedCollectionWithAllMethod() + self.internet_gateways = MockedCollectionWithAllMethod() + self.route_tables = MockedCollectionWithAllMethod() + self.security_groups = MockedCollectionWithAllMethod() + self.network_acls = MockedCollectionWithAllMethod() + + def detach_internet_gateway(self, InternetGatewayId): + MockedVpc.detach_internet_gateway_called = 1 + + +class MockedVpcPeeringConnection: + delete_called = False + + def delete(self): + MockedVpcPeeringConnection.delete_called = True + + +def test_parse_image_name(ec2_patch): + assert ec2_patch.parse_image_name('openqa-SLES12-SP5-EC2.x86_64-0.9.1-BYOS-Build1.55.raw.xz') == { + 'key': '12-SP5-EC2-BYOS-x86_64', + 'build': '0.9.1-1.55' + } + assert ec2_patch.parse_image_name('openqa-SLES15-SP2.x86_64-0.9.3-EC2-HVM-Build1.10.raw.xz') == { + 'key': '15-SP2-EC2-HVM-x86_64', + 'build': '0.9.3-1.10' + } + assert ec2_patch.parse_image_name('openqa-SLES15-SP2-BYOS.x86_64-0.9.3-EC2-HVM-Build1.10.raw.xz') == { + 'key': '15-SP2-EC2-HVM-BYOS-x86_64', + 'build': '0.9.3-1.10' + } + assert ec2_patch.parse_image_name('openqa-SLES15-SP2.aarch64-0.9.3-EC2-HVM-Build1.49.raw.xz') == { + 'key': '15-SP2-EC2-HVM-aarch64', + 'build': '0.9.3-1.49' + } + assert ec2_patch.parse_image_name('openqa-SLES12-SP4-EC2-HVM-BYOS.x86_64-0.9.2-Build2.56.raw.xz') == { + 'key': '12-SP4-EC2-HVM-BYOS-x86_64', + 'build': '0.9.2-2.56' + } + assert ec2_patch.parse_image_name('openqa-SLES15-SP2-CHOST-BYOS.x86_64-0.9.3-EC2-Build1.11.raw.xz') == { + 'key': '15-SP2-EC2-CHOST-BYOS-x86_64', + 'build': '0.9.3-1.11' + } + assert ec2_patch.parse_image_name('do not match') is None + + +def test_cleanup_images_delete_due_time(ec2_patch): + newer_then_min_age = datetime.now(timezone.utc).isoformat() + older_then_max_age = (datetime.now(timezone.utc) - timedelta(hours=max_image_age_hours + 1)).isoformat() + MockedEC2Client.deleted_images = list() + MockedEC2Client.response = { + 'Images': [ + {'Name': 'SomeThingElse', + 'CreationDate': older_then_max_age, 'ImageId': 0}, + {'Name': 'openqa-SLES15-SP2-BYOS.x86_64-0.9.3-EC2-HVM-Build1.10.raw.xz', + 'CreationDate': newer_then_min_age, 'ImageId': 1}, + {'Name': 'openqa-SLES15-SP2-BYOS.x86_64-0.9.3-EC2-HVM-Build1.11.raw.xz', + 'CreationDate': older_then_min_age, 'ImageId': 2}, + {'Name': 'openqa-SLES15-SP2-BYOS.x86_64-0.9.3-EC2-HVM-Build1.12.raw.xz', + 'CreationDate': older_then_min_age, 'ImageId': 3}, + {'Name': 'openqa-SLES15-SP2-BYOS.x86_64-0.9.3-EC2-HVM-Build1.13.raw.xz', + 'CreationDate': older_then_max_age, 'ImageId': 4}, + ] + } + ec2_patch.cleanup_images() + assert MockedEC2Client.deleted_images == [2, 3, 4] + + +def test_cleanup_images_delete_due_quantity(ec2_patch): + MockedEC2Client.deleted_images = list() + MockedEC2Client.response = { 'Images': [ {'Name': 'openqa-SLES15-SP2-BYOS.x86_64-0.9.3-EC2-HVM-Build1.12.raw.xz', 'CreationDate': older_then_min_age, 'ImageId': 3}, @@ -88,8 +271,8 @@ def mocked_ec2_client(): 'CreationDate': older_then_min_age, 'ImageId': 4}, ] } - ec2.cleanup_images() - assert deleted_images == [3] + ec2_patch.cleanup_images() + assert MockedEC2Client.deleted_images == [3] def test_needs_to_delete_snapshot(): @@ -106,88 +289,173 @@ def test_needs_to_delete_snapshot(): assert not EC2.needs_to_delete_snapshot(not_old_enough, days_to_delete) assert not EC2.needs_to_delete_snapshot(wrong_description, days_to_delete) -def test_cleanup_snapshots(monkeypatch): - def mocked_ec2_client(): - pass - snapshotid_to_delete = 'delete_me' - snapshotid_i_have_ami = 'you_can_not_delete_me' +def test_cleanup_snapshots_no_cleanup_due_to_config(ec2_patch, monkeypatch): + monkeypatch.setattr(PCWConfig, 'get_feature_property', lambda *args, **kwargs: -1) + ec2_patch.cleanup_snapshots() + # deletion did not happened because cfgGet returned -1 ( setting not defined in pcw.ini ) + assert MockedEC2Client.snapshotid_to_delete in MockedEC2Client.ec2_snapshots - ec2_snapshots = {snapshotid_to_delete: 'snapshot', snapshotid_i_have_ami: 'snapshot'} - def delete_snapshot(SnapshotId): - ec2_snapshots.pop(SnapshotId, None) +def test_cleanup_snapshots_cleanup_check(ec2_patch): + MockedEC2Client.response = { + 'Snapshots': [{'SnapshotId': MockedEC2Client.snapshotid_to_delete, 'StartTime': datetime.now()}] + } + ec2_patch.cleanup_snapshots() + # snapshot was deleted because setting **is** defined so whole cleanup start actually working + assert MockedEC2Client.snapshotid_to_delete not in MockedEC2Client.ec2_snapshots - response = { - 'Snapshots': [{'SnapshotId': snapshotid_to_delete,'StartTime': datetime.now()}] - } - monkeypatch.setattr(PCWConfig, 'get_feature_property', lambda self, section, field: -1) - monkeypatch.setattr(EC2, 'check_credentials', lambda *args, **kwargs: True) - monkeypatch.setattr(EC2, 'needs_to_delete_snapshot', lambda *args, **kwargs: True) - monkeypatch.setattr(EC2, 'ec2_client', lambda self, region: mocked_ec2_client) - monkeypatch.setattr(EC2, 'get_all_regions', lambda self: ['eu-central']) +def test_cleanup_snapshots_have_ami(ec2_patch): + MockedEC2Client.response = { + 'Snapshots': [{'SnapshotId': MockedEC2Client.snapshotid_i_have_ami, 'StartTime': datetime.now()}] + } + MockedEC2Client.delete_snapshot_raise_error = True + ec2_patch.cleanup_snapshots() + assert MockedEC2Client.snapshotid_i_have_ami in MockedEC2Client.ec2_snapshots - mocked_ec2_client.describe_snapshots = lambda OwnerIds: response - mocked_ec2_client.delete_snapshot = delete_snapshot - ec2 = EC2('fake') - ec2.cleanup_snapshots() +def test_cleanup_volumes_no_cleanup_due_config(ec2_patch, monkeypatch): + monkeypatch.setattr(PCWConfig, 'get_feature_property', lambda *args, **kwargs: -1) + ec2_patch.cleanup_volumes() + assert len(MockedEC2Client.deleted_volumes) == 0 - # deletion did not happened because cfgGet returned -1 ( setting not defined in pcw.ini ) - assert snapshotid_to_delete in ec2_snapshots - monkeypatch.setattr(PCWConfig, 'get_feature_property', mock_get_feature_property) - ec2.cleanup_snapshots() +def test_cleanup_volumes_cleanupcheck(ec2_patch): + MockedEC2Client.response = { + 'Volumes': [{'VolumeId': MockedEC2Client.volumeid_to_delete, + 'CreateTime': datetime.now(timezone.utc) - timedelta(days=ec2_max_volumes_age_days + 1)}, + {'VolumeId': 'too_young_to_die', 'CreateTime': datetime.now(timezone.utc) - timedelta(days=2)}, + {'VolumeId': MockedEC2Client.volumeid_to_delete, + 'CreateTime': datetime.now(timezone.utc) - timedelta(days=ec2_max_volumes_age_days + 1), + 'Tags': [{'Key': 'DO_NOT_DELETE', 'Value': '1'}]}, ] + } + ec2_patch.cleanup_volumes() + assert len(MockedEC2Client.deleted_volumes) == 1 + assert MockedEC2Client.deleted_volumes[0] == MockedEC2Client.volumeid_to_delete - # snapshot was deleted because setting **is** defined so whole cleanup start actually working - assert snapshotid_to_delete not in ec2_snapshots - def delete_snapshot_raise_error(SnapshotId): - error_response = {'Error': {'Code': 'InvalidSnapshot.InUse','Message': 'Message'}} - raise ClientError(error_response=error_response, operation_name='delete_snapshot') +def test_cleanup_uploader_vpc_no_mail_sent_no_deletion(ec2_patch, monkeypatch): + monkeypatch.setattr(EC2, 'delete_vpc', lambda self, region, vpcId: True) + ec2_patch.cleanup_uploader_vpcs() + assert MockedSMTP.smtp_calls == 0 - response = { - 'Snapshots': [{'SnapshotId': snapshotid_i_have_ami, 'StartTime': datetime.now()}] - } - mocked_ec2_client.describe_snapshots = lambda OwnerIds: response - mocked_ec2_client.delete_snapshot = delete_snapshot_raise_error - ec2.cleanup_snapshots() - assert snapshotid_i_have_ami in ec2_snapshots +def test_cleanup_uploader_vpc_mail_sent_due_deletion(ec2_patch_for_vpc, monkeypatch): + monkeypatch.setattr(EC2, 'delete_vpc', lambda self, region, vpcId: False) + ec2_patch_for_vpc.cleanup_uploader_vpcs() + assert MockedSMTP.smtp_calls == 1 -def test_cleanup_volumes(monkeypatch): - def mocked_ec2_client(): - pass +def test_cleanup_uploader_vpc_no_mail_sent_due_dry_run(ec2_patch_for_vpc, monkeypatch): + monkeypatch.setattr(EC2, 'delete_vpc', lambda self, region, vpcId: True) + ec2_patch_for_vpc.dry_run = True + ec2_patch_for_vpc.cleanup_uploader_vpcs() + assert MockedSMTP.smtp_calls == 1 - monkeypatch.setattr(EC2, 'check_credentials', lambda *args, **kwargs: True) - monkeypatch.setattr(EC2, 'ec2_client', lambda self, region: mocked_ec2_client) - monkeypatch.setattr(EC2, 'get_all_regions', lambda self: ['eu-central']) - monkeypatch.setattr(PCWConfig, 'get_feature_property', lambda self, section, field: -1) - volumeid_to_delete = 'delete_me' - deleted_volumes = [] +def test_delete_vpc_no_delete_due_to_config(ec2_patch, monkeypatch): + def mocked_get_boolean(config_path, field=None): + return config_path == 'cleanup/vpc-notify-only' - def delete_volume(VolumeId): - deleted_volumes.append(VolumeId) + monkeypatch.setattr(PCWConfig, 'getBoolean', mocked_get_boolean) + # delete_vpc returns False because config force to skip it + assert not ec2_patch.delete_vpc('region', 'vpcId') + # we should not request VPC actually because config demanding to skip deletion + assert MockedVpc.cnt_calls == 0 - response = { - 'Volumes': [{'VolumeId': volumeid_to_delete, 'CreateTime': datetime.now(timezone.utc) - timedelta(days=6)}, - {'VolumeId': 'too_young_to_die', 'CreateTime': datetime.now(timezone.utc) - timedelta(days=2)}, - {'VolumeId': volumeid_to_delete, 'CreateTime': datetime.now(timezone.utc) - timedelta(days=6), - 'Tags': [{'Key': 'DO_NOT_DELETE', 'Value': '1'}]}, ] - } - mocked_ec2_client.describe_volumes = lambda *args, **kwargs: response - mocked_ec2_client.delete_volume = delete_volume - ec2 = EC2('fake') - ec2.cleanup_volumes() - assert len(deleted_volumes) == 0 +def test_delete_vpc_no_delete_due_instance_linked(ec2_patch): + # delete_vpc return False because vpc has associated running instances + assert not ec2_patch.delete_vpc('region', 'vpcId') + # nevertheless deletion is skipped we requested VPC object + assert MockedVpc.cnt_calls == 1 + + +def test_delete_vpc_deleting_everything(ec2_patch, monkeypatch): + def mocked_delete_internet_gw(arg1, arg2): + delete_vpc_calls_stack.append('delete_internet_gw') + + def mocked_delete_routing_tables(arg1, arg2): + delete_vpc_calls_stack.append('delete_routing_tables') + + def mocked_delete_vpc_endpoints(arg1, arg2, arg3): + delete_vpc_calls_stack.append('delete_vpc_endpoints') + + def mocked_delete_security_groups(arg1, arg2): + delete_vpc_calls_stack.append('delete_security_groups') + + def mocked_delete_vpc_peering_connections(arg1, arg2, arg3): + delete_vpc_calls_stack.append('delete_vpc_peering_connections') + + def mocked_delete_network_acls(arg1, arg2): + delete_vpc_calls_stack.append('delete_network_acls') + + def mocked_delete_vpc_subnets(arg1, arg2): + delete_vpc_calls_stack.append('delete_vpc_subnets') + + # emulated that there is no linked running instance to VPC which we trying to delete + + MockedInstances.is_empty = True + monkeypatch.setattr(EC2, 'delete_internet_gw', mocked_delete_internet_gw) + monkeypatch.setattr(EC2, 'delete_routing_tables', mocked_delete_routing_tables) + monkeypatch.setattr(EC2, 'delete_vpc_endpoints', mocked_delete_vpc_endpoints) + monkeypatch.setattr(EC2, 'delete_security_groups', mocked_delete_security_groups) + monkeypatch.setattr(EC2, 'delete_vpc_peering_connections', mocked_delete_vpc_peering_connections) + monkeypatch.setattr(EC2, 'delete_network_acls', mocked_delete_network_acls) + monkeypatch.setattr(EC2, 'delete_vpc_subnets', mocked_delete_vpc_subnets) + ec2_patch.delete_vpc('region', 'vpc') + + assert delete_vpc_calls_stack == ['delete_internet_gw', 'delete_routing_tables', 'delete_vpc_endpoints', + 'delete_security_groups', 'delete_vpc_peering_connections', + 'delete_network_acls', + 'delete_vpc_subnets', 'boto3_delete_vpc'] + + +def test_delete_vpc_exception_swallow(ec2_patch_for_vpc, monkeypatch): + def mocked_ec2_resource(self, region): + raise Exception + + monkeypatch.setattr(EC2, 'ec2_resource', mocked_ec2_resource) + # delete_vpc return False because vpc has associated running instances + assert not ec2_patch_for_vpc.delete_vpc('region', 'vpcId') + assert MockedSMTP.smtp_calls == 2 + + +def test_delete_internet_gw(ec2_patch): + ec2_patch.delete_internet_gw(MockedVpc('vpcId')) + assert MockedVpc.detach_internet_gateway_called == 1 + assert MockedCollectionItem.delete_called == 1 + + +def test_delete_routing_tables(ec2_patch): + ec2_patch.delete_routing_tables(MockedVpc('vpcId')) + assert MockedCollectionItem.delete_called == 2 + + +def test_delete_vpc_endpoints(ec2_patch): + MockedEC2Client.response = {'VpcEndpoints': [{'VpcEndpointId': 'id'}]} + ec2_patch.delete_vpc_endpoints('region', 'vpcId') + assert MockedEC2Client.delete_vpc_endpoints_called + + +def test_delete_security_groups(ec2_patch): + ec2_patch.delete_security_groups(MockedVpc('vpcId')) + assert MockedCollectionItem.delete_called == 3 + + +def test_delete_vpc_peering_connections(ec2_patch): + MockedEC2Client.response = {'VpcPeeringConnections': [{'VpcPeeringConnectionId': 'id'}]} + ec2_patch.delete_vpc_peering_connections('region', 'vpcId') + assert MockedVpcPeeringConnection.delete_called + - monkeypatch.setattr(PCWConfig, 'get_feature_property', lambda self, section, field: 5) +def test_delete_network_acls(ec2_patch): + ec2_patch.delete_network_acls(MockedVpc('vpcId')) + assert MockedCollectionItem.delete_called == 4 - ec2.cleanup_volumes() - assert len(deleted_volumes) == 1 - assert deleted_volumes[0] == volumeid_to_delete +def test_delete_vpc_subnets(ec2_patch): + ec2_patch.delete_vpc_subnets(MockedVpc('vpcId')) + assert MockedCollectionItem.delete_called == 5 + assert MockedInterface.delete_called diff --git a/tests/test_pcwconfig.py b/tests/test_pcwconfig.py index 4a7f9344..c152f445 100644 --- a/tests/test_pcwconfig.py +++ b/tests/test_pcwconfig.py @@ -99,3 +99,34 @@ def test_get_providers_for_existed_feature(pcw_file): """) providers = PCWConfig.get_providers_for('providerfeature', 'fake') assert not {'azure'} ^ set(providers) + +def test_getBoolean_notdefined(pcw_file): + assert not PCWConfig.getBoolean('feature/bool_property') + +def test_getBoolean_notdefined_namespace(pcw_file): + assert not PCWConfig.getBoolean('feature/bool_property','random_namespace') + +def test_getBoolean_defined(pcw_file): + set_pcw_ini(pcw_file, """ + [feature] + bool_property = True + """) + assert PCWConfig.getBoolean('feature/bool_property') + +def test_getBoolean_defined_namespace(pcw_file): + set_pcw_ini(pcw_file, """ + [feature] + bool_property = False + [feature.namespace.random_namespace] + bool_property = True + """) + assert PCWConfig.getBoolean('feature/bool_property','random_namespace') + +def test_getBoolean_namespace_but_not_defined(pcw_file): + set_pcw_ini(pcw_file, """ + [feature] + bool_property = True + [feature.namespace.random_namespace] + providers = azure + """) + assert PCWConfig.getBoolean('feature/bool_property','random_namespace') diff --git a/webui/settings.py b/webui/settings.py index 2368d4ea..1969898f 100644 --- a/webui/settings.py +++ b/webui/settings.py @@ -259,8 +259,16 @@ def has(setting: str) -> bool: return False @staticmethod - def getBoolean(config_path: str, default=False) -> bool: - value = ConfigFile().get(config_path, default) + def getBoolean(config_path: str, namespace: str = None, default=False) -> bool: + if namespace: + (feature, property) = config_path.split('/') + setting = '{}.namespace.{}/{}'.format(feature, namespace, property) + if PCWConfig.has(setting): + value = ConfigFile().get(setting) + else: + value = ConfigFile().get(config_path, default) + else: + value = ConfigFile().get(config_path, default) if isinstance(value, bool): return value return bool(re.match("^(true|on|1|yes)$", str(value), flags=re.IGNORECASE))