From 3c05bddb76cb2be0f0fc67315c5092884a4dfaa0 Mon Sep 17 00:00:00 2001 From: asmorodskyi Date: Mon, 18 Jan 2021 17:36:29 +0100 Subject: [PATCH] ec2: add cleanup for volumes --- ocw/lib/EC2.py | 27 +++++++++++++++++++++++++++ requirements_test.txt | 1 - setup.py | 3 ++- tests/test_ec2.py | 39 ++++++++++++++++++++++++++++++++++++--- webui/settings.py | 1 + 5 files changed, 66 insertions(+), 5 deletions(-) diff --git a/ocw/lib/EC2.py b/ocw/lib/EC2.py index 079433cb..820df6ff 100644 --- a/ocw/lib/EC2.py +++ b/ocw/lib/EC2.py @@ -117,6 +117,32 @@ def cleanup_snapshots(self): else: raise ex + def cleanup_volumes(self): + cleanup_ec2_max_volumes_age_days = PCWConfig.get_feature_property('cleanup', 'ec2-max-volumes-age-days', + self._namespace) + if cleanup_ec2_max_volumes_age_days < 0: + return + delete_older_than = date.today() - timedelta(days=cleanup_ec2_max_volumes_age_days) + for region in self.all_regions: + response = self.ec2_client(region).describe_volumes() + for volume in response['Volumes']: + if datetime.date(volume['CreateTime']) < delete_older_than: + self.log_info("Deleting volume {} in region {} with CreateTime={}", volume['VolumeId'], + region, volume['CreateTime']) + if self.volume_protected(volume): + self.log_info('Volume {} has tag DO_NOT_DELETE so protected from deletion', volume['VolumeId']) + elif self.dry_run: + self.log_info("Volume deletion of {} skipped due to dry run mode", volume['VolumeId']) + else: + self.ec2_client(region).delete_volume(VolumeId=volume['VolumeId']) + + def volume_protected(self, volume): + if 'Tags' in volume: + for tag in volume['Tags']: + if tag['Key'] == 'DO_NOT_DELETE': + return True + return False + def list_instances(self, region): return [i for i in self.ec2_resource(region).instances.all()] @@ -189,6 +215,7 @@ def parse_image_name(self, img_name): def cleanup_all(self): self.cleanup_images() self.cleanup_snapshots() + self.cleanup_volumes() def cleanup_images(self): for region in self.all_regions: diff --git a/requirements_test.txt b/requirements_test.txt index 6ec3ba9d..56e13303 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -4,6 +4,5 @@ pytest pytest-django faker flake8 -pytest pytest-cov==2.5.0 codecov diff --git a/setup.py b/setup.py index 8eac4555..0e3a1edd 100644 --- a/setup.py +++ b/setup.py @@ -1,2 +1,3 @@ import setuptools -setuptools.setup(name='pcw', version='1.0') \ No newline at end of file + +setuptools.setup(name='pcw', version='1.0') diff --git a/tests/test_ec2.py b/tests/test_ec2.py index 212f5d61..087f3bfa 100644 --- a/tests/test_ec2.py +++ b/tests/test_ec2.py @@ -145,11 +145,11 @@ def delete_snapshot(SnapshotId): def delete_snapshot_raise_error(SnapshotId): error_response = {'Error': {'Code': 'InvalidSnapshot.InUse','Message': 'Message'}} - raise ClientError(error_response=error_response,operation_name='delete_snapshot') + raise ClientError(error_response=error_response, operation_name='delete_snapshot') response = { - 'Snapshots': [{'SnapshotId': snapshotid_i_have_ami,'StartTime': datetime.now()}] - } + 'Snapshots': [{'SnapshotId': snapshotid_i_have_ami, 'StartTime': datetime.now()}] + } mocked_ec2_client.describe_snapshots = lambda OwnerIds: response mocked_ec2_client.delete_snapshot = delete_snapshot_raise_error @@ -157,4 +157,37 @@ def delete_snapshot_raise_error(SnapshotId): assert snapshotid_i_have_ami in ec2_snapshots +def test_cleanup_volumes(monkeypatch): + def mocked_ec2_client(): + pass + + monkeypatch.setattr(EC2, 'check_credentials', lambda *args, **kwargs: True) + monkeypatch.setattr(EC2, 'ec2_client', lambda self, region: mocked_ec2_client) + monkeypatch.setattr(EC2, 'get_all_regions', lambda self: ['eu-central']) + monkeypatch.setattr(PCWConfig, 'get_feature_property', lambda self, section, field: -1) + volumeid_to_delete = 'delete_me' + + deleted_volumes = [] + + def delete_volume(VolumeId): + deleted_volumes.append(VolumeId) + + response = { + 'Volumes': [{'VolumeId': volumeid_to_delete, 'CreateTime': datetime.now(timezone.utc) - timedelta(days=6)}, + {'VolumeId': 'too_young_to_die', 'CreateTime': datetime.now(timezone.utc) - timedelta(days=2)}, + {'VolumeId': volumeid_to_delete, 'CreateTime': datetime.now(timezone.utc) - timedelta(days=6), + 'Tags': [{'Key': 'DO_NOT_DELETE', 'Value': '1'}]}, ] + } + mocked_ec2_client.describe_volumes = lambda *args, **kwargs: response + mocked_ec2_client.delete_volume = delete_volume + ec2 = EC2('fake') + ec2.cleanup_volumes() + + assert len(deleted_volumes) == 0 + + monkeypatch.setattr(PCWConfig, 'get_feature_property', lambda self, section, field: 5) + + ec2.cleanup_volumes() + assert len(deleted_volumes) == 1 + assert deleted_volumes[0] == volumeid_to_delete diff --git a/webui/settings.py b/webui/settings.py index fc19c169..2368d4ea 100644 --- a/webui/settings.py +++ b/webui/settings.py @@ -217,6 +217,7 @@ def get_feature_property(feature: str, property: str, namespace: str = None): 'cleanup/azure-storage-resourcegroup': {'default': 'openqa-upload', 'return_type': str}, 'cleanup/azure-storage-account-name': {'default': 'openqa', 'return_type': str}, 'cleanup/ec2-max-snapshot-age-days': {'default': -1, 'return_type': int}, + 'cleanup/ec2-max-volumes-age-days': {'default': -1, 'return_type': int}, 'notify/to': {'default': None, 'return_type': str}, 'notify/age-hours': {'default': 12, 'return_type': int}, 'cluster.notify/to': {'default': None, 'return_type': str},