ec2: add cleanup for volumes

ocw/lib/EC2.py

@@ -117,6 +117,32 @@ def cleanup_snapshots(self):
                         else:
                             raise ex
 
+    def cleanup_volumes(self):
+        cleanup_ec2_max_volumes_age_days = PCWConfig.get_feature_property('cleanup', 'ec2-max-volumes-age-days',
+                                                                          self._namespace)
+        if cleanup_ec2_max_volumes_age_days < 0:
+            return
+        delete_older_than = date.today() - timedelta(days=cleanup_ec2_max_volumes_age_days)
+        for region in self.all_regions:
+            response = self.ec2_client(region).describe_volumes()
+            for volume in response['Volumes']:
+                if datetime.date(volume['CreateTime']) < delete_older_than:
+                    self.log_info("Deleting volume {} in region {} with CreateTime={}", volume['VolumeId'],
+                                  region, volume['CreateTime'])
+                    if self.volume_protected(volume):
+                        self.log_info('Volume {} has tag DO_NOT_DELETE so protected from deletion', volume['VolumeId'])
+                    elif self.dry_run:
+                        self.log_info("Volume deletion of {} skipped due to dry run mode", volume['VolumeId'])
+                    else:
+                        self.ec2_client(region).delete_volume(VolumeId=volume['VolumeId'])
+
+    def volume_protected(self, volume):
+        if 'Tags' in volume:
+            for tag in volume['Tags']:
+                if tag['Key'] == 'DO_NOT_DELETE':
+                    return True
+        return False
+
     def list_instances(self, region):
         return [i for i in self.ec2_resource(region).instances.all()]
 
@@ -189,6 +215,7 @@ def parse_image_name(self, img_name):
     def cleanup_all(self):
         self.cleanup_images()
         self.cleanup_snapshots()
+        self.cleanup_volumes()
 
     def cleanup_images(self):
         for region in self.all_regions:

requirements_test.txt

@@ -4,6 +4,5 @@ pytest
 pytest-django
 faker
 flake8
-pytest
 pytest-cov==2.5.0
 codecov

setup.py

@@ -1,2 +1,3 @@
 import setuptools
-setuptools.setup(name='pcw', version='1.0')
+
+setuptools.setup(name='pcw', version='1.0')

tests/test_ec2.py

@@ -145,16 +145,49 @@ def delete_snapshot(SnapshotId):
 
     def delete_snapshot_raise_error(SnapshotId):
         error_response = {'Error': {'Code': 'InvalidSnapshot.InUse','Message': 'Message'}}
-        raise ClientError(error_response=error_response,operation_name='delete_snapshot')
+        raise ClientError(error_response=error_response, operation_name='delete_snapshot')
 
     response = {
-        'Snapshots': [{'SnapshotId': snapshotid_i_have_ami,'StartTime': datetime.now()}]
-        }
+        'Snapshots': [{'SnapshotId': snapshotid_i_have_ami, 'StartTime': datetime.now()}]
+    }
     mocked_ec2_client.describe_snapshots = lambda OwnerIds: response
     mocked_ec2_client.delete_snapshot = delete_snapshot_raise_error
 
     ec2.cleanup_snapshots()
     assert snapshotid_i_have_ami in ec2_snapshots
 
 
+def test_cleanup_volumes(monkeypatch):
+    def mocked_ec2_client():
+        pass
+
+    monkeypatch.setattr(EC2, 'check_credentials', lambda *args, **kwargs: True)
+    monkeypatch.setattr(EC2, 'ec2_client', lambda self, region: mocked_ec2_client)
+    monkeypatch.setattr(EC2, 'get_all_regions', lambda self: ['eu-central'])
+    monkeypatch.setattr(PCWConfig, 'get_feature_property', lambda self, section, field: -1)
+    volumeid_to_delete = 'delete_me'
+
+    deleted_volumes = []
+
+    def delete_volume(VolumeId):
+        deleted_volumes.append(VolumeId)
+
+    response = {
+        'Volumes': [{'VolumeId': volumeid_to_delete, 'CreateTime': datetime.now(timezone.utc) - timedelta(days=6)},
+                    {'VolumeId': 'too_young_to_die', 'CreateTime': datetime.now(timezone.utc) - timedelta(days=2)},
+                    {'VolumeId': volumeid_to_delete, 'CreateTime': datetime.now(timezone.utc) - timedelta(days=6),
+                     'Tags': [{'Key': 'DO_NOT_DELETE', 'Value': '1'}]}, ]
+    }
+    mocked_ec2_client.describe_volumes = lambda *args, **kwargs: response
+    mocked_ec2_client.delete_volume = delete_volume
+    ec2 = EC2('fake')
+    ec2.cleanup_volumes()
+
+    assert len(deleted_volumes) == 0
+
+    monkeypatch.setattr(PCWConfig, 'get_feature_property', lambda self, section, field: 5)
+
+    ec2.cleanup_volumes()
 
+    assert len(deleted_volumes) == 1
+    assert deleted_volumes[0] == volumeid_to_delete

webui/settings.py

@@ -217,6 +217,7 @@ def get_feature_property(feature: str, property: str, namespace: str = None):
             'cleanup/azure-storage-resourcegroup': {'default': 'openqa-upload', 'return_type': str},
             'cleanup/azure-storage-account-name': {'default': 'openqa', 'return_type': str},
             'cleanup/ec2-max-snapshot-age-days': {'default': -1, 'return_type': int},
+            'cleanup/ec2-max-volumes-age-days': {'default': -1, 'return_type': int},
             'notify/to': {'default': None, 'return_type': str},
             'notify/age-hours': {'default': 12, 'return_type': int},
             'cluster.notify/to': {'default': None, 'return_type': str},