Merge pull request #88 from Ensembl/merge_master_june

Merge master june

ensembl_dbcopy/admin.py

@@ -16,7 +16,7 @@
 from django.contrib.admin import SimpleListFilter
 from django.core.paginator import Paginator
 from django.db.models import Count
-from django.db.models import F
+from django.db.models import F,Q
 from django.utils.html import format_html
 
 from ensembl_dbcopy.forms import SubmitForm, GroupForm
@@ -128,14 +128,31 @@ class Media:
         css = {
             'all': ('css/db_copy.css',)
         }
+    actions = ['resubmit_jobs', ]
+
+
+    def resubmit_jobs(self, request, queryset):
+        for query in queryset:
+            newJob = RequestJob.objects.get(pk=query.pk)
+            newJob.pk=None
+            newJob.request_date=None
+            newJob.start_date=None
+            newJob.end_date=None
+            newJob.status=None
+            newJob.save()
+            self.message_user(request, 'Job {} resubmitted [new job_id {}]'.format(query.pk, newJob.pk))
+
+    resubmit_jobs.short_description = 'Resubmit Jobs'
 
     form = SubmitForm
     add_form_template = "admin/dbcopy/submit.html"
     change_form_template = "admin/dbcopy/detail.html"
     list_display = ('job_id', 'src_host', 'src_incl_db', 'src_skip_db', 'tgt_host', 'tgt_db_name', 'user',
-                    'start_date', 'end_date', 'overall_status')
+                    'start_date', 'end_date', 'request_date', 'overall_status')
+    search_fields = ('job_id', 'src_host', 'src_incl_db', 'src_skip_db', 'tgt_host', 'tgt_db_name', 'user',
+                    'start_date', 'end_date', 'request_date')
     list_filter = ('src_host', 'tgt_host', UserFilter, OverallStatusFilter)
-    ordering = ('start_date',)
+    ordering = ('-request_date',)
 
     def has_add_permission(self, request):
         return request.user.is_staff
@@ -157,11 +174,17 @@ def get_form(self, request, obj=None, change=False, **kwargs):
 
     def change_view(self, request, object_id, form_url='', extra_context=None):
         context = extra_context or {}
-        transfers_logs = self.get_object(request, object_id).transfer_logs
-        paginator = Paginator(transfers_logs.order_by(F('end_date').desc(nulls_first=True)), 30)
+        search_query = request.GET.get('search_box')
+        if search_query:
+            transfers_logs = self.get_object(request, object_id).transfer_logs.filter(Q(table_name__contains=search_query) | Q(table_schema__contains=search_query) | Q(tgt_host__contains=search_query) | Q(renamed_table_schema__contains=search_query))
+        else:
+            transfers_logs = self.get_object(request, object_id).transfer_logs
+        paginator = Paginator(transfers_logs.order_by(F('end_date').asc(nulls_first=True),F('auto_id')), 30)
         page_number = request.GET.get('page', 1)
         page = paginator.page(page_number)
         context['transfer_logs'] = page
+        if transfers_logs.filter(end_date__isnull=True):
+            context["running_copy"] = transfers_logs.filter(end_date__isnull=True).order_by(F('end_date').desc(nulls_first=True)).earliest('auto_id')
         return super().change_view(request, object_id, form_url, context)
 
     def overall_status(self, obj):

ensembl_dbcopy/fixtures/ensembl_dbcopy.json

@@ -70,7 +70,8 @@
         "start_date": null,
         "end_date": null,
         "user": "testuser",
-        "status": null
+        "status": null,
+        "request_date" : "2020-06-01T10:00:06Z"
     }
   },
   {
@@ -92,7 +93,8 @@
         "start_date": null,
         "end_date": null,
         "user": "testuser",
-        "status": null
+        "status": null,
+        "request_date": "2020-06-05T10:55:06Z"
     }
   },
   {
@@ -114,7 +116,8 @@
         "start_date": null,
         "end_date": null,
         "user": "testuser",
-        "status": null
+        "status": null,
+        "request_date": "2020-06-08T09:55:06Z"
     }
   },
   {

ensembl_dbcopy/forms.py

@@ -42,35 +42,35 @@ class Meta:
 
     src_host = forms.CharField(
         label="Source Host (e.g: host:port)",
-        max_length=80,
+        max_length=2048,
         required=True)
     tgt_host = forms.CharField(
         label="Target Hosts (e.g: Host1:port1,Host2:port2)",
-        max_length=80,
+        max_length=2048,
         required=True)
     src_incl_db = forms.CharField(
         label="Databases to copy (e.g: db1,db2,.. or %variation_99% )",
-        max_length=80,
+        max_length=2048,
         required=False)
     src_skip_db = forms.CharField(
         label="Databases to exclude (e.g db1,db2 or %mart%)",
-        max_length=80,
+        max_length=2048,
         required=False)
     src_incl_tables = forms.CharField(
         label="Only Copy these tables (e.g: table1,table2,..):",
-        max_length=80,
+        max_length=2048,
         required=False)
     src_skip_tables = forms.CharField(
         label="Skip these tables (e.g: table1,table2)",
-        max_length=80,
+        max_length=2048,
         required=False)
     tgt_db_name = forms.CharField(
-        label="Name of database on Target Hosts",
-        max_length=80,
+        label="Name of databases on Target Hosts (e.g: db1,db2)",
+        max_length=2048,
         required=False)
     email_list = forms.CharField(
         label="Email list",
-        max_length=80,
+        max_length=2048,
         required=True)
     user = forms.CharField(
         widget=forms.HiddenInput())
@@ -125,7 +125,22 @@ def clean_email_list(self):
                 "Email list should be formatted like this [email protected] or [email protected],[email protected]")
         return data
 
+    def clean_tgt_db_name(self):
+        tgt_db_name = self.cleaned_data.get('tgt_db_name', '')
+        src_incl_db = self.cleaned_data.get('src_incl_db', '')
+        if tgt_db_name:
+            if tgt_db_name.count(',') != src_incl_db.count(','):
+                raise forms.ValidationError(
+                    "The number of databases to copy should match the number of databases renamed on target hosts")
+            if '%' in src_incl_db:
+                raise forms.ValidationError(
+                    "You can't rename a pattern")
+        return tgt_db_name
+
+
     def __init__(self, *args, **kwargs):
+        if 'initial' in kwargs and 'from_request_job' in kwargs['initial']:
+            kwargs['instance'] = RequestJob.objects.get(pk=kwargs['initial']['from_request_job'])
         super(SubmitForm, self).__init__(*args, **kwargs)
         self.helper = FormHelper()
         self.fields["email_list"].initial = self.user.email

ensembl_dbcopy/migrations/0002_adding_request_date.py

@@ -0,0 +1,24 @@
+# Generated by Django 2.2.13 on 2020-06-08 10:16
+
+from django.db import migrations, models
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ensembl_dbcopy', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='requestjob',
+            options={'verbose_name': 'Copy job', 'verbose_name_plural': 'Copy jobs'},
+        ),
+        migrations.AddField(
+            model_name='requestjob',
+            name='request_date',
+            field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now),
+            preserve_default=False,
+        ),
+    ]

ensembl_dbcopy/models.py

@@ -64,6 +64,7 @@ class RequestJob(models.Model):
     end_date = models.DateTimeField(blank=True, null=True, editable=False)
     user = models.CharField(max_length=64, blank=True, null=True)
     status = models.CharField(max_length=20, blank=True, null=True, editable=False)
+    request_date = models.DateTimeField(editable=False,auto_now_add=True)
 
     class Meta:
         db_table = 'request_job'
@@ -76,13 +77,14 @@ def __str__(self):
 
     @property
     def overall_status(self):
-        if self.end_date and self.status:
-            if self.transfer_logs.filter(end_date__isnull=True).count() > 0:
-                return 'Failed'
-            else:
-                return 'Complete'
-        elif self.transfer_logs.count() > 0:
-            return 'Running'
+        if self.status:
+            if (self.end_date and self.status=='Transfer Ended') or 'Try:' in self.status:
+                if self.transfer_logs.filter(end_date__isnull=True).count() > 0:
+                    return 'Failed'
+                else:
+                    return 'Complete'
+            elif self.transfer_logs.count() > 0 and self.status=='Processing Requests':
+                return 'Running'
         else:
             return 'Submitted'
 
@@ -97,11 +99,15 @@ def detailed_status(self):
             return {'status_msg': 'Complete', 'table_copied': table_copied, 'total_tables': total_tables,
                     'progress': progress}
         elif total_tables > 0:
-            if self.end_date and self.status:
-                return {'status_msg': 'Failed', 'table_copied': table_copied, 'total_tables': total_tables,
-                        'progress': progress}
+            if self.status:
+                if (self.end_date and self.status=='Transfer Ended') or ('Try:' in self.status):
+                    return {'status_msg': 'Failed', 'table_copied': table_copied, 'total_tables': total_tables,
+                            'progress': progress}
+                if self.status=='Processing Requests':
+                    return {'status_msg': 'Running', 'table_copied': table_copied, 'total_tables': total_tables,
+                            'progress': progress}
             else:
-                return {'status_msg': 'Running', 'table_copied': table_copied, 'total_tables': total_tables,
+                return {'status_msg': 'Submitted', 'table_copied': table_copied, 'total_tables': total_tables,
                         'progress': progress}
         else:
             return {'status_msg': 'Submitted', 'table_copied': table_copied, 'total_tables': total_tables,
@@ -144,10 +150,13 @@ class Meta:
     def table_status(self):
         if self.end_date:
             return 'Complete'
-        elif self.job_id.end_date and self.job_id.status:
-            return 'Failed'
+        elif self.job_id.status:
+            if (self.job_id.end_date and self.job_id.status=='Transfer Ended') or ('Try:' in self.job_id.status):
+                return 'Failed'
+            else:
+                return 'Running'
         else:
-            return 'Running'
+            return 'Submitted'
 
 
 class Host(models.Model):

ensembl_dbcopy/static/css/db_copy.css

@@ -46,9 +46,6 @@
   top:3px;
   white-space: nowrap;
 }
-.btn {
-  margin-bottom: 15px;
-}
 .table{
   table-layout:fixed;
 }

ensembl_dbcopy/templates/admin/dbcopy/detail.html

@@ -7,6 +7,13 @@
       <a href="{% url 'admin:ensembl_dbcopy_requestjob_add' %}" class="btn btn-secondary" role="button">Submit New Copy</a>
       <a href="{% url 'ensembl_dbcopy:reset_failed_jobs' original.job_id %}" class="btn btn-secondary" role="button">Reset failed jobs</a>
       <a href="{% url 'admin:ensembl_dbcopy_requestjob_change' original.job_id %}" class="btn btn-secondary" role="button">Refresh</a>
+      <a href="{% url 'admin:ensembl_dbcopy_requestjob_add' %}?from_request_job={{ original.pk }}" class="btn btn-secondary" role="button">Resubmit job</a>
+    </div>
+    <div class="p-3 my-3 border">
+      <form type="get" action=".">
+        <input  id="search_box" class="btn btn-secondary" type="text" name="search_box"  placeholder="Search..." >
+        <button id="search_submit" class="btn btn-secondary" type="submit" >Submit</button>
+      </form>
     </div>
     <div class="p-3 my-3 border">
       <div class="row">
@@ -43,9 +50,24 @@
           <div class="col-sm-10 tablevalue">All the databases</div>
         </div>
       {% endif %}
+      {% if original.src_incl_tables %}
+        <div class="row">
+          <div class="col-sm-2">
+            <div class="tablekey">Source Tables:</div>
+          </div>
+          <div class="col-sm-10 tablevalue">{{ original.src_incl_tables }}</div>
+        </div>
+      {% elif original.src_skip_tables %}
+        <div class="row">
+          <div class="col-sm-2">
+            <div class="tablekey">Source Tables:</div>
+          </div>
+          <div class="col-sm-10 tablevalue">All the tables excluding {{ original.src_skip_tables }}</div>
+        </div>
+      {% endif %}
       <div class="row">
         <div class="col-sm-2">
-          <div class="tablekey">Target Host:</div>
+          <div class="tablekey">Target Hosts:</div>
         </div>
         <div class="col-sm-10 tablevalue">{{ original.tgt_host }}</div>
       </div>
@@ -57,12 +79,26 @@
           <div class="col-sm-10 tablevalue">{{ original.tgt_db_name }}</div>
         </div>
       {% endif %}
+      <div class="row">
+        <div class="col-sm-2">
+          <div class="tablekey">Copy start:</div>
+        </div>
+        <div class="col-sm-10 tablevalue">{{ original.start_date }}</div>
+      </div>
+      {% if original.overall_status == 'Complete' %}
+       <div class="row">
+        <div class="col-sm-2">
+          <div class="tablekey">Copy end:</div>
+        </div>
+        <div class="col-sm-10 tablevalue">{{ original.start_date }}</div>
+        </div>
+      {% endif %}
       {% if original.detailed_status.status_msg == 'Running' %}
         <div class="row">
             <div class="col-sm-2">
               <div class="tablekey">Status:</div>
             </div>
-            <div class="col-sm-10 tablevalue {{ original.detailed_status.status_msg }}">{{ original.detailed_status.status_msg }}: Copying table {{original.transfer_logs.last.table_name}} for {{original.transfer_logs.last.renamed_table_schema}} in progress...</div>
+            <div class="col-sm-10 tablevalue {{ original.detailed_status.status_msg }}">{{ original.detailed_status.status_msg }}: Copying table {{running_copy.table_name}} for {{running_copy.renamed_table_schema}} to {{running_copy.tgt_host}} in progress...</div>
         </div>
         <div class="row">
             <div class="col-sm-2">
@@ -74,6 +110,13 @@
               </div>
             </div>
         </div>
+       {% elif original.detailed_status.status_msg == 'Failed' and original.failed_msg %}
+        <div class="row">
+            <div class="col-sm-2">
+              <div class="tablekey">Status:</div>
+            </div>
+            <div class="col-sm-10 tablevalue {{ original.detailed_status.status_msg }}">{{ original.detailed_status.status_msg }}: {{ original.failed_msg }}</div>
+        </div>
       {% else %}
         <div class="row">
             <div class="col-sm-2">
@@ -88,6 +131,7 @@
         <table class="table table-bordered">
             <thead>
               <tr>
+                <th>Target Host</th>
                 <th>Database name</th>
                 <th>Table name</th>
                 <th>Copy start</th>
@@ -98,7 +142,19 @@
             </thead>
             <tbody>
             {% for log in transfer_logs %}
+             {% if log.message %}
+               {% if log.table_status != 'Complete' %}
+                <div class="p-3 my-3 border">
+                 <div class="row">
+                  <div class="col-sm-2">
+                   <div class="tablekey">Failure detail:</div>
+                  </div>
+                 <div class="col-sm-10 tablevalue Failed"> {{ log.table_schema }}.{{ log.table_name }}: {{ log.message }}</div>
+                </div>
+               {% endif %}
+              {% endif %}
             <tr>
+              <td class="table-break-all">{{log.tgt_host}}</td>
               <td class="table-break-all">{{log.renamed_table_schema}}</td>
               <td class="table-break-all">{{log.table_name}}</td>
               <td class="table-break-all">{{log.start_date}}</td>
@@ -111,15 +167,25 @@
           </table>
         <span class="pagination">
             {% if transfer_logs.has_previous %}
+              {% if 'search_box' in request.get_full_path %}
+                <a class="page-link" href="{{ request.get_full_path }}&page=1">&laquo; first</a>
+                <a class="page-link" href="{{ request.get_full_path }}&page={{ transfer_logs.previous_page_number }}">previous</a>
+              {% else %}
                 <a class="page-link" href="?page=1">&laquo; first</a>
                 <a class="page-link" href="?page={{ transfer_logs.previous_page_number }}">previous</a>
+              {% endif %}
             {% endif %}
             <span class="page-link">
                 Page {{ transfer_logs.number }} of {{ transfer_logs.paginator.num_pages }}.
             </span>
             {% if transfer_logs.has_next %}
+              {% if 'search_box' in request.get_full_path %}
+                <a class="page-link" href="{{ request.get_full_path }}&page={{ transfer_logs.next_page_number }}">next</a>
+                <a class="page-link" href="{{ request.get_full_path }}&page={{ transfer_logs.paginator.num_pages }}">last &raquo;</a>
+              {% else %}
                 <a class="page-link" href="?page={{ transfer_logs.next_page_number }}">next</a>
                 <a class="page-link" href="?page={{ transfer_logs.paginator.num_pages }}">last &raquo;</a>
+              {% endif %}
             {% endif %}
         </span>
       </div>