Merge pull request #208 from collectionspace/jobs-cleanup

Some jobs cleanup
collectionspace · Mar 22, 2024 · 406efb2 · 406efb2
2 parents 9d74d6e + 05c64c9
commit 406efb2
Show file tree

Hide file tree

Showing 11 changed files with 49 additions and 21 deletions.
diff --git a/app/jobs/reaper_expired_job.rb → app/jobs/batch_expired_job.rb b/app/jobs/reaper_expired_job.rb → app/jobs/batch_expired_job.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class ReaperExpiredJob < ApplicationJob
+class BatchExpiredJob < ApplicationJob
   queue_as :default
   sidekiq_options retry: false
 

diff --git a/app/jobs/reaper_limbo_job.rb → app/jobs/step_in_limbo_job.rb b/app/jobs/reaper_limbo_job.rb → app/jobs/step_in_limbo_job.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class ReaperLimboJob < ApplicationJob
+class StepInLimboJob < ApplicationJob
   queue_as :default
   sidekiq_options retry: false
 

diff --git a/app/jobs/reaper_unused_job.rb → app/jobs/unused_manifest_job.rb b/app/jobs/reaper_unused_job.rb → app/jobs/unused_manifest_job.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class ReaperUnusedJob < ApplicationJob
+class UnusedManifestJob < ApplicationJob
   queue_as :default
   sidekiq_options retry: false
 

diff --git a/app/models/batch.rb b/app/models/batch.rb
@@ -2,7 +2,8 @@
 
 class Batch < ApplicationRecord
   include WorkflowManager
-  ARCHIVED_EXPIRY_INTERVAL = 7 # (days) make configurable?
+  ANY_EXPIRY_INTERVAL = 30 # (days) after which batch in any status will be deleted
+  ARCHIVED_EXPIRY_INTERVAL = 7 # (days) after which archived batch will be deleted
   CONTENT_TYPES = [
     'application/vnd.ms-excel',
     'text/csv'
@@ -33,6 +34,14 @@ class Batch < ApplicationRecord
                     where.not("step_state = 'archiving' AND status_state = 'finished'")
                   }
 
+  def any_expiry_interval
+    ANY_EXPIRY_INTERVAL
+  end
+
+  def archived_expiry_interval
+    ARCHIVED_EXPIRY_INTERVAL
+  end
+
   def archived?
     step_archive&.done?
   end
@@ -46,12 +55,8 @@ def can_reset?
   end
 
   def expired?
-    archived? && (Time.now - step_archive.completed_at) > expiry_interval.days
-  end
-
-  # TODO: configurable?
-  def expiry_interval
-    ARCHIVED_EXPIRY_INTERVAL
+    (archived? && (Time.now - step_archive.completed_at) > archived_expiry_interval.days) ||
+      (Time.now - created_at > any_expiry_interval.days)
   end
 
   def fingerprint

diff --git a/app/views/batches/_field_definitions.html.erb b/app/views/batches/_field_definitions.html.erb
@@ -6,8 +6,8 @@
     <table class="table is-fullwidth is-bordered">
       <thead>
         <tr>
-          <td class="has-text-weight-bold">Field</th>
-          <td class="has-text-weight-bold">Description</th>
+          <th class="has-text-weight-bold">Field</th>
+          <th class="has-text-weight-bold">Description</th>
         </tr>
       </thead>
       <tbody>
@@ -31,11 +31,12 @@
           <td>A CSV file to upload</td>
         </tr>
         <tr>
-          <td class="is-italic"><%= fa_icon 'external-link', class: 'mr-2' %><%= link_to "Config", 'https://github.com/collectionspace/collectionspace-mapper/blob/main/doc/batch_configuration.adoc' %></td>
+          <td class="is-italic"><%= fa_icon 'external-link', class: 'mr-2' %><%= link_to t("batch.config.label"), t("batch.config.url") %></td>
           <td>OPTIONAL/EXPERT: Enter batch-specific data mapping configuration in JSON format; click Config link for documentation</td>
         </tr>
       </tbody>
     </table>
-    <p class="content"><%= fa_icon 'external-link', class: 'mr-2' %><%= link_to "View the complete documentation", 'https://collectionspace.atlassian.net/wiki/spaces/COL/pages/2271936513/User+Manual+CollectionSpace+CSV+Importer' %></p>
+    <p class="content"><%= fa_icon 'external-link', class: 'mr-2' %><%= link_to t("batch.docs.label"), t("batch.docs.url") %></p>
+    <p class="content has-text-danger"><%= fa_icon 'warning', class: 'mr-2' %><%= t("batch.expiry", interval: Batch::ANY_EXPIRY_INTERVAL) %></p>
   </div>
 </article>
diff --git a/app/views/batches/_form.html.erb b/app/views/batches/_form.html.erb
@@ -105,7 +105,7 @@
 
         <%= content_tag :div, class: 'column is-full' do %>
           <%= content_tag :div, class: 'field' do %>
-            <%= form.label t('batch.batch_config'), class: 'label' %>
+            <%= form.label t('batch.config.label'), class: 'label' %>
             <%= form.text_area :batch_config, id: 'batch_config_entry' %>
           <% end %>
         <% end %>

diff --git a/app/views/step/archives/_step.html.erb b/app/views/step/archives/_step.html.erb
@@ -8,7 +8,7 @@
     <ul class="mt-2 ml-6 mb-2">
       <li>Delete all files associated with this batch</li>
       <li>Mark the batch as complete (viewable only via the completed tab)</li>
-      <li>When completed the batch will be deleted after <%= batch.expiry_interval %> days</li>
+      <li>When completed the batch will be deleted after <%= batch.archived_expiry_interval %> days</li>
     </ul>
   </article>
 <% end %>

diff --git a/config/locales/en.yml b/config/locales/en.yml
@@ -63,12 +63,18 @@ en:
       create: Create a batch
       delete: Delete this batch
       return: Return to batches
-    batch_config: Config
+    config:
+      label: Config
+      url: "https://github.com/collectionspace/collectionspace-mapper/blob/main/doc/batch_configuration.adoc"
     connection: Connection
     created_at: Created at
     created_by: Created by
     current_state: State
     current_step: Step
+    docs:
+      label: View the complete documentation
+      url: "https://collectionspace.atlassian.net/wiki/spaces/COL/pages/2271936513/User+Manual+CollectionSpace+CSV+Importer"
+    expiry: "Batches are automatically deleted after %{interval} days."
     file: File
     group: Group
     invalid_profile: Profile version mismatch between connection and mapper

diff --git a/config/sidekiq.yml b/config/sidekiq.yml
@@ -1,10 +1,10 @@
 :schedule:
   reap_expired_batches:
-    class: ReaperExpiredJob
+    class: BatchExpiredJob
     cron: '0 1 * * *'
   reap_steps_in_limbo:
-    class: ReaperLimboJob
+    class: StepInLimboJob
     cron: '0 * * * *'
   reap_unused_manifests:
-    class: ReaperUnusedJob
+    class: UnusedManifestJob
     cron: '0 0 1 * *'
diff --git a/test/fixtures/batches.yml b/test/fixtures/batches.yml
@@ -11,6 +11,18 @@ superuser_batch: # no associated steps
   num_rows: 1
   batch_config: '{}'
 
+superuser_batch_expired: # no associated steps
+  name: superuser_batch
+  user: superuser
+  step_state: preprocessing
+  status_state: ready
+  group: default
+  connection: core_superuser
+  mapper: core_collectionobject_6_0
+  num_rows: 1
+  batch_config: '{}'
+  created_at: "2022-01-24 21:29:29.144051"
+
 # STEP 1. PREPROCESSING
 
 superuser_batch_preprocessing:

diff --git a/test/models/batch_test.rb b/test/models/batch_test.rb
@@ -99,7 +99,11 @@ class BatchTest < ActiveSupport::TestCase
     refute batches(:superuser_batch_transferring).expired?
     refute batches(:superuser_batch_archiving).expired?
 
+    assert batches(:superuser_batch_expired).expired?
     assert batches(:superuser_batch_archived).expired?
-    Batch.expired { |b| assert_equal batches(:superuser_batch_archived), b }
+
+    expired = 0
+    Batch.expired { |b| expired += 1 }
+    assert 2, expired
   end
 end