From 05c64c9e371157e7322d06d75547e315c336e401 Mon Sep 17 00:00:00 2001 From: Mark Cooper Date: Tue, 19 Mar 2024 16:51:22 -0700 Subject: [PATCH] Some jobs cleanup Rename for better clarity. Update batch expired to apply to any batch older than 30 days. --- ...er_expired_job.rb => batch_expired_job.rb} | 2 +- ...aper_limbo_job.rb => step_in_limbo_job.rb} | 2 +- ...r_unused_job.rb => unused_manifest_job.rb} | 2 +- app/models/batch.rb | 19 ++++++++++++------- app/views/batches/_field_definitions.html.erb | 9 +++++---- app/views/batches/_form.html.erb | 2 +- app/views/step/archives/_step.html.erb | 2 +- config/locales/en.yml | 8 +++++++- config/sidekiq.yml | 6 +++--- test/fixtures/batches.yml | 12 ++++++++++++ test/models/batch_test.rb | 6 +++++- 11 files changed, 49 insertions(+), 21 deletions(-) rename app/jobs/{reaper_expired_job.rb => batch_expired_job.rb} (77%) rename app/jobs/{reaper_limbo_job.rb => step_in_limbo_job.rb} (95%) rename app/jobs/{reaper_unused_job.rb => unused_manifest_job.rb} (77%) diff --git a/app/jobs/reaper_expired_job.rb b/app/jobs/batch_expired_job.rb similarity index 77% rename from app/jobs/reaper_expired_job.rb rename to app/jobs/batch_expired_job.rb index 30bb1f1c..8ede17fa 100644 --- a/app/jobs/reaper_expired_job.rb +++ b/app/jobs/batch_expired_job.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class ReaperExpiredJob < ApplicationJob +class BatchExpiredJob < ApplicationJob queue_as :default sidekiq_options retry: false diff --git a/app/jobs/reaper_limbo_job.rb b/app/jobs/step_in_limbo_job.rb similarity index 95% rename from app/jobs/reaper_limbo_job.rb rename to app/jobs/step_in_limbo_job.rb index 2f500dfd..363a36c8 100644 --- a/app/jobs/reaper_limbo_job.rb +++ b/app/jobs/step_in_limbo_job.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class ReaperLimboJob < ApplicationJob +class StepInLimboJob < ApplicationJob queue_as :default sidekiq_options retry: false diff --git a/app/jobs/reaper_unused_job.rb b/app/jobs/unused_manifest_job.rb similarity index 77% rename from app/jobs/reaper_unused_job.rb rename to app/jobs/unused_manifest_job.rb index cfa28ffc..6784267c 100644 --- a/app/jobs/reaper_unused_job.rb +++ b/app/jobs/unused_manifest_job.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class ReaperUnusedJob < ApplicationJob +class UnusedManifestJob < ApplicationJob queue_as :default sidekiq_options retry: false diff --git a/app/models/batch.rb b/app/models/batch.rb index 1d4a5e11..2c1f9a72 100644 --- a/app/models/batch.rb +++ b/app/models/batch.rb @@ -2,7 +2,8 @@ class Batch < ApplicationRecord include WorkflowManager - ARCHIVED_EXPIRY_INTERVAL = 7 # (days) make configurable? + ANY_EXPIRY_INTERVAL = 30 # (days) after which batch in any status will be deleted + ARCHIVED_EXPIRY_INTERVAL = 7 # (days) after which archived batch will be deleted CONTENT_TYPES = [ 'application/vnd.ms-excel', 'text/csv' @@ -33,6 +34,14 @@ class Batch < ApplicationRecord where.not("step_state = 'archiving' AND status_state = 'finished'") } + def any_expiry_interval + ANY_EXPIRY_INTERVAL + end + + def archived_expiry_interval + ARCHIVED_EXPIRY_INTERVAL + end + def archived? step_archive&.done? end @@ -46,12 +55,8 @@ def can_reset? end def expired? - archived? && (Time.now - step_archive.completed_at) > expiry_interval.days - end - - # TODO: configurable? - def expiry_interval - ARCHIVED_EXPIRY_INTERVAL + (archived? && (Time.now - step_archive.completed_at) > archived_expiry_interval.days) || + (Time.now - created_at > any_expiry_interval.days) end def fingerprint diff --git a/app/views/batches/_field_definitions.html.erb b/app/views/batches/_field_definitions.html.erb index 61628b70..c2004b48 100644 --- a/app/views/batches/_field_definitions.html.erb +++ b/app/views/batches/_field_definitions.html.erb @@ -6,8 +6,8 @@ - + @@ -31,11 +31,12 @@ - +
Field - Description + FieldDescription
A CSV file to upload
<%= fa_icon 'external-link', class: 'mr-2' %><%= link_to "Config", 'https://github.com/collectionspace/collectionspace-mapper/blob/main/doc/batch_configuration.adoc' %><%= fa_icon 'external-link', class: 'mr-2' %><%= link_to t("batch.config.label"), t("batch.config.url") %> OPTIONAL/EXPERT: Enter batch-specific data mapping configuration in JSON format; click Config link for documentation
-

<%= fa_icon 'external-link', class: 'mr-2' %><%= link_to "View the complete documentation", 'https://collectionspace.atlassian.net/wiki/spaces/COL/pages/2271936513/User+Manual+CollectionSpace+CSV+Importer' %>

+

<%= fa_icon 'external-link', class: 'mr-2' %><%= link_to t("batch.docs.label"), t("batch.docs.url") %>

+

<%= fa_icon 'warning', class: 'mr-2' %><%= t("batch.expiry", interval: Batch::ANY_EXPIRY_INTERVAL) %>

diff --git a/app/views/batches/_form.html.erb b/app/views/batches/_form.html.erb index 58e00374..03db9758 100644 --- a/app/views/batches/_form.html.erb +++ b/app/views/batches/_form.html.erb @@ -105,7 +105,7 @@ <%= content_tag :div, class: 'column is-full' do %> <%= content_tag :div, class: 'field' do %> - <%= form.label t('batch.batch_config'), class: 'label' %> + <%= form.label t('batch.config.label'), class: 'label' %> <%= form.text_area :batch_config, id: 'batch_config_entry' %> <% end %> <% end %> diff --git a/app/views/step/archives/_step.html.erb b/app/views/step/archives/_step.html.erb index acb4ec31..6d96ad8b 100644 --- a/app/views/step/archives/_step.html.erb +++ b/app/views/step/archives/_step.html.erb @@ -8,7 +8,7 @@ <% end %> diff --git a/config/locales/en.yml b/config/locales/en.yml index ef4dbeb3..9e92863a 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -63,12 +63,18 @@ en: create: Create a batch delete: Delete this batch return: Return to batches - batch_config: Config + config: + label: Config + url: "https://github.com/collectionspace/collectionspace-mapper/blob/main/doc/batch_configuration.adoc" connection: Connection created_at: Created at created_by: Created by current_state: State current_step: Step + docs: + label: View the complete documentation + url: "https://collectionspace.atlassian.net/wiki/spaces/COL/pages/2271936513/User+Manual+CollectionSpace+CSV+Importer" + expiry: "Batches are automatically deleted after %{interval} days." file: File group: Group invalid_profile: Profile version mismatch between connection and mapper diff --git a/config/sidekiq.yml b/config/sidekiq.yml index 347547b2..0c38c662 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -1,10 +1,10 @@ :schedule: reap_expired_batches: - class: ReaperExpiredJob + class: BatchExpiredJob cron: '0 1 * * *' reap_steps_in_limbo: - class: ReaperLimboJob + class: StepInLimboJob cron: '0 * * * *' reap_unused_manifests: - class: ReaperUnusedJob + class: UnusedManifestJob cron: '0 0 1 * *' diff --git a/test/fixtures/batches.yml b/test/fixtures/batches.yml index 4bf6aed0..d95af897 100644 --- a/test/fixtures/batches.yml +++ b/test/fixtures/batches.yml @@ -11,6 +11,18 @@ superuser_batch: # no associated steps num_rows: 1 batch_config: '{}' +superuser_batch_expired: # no associated steps + name: superuser_batch + user: superuser + step_state: preprocessing + status_state: ready + group: default + connection: core_superuser + mapper: core_collectionobject_6_0 + num_rows: 1 + batch_config: '{}' + created_at: "2022-01-24 21:29:29.144051" + # STEP 1. PREPROCESSING superuser_batch_preprocessing: diff --git a/test/models/batch_test.rb b/test/models/batch_test.rb index cfc75643..8c7aec80 100644 --- a/test/models/batch_test.rb +++ b/test/models/batch_test.rb @@ -99,7 +99,11 @@ class BatchTest < ActiveSupport::TestCase refute batches(:superuser_batch_transferring).expired? refute batches(:superuser_batch_archiving).expired? + assert batches(:superuser_batch_expired).expired? assert batches(:superuser_batch_archived).expired? - Batch.expired { |b| assert_equal batches(:superuser_batch_archived), b } + + expired = 0 + Batch.expired { |b| expired += 1 } + assert 2, expired end end