diff --git a/lib/bin/purge-forms.js b/lib/bin/purge.js similarity index 57% rename from lib/bin/purge-forms.js rename to lib/bin/purge.js index cc5561d3e..20909d063 100644 --- a/lib/bin/purge-forms.js +++ b/lib/bin/purge.js @@ -7,20 +7,28 @@ // including this file, may be copied, modified, propagated, or distributed // except according to the terms contained in the LICENSE file. // -// This script checks for (soft-)deleted forms and purges any that were deleted -// over 30 days ago. +// This script checks for (soft-)deleted forms and submissions and purges +// any that were deleted over 30 days ago. +// +// It also accepts command line arguments that can force the purging of +// forms and submissions that were deleted less than 30 days ago. +// +// It can also be used to purge a specific form or submission +// (that has already been marked deleted). const { run } = require('../task/task'); -const { purgeForms } = require('../task/purge'); +const { purgeTask } = require('../task/purge'); const { program } = require('commander'); program.option('-f, --force', 'Force any soft-deleted form to be purged right away.'); +program.option('-m, --mode ', 'Mode of purging. Can be "forms", "submissions", or "all". Default is "all".', 'all'); program.option('-i, --formId ', 'Purge a specific form based on its id.', parseInt); program.option('-p, --projectId ', 'Restrict purging to a specific project.', parseInt); -program.option('-x, --xmlFormId ', 'Restrict purging to specific form based on xmlFormId (must be used with project id).'); +program.option('-x, --xmlFormId ', 'Restrict purging to specific form based on xmlFormId (must be used with projectId).'); +program.option('-s, --instanceId ', 'Restrict purging to a specific submission based on instanceId (use with projectId and xmlFormId).'); + program.parse(); const options = program.opts(); -run(purgeForms(options.force, options.formId, options.projectId, options.xmlFormId) - .then((count) => `Forms purged: ${count}`)); +run(purgeTask(options)); diff --git a/lib/model/migrations/20240914-01-add-submission-delete-verb.js b/lib/model/migrations/20240914-01-add-submission-delete-verb.js new file mode 100644 index 000000000..d9bf1f7bd --- /dev/null +++ b/lib/model/migrations/20240914-01-add-submission-delete-verb.js @@ -0,0 +1,23 @@ +// Copyright 2024 ODK Central Developers +// See the NOTICE file at the top-level directory of this distribution and at +// https://github.com/getodk/central-backend/blob/master/NOTICE. +// This file is part of ODK Central. It is subject to the license terms in +// the LICENSE file found in the top-level directory of this distribution and at +// https://www.apache.org/licenses/LICENSE-2.0. No part of ODK Central, +// including this file, may be copied, modified, propagated, or distributed +// except according to the terms contained in the LICENSE file. + +const up = (db) => db.raw(` +UPDATE roles +SET verbs = verbs || '["submission.delete", "submission.restore"]'::jsonb +WHERE system in ('admin', 'manager') +`); + +const down = (db) => db.raw(` +UPDATE roles +SET verbs = (verbs - 'submission.delete' - 'submission.restore') +WHERE system in ('admin', 'manager') +`); + +module.exports = { up, down }; + diff --git a/lib/model/migrations/20240914-02-remove-orphaned-client-audits.js b/lib/model/migrations/20240914-02-remove-orphaned-client-audits.js new file mode 100644 index 000000000..9dd633a71 --- /dev/null +++ b/lib/model/migrations/20240914-02-remove-orphaned-client-audits.js @@ -0,0 +1,27 @@ +// Copyright 2024 ODK Central Developers +// See the NOTICE file at the top-level directory of this distribution and at +// https://github.com/getodk/central-backend/blob/master/NOTICE. +// This file is part of ODK Central. It is subject to the license terms in +// the LICENSE file found in the top-level directory of this distribution and at +// https://www.apache.org/licenses/LICENSE-2.0. No part of ODK Central, +// including this file, may be copied, modified, propagated, or distributed +// except according to the terms contained in the LICENSE file. + +// From earlier form-purging, there could have been leftover client audits +// that were not properly purged because of how they joined to submissions. +// This migration deletes those client audit rows, allowing the blobs to finally +// be de-referenced and also eventually purged (by the puring cron job). + +const up = (db) => db.raw(` +DELETE FROM client_audits +WHERE "blobId" NOT IN ( + SELECT "blobId" FROM submission_attachments + WHERE "blobId" IS NOT NULL + AND "isClientAudit" IS true +); +`); + +const down = () => {}; + +module.exports = { up, down }; + diff --git a/lib/model/query/audits.js b/lib/model/query/audits.js index f8e75b5bf..2aa47e0d0 100644 --- a/lib/model/query/audits.js +++ b/lib/model/query/audits.js @@ -36,7 +36,7 @@ const actionCondition = (action) => { // The backup action was logged by a backup script that has been removed. // Even though the script has been removed, the audit log entries it logged // have not, so we should continue to exclude those. - return sql`action not in ('entity.create', 'entity.bulk.create', 'entity.error', 'entity.update.version', 'entity.update.resolve', 'entity.delete', 'submission.create', 'submission.update', 'submission.update.version', 'submission.attachment.update', 'submission.reprocess', 'backup', 'analytics')`; + return sql`action not in ('entity.create', 'entity.bulk.create', 'entity.error', 'entity.update.version', 'entity.update.resolve', 'entity.delete', 'submission.create', 'submission.update', 'submission.update.version', 'submission.attachment.update', 'submission.reprocess', 'submission.delete', 'submission.restore', 'backup', 'analytics')`; else if (action === 'user') return sql`action in ('user.create', 'user.update', 'user.delete', 'user.assignment.create', 'user.assignment.delete', 'user.session.create')`; else if (action === 'field_key') @@ -48,7 +48,7 @@ const actionCondition = (action) => { else if (action === 'form') return sql`action in ('form.create', 'form.update', 'form.delete', 'form.restore', 'form.purge', 'form.attachment.update', 'form.submission.export', 'form.update.draft.set', 'form.update.draft.delete', 'form.update.publish')`; else if (action === 'submission') - return sql`action in ('submission.create', 'submission.update', 'submission.update.version', 'submission.attachment.update', 'submission.reprocess')`; + return sql`action in ('submission.create', 'submission.update', 'submission.update.version', 'submission.attachment.update', 'submission.reprocess', 'submission.delete', 'submission.restore', 'submission.purge')`; else if (action === 'dataset') return sql`action in ('dataset.create', 'dataset.update')`; else if (action === 'entity') diff --git a/lib/model/query/entities.js b/lib/model/query/entities.js index 0238ea98f..568af77dc 100644 --- a/lib/model/query/entities.js +++ b/lib/model/query/entities.js @@ -592,7 +592,12 @@ const _get = (includeSource) => { `} ${!includeSource ? sql`` : sql` LEFT JOIN entity_def_sources ON entity_defs."sourceId" = entity_def_sources."id" - LEFT JOIN submission_defs ON submission_defs.id = entity_def_sources."submissionDefId" + LEFT JOIN ( + SELECT submission_defs.* FROM submission_defs + JOIN submissions ON submission_defs."submissionId" = submissions.id + JOIN forms ON submissions."formId" = forms.id + WHERE submissions."deletedAt" IS NULL AND forms."deletedAt" IS NULL + ) as submission_defs on submission_defs.id = entity_def_sources."submissionDefId" LEFT JOIN ( SELECT submissions.*, submission_defs."userAgent" FROM submissions JOIN submission_defs ON submissions.id = submission_defs."submissionId" AND root diff --git a/lib/model/query/forms.js b/lib/model/query/forms.js index 476f3ce62..c609e4b7f 100644 --- a/lib/model/query/forms.js +++ b/lib/model/query/forms.js @@ -406,7 +406,7 @@ const _trashedFilter = (force, id, projectId, xmlFormId) => { // 3. Update actees table for the specific form to leave some useful information behind // 4. Delete the forms and their resources from the database // 5. Purge unattached blobs -const purge = (force = false, id = null, projectId = null, xmlFormId = null) => ({ oneFirst, Blobs }) => { +const purge = (force = false, id = null, projectId = null, xmlFormId = null) => ({ oneFirst }) => { if (xmlFormId != null && projectId == null) throw Problem.internal.unknown({ error: 'Must also specify projectId when using xmlFormId' }); return oneFirst(sql` @@ -415,6 +415,15 @@ with redacted_audits as ( from forms where audits."acteeId" = forms."acteeId" and ${_trashedFilter(force, id, projectId, xmlFormId)} + ), deleted_client_audits as ( + delete from client_audits + using submission_attachments, submission_defs, submissions, forms + where client_audits."blobId" = submission_attachments."blobId" + and submission_attachments."submissionDefId" = submission_defs.id + and submission_attachments."isClientAudit" = true + and submission_defs."submissionId" = submissions.id + and submissions."formId" = forms.id + and ${_trashedFilter(force, id, projectId, xmlFormId)} ), purge_audits as ( insert into audits ("action", "acteeId", "loggedAt", "processed") select 'form.purge', "acteeId", clock_timestamp(), clock_timestamp() @@ -437,9 +446,7 @@ with redacted_audits as ( where ${_trashedFilter(force, id, projectId, xmlFormId)} returning 1 ) -select count(*) from deleted_forms`) - .then((count) => Blobs.purgeUnattached() - .then(() => Promise.resolve(count))); +select count(*) from deleted_forms`); }; //////////////////////////////////////////////////////////////////////////////// diff --git a/lib/model/query/submissions.js b/lib/model/query/submissions.js index 588dc0e93..907210842 100644 --- a/lib/model/query/submissions.js +++ b/lib/model/query/submissions.js @@ -7,13 +7,14 @@ // including this file, may be copied, modified, propagated, or distributed // except according to the terms contained in the LICENSE file. +const config = require('config'); const { map } = require('ramda'); const { sql } = require('slonik'); const { Frame, table } = require('../frame'); const { Actor, Form, Submission } = require('../frames'); const { odataFilter, odataOrderBy } = require('../../data/odata-filter'); const { odataToColumnMap, odataSubTableToColumnMap } = require('../../data/submission'); -const { unjoiner, extender, equals, page, updater, QueryOptions, insertMany } = require('../../util/db'); +const { unjoiner, extender, equals, page, updater, QueryOptions, insertMany, markDeleted, markUndeleted } = require('../../util/db'); const { blankStringToNull, construct } = require('../../util/util'); const Problem = require('../../util/problem'); const { streamEncBlobs } = require('../../util/blob'); @@ -200,6 +201,17 @@ const getById = (submissionId) => ({ maybeOne }) => maybeOne(sql`select * from submissions where id=${submissionId} and "deletedAt" is null`) .then(map(construct(Submission))); +const getDeleted = (projectId, formId, instanceId) => ({ maybeOne }) => + maybeOne(sql` select submissions.* + from submissions + join forms on submissions."formId" = forms.id + where forms."projectId" = ${projectId} + and submissions."formId" = ${formId} + and submissions."instanceId" = ${instanceId} + and submissions."deletedAt" IS NOT NULL +`) + .then(map(construct(Submission))); + const joinOnSkiptoken = (skiptoken, formId, draft) => { if (skiptoken == null) return sql``; // In the case of a subtable, we fetch all submissions without pagination: we @@ -386,14 +398,86 @@ const getForExport = (formId, instanceId, draft, options = QueryOptions.none) => maybeOne(_export(formId, draft, [], options.withCondition({ 'submissions.instanceId': instanceId }))) .then(map(_exportUnjoiner)); +//////////////////////////////////////////////////////////////////////////////// +// DELETE SUBMISSION + +const del = (submission) => ({ run }) => + run(markDeleted(submission)); +del.audit = (submission, form) => (log) => log('submission.delete', { acteeId: form.acteeId }, { instanceId: submission.instanceId }); + +const restore = (submission) => ({ run }) => + run(markUndeleted(submission)); +restore.audit = (submission, form) => (log) => log('submission.restore', { acteeId: form.acteeId }, { instanceId: submission.instanceId }); + +//////////////////////////////////////////////////////////////////////////////// +// PURGING SOFT-DELETED SUBMISSIONS + +const DAY_RANGE = config.has('default.taskSchedule.purge') + ? config.get('default.taskSchedule.purge') + : 30; // Default is 30 days + + +// Submission purging and the trash filter can target a specific submission +// or all deleted submissions, but can't be used to filter-purge by project or form. +const _trashedFilter = (force, projectId, xmlFormId, instanceId) => { + const idFilter = ((instanceId != null) && (projectId != null) && (xmlFormId != null) + ? sql`and submissions."instanceId" = ${instanceId} + and forms."projectId" = ${projectId} + and forms."xmlFormId" = ${xmlFormId}` + : sql``); + return (force + ? sql`submissions."deletedAt" is not null ${idFilter}` + : sql`submissions."deletedAt" < current_date - cast(${DAY_RANGE} as int) ${idFilter}`); +}; + +const purge = (force = false, projectId = null, xmlFormId = null, instanceId = null) => ({ oneFirst }) => { + if ((instanceId != null || projectId != null || xmlFormId != null) && !(instanceId != null && projectId != null && xmlFormId != null)) { + throw Problem.internal.unknown({ error: 'Must specify either all or none of projectId, xmlFormId, and instanceId' }); + } + return oneFirst(sql` +with redacted_audits as ( + update audits set notes = '' + from submissions + join forms on submissions."formId" = forms.id + where (audits.details->>'submissionId')::int = submissions.id + and ${_trashedFilter(force, projectId, xmlFormId, instanceId)} + ), deleted_client_audits as ( + delete from client_audits + using submission_attachments, submission_defs, submissions, forms + where client_audits."blobId" = submission_attachments."blobId" + and submission_attachments."submissionDefId" = submission_defs.id + and submission_attachments."isClientAudit" = true + and submission_defs."submissionId" = submissions.id + and submissions."formId" = forms.id + and ${_trashedFilter(force, projectId, xmlFormId, instanceId)} + ), purge_audits as ( + insert into audits ("action", "loggedAt", "processed", "details") + select 'submission.purge', clock_timestamp(), clock_timestamp(), jsonb_build_object('submissions_deleted', "count") + from ( + select count(*) as count + from submissions + join forms on forms.id = submissions."formId" + where ${_trashedFilter(force, projectId, xmlFormId, instanceId)} + ) as del_sub_count + where del_sub_count.count > 0 + ), deleted_submissions as ( + delete from submissions + using forms + where submissions."formId" = forms.id + and ${_trashedFilter(force, projectId, xmlFormId, instanceId)} + returning 1 + ) +select count(*) from deleted_submissions`); +}; module.exports = { createNew, createVersion, - update, clearDraftSubmissions, clearDraftSubmissionsForProject, + update, del, restore, purge, clearDraftSubmissions, clearDraftSubmissionsForProject, setSelectMultipleValues, getSelectMultipleValuesForExport, getByIdsWithDef, getSubAndDefById, getByIds, getAllForFormByIds, getById, countByFormId, verifyVersion, getDefById, getCurrentDefByIds, getAnyDefByFormAndInstanceId, getDefsByFormAndLogicalId, getDefBySubmissionAndInstanceId, getRootForInstanceId, + getDeleted, streamForExport, getForExport }; diff --git a/lib/resources/entities.js b/lib/resources/entities.js index fe2b1f42e..4f8b3e132 100644 --- a/lib/resources/entities.js +++ b/lib/resources/entities.js @@ -156,13 +156,13 @@ module.exports = (service, endpoint) => { })); - service.delete('/projects/:projectId/datasets/:name/entities/:uuid', endpoint(async ({ Datasets, Entities }, { auth, params, queryOptions }) => { + service.delete('/projects/:projectId/datasets/:name/entities/:uuid', endpoint(async ({ Datasets, Entities }, { auth, params }) => { const dataset = await Datasets.get(params.projectId, params.name, true).then(getOrNotFound); await auth.canOrReject('entity.delete', dataset); - const entity = await Entities.getById(dataset.id, params.uuid, queryOptions).then(getOrNotFound); + const entity = await Entities.getById(dataset.id, params.uuid).then(getOrNotFound); await Entities.del(entity, dataset); diff --git a/lib/resources/submissions.js b/lib/resources/submissions.js index f3d1eaa7e..69fa97821 100644 --- a/lib/resources/submissions.js +++ b/lib/resources/submissions.js @@ -267,6 +267,22 @@ module.exports = (service, endpoint) => { .then(getOrNotFound) .then((submission) => Submissions.update(form, submission, Submission.fromApi(body)))))); + service.delete('/projects/:projectId/forms/:formId/submissions/:instanceId', endpoint(async ({ Forms, Submissions }, { params, auth }) => { + const form = await Forms.getByProjectAndXmlFormId(params.projectId, params.formId, false).then(getOrNotFound); + await auth.canOrReject('submission.delete', form); + const submission = await Submissions.getByIdsWithDef(params.projectId, params.formId, params.instanceId, false).then(getOrNotFound); + await Submissions.del(submission, form); + return success(); + })); + + service.post('/projects/:projectId/forms/:formId/submissions/:instanceId/restore', endpoint(async ({ Forms, Submissions }, { params, auth }) => { + const form = await Forms.getByProjectAndXmlFormId(params.projectId, params.formId, false).then(getOrNotFound); + await auth.canOrReject('submission.restore', form); + const submission = await Submissions.getDeleted(params.projectId, form.id, params.instanceId).then(getOrNotFound); + await Submissions.restore(submission, form); + return success(); + })); + const dataOutputs = (base, draft, getForm) => { diff --git a/lib/task/purge.js b/lib/task/purge.js index 974b12c1b..e83ba52ce 100644 --- a/lib/task/purge.js +++ b/lib/task/purge.js @@ -9,7 +9,31 @@ const { task } = require('./task'); -const purgeForms = task.withContainer(({ Forms }) => (force = false, formId = null, projectId = null, xmlFormId = null) => - Forms.purge(force, formId, projectId, xmlFormId)); +const purgeTask = task.withContainer((container) => async (options = {}) => { + // Form/submission purging will happen within its own transaction + const message = await container.db.transaction(async trxn => { + const { Forms, Submissions } = container.with({ db: trxn }); + try { + if (options.mode === 'submissions' || options.instanceId) { + const count = await Submissions.purge(options.force, options.projectId, options.xmlFormId, options.instanceId); + return `Submissions purged: ${count}`; + } else if (options.mode === 'forms' || (options.formId || options.xmlFormId)) { + const count = await Forms.purge(options.force, options.formId, options.projectId, options.xmlFormId); + return `Forms purged: ${count}`; + } else { + const formCount = await Forms.purge(options.force, options.formId, options.projectId, options.xmlFormId); + const submissionCount = await Submissions.purge(options.force, options.projectId, options.xmlFormId, options.instanceId); + return `Forms purged: ${formCount}, Submissions purged: ${submissionCount}`; + } + } catch (error) { + return error?.problemDetails?.error; + } + }); -module.exports = { purgeForms }; + // Purging unattached blobs is outside of the above transaction because it + // may interact with an external blob store. + await container.Blobs.purgeUnattached(); + return message; +}); + +module.exports = { purgeTask }; diff --git a/test/integration/api/odata.js b/test/integration/api/odata.js index 0d0f9b417..30ee42323 100644 --- a/test/integration/api/odata.js +++ b/test/integration/api/odata.js @@ -126,6 +126,13 @@ describe('api: /forms/:id.svc', () => { withSubmission(service, (asAlice) => asAlice.get("/v1/projects/1/forms/doubleRepeat.svc/Submissions('nonexistent')").expect(404)))); + it('should reject if the submission has been soft-deleted', testService((service) => + withSubmission(service, (asAlice) => + asAlice.delete('/v1/projects/1/forms/doubleRepeat/submissions/double') + .expect(200) // soft-delete + .then(() => asAlice.get("/v1/projects/1/forms/doubleRepeat.svc/Submissions('double')") + .expect(404))))); + it('should return a single row result', testService((service) => withSubmission(service, (asAlice) => asAlice.get("/v1/projects/1/forms/doubleRepeat.svc/Submissions('double')") @@ -649,6 +656,65 @@ describe('api: /forms/:id.svc', () => { }); })))); + it('should exclude a deleted submission from rows', testService((service) => + withSubmissions(service, (asAlice) => + asAlice.delete('/v1/projects/1/forms/withrepeat/submissions/rthree') + .expect(200) + .then(() => asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions') + .expect(200) + .then(({ body }) => { + for (const idx of [0, 1]) { + body.value[idx].__system.submissionDate.should.be.an.isoDate(); + // eslint-disable-next-line no-param-reassign + delete body.value[idx].__system.submissionDate; + } + + body.should.eql({ + '@odata.context': 'http://localhost:8989/v1/projects/1/forms/withrepeat.svc/$metadata#Submissions', + value: [{ + __id: 'rtwo', + __system: { + // submissionDate is checked above, + updatedAt: null, + submitterId: '5', + submitterName: 'Alice', + attachmentsPresent: 0, + attachmentsExpected: 0, + status: null, + reviewState: null, + deviceId: null, + edits: 0, + formVersion: '1.0' + }, + meta: { instanceID: 'rtwo' }, + name: 'Bob', + age: 34, + children: { + 'child@odata.navigationLink': "Submissions('rtwo')/children/child" + } + }, { + __id: 'rone', + __system: { + // submissionDate is checked above, + updatedAt: null, + submitterId: '5', + submitterName: 'Alice', + attachmentsPresent: 0, + attachmentsExpected: 0, + status: null, + reviewState: null, + deviceId: null, + edits: 0, + formVersion: '1.0' + }, + meta: { instanceID: 'rone' }, + name: 'Alice', + age: 30, + children: {} + }] + }); + }))))); + it('should return a count even if there are no rows', testService((service) => service.login('alice', (asAlice) => asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions?$count=true') @@ -803,7 +869,7 @@ describe('api: /forms/:id.svc', () => { }); })); - it('should support $skiptoken even if associated submission is deleted', testService(async (service, { run }) => { + it('should support $skiptoken even if associated submission is deleted', testService(async (service) => { const asAlice = await service.login('alice'); await asAlice.post('/v1/projects/1/forms/simple/submissions') .send(testData.instances.simple.one) @@ -817,9 +883,7 @@ describe('api: /forms/:id.svc', () => { .expect(200) .then(({ body }) => new URL(body['@odata.nextLink']).searchParams.get('$skiptoken')); QueryOptions.parseSkiptoken(skiptoken).instanceId.should.equal('two'); - // We don't have a submission delete endpoint yet, but we should soon. - await run(sql`UPDATE submissions SET "deletedAt" = clock_timestamp() -WHERE "instanceId" = 'two'`); + await asAlice.delete('/v1/projects/1/forms/simple/submissions/two'); const { body: odata } = await asAlice.get(url`/v1/projects/1/forms/simple.svc/Submissions?%24skiptoken=${skiptoken}`) .expect(200); odata.value.length.should.equal(1); diff --git a/test/integration/api/submissions.js b/test/integration/api/submissions.js index 07f1c06cc..b63464cdc 100644 --- a/test/integration/api/submissions.js +++ b/test/integration/api/submissions.js @@ -3,7 +3,7 @@ const should = require('should'); const uuid = require('uuid').v4; const { sql } = require('slonik'); const { createReadStream, readFileSync } = require('fs'); -const { testService } = require('../setup'); +const { testService, testServiceFullTrx } = require('../setup'); const testData = require('../../data/xml'); const { pZipStreamToFiles } = require('../../util/zip'); const { map } = require('ramda'); @@ -1399,6 +1399,158 @@ describe('api: /forms/:id/submissions', () => { .then(({ body }) => { body.reviewState.should.equal('approved'); }))))); }); + describe('/:instanceId DELETE', () => { + it('should reject notfound if the submission does not exist', testService((service) => + service.login('alice', (asAlice) => + asAlice.delete('/v1/projects/1/forms/simple/submissions/one') + .expect(404)))); + + it('should reject if the user cannot delete', testService((service) => + service.login('alice', (asAlice) => + asAlice.post('/v1/projects/1/forms/simple/submissions') + .send(testData.instances.simple.one) + .set('Content-Type', 'application/xml') + .expect(200) + .then(() => service.login('chelsea', (asChelsea) => + asChelsea.delete('/v1/projects/1/forms/simple/submissions/one') + .expect(403)))))); + + it('should soft-delete the submission and not be able to access it again', testService((service) => + service.login('alice', (asAlice) => + asAlice.post('/v1/projects/1/forms/simple/submissions') + .send(testData.instances.simple.one) + .set('Content-Type', 'application/xml') + .expect(200) + .then(() => asAlice.delete('/v1/projects/1/forms/simple/submissions/one') + .expect(200)) + .then(() => asAlice.get('/v1/projects/1/forms/simple/submissions/one') + .expect(404))))); + + it('should not let a draft submission be deleted', testService(async (service) => { + const asAlice = await service.login('alice'); + await asAlice.post('/v1/projects/1/forms/simple/draft'); + await asAlice.post('/v1/projects/1/forms/simple/draft/submissions') + .send(testData.instances.simple.one) + .set('Content-Type', 'application/xml') + .expect(200); + // draft submission delete resource does not exist + await asAlice.delete('/v1/projects/1/forms/simple/draft/submissions/one') + .expect(404); + })); + + it('should not let a submission with the same instanceId as a deleted submission be sent', testServiceFullTrx(async (service, { Submissions }) => { + const asAlice = await service.login('alice'); + + await asAlice.post('/v1/projects/1/forms/simple/submissions') + .send(testData.instances.simple.one) + .set('Content-Type', 'application/xml') + .expect(200); + + await asAlice.delete('/v1/projects/1/forms/simple/submissions/one') + .expect(200); + + await asAlice.post('/v1/projects/1/forms/simple/submissions') + .send(testData.instances.simple.one) + .set('Content-Type', 'application/xml') + .expect(409); + + // once purged, the submission can be sent in again + await Submissions.purge(true); + + await asAlice.post('/v1/projects/1/forms/simple/submissions') + .send(testData.instances.simple.one) + .set('Content-Type', 'application/xml') + .expect(200); + })); + + it('should delete and restore non-draft submission even when draft submission with same instanceId exists', testService(async (service) => { + const asAlice = await service.login('alice'); + + await asAlice.post('/v1/projects/1/forms/simple/submissions') + .send(testData.instances.simple.one) + .set('Content-Type', 'application/xml') + .expect(200); + + // make a draft submission with the same instanceId + await asAlice.post('/v1/projects/1/forms/simple/draft'); + await asAlice.post('/v1/projects/1/forms/simple/draft/submissions') + .send(testData.instances.simple.one) + .set('Content-Type', 'application/xml') + .expect(200); + + // does not effect delete and restore of non-draft submission + await asAlice.delete('/v1/projects/1/forms/simple/submissions/one') + .expect(200); + + await asAlice.post('/v1/projects/1/forms/simple/submissions/one/restore') + .expect(200); + + await asAlice.get('/v1/projects/1/forms/simple/submissions/one') + .expect(200); + })); + }); + + describe('/:instanceId RESTORE', () => { + it('should reject if the submission has not been deleted', testService((service) => + service.login('alice', (asAlice) => + asAlice.post('/v1/projects/1/forms/simple/submissions') + .send(testData.instances.simple.one) + .set('Content-Type', 'application/xml') + .expect(200) + .then(() => asAlice.post('/v1/projects/1/forms/simple/submissions/one/restore') + .expect(404))))); + + it('should reject if the submission does not exist', testService((service) => + service.login('alice', (asAlice) => + asAlice.post('/v1/projects/1/forms/simple/submissions/nonexistant/restore') + .expect(404)))); + + it('should reject if the user cannot restore', testService(async (service) => { + const asAlice = await service.login('alice'); + const asChelsea = await service.login('chelsea'); + + // Create a submission + await asAlice.post('/v1/projects/1/forms/simple/submissions') + .send(testData.instances.simple.one) + .set('Content-Type', 'application/xml') + .expect(200); + + // Delete the submission + await asAlice.delete('/v1/projects/1/forms/simple/submissions/one') + .expect(200); + + // Chelsea cannot restore + await asChelsea.post('/v1/projects/1/forms/simple/submissions/one/restore') + .expect(403); + })); + + it('should soft-delete the submission and then restore it', testService(async (service) => { + const asAlice = await service.login('alice'); + + // Create a submission + await asAlice.post('/v1/projects/1/forms/simple/submissions') + .send(testData.instances.simple.one) + .set('Content-Type', 'application/xml') + .expect(200); + + // Delete the submission + await asAlice.delete('/v1/projects/1/forms/simple/submissions/one') + .expect(200); + + // Accessing the submission should 404 + await asAlice.get('/v1/projects/1/forms/simple/submissions/one') + .expect(404); + + // Restore the submission + await asAlice.post('/v1/projects/1/forms/simple/submissions/one/restore') + .expect(200); + + // Accessing the submission should 200 + await asAlice.get('/v1/projects/1/forms/simple/submissions/one') + .expect(200); + })); + }); + describe('.csv.zip GET', () => { // NOTE: tests related to decryption of .csv.zip export are located in test/integration/other/encryption.js @@ -1502,6 +1654,39 @@ describe('api: /forms/:id/submissions', () => { csv[3].should.eql([ '' ]); }))))); + it('should not return data from deleted submissions in csv export', testService(async (service) => { + const asAlice = await service.login('alice'); + + await asAlice.post('/v1/projects/1/forms/simple/submissions') + .send(testData.instances.simple.one) + .set('Content-Type', 'text/xml') + .expect(200); + + await asAlice.post('/v1/projects/1/forms/simple/submissions') + .send(testData.instances.simple.two) + .set('Content-Type', 'text/xml') + .expect(200); + + await asAlice.post('/v1/projects/1/forms/simple/submissions') + .send(testData.instances.simple.three) + .set('Content-Type', 'text/xml') + .expect(200); + + await asAlice.delete('/v1/projects/1/forms/simple/submissions/two'); + + const result = await pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip')); + const csv = result['simple.csv'].split('\n').map((row) => row.split(',')); + csv.length.should.equal(4); // header + 2 data rows + newline + csv[0].should.eql([ 'SubmissionDate', 'meta-instanceID', 'name', 'age', 'KEY', 'SubmitterID', 'SubmitterName', 'AttachmentsPresent', 'AttachmentsExpected', 'Status', 'ReviewState', 'DeviceID', 'Edits', 'FormVersion' ]); + csv[1].shift().should.be.an.recentIsoDate(); + // eslint-disable-next-line comma-spacing + csv[1].should.eql([ 'three','Chelsea','38','three','5','Alice','0','0','','','','0','' ]); + csv[2].shift().should.be.an.recentIsoDate(); + // eslint-disable-next-line comma-spacing + csv[2].should.eql([ 'one','Alice','30','one','5','Alice','0','0','','','','0','' ]); + csv[3].should.eql([ '' ]); + })); + it('should return a submitter-filtered zipfile with the relevant data', testService((service) => service.login('alice', (asAlice) => service.login('bob', (asBob) => diff --git a/test/integration/other/blobs.js b/test/integration/other/blobs.js index 26ffb61d8..e17e1695f 100644 --- a/test/integration/other/blobs.js +++ b/test/integration/other/blobs.js @@ -43,6 +43,7 @@ describe('blob query module', () => { .expect(201)) .then(() => asAlice.delete('/v1/projects/1/forms/binaryType')) .then(() => container.Forms.purge(true)) + .then(() => container.Blobs.purgeUnattached()) .then(() => container.oneFirst(sql`select count(*) from blobs`)) .then((count) => count.should.equal(1))))); // }); diff --git a/test/integration/other/form-purging.js b/test/integration/other/form-purging.js index db498225b..464baad44 100644 --- a/test/integration/other/form-purging.js +++ b/test/integration/other/form-purging.js @@ -1,6 +1,7 @@ const { createReadStream, readFileSync } = require('fs'); const appPath = require('app-root-path'); const { sql } = require('slonik'); +const assert = require('assert'); const { testService } = require('../setup'); const testData = require('../../data/xml'); const { exhaust } = require(appPath + '/lib/worker/worker'); @@ -59,25 +60,6 @@ describe('query module form purge', () => { counts.should.eql([ 0, 0 ]); }))))); - it('should purge a deleted form by ID', testService((service, container) => - service.login('alice', (asAlice) => - asAlice.delete('/v1/projects/1/forms/simple') - .expect(200) - .then(() => asAlice.post('/v1/projects/1/forms') - .send(testData.forms.withAttachments) - .set('Content-Type', 'application/xml') - .expect(200)) - .then(() => container.Forms.getByProjectAndXmlFormId(1, 'withAttachments').then((o) => o.get())) - .then((ghostForm) => asAlice.delete('/v1/projects/1/withAttachments') - .then(() => container.Forms.purge(true, 1)) // force delete a single form - .then(() => Promise.all([ - container.oneFirst(sql`select count(*) from forms where id = ${ghostForm.id}`), - container.oneFirst(sql`select count(*) from forms where id = 1`), // deleted form id - ]) - .then((counts) => { - counts.should.eql([ 1, 0 ]); - })))))); - it('should log the purge action in the audit log', testService((service, container) => service.login('alice', (asAlice) => container.Forms.getByProjectAndXmlFormId(1, 'simple').then((o) => o.get()) // get the form before we delete it @@ -90,6 +72,26 @@ describe('query module form purge', () => { audit.get().acteeId.should.equal(form.acteeId); }))))); + it('should log purge action in the audit log for each form', testService(async (service, container) => { + const asAlice = await service.login('alice'); + + const simpleForm = await container.Forms.getByProjectAndXmlFormId(1, 'simple').then((o) => o.get()); + const repeatForm = await container.Forms.getByProjectAndXmlFormId(1, 'withrepeat').then((o) => o.get()); + + await asAlice.delete('/v1/projects/1/forms/simple') + .expect(200); + + await asAlice.delete('/v1/projects/1/forms/withrepeat') + .expect(200); + + await container.Forms.purge(true); + + await asAlice.get('/v1/audits') + .then(({ body }) => { + body.filter((a) => a.action === 'form.purge').map(a => a.acteeId).should.eqlInAnyOrder([simpleForm.acteeId, repeatForm.acteeId]); + }); + })); + it('should update the actee table with purgedAt details', testService((service, container) => service.login('alice', (asAlice) => container.Forms.getByProjectAndXmlFormId(1, 'simple').then((o) => o.get()) // get the form before we delete it @@ -142,6 +144,7 @@ describe('query module form purge', () => { .then((ghostForm) => asAlice.delete('/v1/projects/1/forms/withAttachments') .expect(200) .then(() => container.Forms.purge(true)) + .then(() => container.Blobs.purgeUnattached()) .then(() => Promise.all([ container.oneFirst(sql`select count(*) from forms where id = ${ghostForm.id}`), container.oneFirst(sql`select count(*) from form_defs where "formId" = ${ghostForm.id}`), @@ -166,6 +169,7 @@ describe('query module form purge', () => { .then((ghostForm) => asAlice.delete('/v1/projects/1/forms/withAttachments') .expect(200) .then(() => container.Forms.purge(true)) + .then(() => container.Blobs.purgeUnattached()) .then(() => Promise.all([ container.oneFirst(sql`select count(*) from forms where id = ${ghostForm.id}`), container.oneFirst(sql`select count(*) from form_defs where "formId" = ${ghostForm.id}`), @@ -195,6 +199,7 @@ describe('query module form purge', () => { .expect(200) .then(() => container.Blobs.s3UploadPending()) .then(() => container.Forms.purge(true)) + .then(() => container.Blobs.purgeUnattached()) .then(() => Promise.all([ container.oneFirst(sql`select count(*) from forms where id = ${ghostForm.id}`), container.oneFirst(sql`select count(*) from form_defs where "formId" = ${ghostForm.id}`), @@ -241,6 +246,97 @@ describe('query module form purge', () => { .then(() => container.oneFirst(sql`select count(*) from form_field_values`)) .then((count) => count.should.eql(0))))); + describe('purging specific forms via specific arguments', () => { + it('should purge a deleted form by ID', testService((service, container) => + service.login('alice', (asAlice) => + asAlice.delete('/v1/projects/1/forms/simple') + .expect(200) + .then(() => asAlice.post('/v1/projects/1/forms') + .send(testData.forms.withAttachments) + .set('Content-Type', 'application/xml') + .expect(200)) + .then(() => container.Forms.getByProjectAndXmlFormId(1, 'withAttachments').then((o) => o.get())) + .then((ghostForm) => asAlice.delete('/v1/projects/1/withAttachments') + .then(() => container.Forms.purge(true, 1)) // force delete a single form + .then(() => Promise.all([ + container.oneFirst(sql`select count(*) from forms where id = ${ghostForm.id}`), + container.oneFirst(sql`select count(*) from forms where id = 1`), // deleted form id + ]) + .then((counts) => { + counts.should.eql([ 1, 0 ]); + })))))); + + it('should purge all versions of deleted form in project', testService(async (service, container) => { + const asAlice = await service.login('alice'); + + await asAlice.delete('/v1/projects/1/forms/simple') + .expect(200); + + // new version (will be v2) + await asAlice.post('/v1/projects/1/forms?ignoreWarnings=true') + .send(testData.forms.simple) + .set('Content-Type', 'application/xml') + .expect(200); + + // publish new version v2 + await asAlice.post('/v1/projects/1/forms/simple/draft/publish?ignoreWarnings=true&version=v2') + .expect(200); + + // delete new version v2 + await asAlice.delete('/v1/projects/1/forms/simple') + .expect(200); + + // new version (will be v3) + await asAlice.post('/v1/projects/1/forms?ignoreWarnings=true') + .send(testData.forms.simple) + .set('Content-Type', 'application/xml') + .expect(200); + + // publish new version v3 but don't delete + await asAlice.post('/v1/projects/1/forms/simple/draft/publish?ignoreWarnings=true&version=v3') + .expect(200); + + const count = await container.Forms.purge(true, null, 1, 'simple'); + count.should.equal(2); + })); + + it('should purge named form only from specified project', testService(async (service, container) => { + const asAlice = await service.login('alice'); + + // delete simple form in project 1 (but don't purge it) + await asAlice.delete('/v1/projects/1/forms/simple') + .expect(200); + + const newProjectId = await asAlice.post('/v1/projects') + .send({ name: 'Project Two' }) + .then(({ body }) => body.id); + + await asAlice.post(`/v1/projects/${newProjectId}/forms?publish=true`) + .send(testData.forms.simple) + .set('Content-Type', 'application/xml') + .expect(200); + + await asAlice.delete(`/v1/projects/${newProjectId}/forms/simple`) + .expect(200); + + const count = await container.Forms.purge(true, null, newProjectId, 'simple'); + count.should.equal(1); + })); + + it('should throw an error when xmlFormId specified without project ID', testService(async (service, container) => { + const asAlice = await service.login('alice'); + + await asAlice.delete('/v1/projects/1/forms/simple') + .expect(200); + + await assert.throws(() => { container.Forms.purge(true, null, null, 'simple'); }, (err) => { + err.problemCode.should.equal(500.1); + err.problemDetails.error.should.equal('Must also specify projectId when using xmlFormId'); + return true; + }); + })); + }); + describe('purging form submissions', () => { const withSimpleIds = (deprecatedId, instanceId) => testData.instances.simple.one .replace('one${deprecatedId} { })) .then(() => asAlice.delete('/v1/projects/1/forms/binaryType')) .then(() => container.Forms.purge(true)) + .then(() => container.Blobs.purgeUnattached()) .then(() => container.oneFirst(sql`select count(*) from submission_attachments`) .then((count) => count.should.equal(0))) .then(() => container.oneFirst(sql`select count(*) from blobs`) @@ -323,7 +420,7 @@ describe('query module form purge', () => { // eslint-disable-next-line space-in-parens .then((audit) => audit.get().notes.should.equal('') ))))); - it('should purge client audit log attachments', testService((service, container) => + it('should purge client audit log attachments (that have been processed into database)', testService((service, container) => service.login('alice', (asAlice) => asAlice.post('/v1/projects/1/forms?publish=true') .set('Content-Type', 'application/xml') @@ -334,8 +431,10 @@ describe('query module form purge', () => { .attach('audit.csv', createReadStream(appPath + '/test/data/audit.csv'), { filename: 'audit.csv' }) .attach('xml_submission_file', Buffer.from(testData.instances.clientAudits.one), { filename: 'data.xml' }) .expect(201)) + .then(() => exhaust(container)) .then(() => asAlice.delete('/v1/projects/1/forms/audits')) .then(() => container.Forms.purge(true)) + .then(() => container.Blobs.purgeUnattached()) .then(() => Promise.all([ container.oneFirst(sql`select count(*) from client_audits`), container.oneFirst(sql`select count(*) from blobs`) @@ -350,6 +449,7 @@ describe('query module form purge', () => { .then(() => asAlice.delete('/v1/projects/1/forms/simple2') // Delete form .expect(200)) .then(() => container.Forms.purge(true)) + .then(() => container.Blobs.purgeUnattached()) .then(() => container.oneFirst(sql`select count(*) from blobs`)) .then((count) => count.should.equal(0))))); }); diff --git a/test/integration/other/migrations.js b/test/integration/other/migrations.js index 4ed27339a..8b6354a4e 100644 --- a/test/integration/other/migrations.js +++ b/test/integration/other/migrations.js @@ -4,8 +4,11 @@ const uuid = require('uuid').v4; const config = require('config'); const { testContainerFullTrx, testServiceFullTrx } = require('../setup'); const { sql } = require('slonik'); +const { createReadStream } = require('fs'); const { Actor, Config } = require(appRoot + '/lib/model/frames'); const { withDatabase } = require(appRoot + '/lib/model/migrate'); +const { exhaust } = require(appRoot + '/lib/worker/worker'); + const testData = require('../../data/xml'); const populateUsers = require('../fixtures/01-users'); const populateForms = require('../fixtures/02-forms'); @@ -988,3 +991,75 @@ testMigration('20240215-02-dedupe-verbs.js', () => { for (const { verbs } of roles) verbs.should.eql([...new Set(verbs)]); })); }); + +testMigration('20240914-02-remove-orphaned-client-audits.js', () => { + it('should remove orphaned client audits', testServiceFullTrx(async (service, container) => { + await populateUsers(container); + await populateForms(container); + + const asAlice = await service.login('alice'); + + await asAlice.post('/v1/projects/1/forms?publish=true') + .send(testData.forms.clientAudits) + .expect(200); + + // Send the submission with the client audit attachment + await asAlice.post('/v1/projects/1/submission') + .set('X-OpenRosa-Version', '1.0') + .attach('audit.csv', createReadStream(appRoot + '/test/data/audit.csv'), { filename: 'audit.csv' }) + .attach('xml_submission_file', Buffer.from(testData.instances.clientAudits.one), { filename: 'data.xml' }) + .expect(201); + + await asAlice.post('/v1/projects/1/submission') + .set('X-OpenRosa-Version', '1.0') + .attach('log.csv', createReadStream(appRoot + '/test/data/audit2.csv'), { filename: 'log.csv' }) + .attach('xml_submission_file', Buffer.from(testData.instances.clientAudits.two), { filename: 'data.xml' }) + .expect(201); + + await exhaust(container); + + // there should be 8 total rows (5 + 3) + let numClientAudits = await container.oneFirst(sql`select count(*) from client_audits`); + numClientAudits.should.equal(8); + + // there should be 2 blobs + let blobCount = await container.oneFirst(sql`select count(*) from blobs`); + blobCount.should.equal(2); + + // delete one of the submissions + await asAlice.delete('/v1/projects/1/forms/audits/submissions/one') + .expect(200); + + // simulate purge without client audit purge + await container.run(sql`delete from submissions + where submissions."deletedAt" is not null`); + + // purge unattached blobs (will not purge any because one is still referenced) + await container.Blobs.purgeUnattached(true); + + // blobs count should still be 2 + blobCount = await container.oneFirst(sql`select count(*) from blobs`); + blobCount.should.equal(2); + + // client audits still equals 8 after purge (3 orphaned) + numClientAudits = await container.oneFirst(sql`select count(*) from client_audits`); + numClientAudits.should.equal(8); + + // clean up orphaned client audits + await up(); + + numClientAudits = await container.oneFirst(sql`select count(*) from client_audits`); + numClientAudits.should.equal(3); + + // blob count will still be two + blobCount = await container.oneFirst(sql`select count(*) from blobs`); + blobCount.should.equal(2); + + // but next run of purging unattached blobs will purge one + await container.Blobs.purgeUnattached(true); + + // blobs count should finally be 1 + blobCount = await container.oneFirst(sql`select count(*) from blobs`); + blobCount.should.equal(1); + })); +}); diff --git a/test/integration/other/submission-purging.js b/test/integration/other/submission-purging.js new file mode 100644 index 000000000..8cf1ca765 --- /dev/null +++ b/test/integration/other/submission-purging.js @@ -0,0 +1,580 @@ +const assert = require('assert'); +const { sql } = require('slonik'); +const { testService } = require('../setup'); +const testData = require('../../data/xml'); +const should = require('should'); +const { createReadStream } = require('fs'); +const { pZipStreamToFiles } = require('../../util/zip'); + +const appPath = require('app-root-path'); +const { exhaust } = require(appPath + '/lib/worker/worker'); + +describe('query module submission purge', () => { + describe('submission purge arguments', () => { + it('should purge a specific submission', testService(async (service, { Submissions, oneFirst }) => { + const asAlice = await service.login('alice'); + + // Create two submissions + await asAlice.post('/v1/projects/1/forms/simple/submissions') + .send(testData.instances.simple.one) + .set('Content-Type', 'application/xml') + .expect(200); + + await asAlice.post('/v1/projects/1/forms/simple/submissions') + .send(testData.instances.simple.two) + .set('Content-Type', 'application/xml') + .expect(200); + + // Delete both submissions + await asAlice.delete('/v1/projects/1/forms/simple/submissions/one') + .expect(200); + await asAlice.delete('/v1/projects/1/forms/simple/submissions/two') + .expect(200); + + // Purge submissions here should not purge anything because they were in the trash less than 30 days + let purgeCount = await Submissions.purge(); + purgeCount.should.equal(0); + + // But we should be able to force purge a submission + // specified by projectId, xmlFormId, and instanceId + purgeCount = await Submissions.purge(true, 1, 'simple', 'one'); + purgeCount.should.equal(1); + + // One (soft-deleted) submission should still be in the database + const submissionCount = await oneFirst(sql`select count(*) from submissions`); + submissionCount.should.equal(1); + })); + + it('should throw an error when instanceId specified without project ID and xmlFormId', testService(async (service, container) => { + const asAlice = await service.login('alice'); + + await asAlice.post('/v1/projects/1/forms/simple/submissions') + .send(testData.instances.simple.one) + .set('Content-Type', 'application/xml') + .expect(200); + + await asAlice.delete('/v1/projects/1/forms/simple/submissions/one') + .expect(200); + + await assert.throws(() => { container.Submissions.purge(true, null, null, 'one'); }, (err) => { + err.problemCode.should.equal(500.1); + err.problemDetails.error.should.equal('Must specify either all or none of projectId, xmlFormId, and instanceId'); + return true; + }); + })); + + it('should throw an error when project ID or xmlFormId is non-null but there is no instance id', testService(async (service, container) => { + const asAlice = await service.login('alice'); + + await asAlice.post('/v1/projects/1/forms/simple/submissions') + .send(testData.instances.simple.one) + .set('Content-Type', 'application/xml') + .expect(200); + + await asAlice.delete('/v1/projects/1/forms/simple/submissions/one') + .expect(200); + + await assert.throws(() => { container.Submissions.purge(true, null, 'simple', null); }, (err) => { + err.problemCode.should.equal(500.1); + err.problemDetails.error.should.equal('Must specify either all or none of projectId, xmlFormId, and instanceId'); + return true; + }); + })); + }); + + describe('30 day time limit', () => { + it('should purge a submission deleted over 30 days ago', testService(async (service, { Submissions, oneFirst, run }) => { + const asAlice = await service.login('alice'); + + await asAlice.post('/v1/projects/1/forms/simple/submissions') + .send(testData.instances.simple.one) + .set('Content-Type', 'application/xml') + .expect(200); + + await asAlice.delete('/v1/projects/1/forms/simple/submissions/one'); + + await run(sql`update submissions set "deletedAt" = '1999-1-1' where "deletedAt" is not null`); + + const purgeCount = await Submissions.purge(); + purgeCount.should.equal(1); + + const counts = await Promise.all([ + oneFirst(sql`select count(*) from submissions`), + oneFirst(sql`select count(*) from submission_defs`) + ]); + + counts.should.eql([ 0, 0 ]); + })); + + it('should purge multiple submissions deleted over 30 days ago', testService(async (service, { Submissions, oneFirst, run }) => { + const asAlice = await service.login('alice'); + + await asAlice.post('/v1/projects/1/forms/simple/submissions') + .send(testData.instances.simple.one) + .set('Content-Type', 'application/xml') + .expect(200); + + await asAlice.post('/v1/projects/1/forms/simple/submissions') + .send(testData.instances.simple.two) + .set('Content-Type', 'application/xml') + .expect(200); + + await asAlice.post('/v1/projects/1/forms/simple/submissions') + .send(testData.instances.simple.three) + .set('Content-Type', 'application/xml') + .expect(200); + + await asAlice.delete('/v1/projects/1/forms/simple/submissions/one'); + await asAlice.delete('/v1/projects/1/forms/simple/submissions/two'); + // Mark two as deleted a long time ago + await run(sql`update submissions set "deletedAt" = '1999-1-1' where "deletedAt" is not null`); + + // More recent delete, within 30 day window + await asAlice.delete('/v1/projects/1/forms/simple/submissions/three'); + + const purgeCount = await Submissions.purge(); + purgeCount.should.equal(2); + + // Remaining submissions not recently deleted + const counts = await Promise.all([ + oneFirst(sql`select count(*) from submissions`), + oneFirst(sql`select count(*) from submission_defs`) + ]); + counts.should.eql([ 1, 1 ]); + })); + + it('should purge recently deleted submission when forced', testService(async (service, { Submissions }) => { + const asAlice = await service.login('alice'); + + await asAlice.post('/v1/projects/1/forms/simple/submissions') + .send(testData.instances.simple.one) + .set('Content-Type', 'application/xml') + .expect(200); + + await asAlice.delete('/v1/projects/1/forms/simple/submissions/one'); + const purgeCount = await Submissions.purge(true); + purgeCount.should.equal(1); + })); + }); + + describe('deep cleanup of all submission artifacts', () => { + it('should purge attachments with submission', testService(async (service, { Submissions, oneFirst }) => { + const asAlice = await service.login('alice'); + + await asAlice.post('/v1/projects/1/forms?publish=true') + .set('Content-Type', 'application/xml') + .send(testData.forms.binaryType) + .expect(200); + + await asAlice.post('/v1/projects/1/submission') + .set('X-OpenRosa-Version', '1.0') + .attach('xml_submission_file', Buffer.from(testData.instances.binaryType.both), { filename: 'data.xml' }) + .attach('here_is_file2.jpg', Buffer.from('this is test file two'), { filename: 'here_is_file2.jpg' }) + .expect(201); + + let attachments = await oneFirst(sql`select count(*) from submission_attachments`); + attachments.should.equal(2); + + await asAlice.delete('/v1/projects/1/forms/binaryType/submissions/both'); + await Submissions.purge(true); + + attachments = await oneFirst(sql`select count(*) from submission_attachments`); + attachments.should.equal(0); + })); + + it('should purge blobs associated with attachments when purging submission', testService(async (service, { Blobs, Submissions, oneFirst }) => { + const asAlice = await service.login('alice'); + + await asAlice.post('/v1/projects/1/forms?publish=true') + .set('Content-Type', 'application/xml') + .send(testData.forms.binaryType) + .expect(200); + + // Submission has 2 attachments + await asAlice.post('/v1/projects/1/submission') + .set('X-OpenRosa-Version', '1.0') + .attach('xml_submission_file', Buffer.from(testData.instances.binaryType.both), { filename: 'data.xml' }) + .attach('my_file1.mp4', Buffer.from('this is test file one'), { filename: 'my_file1.mp4' }) + .attach('here_is_file2.jpg', Buffer.from('this is test file two'), { filename: 'here_is_file2.jpg' }) + .expect(201); + + // Submission has 1 attachment with same content as one attachment above + await asAlice.post('/v1/projects/1/submission') + .set('X-OpenRosa-Version', '1.0') + .attach('xml_submission_file', Buffer.from(testData.instances.binaryType.one), { filename: 'data.xml' }) + .attach('my_file1.mp4', Buffer.from('this is test file one'), { filename: 'my_file1.mp4' }) + .expect(201); + + let blobCount = await oneFirst(sql`select count(*) from blobs`); + blobCount.should.equal(2); + + // Delete submission with 2 attachments + await asAlice.delete('/v1/projects/1/forms/binaryType/submissions/both'); + await Submissions.purge(true); + + // Purge unattached blobs + await Blobs.purgeUnattached(); + + // One blob still remains from first submission which was not deleted + blobCount = await oneFirst(sql`select count(*) from blobs`); + blobCount.should.equal(1); + })); + + it('should purge all versions of a deleted submission', testService(async (service, { Submissions, oneFirst }) => { + const asAlice = await service.login('alice'); + + // Create a submission on an existing form (simple) + await asAlice.post('/v1/projects/1/forms/simple/submissions') + .send(testData.instances.simple.one) + .set('Content-Type', 'application/xml') + .expect(200); + + // Edit the submission + await asAlice.patch('/v1/projects/1/forms/simple/submissions/one') + .send(testData.instances.simple.one + .replace('one', 'oneone2') + .replace('30', '99')) + .set('Content-Type', 'application/xml') + .expect(200); + + // Delete the submission + await asAlice.delete('/v1/projects/1/forms/simple/submissions/one'); + + // Purge the submission + await Submissions.purge(true); + + // Check that the submission is deleted + const submissionCount = await oneFirst(sql`select count(*) from submissions`); + submissionCount.should.equal(0); + + // Check that submission defs are also deleted + const submissionDefCount = await oneFirst(sql`select count(*) from submission_defs`); + submissionDefCount.should.equal(0); + })); + + it('should purge comments of a deleted submission', testService(async (service, { Submissions, oneFirst }) => { + const asAlice = await service.login('alice'); + + // Create a submission + await asAlice.post('/v1/projects/1/forms/simple/submissions') + .send(testData.instances.simple.one) + .set('Content-Type', 'application/xml') + .expect(200); + + // Add a comment to the submission + await asAlice.post('/v1/projects/1/forms/simple/submissions/one/comments') + .send({ body: 'new comment here' }) + .expect(200); + + let commentCount = await oneFirst(sql`select count(*) from comments`); + commentCount.should.equal(1); + + // Delete the submission + await asAlice.delete('/v1/projects/1/forms/simple/submissions/one'); + + // Purge the submission + await Submissions.purge(true); + + // Check that the comment is deleted + commentCount = await oneFirst(sql`select count(*) from comments`); + commentCount.should.equal(0); + })); + + it('should redact notes of a deleted submission sent with x-action-notes', testService(async (service, { Submissions, oneFirst }) => { + const asAlice = await service.login('alice'); + + // Create a submission + await asAlice.post('/v1/projects/1/forms/simple/submissions') + .send(testData.instances.simple.one) + .set('X-Action-Notes', 'a note about the submission') + .set('Content-Type', 'application/xml') + .expect(200); + + // Check that the note exists in the submission's audit log + await asAlice.get('/v1/projects/1/forms/simple/submissions/one/audits') + .expect(200) + .then(({ body }) => { + body.length.should.equal(1); + body[0].notes.should.equal('a note about the submission'); + }); + + // Delete the submission + await asAlice.delete('/v1/projects/1/forms/simple/submissions/one'); + + // Purge the submission + await Submissions.purge(true); + + // Look at what is in the audit log via the database because the submission is deleted + const auditNotes = await oneFirst(sql`select notes from audits where action = 'submission.create'`); + + // Check that the note is redacted + auditNotes.should.equal(''); + })); + + it('should purge form field values of a deleted submission', testService(async (service, container) => { + const asAlice = await service.login('alice'); + + // Upload the selectMultiple form + await asAlice.post('/v1/projects/1/forms?publish=true') + .set('Content-Type', 'application/xml') + .send(testData.forms.selectMultiple) + .expect(200); + + // Create a submission + await asAlice.post('/v1/projects/1/forms/selectMultiple/submissions') + .send(testData.instances.selectMultiple.one) + .set('Content-Type', 'application/xml') + .expect(200); + + // Exhaust worker to update select multiple database values + await exhaust(container); + + // Check that the form field values are in the database + const numFieldValues = await container.oneFirst(sql`select count(*) from form_field_values`); + numFieldValues.should.equal(5); + + // Delete submission + await asAlice.delete('/v1/projects/1/forms/selectMultiple/submissions/one'); + + // Purge the submission + await container.Submissions.purge(true); + + // Check that the form field values are deleted from the database + const count = await container.oneFirst(sql`select count(*) from form_field_values`); + count.should.equal(0); + })); + + it('should purge client audit blobs attachments for a deleted submission', testService(async (service, container) => { + const asAlice = await service.login('alice'); + + // Create the form + await asAlice.post('/v1/projects/1/forms?publish=true') + .send(testData.forms.clientAudits) + .expect(200); + + // Send the submission with the client audit attachment + await asAlice.post('/v1/projects/1/submission') + .set('X-OpenRosa-Version', '1.0') + .attach('audit.csv', createReadStream(appPath + '/test/data/audit.csv'), { filename: 'audit.csv' }) + .attach('xml_submission_file', Buffer.from(testData.instances.clientAudits.one), { filename: 'data.xml' }) + .expect(201); + + // Send a second submission + await asAlice.post('/v1/projects/1/submission') + .set('X-OpenRosa-Version', '1.0') + .attach('log.csv', createReadStream(appPath + '/test/data/audit2.csv'), { filename: 'log.csv' }) + .attach('xml_submission_file', Buffer.from(testData.instances.clientAudits.two), { filename: 'data.xml' }) + .expect(201); + + // Process the client audit attachment + await exhaust(container); + + // Check that the client audit events are in the database + const clientAuditEventCount = await container.all(sql`select count(*) from client_audits group by "blobId" order by count(*) desc`); + clientAuditEventCount.should.eql([{ count: 5 }, { count: 3 }]); // 1 blob with 5 events in it, another with 3 + + // Check that the blobs are in the database + const numBlobs = await container.oneFirst(sql`select count(*) from blobs`); + numBlobs.should.equal(2); + + // Delete one of the submissions + await asAlice.delete('/v1/projects/1/forms/audits/submissions/one') + .expect(200); + + // Purge the submission + await container.Submissions.purge(true); + + // Purge unattached blobs + await container.Blobs.purgeUnattached(); + + // Check that some of the client audit events are deleted from the database + const numClientAudits = await container.oneFirst(sql`select count(*) from client_audits`); + numClientAudits.should.equal(3); // from the non-deleted submission + + // Check one blob is deleted from the database + const count = await container.oneFirst(sql`select count(*) from blobs`); + count.should.equal(1); + })); + + it('should purge client audit blobs when two submissions have same client audit file', testService(async (service, container) => { + const asAlice = await service.login('alice'); + + // Create the form + await asAlice.post('/v1/projects/1/forms?publish=true') + .send(testData.forms.clientAudits) + .expect(200); + + // Send the submission with the client audit attachment + await asAlice.post('/v1/projects/1/submission') + .set('X-OpenRosa-Version', '1.0') + .attach('audit.csv', createReadStream(appPath + '/test/data/audit.csv'), { filename: 'audit.csv' }) + .attach('xml_submission_file', Buffer.from(testData.instances.clientAudits.one), { filename: 'data.xml' }) + .expect(201); + + // Send a second submission + await asAlice.post('/v1/projects/1/submission') + .set('X-OpenRosa-Version', '1.0') + .attach('log.csv', createReadStream(appPath + '/test/data/audit.csv'), { filename: 'log.csv' }) + .attach('xml_submission_file', Buffer.from(testData.instances.clientAudits.two), { filename: 'data.xml' }) + .expect(201); + + // Process the client audit attachment + await exhaust(container); + + // Both submissions share the same client audit data blob so there is only 1 blob + const numBlobs = await container.oneFirst(sql`select count(*) from blobs`); + numBlobs.should.equal(1); + + // There is only one blob with shared events + const clientAuditEventCount = await container.all(sql`select count(*) from client_audits group by "blobId" order by count(*) desc`); + clientAuditEventCount.should.eql([{ count: 5 }]); // 1 blob with 5 events in it + + // Delete one of the submissions (instanceId two) + await asAlice.delete('/v1/projects/1/forms/audits/submissions/two') + .expect(200); + + // Purge the submission + await container.Submissions.purge(true); + + // Purge unattached blobs + await container.Blobs.purgeUnattached(); + + // The one blob is still in the database because it is still referenced by the other submission + const count = await container.oneFirst(sql`select count(*) from blobs`); + count.should.equal(1); + + // Unfortunately, the client audits all get deleted + const numClientAudits = await container.oneFirst(sql`select count(*) from client_audits`); + numClientAudits.should.equal(0); + + // But the export still works (adhoc-processing of client audits) + const result = await pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip')); + + result.filenames.should.eql([ + 'audits.csv', + 'audits - audit.csv' + ]); + + result['audits - audit.csv'].should.equal(`instance ID,event,node,start,end,latitude,longitude,accuracy,old-value,new-value +one,a,/data/a,2000-01-01T00:01,2000-01-01T00:02,1,2,3,aa,bb +one,b,/data/b,2000-01-01T00:02,2000-01-01T00:03,4,5,6,cc,dd +one,c,/data/c,2000-01-01T00:03,2000-01-01T00:04,7,8,9,ee,ff +one,d,/data/d,2000-01-01T00:10,,10,11,12,gg, +one,e,/data/e,2000-01-01T00:11,,,,,hh,ii +`); + })); + }); + + describe('submissions as entity sources', () => { + it('should set submission def id on entity source to null when submission deleted', testService(async (service, container) => { + const asAlice = await service.login('alice'); + + // Create the form + await asAlice.post('/v1/projects/1/forms?publish=true') + .send(testData.forms.simpleEntity) + .expect(200); + + // Send the submission + await asAlice.post('/v1/projects/1/forms/simpleEntity/submissions') + .send(testData.instances.simpleEntity.one) + .set('Content-Type', 'application/xml') + .expect(200); + + // Process the submission + await exhaust(container); + + // Delete the submission + await asAlice.delete('/v1/projects/1/forms/simpleEntity/submissions/one'); + + // Check the submission in the entity source while it is soft-deleted + await asAlice.get('/v1/projects/1/datasets/people/entities/12345678-1234-4123-8234-123456789abc/audits') + .expect(200) + .then(({ body: logs }) => { + logs[0].should.be.an.Audit(); + logs[0].action.should.be.eql('entity.create'); + logs[0].actor.displayName.should.be.eql('Alice'); + + logs[0].details.source.event.should.be.an.Audit(); + logs[0].details.source.event.actor.displayName.should.be.eql('Alice'); + logs[0].details.source.event.loggedAt.should.be.isoDate(); + + logs[0].details.source.submission.instanceId.should.be.eql('one'); + logs[0].details.source.submission.submitter.displayName.should.be.eql('Alice'); + logs[0].details.source.submission.createdAt.should.be.isoDate(); + + // submission is only a stub so it shouldn't have currentVersion + logs[0].details.source.submission.should.not.have.property('currentVersion'); + }); + + // Purge the submission + await container.Submissions.purge(true); + + // Check the source def in the database has been set to null + const sourceDef = await container.oneFirst(sql`select "submissionDefId" from entity_def_sources where details -> 'submission' ->> 'instanceId' = 'one'`); + should.not.exist(sourceDef); + + // Check the submission in the entity source after it is purged + await asAlice.get('/v1/projects/1/datasets/people/entities/12345678-1234-4123-8234-123456789abc/audits') + .expect(200) + .then(({ body: logs }) => { + logs[0].should.be.an.Audit(); + logs[0].action.should.be.eql('entity.create'); + logs[0].actor.displayName.should.be.eql('Alice'); + + logs[0].details.source.event.should.be.an.Audit(); + logs[0].details.source.event.actor.displayName.should.be.eql('Alice'); + logs[0].details.source.event.loggedAt.should.be.isoDate(); + + logs[0].details.source.submission.instanceId.should.be.eql('one'); + logs[0].details.source.submission.submitter.displayName.should.be.eql('Alice'); + logs[0].details.source.submission.createdAt.should.be.isoDate(); + + // submission is only a stub so it shouldn't have currentVersion + logs[0].details.source.submission.should.not.have.property('currentVersion'); + }); + })); + }); + + describe('submission.purge audit event', () => { + it('should log a purge event in the audit log when purging submissions', testService(async (service, { Submissions }) => { + const asAlice = await service.login('alice'); + + // Create two submissions + await asAlice.post('/v1/projects/1/forms/simple/submissions') + .send(testData.instances.simple.one) + .set('Content-Type', 'application/xml') + .expect(200); + + await asAlice.post('/v1/projects/1/forms/simple/submissions') + .send(testData.instances.simple.two) + .set('Content-Type', 'application/xml') + .expect(200); + + // Delete both submissions + await asAlice.delete('/v1/projects/1/forms/simple/submissions/one') + .expect(200); + await asAlice.delete('/v1/projects/1/forms/simple/submissions/two') + .expect(200); + + // Purge submissions + await Submissions.purge(true); + + await asAlice.get('/v1/audits') + .then(({ body }) => { + body.filter((a) => a.action === 'submission.purge').length.should.equal(1); + body[0].details.should.eql({ submissions_deleted: 2 }); + }); + })); + + it('should not log event if no submissions purged', testService(async (service, { Submissions }) => { + const asAlice = await service.login('alice'); + // No deleted submissions exist here to purge + await Submissions.purge(true); + + await asAlice.get('/v1/audits') + .then(({ body }) => { + body.filter((a) => a.action === 'submission.purge').length.should.equal(0); + }); + })); + }); +}); diff --git a/test/integration/task/purge.js b/test/integration/task/purge.js index 66e387d4e..5e8b2e25e 100644 --- a/test/integration/task/purge.js +++ b/test/integration/task/purge.js @@ -1,207 +1,210 @@ const appRoot = require('app-root-path'); -const assert = require('assert'); -const { testTask, testService } = require('../setup'); -const { purgeForms } = require(appRoot + '/lib/task/purge'); -const testData = require('../../data/xml'); +const { testTask } = require('../setup'); +const { purgeTask } = require(appRoot + '/lib/task/purge'); -// The basics of this task are tested here, including returning the count +// The basics of this task are tested here, including returning the message // of purged forms, but the full functionality is more thoroughly tested in -// test/integration/other/form-purging.js - -describe('task: purge deleted forms', () => { - it('should not purge recently deleted forms by default', testTask(({ Forms }) => - Forms.getByProjectAndXmlFormId(1, 'simple') - .then((form) => Forms.del(form.get()) - .then(() => purgeForms()) - .then((count) => { - count.should.equal(0); - })))); - - it('should purge recently deleted form if forced', testTask(({ Forms }) => - Forms.getByProjectAndXmlFormId(1, 'simple') - .then((form) => Forms.del(form.get()) - .then(() => purgeForms(true)) - .then((count) => { - count.should.equal(1); - })))); - - it('should return count for multiple forms purged', testTask(({ Forms }) => - Forms.getByProjectAndXmlFormId(1, 'simple') - .then((form) => Forms.del(form.get())) - .then(() => Forms.getByProjectAndXmlFormId(1, 'withrepeat') - .then((form) => Forms.del(form.get()))) - .then(() => purgeForms(true) - .then((count) => { - count.should.equal(2); - })))); - - it('should not purge specific recently deleted form', testTask(({ Forms }) => - Forms.getByProjectAndXmlFormId(1, 'simple') - .then((form) => Forms.del(form.get()) - .then(() => purgeForms(false, 1)) - .then((count) => { - count.should.equal(0); - })))); - - it('should purge specific recently deleted form if forced', testTask(({ Forms }) => - Forms.getByProjectAndXmlFormId(1, 'simple') - .then((form) => Forms.del(form.get()) - .then(() => purgeForms(true, 1)) - .then((count) => { - count.should.equal(1); - })))); - - it('should force purge only specific form', testTask(({ Forms }) => - Forms.getByProjectAndXmlFormId(1, 'simple') - .then((form) => Forms.del(form.get())) - .then(() => Forms.getByProjectAndXmlFormId(1, 'withrepeat') - .then((form) => Forms.del(form.get()) - .then(() => purgeForms(true, 1)) - .then((count) => { - count.should.equal(1); - }))))); - - describe('with projectId', () => { - it('should not purge recently deleted forms even if projectId is matched', testTask(({ Forms }) => - Forms.getByProjectAndXmlFormId(1, 'simple') - .then((form) => Forms.del(form.get()) - .then(() => purgeForms(null, null, 1)) - .then((count) => { - count.should.equal(0); - })))); +// test/integration/other/form-purging.js and test/integration/other/submission-purging.js. - it('should not purge recently deleted forms even if projectId AND formId is matched', testTask(({ Forms }) => - Forms.getByProjectAndXmlFormId(1, 'simple') - .then((form) => Forms.del(form.get()) - .then(() => purgeForms(null, 1, 1)) - .then((count) => { - count.should.equal(0); - })))); +describe('task: purge deleted resources (forms and submissions)', () => { + describe('forms', () => { + describe('force flag', () => { + it('should not purge recently deleted forms by default', testTask(({ Forms }) => + Forms.getByProjectAndXmlFormId(1, 'simple') + .then((form) => Forms.del(form.get()) + .then(() => purgeTask({ mode: 'forms' })) + .then((message) => { + message.should.equal('Forms purged: 0'); + })))); - it('should purge specific form', testTask(({ Forms }) => - Forms.getByProjectAndXmlFormId(1, 'simple') - .then((form) => Forms.del(form.get())) - .then(() => Forms.getByProjectAndXmlFormId(1, 'withrepeat') + it('should purge recently deleted form if forced', testTask(({ Forms }) => + Forms.getByProjectAndXmlFormId(1, 'simple') + .then((form) => Forms.del(form.get()) + .then(() => purgeTask({ mode: 'forms', force: true })) + .then((message) => { + message.should.equal('Forms purged: 1'); + })))); + + it('should return message for multiple forms purged', testTask(({ Forms }) => + Forms.getByProjectAndXmlFormId(1, 'simple') + .then((form) => Forms.del(form.get())) + .then(() => Forms.getByProjectAndXmlFormId(1, 'withrepeat') + .then((form) => Forms.del(form.get()))) + .then(() => purgeTask({ mode: 'forms', force: true }) + .then((message) => { + message.should.equal('Forms purged: 2'); + })))); + }); + + describe('form specified by formId', () => { + it('should not purge specific recently deleted form', testTask(({ Forms }) => + Forms.getByProjectAndXmlFormId(1, 'simple') .then((form) => Forms.del(form.get()) - .then(() => purgeForms(true, 1, 1)) - .then((count) => { - count.should.equal(1); - }))))); + .then(() => purgeTask({ mode: 'forms', force: false, formId: 1 })) + .then((message) => { + message.should.equal('Forms purged: 0'); + })))); - it('should not purge specific form if tied to a different project', testTask(({ Forms }) => - Forms.getByProjectAndXmlFormId(1, 'simple') - .then((form) => Forms.del(form.get())) - .then(() => Forms.getByProjectAndXmlFormId(1, 'withrepeat') + it('should purge specific recently deleted form if forced', testTask(({ Forms }) => + Forms.getByProjectAndXmlFormId(1, 'simple') .then((form) => Forms.del(form.get()) - .then(() => purgeForms(true, 1, 2)) - .then((count) => { - count.should.equal(0); - }))))); + .then(() => purgeTask({ mode: 'forms', force: true, formId: 1 })) + .then((message) => { + message.should.equal('Forms purged: 1'); + })))); + + it('should force purge only specific form', testTask(({ Forms }) => + Forms.getByProjectAndXmlFormId(1, 'simple') + .then((form) => Forms.del(form.get())) + .then(() => Forms.getByProjectAndXmlFormId(1, 'withrepeat') + .then((form) => Forms.del(form.get()) + .then(() => purgeTask({ mode: 'forms', force: true, formId: 1 })) + .then((message) => { + message.should.equal('Forms purged: 1'); + }))))); + }); + + describe('form specified with projectId', () => { + it('should not purge recently deleted forms even if projectId is matched (when not forced', testTask(({ Forms }) => + Forms.getByProjectAndXmlFormId(1, 'simple') + .then((form) => Forms.del(form.get()) + .then(() => purgeTask({ mode: 'forms', projectId: 1 })) + .then((message) => { + message.should.equal('Forms purged: 0'); + })))); - it('should purge all forms if no form ID supplied', testTask(({ Forms }) => - Forms.getByProjectAndXmlFormId(1, 'simple') - .then((form) => Forms.del(form.get())) - .then(() => Forms.getByProjectAndXmlFormId(1, 'withrepeat') + it('should not purge recently deleted forms even if projectId AND formId is matched (when not forced)', testTask(({ Forms }) => + Forms.getByProjectAndXmlFormId(1, 'simple') .then((form) => Forms.del(form.get()) - .then(() => purgeForms(true, null, 1)) - .then((count) => { - count.should.equal(2); - }))))); + .then(() => purgeTask({ mode: 'forms', projectId: 1, formId: 1 })) + .then((message) => { + message.should.equal('Forms purged: 0'); + })))); + + it('should purge specific form', testTask(({ Forms }) => + Forms.getByProjectAndXmlFormId(1, 'simple') + .then((form) => Forms.del(form.get())) + .then(() => Forms.getByProjectAndXmlFormId(1, 'withrepeat') + .then((form) => Forms.del(form.get()) + .then(() => purgeTask({ mode: 'forms', force: true, projectId: 1, formId: 1 })) + .then((message) => { + message.should.equal('Forms purged: 1'); + }))))); + + it('should not purge specific form if tied to a different project', testTask(({ Forms }) => + Forms.getByProjectAndXmlFormId(1, 'simple') + .then((form) => Forms.del(form.get())) + .then(() => Forms.getByProjectAndXmlFormId(1, 'withrepeat') + .then((form) => Forms.del(form.get()) + .then(() => purgeTask({ mode: 'forms', force: true, projectId: 2, formId: 1 })) + .then((message) => { + message.should.equal('Forms purged: 0'); + }))))); + + it('should purge all forms in project if no form ID supplied', testTask(({ Forms }) => + Forms.getByProjectAndXmlFormId(1, 'simple') + .then((form) => Forms.del(form.get())) + .then(() => Forms.getByProjectAndXmlFormId(1, 'withrepeat') + .then((form) => Forms.del(form.get()) + .then(() => purgeTask({ mode: 'forms', force: true, projectId: 1 })) + .then((message) => { + message.should.equal('Forms purged: 2'); + }))))); + + it('should not purge multiple forms if tied to a different project', testTask(({ Forms }) => + Forms.getByProjectAndXmlFormId(1, 'simple') + .then((form) => Forms.del(form.get())) + .then(() => Forms.getByProjectAndXmlFormId(1, 'withrepeat') + .then((form) => Forms.del(form.get()) + .then(() => purgeTask({ mode: 'forms', force: true, projectId: 2 })) + .then((message) => { + message.should.equal('Forms purged: 0'); + }))))); + }); + + describe('with xmlFormId', () => { + it('should throw error if xmlFormId specified without projectId', testTask(async ({ Forms }) => { + const form = await Forms.getByProjectAndXmlFormId(1, 'simple'); + await Forms.del(form.get()); + const message = await purgeTask({ mode: 'forms', force: true, xmlFormId: 'simple' }); + message.should.equal('Must also specify projectId when using xmlFormId'); + })); + + it('should force purge form by project and xmlFormId', testTask(({ Forms }) => + Forms.getByProjectAndXmlFormId(1, 'simple') + .then((form) => Forms.del(form.get()) + .then(() => purgeTask({ mode: 'forms', force: true, projectId: 1, xmlFormId: 'simple' })) + .then((message) => { + message.should.equal('Forms purged: 1'); + })))); - it('should not purge multiple forms if tied to a different project', testTask(({ Forms }) => - Forms.getByProjectAndXmlFormId(1, 'simple') - .then((form) => Forms.del(form.get())) - .then(() => Forms.getByProjectAndXmlFormId(1, 'withrepeat') + it('should not purge form by project and xmlFormId if form deleted recently and not forced', testTask(({ Forms }) => + Forms.getByProjectAndXmlFormId(1, 'simple') .then((form) => Forms.del(form.get()) - .then(() => purgeForms(true, null, 2)) - .then((count) => { - count.should.equal(0); - }))))); + .then(() => purgeTask({ mode: 'forms', force: false, projectId: 1, xmlFormId: 'simple' })) + .then((message) => { + message.should.equal('Forms purged: 0'); + })))); + }); + }); + + describe('submissions', () => { + // Can't set up more data in this task test setup but we can still test the function args + it('should call submission purge if mode is specified as submissions', testTask(() => + purgeTask({ mode: 'submissions' }) + .then((message) => { + message.should.equal('Submissions purged: 0'); + }))); + + it('should call submission purge if submission instance id is specified', testTask(() => + purgeTask({ instanceId: 'abc', projectId: 1, xmlFormId: 'simple' }) + .then((message) => { + message.should.equal('Submissions purged: 0'); + }))); + + it('should complain if instance id specified without project and form', testTask(() => + purgeTask({ instanceId: 'abc' }) + .then((message) => { + message.should.equal('Must specify either all or none of projectId, xmlFormId, and instanceId'); + }))); + + it('should complain if instance id specified without project', testTask(() => + purgeTask({ instanceId: 'abc', xmlFormId: 'simple' }) + .then((message) => { + message.should.equal('Must specify either all or none of projectId, xmlFormId, and instanceId'); + }))); + + it('should complain if instance id specified without form', testTask(() => + purgeTask({ instanceId: 'abc', projectId: 1 }) + .then((message) => { + message.should.equal('Must specify either all or none of projectId, xmlFormId, and instanceId'); + }))); }); - describe('with xmlFormId', () => { - it('should throw error if xmlFormId specified without projectId', testTask(async ({ Forms }) => { - const form = await Forms.getByProjectAndXmlFormId(1, 'simple'); - await Forms.del(form.get()); - await assert.throws(() => { purgeForms(true, null, null, 'simple'); }, (err) => { - err.problemCode.should.equal(500.1); - err.problemDetails.error.should.equal('Must also specify projectId when using xmlFormId'); - return true; - }); - })); - - it('should force purge form by project and xmlFormId', testTask(({ Forms }) => + describe('all', () => { + it('should purge both forms and submissions when neither mode is specified (not forced)', testTask(({ Forms }) => Forms.getByProjectAndXmlFormId(1, 'simple') .then((form) => Forms.del(form.get()) - .then(() => purgeForms(true, null, 1, 'simple')) - .then((count) => { - count.should.equal(1); + .then(() => purgeTask()) + .then((message) => { + message.should.equal('Forms purged: 0, Submissions purged: 0'); })))); - it('should not purge form by project and xmlFormId if form deleted recently and not forced', testTask(({ Forms }) => + + it('should purge both forms and submissions when neither mode is specified (forced)', testTask(({ Forms }) => Forms.getByProjectAndXmlFormId(1, 'simple') .then((form) => Forms.del(form.get()) - .then(() => purgeForms(false, null, 1, 'simple')) - .then((count) => { - count.should.equal(0); + .then(() => purgeTask({ force: true })) + .then((message) => { + message.should.equal('Forms purged: 1, Submissions purged: 0'); })))); - it('should purge all versions of deleted form in project', testService(async (service, container) => { - const asAlice = await service.login('alice'); - - await asAlice.delete('/v1/projects/1/forms/simple') - .expect(200); - - // new version (will be v2) - await asAlice.post('/v1/projects/1/forms?ignoreWarnings=true') - .send(testData.forms.simple) - .set('Content-Type', 'application/xml') - .expect(200); - - // publish new version v2 - await asAlice.post('/v1/projects/1/forms/simple/draft/publish?ignoreWarnings=true&version=v2') - .expect(200); - - // delete new version v2 - await asAlice.delete('/v1/projects/1/forms/simple') - .expect(200); - - // new version (will be v3) - await asAlice.post('/v1/projects/1/forms?ignoreWarnings=true') - .send(testData.forms.simple) - .set('Content-Type', 'application/xml') - .expect(200); - - // publish new version v3 but don't delete - await asAlice.post('/v1/projects/1/forms/simple/draft/publish?ignoreWarnings=true&version=v3') - .expect(200); - - const count = await container.Forms.purge(true, null, 1, 'simple'); - count.should.equal(2); - })); - - it('should purge named form only from specified project', testService(async (service, container) => { - const asAlice = await service.login('alice'); - - // delete simple form in project 1 (but don't purge it) - await asAlice.delete('/v1/projects/1/forms/simple') - .expect(200); - - const newProjectId = await asAlice.post('/v1/projects') - .send({ name: 'Project Two' }) - .then(({ body }) => body.id); - - await asAlice.post(`/v1/projects/${newProjectId}/forms?publish=true`) - .send(testData.forms.simple) - .set('Content-Type', 'application/xml') - .expect(200); - - await asAlice.delete(`/v1/projects/${newProjectId}/forms/simple`) - .expect(200); - - const count = await container.Forms.purge(true, null, newProjectId, 'simple'); - count.should.equal(1); - })); + it('should accept other mode and treat as "all"', testTask(({ Forms }) => + Forms.getByProjectAndXmlFormId(1, 'simple') + .then((form) => Forms.del(form.get()) + .then(() => purgeTask({ force: true, mode: 'something_else' })) + .then((message) => { + message.should.equal('Forms purged: 1, Submissions purged: 0'); + })))); }); - });