Merge pull request #1231 from sadiqkhoja/features/get-deleted-submiss…

…ions

Update Submissions OData to include `deletedAt`

lib/data/odata-filter.js

@@ -87,6 +87,50 @@ const odataFilter = (expr, odataToColumnMap) => {
   return op(ast);
 };
 
+// Returns sql expression to exclude deleted records if provided OData filter
+// expression doesn't use `__system/deletedAt` field. Logic of this function
+// can easily be merged with `OdataFilter()` but it is kept separate for the
+// for the sake of simplicity and for separation of concerns.
+const odataExcludeDeleted = (expr, odataToColumnMap) => {
+  const deleteAtColumn = odataToColumnMap.get('__system/deletedAt')
+    ?? odataToColumnMap.get('$root/Submissions/__system/deletedAt');
+
+  const filterOutDeletedRecordsExp = sql`(${sql.identifier(deleteAtColumn.split('.'))} is null)`;
+
+  if (expr == null) return filterOutDeletedRecordsExp;
+
+  let ast; // still hate this.
+  try { ast = odataParser.filter(expr); } // eslint-disable-line brace-style
+  catch (ex) { throw Problem.user.unparseableODataExpression({ reason: ex.message }); }
+
+  const hasDeletedAtClause = (node) => {
+    if (node.type === 'FirstMemberExpression' || node.type === 'RootExpression') {
+      if (node.raw === '__system/deletedAt'
+        || node.raw === '$root/Submissions/__system/deletedAt') return true;
+    } else if (node.type === 'MethodCallExpression') {
+      // n.b. we only support Unary (single-arity functions)
+      return hasDeletedAtClause(node.value.parameters[0]);
+    } else if (node.type === 'EqualsExpression'
+            || node.type === 'NotEqualsExpression'
+            || node.type === 'LesserThanExpression'
+            || node.type === 'LesserOrEqualsExpression'
+            || node.type === 'GreaterThanExpression'
+            || node.type === 'GreaterOrEqualsExpression'
+            || node.type === 'AndExpression'
+            || node.type === 'OrExpression') {
+      return hasDeletedAtClause(node.value.left) || hasDeletedAtClause(node.value.right);
+    } else if (node.type === 'BoolParenExpression'
+            || node.type === 'NotExpression') {
+      return hasDeletedAtClause(node.value);
+    }
+    return false;
+  };
+
+  if (!hasDeletedAtClause(ast)) return filterOutDeletedRecordsExp;
+
+  return sql`true`;
+};
+
 const odataOrderBy = (expr, odataToColumnMap, stableOrderColumn = null) => {
   let initialOrder = null;
   const clauses = expr.split(',').map((exp) => {
@@ -115,5 +159,4 @@ const odataOrderBy = (expr, odataToColumnMap, stableOrderColumn = null) => {
   return sql`ORDER BY ${sql.join(clauses, sql`,`)}`;
 };
 
-module.exports = { odataFilter, odataOrderBy };
-
+module.exports = { odataFilter, odataOrderBy, odataExcludeDeleted };

lib/data/odata.js

@@ -150,7 +150,8 @@ const submissionToOData = (fields, table, submission, options = {}) => new Promi
       deviceId: submission.deviceId || null,
       // because of how we compute this value we don't need to default it to 0:
       edits: submission.aux.edit.count,
-      formVersion: submission.aux.exports.formVersion
+      formVersion: submission.aux.exports.formVersion,
+      deletedAt: submission.deletedAt
     };
 
     if (!options.metadata || options.metadata.__system) {
@@ -363,6 +364,7 @@ const systemFields = new Set([
   '__system/deviceId',
   '__system/edits',
   '__system/formVersion',
+  '__system/deletedAt'
 ]);
 
 module.exports = { submissionToOData, systemFields };

lib/data/submission.js

@@ -342,7 +342,8 @@ const filterableFields = [
   ['__system/submissionDate', 'submissions.createdAt'],
   ['__system/updatedAt', 'submissions.updatedAt'],
   ['__system/submitterId', 'submissions.submitterId'],
-  ['__system/reviewState', 'submissions.reviewState']
+  ['__system/reviewState', 'submissions.reviewState'],
+  ['__system/deletedAt', 'submissions.deletedAt']
 ];
 
 const odataToColumnMap = new Map(filterableFields.concat(filterableFields.map(f => ([`$root/Submissions/${f[0]}`, f[1]]))));

lib/formats/odata.js

@@ -126,6 +126,7 @@ const edmxTemplater = template(`<?xml version="1.0" encoding="UTF-8"?>
       <ComplexType Name="metadata">
         <Property Name="submissionDate" Type="Edm.DateTimeOffset"/>
         <Property Name="updatedAt" Type="Edm.DateTimeOffset"/>
+        <Property Name="deletedAt" Type="Edm.DateTimeOffset"/>
         <Property Name="submitterId" Type="Edm.String"/>
         <Property Name="submitterName" Type="Edm.String"/>
         <Property Name="attachmentsPresent" Type="Edm.Int64"/>

lib/model/frames/submission.js

@@ -25,6 +25,7 @@ class Submission extends Frame.define(
   'deviceId',   readable,               'createdAt',    readable,
   'updatedAt',  readable,               'reviewState',  readable, writable,
   'userAgent',  readable,               'draft',
+  'deletedAt',  readable,
   embedded('submitter'),
   embedded('currentVersion')
 ) {

lib/model/query/submissions.js

@@ -12,7 +12,7 @@ const { map } = require('ramda');
 const { sql } = require('slonik');
 const { Frame, table } = require('../frame');
 const { Actor, Form, Submission } = require('../frames');
-const { odataFilter, odataOrderBy } = require('../../data/odata-filter');
+const { odataFilter, odataOrderBy, odataExcludeDeleted } = require('../../data/odata-filter');
 const { odataToColumnMap, odataSubTableToColumnMap } = require('../../data/submission');
 const { unjoiner, extender, equals, page, updater, QueryOptions, insertMany, markDeleted, markUndeleted } = require('../../util/db');
 const { blankStringToNull, construct } = require('../../util/util');
@@ -231,7 +231,7 @@ const joinOnSkiptoken = (skiptoken, formId, draft) => {
 
 const countByFormId = (formId, draft, options = QueryOptions.none) => ({ one }) => {
   const filter = sql`${equals({ formId, draft })} AND
-"deletedAt" IS NULL AND
+${odataExcludeDeleted(options.filter, odataToColumnMap)} AND
 ${odataFilter(options.filter, odataToColumnMap)}`;
   return one(sql`
 SELECT * FROM
@@ -382,7 +382,7 @@ where
   ${encrypted ? sql`(form_defs."encKeyId" is null or form_defs."encKeyId" in (${sql.join(keyIds, sql`,`)})) and` : sql``}
   ${odataFilter(options.filter, options.isSubmissionsTable ? odataToColumnMap : odataSubTableToColumnMap)} and
   ${equals(options.condition)}
-  and submission_defs.current=true and submissions."formId"=${formId} and submissions."deletedAt" is null
+  and submission_defs.current=true and submissions."formId"=${formId} and ${odataExcludeDeleted(options.filter, options.isSubmissionsTable ? odataToColumnMap : odataSubTableToColumnMap)}
 ${options.orderby ? sql`
   ${odataOrderBy(options.orderby, odataToColumnMap, 'submissions.id')}`
     : sql`order by submissions."createdAt" desc, submissions.id desc`}