Speed up entities query for dataset CSV export and entity OData (#1271)

* Speeding up entities query by removing updates count subquery

* Change entity data structure in unit test

* small changes in response to code review

lib/data/entity.js

@@ -238,7 +238,7 @@ const streamEntityCsv = (inStream, properties) => {
 
   // System properties
   header.push(...[ '__createdAt', '__creatorId', '__creatorName', '__updates', '__updatedAt', '__version']);
-  props.push(...['createdAt', 'creatorId', 'aux.creator.displayName', 'aux.stats.updates', 'updatedAt', 'def.version'].map(e => e.split('.')));
+  props.push(...['createdAt', 'creatorId', 'aux.creator.displayName', 'updates', 'updatedAt', 'def.version'].map(e => e.split('.')));
 
   const entityStream = _entityTransformer(header, props);
 
@@ -282,7 +282,7 @@ const selectFields = (entity, properties, selectedProperties) => {
     createdAt: entity.createdAt,
     creatorId: entity.aux.creator.id.toString(),
     creatorName: entity.aux.creator.displayName,
-    updates: entity.aux.stats.updates,
+    updates: entity.updates,
     updatedAt: entity.updatedAt,
     version: entity.def.version,
     conflict: entity.conflict

lib/model/frames/entity.js

@@ -33,6 +33,14 @@ class Entity extends Frame.define(
 ) {
   get def() { return this.aux.def; }
 
+  // Note: we introduced this 'updates' count to entities in datasets (via odata and csv export)
+  // before we exposed a similar idea through the 'version' property.
+  // This version property increments with each entity update so it is safe to compute 'updates'
+  // from version instead of calculating it in less efficient ways.
+  // Note that 'updates' is used in too many places in frontend code and exposed in the odata/csv
+  // exports to remove it from the system properties.
+  get updates() { return this.aux.def.version - 1; }
+
   static fromParseEntityData(entityData, options = { create: true, update: false }) {
     const { dataset } = entityData.system;
     const { data } = entityData;
@@ -80,10 +88,6 @@ class Entity extends Frame.define(
 
 Entity.Partial = class extends Entity {};
 
-Entity.Extended = class extends Frame.define(
-  'updates'
-) { };
-
 Entity.Def = Frame.define(
   table('entity_defs', 'def'),
   'id',                             'entityId',

lib/model/query/entities.js

@@ -744,16 +744,12 @@ resolveConflict.audit = (entity, dataset) => (log) => log('entity.update.resolve
 ////////////////////////////////////////////////////////////////////////////////
 // SERVING ENTITIES
 
-const _exportUnjoiner = unjoiner(Entity, Entity.Def, Entity.Extended.into('stats'), Actor.alias('actors', 'creator'));
+const _exportUnjoiner = unjoiner(Entity, Entity.Def, Actor.alias('actors', 'creator'));
 
 const streamForExport = (datasetId, options = QueryOptions.none) => ({ stream }) =>
   stream(sql`
 SELECT ${_exportUnjoiner.fields} FROM entity_defs
 INNER JOIN entities ON entities.id = entity_defs."entityId"
-INNER JOIN
-  (
-    SELECT "entityId", (COUNT(id) - 1) AS "updates" FROM entity_defs GROUP BY "entityId"
-  ) stats ON stats."entityId"=entity_defs."entityId"
 LEFT JOIN actors ON entities."creatorId"=actors.id
 ${options.skiptoken ? sql`
   INNER JOIN

test/unit/data/entity.js

@@ -660,11 +660,9 @@ describe('extracting and validating entities', () => {
         creator: {
           id: 'id',
           displayName: 'displayName'
-        },
-        stats: {
-          updates: 0
         }
-      }
+      },
+      updates: 0
     };
     const properties = [{ name: 'firstName' }, { name: 'lastName' }];
 
@@ -743,11 +741,9 @@ describe('extracting and validating entities', () => {
           creator: {
             id: 'id',
             displayName: 'displayName'
-          },
-          stats: {
-            updates: 0
           }
-        }
+        },
+        updates: 0
       };
       const selectedProperties = null;
       const result = selectFields(data, properties, selectedProperties);