diff --git a/src/org/labkey/snd/SNDManager.java b/src/org/labkey/snd/SNDManager.java index 07842fdf..a2bf261e 100644 --- a/src/org/labkey/snd/SNDManager.java +++ b/src/org/labkey/snd/SNDManager.java @@ -92,6 +92,7 @@ import java.sql.SQLException; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -101,6 +102,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.atomic.AtomicInteger; @@ -564,6 +566,56 @@ private Map getPackageCategories(Container c, User u, int pkgId return categories; } + /** + * Return a cached map of Package Categories by PkgId + * If pkgIds is not null, get for list of pkgIds, else get for entire database. + * + * @param c + * @param u + * @param pkgIds + * @param errors + * @return + */ + private Map> getBulkPackageCategories(Container c, User u, @Nullable List pkgIds, BatchValidationException errors) + { + UserSchema schema = getSndUserSchema(c, u); + + SQLFragment sql = new SQLFragment("SELECT cj.CategoryId, ca.Description, PkgId FROM "); + sql.append(schema.getTable(SNDSchema.PKGCATEGORYJUNCTION_TABLE_NAME), "cj"); + sql.append(" JOIN "); + sql.append(schema.getTable(SNDSchema.PKGCATEGORIES__TABLE_NAME), "ca"); + sql.append(" ON cj.CategoryId = ca.CategoryId"); + if (pkgIds != null) { + sql.append(" WHERE PkgId IN ( "); + ArrayDeque pkgIdsQueue = new ArrayDeque<>(pkgIds); + while (pkgIdsQueue.size() > 1) { + sql.append("?, ").add(pkgIdsQueue.pop()); + } + sql.append("?) ").add(pkgIdsQueue.pop()); + } + SqlSelector selector = new SqlSelector(schema.getDbSchema(), sql); + + Map> pkgCategoriesByPkgId = new HashMap<>(); + + try (TableResultSet resultSet = selector.getResultSet()) + { + resultSet.forEach(row -> { + Integer pkgId = (Integer) row.get("PkgId"); + Integer categoryId = (Integer) row.get("CategoryId"); + String description = (String) row.get ("Description"); + if (!pkgCategoriesByPkgId.containsKey(pkgId)) { + pkgCategoriesByPkgId.put(pkgId, new HashMap<>()); + } + pkgCategoriesByPkgId.get(pkgId).put(categoryId, description); + }); + } + catch (SQLException e) + { + errors.addRowError(new ValidationException(e.getMessage())); + } + return pkgCategoriesByPkgId; + } + /** * Gets the extensible fields for a given table */ @@ -600,6 +652,32 @@ public Package addExtraFieldsToPackage(Container c, User u, Package pkg, @Nullab return pkg; } + /** + * Retrieve extensible fields as an argument and add them as extraFields to pkg object + * @param pkg + * @param extraFields + * @param row + * @return + */ + public Package addExtraFieldsToPackage(Package pkg, List extraFields, @Nullable Map row) + { + Map extras = new HashMap<>(); + for (GWTPropertyDescriptor extraField : extraFields) + { + if (row == null) + { + extras.put(extraField, ""); + } + else + { + extras.put(extraField, row.get(extraField.getName())); + } + } + pkg.setExtraFields(extras); + + return pkg; + } + /** * Recursive call to iterate through the hierarchy of a super package to get its subpackages */ @@ -853,6 +931,9 @@ private SuperPackage getFullSuperPackage(Container c, User u, int superPkgId, bo { UserSchema schema = getSndUserSchema(c, u); + TableInfo pkgsTable = getTableInfo(schema, SNDSchema.PKGS_TABLE_NAME); + QueryUpdateService pkgQus = getQueryUpdateService(pkgsTable); + SQLFragment sql = new SQLFragment("SELECT sp.SuperPkgId, sp.PkgId, sp.SortOrder, sp.Required, pkg.PkgId, pkg.Description, pkg.Active, pkg.Narrative, pkg.Repeatable FROM "); sql.append(schema.getTable(SNDSchema.SUPERPKGS_TABLE_NAME), "sp"); sql.append(" JOIN " + SNDSchema.NAME + "." + SNDSchema.PKGS_TABLE_NAME + " pkg"); @@ -962,6 +1043,38 @@ public Package addLookupsToPkg(Container c, User u, Package pkg) return pkg; } + /** + * Retrieve lookups for packages. Separates this logic out of being called for every single package + */ + public Map getLookups(Container c, User u) { + UserSchema schema = getSndUserSchema(c, u); + Map> sndLookups = ((SNDUserSchema) schema).getLookupSets(); + Map lookups = new TreeMap<>(Comparator.comparing((String o) -> o.split("\\.")[1])); + + String key, label; + for (String sndLookup : sndLookups.keySet()) + { + key = "snd." + sndLookup; + label = ((String) sndLookups.get(sndLookup).get("Label")); + if (label != null) + { + lookups.put(key, label); + } + else + { + lookups.put(key, sndLookup); + } + } + + for (TableInfo ti : _attributeLookups) + { + key = ti.getSchema().getName() + "." + ti.getName(); + lookups.put(key, ti.getTitle()); + } + + return lookups; + } + /** * Given a row from the snd.Pkgs table, this creates the Package object. Options to include extensible columns, lookup values * and attributes of subpackages @@ -997,6 +1110,64 @@ private Package createPackage(Container c, User u, Map row, bool return pkg; } + /** + * Does same as createPackage but retrieves superPackage and package info from cached map objects from arguments instead of database queries + * @param c + * @param u + * @param row + * @param packageExtraFields + * @param superPackages + * @param childrenByParentId + * @param pkgCategoriesByPkgId + * @param lookups + * @param includeExtraFields + * @param includeLookups + * @param includeFullSubpackages + * @param errors + * @return + */ + private Package getBulkPackage(Container c, User u, Map row, List packageExtraFields, + List superPackages, Map> childrenByParentId, + Map> pkgCategoriesByPkgId, Map lookups, boolean includeExtraFields, + boolean includeLookups, boolean includeFullSubpackages, BatchValidationException errors) + { + Package pkg = new Package(); + if (row != null) + { + pkg.setPkgId((Integer) row.get(Package.PKG_ID)); + pkg.setDescription((String) row.get(Package.PKG_DESCRIPTION)); + pkg.setActive((boolean) row.get(Package.PKG_ACTIVE)); + pkg.setRepeatable((boolean) row.get(Package.PKG_REPEATABLE)); + pkg.setNarrative((String) row.get(Package.PKG_NARRATIVE)); + pkg.setQcState((Integer) row.get(Package.PKG_QCSTATE)); + pkg.setHasEvent((boolean) row.get(Package.PKG_HASEVENT)); + pkg.setHasProject((boolean) row.get(Package.PKG_HASPROJECT)); + pkg.setModified((Date) row.get(Package.PKG_MODIFIED)); + pkg.setCategories(pkgCategoriesByPkgId.get(pkg.getPkgId())); + pkg.setAttributes(getPackageAttributes(c, u, pkg.getPkgId())); + + Optional topLevelSuperPkg = superPackages.stream().filter(s -> s.getPkgId().equals(pkg.getPkgId()) && s.getParentSuperPkgId() == null) + .findFirst(); + + topLevelSuperPkg.ifPresent(superPackage -> pkg.setTopLevelSuperPkgId(superPackage.getSuperPkgId())); + + if(includeFullSubpackages) { + Integer superPkgId = superPackages.stream().filter(s -> s.getPkgId().equals(pkg.getPkgId())) + .findFirst().get().getSuperPkgId(); + List subPackages = childrenByParentId.getOrDefault(superPkgId, new ArrayList<>()); + pkg.setSubpackages(subPackages); + } + + if (includeExtraFields) + addExtraFieldsToPackage(pkg, packageExtraFields, row); + if (includeLookups) + pkg.setLookups(lookups); + } + + return pkg; + } + + /** * Gets a list of full packages given a list of package Ids. Options to include extensible columns, lookup values and * all attributes for sub packages. @@ -1040,6 +1211,61 @@ public List getPackages(Container c, User u, List pkgIds, bool return packages; } + /** + * Does same as getPackages but retrieves the superPackage and package info from cached object maps instead of database queries + * @param c + * @param u + * @param pkgIds + * @param pkgQus + * @param packageExtraFields + * @param superPackages + * @param childrenByParentId + * @param pkgCategoriesByPkgId + * @param lookups + * @param includeExtraFields + * @param includeLookups + * @param includeFullSubpackages + * @param errors + * @return + */ + public List getBulkPackages(Container c, User u, List pkgIds, QueryUpdateService pkgQus, + List packageExtraFields, List superPackages, + Map> childrenByParentId, + Map> pkgCategoriesByPkgId, + Map lookups, + boolean includeExtraFields, boolean includeLookups, boolean includeFullSubpackages, BatchValidationException errors) + { + List> rows = null; + List> keys = new ArrayList<>(); + Map key; + for (Integer pkgId : pkgIds) + { + key = new HashMap<>(); + key.put("PkgId", pkgId); + keys.add(key); + } + + List packages = new ArrayList<>(); + try + { + rows = pkgQus.getRows(u, c, keys); + } + catch (InvalidKeyException | QueryUpdateServiceException | SQLException e) + { + errors.addRowError(new ValidationException(e.getMessage())); + } + + if (!errors.hasErrors() && rows != null && !rows.isEmpty()) + { + for (Map row : rows) + { + packages.add(getBulkPackage(c, u, row, packageExtraFields, superPackages, childrenByParentId, pkgCategoriesByPkgId, lookups, includeExtraFields, includeLookups, includeFullSubpackages, errors)); + } + } + + return packages; + } + /** * Used for validation to ensure there are no later project revisions */ @@ -2588,9 +2814,7 @@ public Event createEvent(Container c, User u, Event event, boolean validateOnly) QueryUpdateService eventQus = getQueryUpdateService(eventTable); QueryUpdateService eventsCacheQus = getNewQueryUpdateService(schema, SNDSchema.EVENTSCACHE_TABLE_NAME); - String htmlEventNarrative = generateEventNarrative(c, u, event, topLevelEventDataPkgs, true, false); - Map eventsCacheRow = getEventsCacheRow(c, u, event.getEventId(), htmlEventNarrative); - String textEventNarrative = PlainTextNarrativeDisplayColumn.removeHtmlTagsFromNarrative(htmlEventNarrative); + BatchValidationException errors = new BatchValidationException(); QueryUpdateService eventNotesQus = null; @@ -2610,8 +2834,10 @@ public Event createEvent(Container c, User u, Event event, boolean validateOnly) } insertEventDatas(c, u, event, errors); + String htmlEventNarrative = generateEventNarrative(c, u, event, topLevelEventDataPkgs, true, false); + Map eventsCacheRow = getEventsCacheRow(c, u, event.getEventId(), htmlEventNarrative); + String textEventNarrative = PlainTextNarrativeDisplayColumn.removeHtmlTagsFromNarrative(htmlEventNarrative); eventsCacheQus.insertRows(u, c, Collections.singletonList(eventsCacheRow), errors, null, null); - generateEventNarrative(c, u, event, topLevelEventDataPkgs, true, false); NarrativeAuditProvider.addAuditEntry(c, u, event.getEventId(), event.getSubjectId(), event.getDate(), textEventNarrative, event.getQcState(), "Create event"); tx.commit(); } @@ -2919,8 +3145,14 @@ public void populateNarrativeCache(Container c, User u, List eventIds, UserSchema sndSchema = getSndUserSchemaAdminRole(c, u); QueryUpdateService eventsCacheQus = getNewQueryUpdateService(sndSchema, SNDSchema.EVENTSCACHE_TABLE_NAME); + TableInfo pkgsTable = getTableInfo(sndSchema, SNDSchema.PKGS_TABLE_NAME); + QueryUpdateService pkgQus = getQueryUpdateService(pkgsTable); + List eventExtraFields = getExtraFields(c, u, SNDSchema.EVENTS_TABLE_NAME); List eventDataExtraFields = getExtraFields(c, u, SNDSchema.EVENTDATA_TABLE_NAME); + List packageExtraFields = getExtraFields(c, u, SNDSchema.PKGS_TABLE_NAME); + + Map lookups = getLookups(c, u); // Logger for pipeline if (logger != null) @@ -2928,7 +3160,7 @@ public void populateNarrativeCache(Container c, User u, List eventIds, logger.info("Generating narratives."); } - Map superPackages = getAllFullSuperPackages(c, u, errors); + Map superPackages = getBulkSuperPkgs(c, u, packageExtraFields, pkgQus, null, lookups, errors); AtomicInteger count = new AtomicInteger(0); @@ -2939,10 +3171,10 @@ public void populateNarrativeCache(Container c, User u, List eventIds, List> rows = new ArrayList<>(); //Top Level SuperPkgs grouped by EventDataId and grouped by EventId - Map> topLevelSuperPkgs = getAllTopLevelEventDataSuperPkgs(c, u, partitionEventIds, superPackages); + Map> topLevelSuperPkgs = getBulkTopLevelEventDataSuperPkgs(c, u, partitionEventIds, superPackages); //Events grouped by EventId - Map events = getAllEvents(c, u, partitionEventIds, null, topLevelSuperPkgs, true, errors, eventExtraFields, eventDataExtraFields); + Map events = getBulkEvents(c, u, partitionEventIds, null, topLevelSuperPkgs, true, errors, eventExtraFields, eventDataExtraFields, false); partitionEventIds.forEach((Integer eventId) -> { Map row = new ArrayListMap<>(); @@ -3195,6 +3427,7 @@ private String generateEventDataNarrative(Container c, User u, Event event, Even { eventDataNarrative.append(generateEventDataNarrative(c, u, event, data, getSuperPackage(data.getSuperPkgId(), superPackage.getChildPackages()), tabIndex, genHtml, genRedacted)); + } } } @@ -3469,12 +3702,13 @@ public List> getProjectItemsList(Container c, User u, int pr * @param c * @param u * @param eventIds + * @param superPackages * @return */ - private Map> getAllTopLevelEventDataSuperPkgs(Container c, User u, List eventIds, Map superPackages) { + private Map> getBulkTopLevelEventDataSuperPkgs(Container c, User u, List eventIds, Map superPackages) { // EventData from query - SELECT * FROM EventData WHERE EventId IN {eventIds} AND ParentEventId IS NULL ORDER BY EventDataId - TableSelector eventDataSelector = getTableSelector(c, u, eventIds, SNDSchema.EVENTDATA_TABLE_NAME, "EventId", "EventDataId", "ParentEventDataId"); + TableSelector eventDataSelector = getTableSelector(c, u, eventIds, SNDSchema.EVENTDATA_TABLE_NAME, Event.EVENT_ID, EventData.EVENT_DATA_ID, EventData.EVENT_DATA_PARENT_EVENTDATAID); List allEventData = eventDataSelector.getArrayList(EventData.class); // Get SuperPackages for eventData and group by EventId @@ -3492,6 +3726,40 @@ private Map> getAllTopLevelEventDataSuperPkg } + /** + * getEvent method that uses cache-optimized methods for bulk loading to more quickly build a single event + * @param c + * @param u + * @param eventId + * @param narrativeOptions + * @param skipPermissionCheck + * @param errors + * @return + */ + @Nullable + public Event getEvent(Container c, User u, int eventId, Set narrativeOptions, boolean skipPermissionCheck, BatchValidationException errors) { + + UserSchema schema = getSndUserSchema(c, u); + + Map lookups = getLookups(c, u); + + TableInfo pkgsTable = getTableInfo(schema, SNDSchema.PKGS_TABLE_NAME); + QueryUpdateService pkgQus = getQueryUpdateService(pkgsTable); + + List eventExtraFields = getExtraFields(c, u, SNDSchema.EVENTS_TABLE_NAME); + List eventDataExtraFields = getExtraFields(c, u, SNDSchema.EVENTDATA_TABLE_NAME); + List packageExtraFields = getExtraFields(c, u, SNDSchema.PKGS_TABLE_NAME); + + Map superPackages = getBulkSuperPkgs(c, u, packageExtraFields, pkgQus, eventId, lookups, errors); + + Map> topLevelEventDataSuperPkgs = getBulkTopLevelEventDataSuperPkgs(c, u, Collections.singletonList(eventId), superPackages); + + Event event = getBulkEvents(c, u, Collections.singletonList(eventId), narrativeOptions, topLevelEventDataSuperPkgs, skipPermissionCheck, errors, eventExtraFields, eventDataExtraFields, true).get(eventId); + + return event; + + } + /** * Query the Event table and retrieve rows for a set of eventIds and populate data/create narratives * @@ -3502,26 +3770,33 @@ private Map> getAllTopLevelEventDataSuperPkg * @param topLevelSuperPkgs * @param skipPermissionCheck * @param errors + * @param eventExtraFields + * @param eventDataExtraFields * @return */ @Nullable - public Map getAllEvents(Container c, User u, List eventIds, Set narrativeOptions, + public Map getBulkEvents(Container c, User u, List eventIds, Set narrativeOptions, @Nullable Map> topLevelSuperPkgs, boolean skipPermissionCheck, BatchValidationException errors, - List eventExtraFields, List eventDataExtraFields) { + List eventExtraFields, List eventDataExtraFields, + boolean includeEmptySubPackges) { // Events from query - SELECT * FROM Events WHERE EventId IN {eventIds} - TableSelector eventSelector = getTableSelector(c, u, eventIds, SNDSchema.EVENTS_TABLE_NAME, "EventId", null, null); + TableSelector eventSelector = getTableSelector(c, u, eventIds, SNDSchema.EVENTS_TABLE_NAME, Event.EVENT_ID, null, null); List events = eventSelector.getArrayList(Event.class); Collection> eventsExtensibleFields = eventSelector.getMapCollection(); + TableSelector eventDataSelector = getTableSelector(c, u, eventIds, SNDSchema.EVENTDATA_TABLE_NAME, Event.EVENT_ID, null, null); + List rawEventData = eventDataSelector.getArrayList(EventData.class); + Collection> eventDataExtensibleFields = eventDataSelector.getMapCollection(); + //EventNotes grouped by EventId - Map eventNotes = getEventNotes(c, u, eventIds); + Map eventNotes = getBulkEventNotes(c, u, eventIds); //ProjectIdRev strings grouped by Event ObjectId - Map projectIdRevs = getProjectIdRevs(c, u, events.stream().map(Event::getParentObjectId).collect(Collectors.toList())); + Map projectIdRevs = getBulkProjectIdRevs(c, u, events.stream().map(Event::getParentObjectId).collect(Collectors.toList())); //EventData grouped by EventId - Map> eventData = getAllEventData(c, u, topLevelSuperPkgs, eventDataExtraFields); + Map> eventData = getBulkEventData(c, topLevelSuperPkgs, rawEventData, eventDataExtensibleFields, eventDataExtraFields, includeEmptySubPackges); // Build events from eventData, eventNotes, and project data and group by EventId Map eventsById = events.stream().collect(Collectors.toMap(Event::getEventId, (Event event) -> { @@ -3530,7 +3805,7 @@ public Map getAllEvents(Container c, User u, List event Map extraFields = eventsExtensibleFields .stream() - .filter((Map map) -> event.getEventId().equals(map.get("eventId"))) + .filter((Map map) -> event.getEventId().equals(map.get(Event.EVENT_ID))) .findFirst() .orElse(Collections.emptyMap()); @@ -3562,15 +3837,20 @@ public Map getAllEvents(Container c, User u, List event } /** - * Query the SuperPackages table and retrieve all rows for a set of superPkgIds - * + * Return a cached map of SuperPackage objects by superPkgId + * Builds each underlying SuperPackage object with child superPackages + * Retrieves SuperPackages for an eventId if it is not null, else retrieves SuperPackages for entire database * @param c * @param u + * @param packageExtraFields + * @param pkgQus + * @param eventId + * @param lookups * @param errors * @return */ - @Nullable - private Map getAllFullSuperPackages(Container c, User u, BatchValidationException errors) { + private Map getBulkSuperPkgs(Container c, User u, List packageExtraFields, QueryUpdateService pkgQus, @Nullable Integer eventId, + Map lookups, BatchValidationException errors) { UserSchema schema = getSndUserSchema(c, u); @@ -3580,26 +3860,63 @@ private Map getAllFullSuperPackages(Container c, User u, sql.append(schema.getTable(SNDSchema.SUPERPKGS_TABLE_NAME), "sp"); sql.append(" JOIN " + SNDSchema.NAME + "." + SNDSchema.PKGS_TABLE_NAME + " pkg"); sql.append(" ON sp.PkgId = pkg.PkgId "); + if (eventId != null) { + sql.append(" INNER JOIN " + SNDSchema.NAME + "." + SNDSchema.EVENTDATA_TABLE_NAME + " ed"); + sql.append(" ON ed.SuperPkgId = sp.SuperPkgId "); + sql.append(" WHERE ed.EventId = ? ").add(eventId); + } sql.append(" ORDER BY sp.SuperPkgId "); SqlSelector superPkgSelector = new SqlSelector(schema.getDbSchema(), sql); List superPackages = superPkgSelector.getArrayList(SuperPackage.class); - // Group All SuperPkgs by SuperPkgId - Map superPackagesById = superPackages.stream().collect(Collectors.toMap(SuperPackage::getSuperPkgId, superPackage -> superPackage)); + List pkgIds = superPackages.stream().map(SuperPackage::getPkgId).toList(); + + List fullTreeSuperPkgs; + + if (eventId != null) { + fullTreeSuperPkgs = new ArrayList(); + pkgIds.forEach(pkgId -> { + SQLFragment packageSql = new SQLFragment("SELECT * FROM "); + packageSql.append(SNDSchema.NAME + "." + SNDSchema.SUPERPKGS_FUNCTION_NAME + "(?)").add(pkgId); + SqlSelector childSuperPkgSelector = new SqlSelector(schema.getDbSchema(), packageSql); + fullTreeSuperPkgs.addAll(childSuperPkgSelector.getArrayList(SuperPackage.class)); + }); + + } else { + SQLFragment superPkgSql = new SQLFragment("SELECT * FROM "); + superPkgSql.append(SNDSchema.NAME + "." + SNDSchema.ALL_SUPERPKGS_FUNCTION_NAME + "()"); + SqlSelector childSuperPkgSelector = new SqlSelector(schema.getDbSchema(), superPkgSql); + fullTreeSuperPkgs = childSuperPkgSelector.getArrayList(SuperPackage.class); + pkgIds = null; + } + + Map> pkgCategoriesByPkgId; + if (pkgIds == null || !pkgIds.isEmpty()) { + pkgCategoriesByPkgId = getBulkPackageCategories(c, u, pkgIds, errors); + } else { + pkgCategoriesByPkgId = new HashMap<>(); + } + + Map> childrenByParentId = fullTreeSuperPkgs.stream().filter(sp -> sp.getParentSuperPkgId() != null).collect(Collectors.groupingBy(SuperPackage::getParentSuperPkgId)); - // All Child SuperPkgs grouped by TopLevelPkgId - Map> childSuperPackages = getAllFullChildSuperPkgs(c, u, true, errors); + Map superPackagesById = superPackages.stream().collect(Collectors.toMap( + SuperPackage::getSuperPkgId, + (SuperPackage superPackage) -> superPackage, + (s1, s2) -> s1 + )); + + Map> childrenByTopLevelPkgId = getBulkChildSuperPkgs(c, u, fullTreeSuperPkgs, childrenByParentId, pkgCategoriesByPkgId, packageExtraFields, pkgQus, lookups, true, errors); superPackagesById.forEach((superPkgId, superPackage) -> { Integer pkgId = superPackage.getPkgId(); if (pkgId != null) { - List pkgIds = Collections.singletonList(pkgId); - List packages = getPackages(c, u, pkgIds, true, true, true, errors); + List packageIds = Collections.singletonList(pkgId); + List packages = getBulkPackages(c, u, packageIds, pkgQus, packageExtraFields, fullTreeSuperPkgs, childrenByParentId, pkgCategoriesByPkgId, lookups, true, true, true, errors); if (!packages.isEmpty()) { superPackage.setPkg(packages.get(0)); } - superPackage.setChildPackages(childSuperPackages.get(pkgId)); + superPackage.setChildPackages(childrenByTopLevelPkgId.get(pkgId)); } }); @@ -3608,19 +3925,26 @@ private Map getAllFullSuperPackages(Container c, User u, } /** - * Recursively get all children for the super package which corresponds to pkgId + * Retrieve the child SuperPackage objects for a list of SuperPackages + * @param c + * @param u + * @param allSuperPackages + * @param childrenByParentId + * @param pkgCategoriesByPkgId + * @param packageExtraFields + * @param pkgQus + * @param lookups + * @param includeFullSubpackages + * @param errors + * @return */ - private Map> getAllFullChildSuperPkgs(Container c, User u, boolean includeFullSubpackages, BatchValidationException errors) { + private Map> getBulkChildSuperPkgs (Container c, User u, List allSuperPackages, + Map> childrenByParentId, + Map> pkgCategoriesByPkgId, + List packageExtraFields, QueryUpdateService pkgQus, + Map lookups, + boolean includeFullSubpackages, BatchValidationException errors) { - UserSchema schema = getSndUserSchema(c, u); - - SQLFragment childSql = new SQLFragment("SELECT * FROM "); - childSql.append(SNDSchema.NAME + "." + SNDSchema.ALL_SUPERPKGS_FUNCTION_NAME + "()"); - - SqlSelector selector = new SqlSelector(schema.getDbSchema(), childSql); - List allSuperPackages = selector.getArrayList(SuperPackage.class); - - // Group results of TopLevelSuperPkgs by TopLevelPkgId Map> superPackagesByTopLevelPkgId = allSuperPackages.stream().collect(Collectors.groupingBy(SuperPackage::getTopLevelPkgId)); superPackagesByTopLevelPkgId.forEach((topLevelPkgId, superPackages) -> { @@ -3632,8 +3956,8 @@ private Map> getAllFullChildSuperPkgs(Container c, U .filter(sp -> sp.getParentSuperPkgId() != null) .forEach((SuperPackage superPackage) -> { if (includeFullSubpackages) { - List pkgIds = Collections.singletonList(superPackage.getPkgId()); - superPackage.setPkg(getPackages(c, u, pkgIds, true, true, true, errors).get(0)); + List packageIds = Collections.singletonList(superPackage.getPkgId()); + superPackage.setPkg(getBulkPackages(c, u, packageIds, pkgQus, packageExtraFields, allSuperPackages, childrenByParentId, pkgCategoriesByPkgId, lookups, true, true, true, errors).get(0)); } if (superPackage.getParentSuperPkgId().intValue() == root.getSuperPkgId().intValue()) { children.add(addChildren(superPackage, superPackages)); @@ -3645,19 +3969,21 @@ private Map> getAllFullChildSuperPkgs(Container c, U }); return superPackagesByTopLevelPkgId; - } /** - * Query EventData table and populate attribute data for a set of top level SuperPackages - * + * Populate eventData and attribute data for a set of top level SuperPackages * @param c - * @param u * @param currentLevelSuperPkgs + * @param currentEventData + * @param eventDataExtensibleFields + * @param eventDataExtraFields * @return */ - private Map> getAllEventData(Container c, User u, Map> currentLevelSuperPkgs, List eventDataExtraFields) { + private Map> getBulkEventData(Container c, Map> currentLevelSuperPkgs, + List currentEventData, Collection> eventDataExtensibleFields, + List eventDataExtraFields, boolean includeEmptySubPackages) { if (currentLevelSuperPkgs == null) { return null; @@ -3668,20 +3994,36 @@ private Map> getAllEventData(Container c, User u, Map map) -> map.keySet().stream()) .collect(Collectors.toList()); - // EventData from query - SELECT * FROM EventData WHERE EventDataId IN {eventDataIds} - TableSelector eventDataSelector = getTableSelector(c, u, eventDataIds, SNDSchema.EVENTDATA_TABLE_NAME, "EventDataId", null, null); - List allEventData = eventDataSelector.getArrayList(EventData.class); - Collection> eventDataExtensibleFields = eventDataSelector.getMapCollection(); + List originalEventDataIds = currentEventData.stream().filter(e -> e.getEventDataId() != null).map(EventData::getEventDataId).toList(); + + List allEventData = new ArrayList<>(currentEventData); + + if (includeEmptySubPackages) { + List emptyEventData = currentLevelSuperPkgs.entrySet().stream() + .flatMap(outerEntry -> outerEntry.getValue().entrySet().stream().map(innerEntry -> Map.entry(outerEntry.getKey(), innerEntry))) + .filter(entry -> !originalEventDataIds.contains(entry.getValue().getKey())) + .map(entry -> { + EventData eventData = new EventData(); + eventData.setEventDataId(entry.getValue().getKey()); + eventData.setEventId(entry.getKey()); + eventData.setSuperPkgId(entry.getValue().getValue().getSuperPkgId()); + return eventData; + }) + .toList(); + + + allEventData.addAll(emptyEventData); + } // Child EventData grouped by EventId - Map> childEventData = getChildEventData(c, u, eventDataIds); + Map> childEventData = getBulkChildEventData(allEventData, eventDataIds); // Build eventData from attributes and superPkgs and group by eventId - Map> eventDataByEventId = allEventData.stream().map((EventData eventData) -> { + Map> eventDataByEventId = allEventData.stream().filter(e -> eventDataIds.contains(e.getEventDataId())).map((EventData eventData) -> { Map properties = OntologyManager.getPropertyObjects(c, eventData.getObjectURI()); - Map superPackagesById = currentLevelSuperPkgs.get(eventData.getEventId()); - SuperPackage superPackage = superPackagesById.get(eventData.getEventDataId()); + Map superPackagesByEventDataId = currentLevelSuperPkgs.get(eventData.getEventId()); + SuperPackage superPackage = superPackagesByEventDataId.get(eventData.getEventDataId()); if (superPackage != null) { List attributeData = superPackage.getPkg().getAttributes() @@ -3696,20 +4038,35 @@ private Map> getAllEventData(Container c, User u, Map extraFields = eventDataExtensibleFields .stream() - .filter((Map map) -> eventData.getEventDataId().equals(map.get("eventDataId"))) + .filter((Map map) -> eventData.getEventDataId().equals(map.get(EventData.EVENT_DATA_ID))) .findFirst() .orElse(Collections.emptyMap()); addExtraFieldsToEventData(eventData, eventDataExtraFields, extraFields); - Map> nextLevelSuperPkgs = getNextLevelEventDataSuperPkgs(eventData, childEventData, currentLevelSuperPkgs); + boolean isEmptyEventDataAndHasChildPackages = + includeEmptySubPackages + && + !originalEventDataIds.contains(eventData.getEventDataId()) + && + !superPackagesByEventDataId.get(eventData.getEventDataId()).getChildPackages().isEmpty(); + + Map> nextLevelSuperPkgs = getNextLevelEventDataSuperPkgs(eventData, childEventData, currentLevelSuperPkgs, includeEmptySubPackages, isEmptyEventDataAndHasChildPackages); if (nextLevelSuperPkgs != null && !nextLevelSuperPkgs.isEmpty()) { // Recursion for next child level of sub packages - Map> subEventData = getAllEventData(c, u, nextLevelSuperPkgs, eventDataExtraFields); + Map> subEventData = getBulkEventData(c, nextLevelSuperPkgs, currentEventData, eventDataExtensibleFields, eventDataExtraFields, includeEmptySubPackages); if (subEventData != null) { List sorted = subEventData.get(eventData.getEventId()).stream().sorted(Comparator.comparing( (EventData child) -> nextLevelSuperPkgs.get(child.getEventId()).get(child.getEventDataId()).getTreePath())) + .filter(ed -> ed.getParentEventDataId() == null || ed.getParentEventDataId().equals(eventData.getEventDataId())) + .map(ed -> { + if (!originalEventDataIds.contains(ed.getEventDataId())) { + ed.setEventId(0); + ed.setEventDataId(null); + } + return ed; + }) .collect(Collectors.toList()); eventData.setSubPackages(sorted); } @@ -3724,21 +4081,17 @@ private Map> getAllEventData(Container c, User u, Map> getChildEventData(Container c, User u, List parentEventDataIds) { + private Map> getBulkChildEventData(List allEventData, List parentEventDataIds) { if (parentEventDataIds.isEmpty()) { return Collections.emptyMap(); } - // EventData from query - SELECT * FROM EventData WHERE ParentEventDataId IN {parentEventDataIds} - TableSelector eventDataSelector = getTableSelector(c, u, parentEventDataIds, SNDSchema.EVENTDATA_TABLE_NAME, "ParentEventDataId", null, null); - List childEventData = eventDataSelector.getArrayList(EventData.class); + List childEventData = allEventData.stream().filter(e -> parentEventDataIds.contains(e.getParentEventDataId())).toList(); // Group childEventData by eventId Map> childEventDataByEventId = childEventData.stream().collect(Collectors.groupingBy(EventData::getEventId)); @@ -3786,9 +4139,10 @@ private AttributeData getAttributeData(GWTPropertyDescriptor propertyDescriptor, * @return */ private Map> getNextLevelEventDataSuperPkgs(EventData eventData, Map> childEventData, Map> currentLevelSuperPkgs) { + List> childEventData, Map> currentLevelSuperPkgs, + boolean includeEmptySubPackages, boolean hasChildPackages) { - if (!childEventData.containsKey(eventData.getEventId())) { + if (!childEventData.containsKey(eventData.getEventId()) && !hasChildPackages) { return null; } @@ -3805,18 +4159,43 @@ private Map> getNextLevelEventDataSuperPkgs( )); // Get superPkg for eventData and group by eventId and then by eventId - Map> nextLevelEventDataSuperPkgs = childEventData.get(eventData.getEventId()) - .stream() - .filter((EventData child) -> childSuperPkgs.containsKey(child.getSuperPkgId())) - .collect( - Collectors.groupingBy( - EventData::getEventId, - Collectors.toMap( - EventData::getEventDataId, - (EventData child) -> childSuperPkgs.get(child.getSuperPkgId()) - ) - ) - ); + Map> nextLevelEventDataSuperPkgs; + List eventDataSuperPkgIds; + + if (!childEventData.isEmpty()) { + Map children = childSuperPkgs; + nextLevelEventDataSuperPkgs = childEventData.get(eventData.getEventId()) + .stream() + .filter((EventData child) -> children.containsKey(child.getSuperPkgId())) + .collect( + Collectors.groupingBy( + EventData::getEventId, + Collectors.toMap( + EventData::getEventDataId, + (EventData child) -> children.get(child.getSuperPkgId()) + ) + ) + ); + eventDataSuperPkgIds = nextLevelEventDataSuperPkgs.containsKey(eventData.getEventId()) + ? nextLevelEventDataSuperPkgs.get(eventData.getEventId()).values().stream().map(SuperPackage::getSuperPkgId).toList() + : new ArrayList<>(); + } + else + { + nextLevelEventDataSuperPkgs = new HashMap<>(); + nextLevelEventDataSuperPkgs.put(eventData.getEventId(), new HashMap<>()); + eventDataSuperPkgIds = new ArrayList<>(); + } + + AtomicInteger emptyEventDataId = new AtomicInteger(0); + + if (includeEmptySubPackages && nextLevelEventDataSuperPkgs.containsKey(eventData.getEventId())) { + childSuperPkgs.values().stream() + .filter(superPkg -> !eventDataSuperPkgIds.contains(superPkg.getSuperPkgId())) + .forEach(spkg -> { + nextLevelEventDataSuperPkgs.get(eventData.getEventId()).put(emptyEventDataId.getAndAdd(1), spkg); + }); + } return nextLevelEventDataSuperPkgs; } @@ -3829,10 +4208,10 @@ private Map> getNextLevelEventDataSuperPkgs( * @param eventIds * @return */ - private Map getEventNotes(Container c, User u, List eventIds) { + private Map getBulkEventNotes(Container c, User u, List eventIds) { // EventNotes from query - SELECT * FROM EventNotes WHERE EventId IN {eventIds} - TableSelector eventNoteSelector = getTableSelector(c, u, eventIds, SNDSchema.EVENTNOTES_TABLE_NAME, "EventId", null, null); + TableSelector eventNoteSelector = getTableSelector(c, u, eventIds, SNDSchema.EVENTNOTES_TABLE_NAME, Event.EVENT_ID, null, null); List eventNotes = eventNoteSelector.getArrayList(EventNote.class); // Group eventNotes by eventId @@ -3853,10 +4232,10 @@ private Map getEventNotes(Container c, User u, List ev * @param objectIds * @return */ - private Map getProjectIdRevs(Container c, User u, List objectIds) { + private Map getBulkProjectIdRevs(Container c, User u, List objectIds) { // Projects from query - SELECT * FROM Projects WHERE ObjectId IN {objectIds} - TableSelector projectSelector = getTableSelector(c, u, objectIds, SNDSchema.PROJECTS_TABLE_NAME, "ObjectId", null, null); + TableSelector projectSelector = getTableSelector(c, u, objectIds, SNDSchema.PROJECTS_TABLE_NAME, Project.PROJECT_OBJECTID, null, null); List projects = projectSelector.getArrayList(Project.class); // Concat string of ProjectId + RevisionNum and Group by ObjectId diff --git a/src/org/labkey/snd/SNDServiceImpl.java b/src/org/labkey/snd/SNDServiceImpl.java index 2ef328f9..43f5794f 100644 --- a/src/org/labkey/snd/SNDServiceImpl.java +++ b/src/org/labkey/snd/SNDServiceImpl.java @@ -336,7 +336,7 @@ public Event getEvent(Container c, User u, int eventId, Set