diff --git a/api-src/org/labkey/api/snd/Event.java b/api-src/org/labkey/api/snd/Event.java index 665b5090..a1ac4df9 100644 --- a/api-src/org/labkey/api/snd/Event.java +++ b/api-src/org/labkey/api/snd/Event.java @@ -79,6 +79,7 @@ public class Event public static final String SND_EVENT_DATE_CSS_CLASS = "snd-event-date"; public static final String SND_EVENT_SUBJECT_CSS_CLASS = "snd-event-subject"; + public static final String SND_EVENT_NOTE_CSS_CLASS = "snd-event-note"; public static final String SND_EXCEPTION_MSG_JSON = "message"; public static final String SND_EXCEPTION_SEVERITY_JSON = "severity"; diff --git a/api-src/org/labkey/api/snd/EventNote.java b/api-src/org/labkey/api/snd/EventNote.java new file mode 100644 index 00000000..f13a99d6 --- /dev/null +++ b/api-src/org/labkey/api/snd/EventNote.java @@ -0,0 +1,41 @@ +package org.labkey.api.snd; + +public class EventNote +{ + private Integer _eventId; + private String _note; + private Integer _eventNoteId; + private String _container; + + public static final String EVENT_ID = "eventId"; + public static final String NOTE = "note"; + public static final String EVENT_NOTE_ID = "eventNoteId"; + public static final String CONTAINER = "Container"; + + public EventNote(Integer eventId, String note, Integer eventNoteId) { + _eventId = eventId; + _note = note; + _eventNoteId = eventNoteId; + } + + public EventNote () {} + + public Integer getEventId() + { + return _eventId; + } + + public void setEventId(Integer eventId) + { + _eventId = eventId; + } + + public String getNote() { return _note; } + + public void setNote(String note) { _note = note; } + + public Integer getEventNoteId() { return _eventNoteId; } + + public void setEventNoteId(Integer eventNoteId) { _eventNoteId = eventNoteId; } + +} diff --git a/api-src/org/labkey/api/snd/SNDService.java b/api-src/org/labkey/api/snd/SNDService.java index 84709782..b65d2f20 100644 --- a/api-src/org/labkey/api/snd/SNDService.java +++ b/api-src/org/labkey/api/snd/SNDService.java @@ -66,7 +66,7 @@ static SNDService get() void fillInNarrativeCache(Container c, User u, Logger logger); void clearNarrativeCache(Container c, User u); void deleteNarrativeCacheRows(Container c, User u, List> eventIds); - void populateNarrativeCache(Container c, User u, List> eventIds, Logger logger); + void populateNarrativeCache(Container c, User u, List eventIds, Logger logger); Map getAllCategories(Container c, User u); Integer getQCStateId(Container c, User u, QCStateEnum qcState); QCStateEnum getQCState(Container c, User u, int qcStateId); diff --git a/api-src/org/labkey/api/snd/SuperPackage.java b/api-src/org/labkey/api/snd/SuperPackage.java index 3a1f1715..2bf9cd95 100644 --- a/api-src/org/labkey/api/snd/SuperPackage.java +++ b/api-src/org/labkey/api/snd/SuperPackage.java @@ -26,6 +26,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.function.Function; import static org.labkey.api.snd.Package.PKG_ATTRIBUTES; @@ -41,7 +42,9 @@ public class SuperPackage private Integer _pkgId; private Package _pkg; private String _superPkgPath; + private String _treePath; private Integer _parentSuperPkgId; + private Integer _topLevelPkgId; private String _description; // From referenced package private String _narrative; // From referenced package private Integer _sortOrder; @@ -51,13 +54,16 @@ public class SuperPackage public static final String SUPERPKG_ID = "superPkgId"; public static final String SUPERPKG_PARENTID = "parentSuperPkgId"; + public static final String SUPERPKG_TOP_LEVEL_PKGID = "topLevelPkgId"; public static final String SUPERPKG_PKGID = "pkgId"; public static final String SUPERPKG_DESCRIPTION = "description"; public static final String SUPERPKG_NARRATIVE = "narrative"; public static final String SUPERPKG_ORDER = "sortOrder"; public static final String SUPERPKG_REPEATABLE = "repeatable"; public static final String SUPERPKG_PATH = "superPkgPath"; + public static final String SUPERPKG_TREEPATH = "treePath"; public static final String SUPERPKG_REQUIRED = "required"; + public static final String SUPERPKG_PKGID_CSS_CLASS = "snd-superpkg-pkg-id"; public SuperPackage() { @@ -96,6 +102,11 @@ public void setParentSuperPkgId(Integer parentSuperPkgId) _parentSuperPkgId = parentSuperPkgId; } + @Nullable + public Integer getTopLevelPkgId() { return _topLevelPkgId; } + + public void setTopLevelPkgId(Integer topLevelPkgId) { _topLevelPkgId = topLevelPkgId; } + @Nullable public List getChildPackages() { @@ -156,6 +167,11 @@ public void setSuperPkgPath(String superPkgPath) _superPkgPath = superPkgPath; } + @Nullable + public String getTreePath() { return _treePath; } + + public void setTreePath(String treePath) { _treePath = treePath; } + @Nullable public Integer getSuperPkgId() { @@ -211,6 +227,8 @@ public Map getSuperPackageRow(Container c) superPkgValues.put("container", c); superPkgValues.put(SUPERPKG_PATH, getSuperPkgPath()); superPkgValues.put(SUPERPKG_REQUIRED, getRequired()); + superPkgValues.put(SUPERPKG_TREEPATH, getTreePath()); + superPkgValues.put(SUPERPKG_TOP_LEVEL_PKGID, getTopLevelPkgId()); return superPkgValues; } diff --git a/src/org/labkey/snd/SNDController.java b/src/org/labkey/snd/SNDController.java index 4c677290..1c7fc01a 100644 --- a/src/org/labkey/snd/SNDController.java +++ b/src/org/labkey/snd/SNDController.java @@ -1123,9 +1123,9 @@ private List parseAttributeData(JSONArray attributesDataJson) } else { - propertyId = attributeJson.getInt("propertyId"); + propertyId = attributeJson.optInt("propertyId"); } - String value = attributeJson.getString("value"); + String value = attributeJson.optString("value"); // propertyDescriptor not used for saving, so make it null if (propertyName != null) diff --git a/src/org/labkey/snd/SNDManager.java b/src/org/labkey/snd/SNDManager.java index 16edff8c..4bb3677c 100644 --- a/src/org/labkey/snd/SNDManager.java +++ b/src/org/labkey/snd/SNDManager.java @@ -20,6 +20,7 @@ import com.google.common.collect.Maps; import org.apache.commons.beanutils.ConversionException; import org.apache.commons.beanutils.ConvertUtils; +import org.apache.commons.collections4.ListUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -35,6 +36,7 @@ import org.labkey.api.data.Results; import org.labkey.api.data.SQLFragment; import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Sort; import org.labkey.api.data.SqlExecutor; import org.labkey.api.data.SqlSelector; import org.labkey.api.data.TableInfo; @@ -71,6 +73,7 @@ import org.labkey.api.snd.Event; import org.labkey.api.snd.EventData; import org.labkey.api.snd.EventNarrativeOption; +import org.labkey.api.snd.EventNote; import org.labkey.api.snd.Package; import org.labkey.api.snd.PackageDomainKind; import org.labkey.api.snd.Project; @@ -90,6 +93,7 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.Date; @@ -99,7 +103,8 @@ import java.util.Map; import java.util.Set; import java.util.TreeMap; -import java.util.TreeSet; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import static org.labkey.api.snd.EventNarrativeOption.HTML_NARRATIVE; import static org.labkey.api.snd.EventNarrativeOption.REDACTED_HTML_NARRATIVE; @@ -297,7 +302,7 @@ public Object normalizeLookupValue(User u, Container c, String schema, String ta * Called from SNDService.savePackage when saving updates to an already existing package. */ public void updatePackage(User u, Container c, @NotNull Package pkg, @Nullable SuperPackage superPkg, BatchValidationException errors) { - updatePackage(u, c, pkg, superPkg, errors, false); + updatePackage(u, c, pkg, superPkg, errors, false); } public void updatePackage(User u, Container c, @NotNull Package pkg, @Nullable SuperPackage superPkg, BatchValidationException errors, boolean isPipelineJob) @@ -2148,6 +2153,28 @@ public Event addExtraFieldsToEvent(Container c, User u, Event event, @Nullable M return event; } + /** + * Add extensible columns to an event. + */ + public Event addExtraFieldsToEvent(Event event, List extraFields, @Nullable Map row) + { + Map extras = new HashMap<>(); + for (GWTPropertyDescriptor extraField : extraFields) + { + if (row == null) + { + extras.put(extraField, ""); + } + else + { + extras.put(extraField, row.get(extraField.getName())); + } + } + event.setExtraFields(extras); + + return event; + } + /** * Add extensible columns to event data. */ @@ -2171,6 +2198,28 @@ private EventData addExtraFieldsToEventData(Container c, User u, EventData event return eventData; } + /** + * Add extensible columns to event data. + */ + private EventData addExtraFieldsToEventData(EventData eventData, List extraFields, @Nullable Map row) + { + Map extras = new HashMap<>(); + for (GWTPropertyDescriptor extraField : extraFields) + { + if (row == null) + { + extras.put(extraField, null); + } + else + { + extras.put(extraField, row.get(extraField.getName())); + } + } + eventData.setExtraFields(extras); + + return eventData; + } + /** * Get an empty event with just event meta data. Used for creating the initial forms. Called from SND GetEvent API */ @@ -2775,10 +2824,11 @@ public int updateNarrativeCache(Container container, User user, Setcach } else { - for (Integer cacheDatum : cacheData) + List eventIds = new ArrayList<>(cacheData); + for (Integer eventId : eventIds) { row = new HashMap<>(); - row.put("EventId", cacheDatum); + row.put("EventId", eventId); rows.add(row); } @@ -2788,7 +2838,7 @@ public int updateNarrativeCache(Container container, User user, Setcach if (isUpdate) { log.info("Repopulate affected rows in narrative cache."); - populateNarrativeCache(container, user, rows, errors, log); + populateNarrativeCache(container, user, eventIds, errors, log); } } @@ -2854,28 +2904,19 @@ public void fillInNarrativeCache(Container c, User u, BatchValidationException e List eventIds = selector.getArrayList(Integer.class); - Map row; - List> rows = new ArrayList<>(); - for (Integer eventId : eventIds) - { - row = new HashMap<>(); - row.put("EventId", eventId); - rows.add(row); - } - - populateNarrativeCache(c, u, rows, errors, logger); + populateNarrativeCache(c, u, eventIds, errors, logger); } /** * Populate specific event narratives in narrative cache. */ - public void populateNarrativeCache(Container c, User u, List> eventIds, BatchValidationException errors, @Nullable Logger logger) + public void populateNarrativeCache(Container c, User u, List eventIds, BatchValidationException errors, @Nullable Logger logger) { UserSchema sndSchema = getSndUserSchemaAdminRole(c, u); QueryUpdateService eventsCacheQus = getNewQueryUpdateService(sndSchema, SNDSchema.EVENTSCACHE_TABLE_NAME); - Map eventDataTopLevelSuperPkgs; - List> rows = new ArrayList<>(); + List eventExtraFields = getExtraFields(c, u, SNDSchema.EVENTS_TABLE_NAME); + List eventDataExtraFields = getExtraFields(c, u, SNDSchema.EVENTDATA_TABLE_NAME); // Logger for pipeline if (logger != null) @@ -2883,45 +2924,43 @@ public void populateNarrativeCache(Container c, User u, List logger.info("Generating narratives."); } - int count = 0; - Integer eventId; - Map row; - for (Map eventRow : eventIds) - { - eventId = (Integer) eventRow.get("EventId"); - if (eventId != null) - { - row = new ArrayListMap<>(); - eventDataTopLevelSuperPkgs = getTopLevelEventDataSuperPkgs(c, u, eventId, errors); - Event event = getEvent(c, u, eventId, null, eventDataTopLevelSuperPkgs, true, errors); // don't populate narratives or set narrative options, since we're repopulating the narrative cache - String eventNarrative = generateEventNarrative(c, u, event, eventDataTopLevelSuperPkgs, true, false); + Map superPackages = getAllFullSuperPackages(c, u, errors); + + AtomicInteger count = new AtomicInteger(0); + + // Partition eventIds for batching + List> eventIdsPartitioned = ListUtils.partition(eventIds, 2100); + + eventIdsPartitioned.forEach((List partitionEventIds) -> { + List> rows = new ArrayList<>(); + + //Top Level SuperPkgs grouped by EventDataId and grouped by EventId + Map> topLevelSuperPkgs = getAllTopLevelEventDataSuperPkgs(c, u, partitionEventIds, superPackages); + + //Events grouped by EventId + Map events = getAllEvents(c, u, partitionEventIds, null, topLevelSuperPkgs, true, errors, eventExtraFields, eventDataExtraFields); + + partitionEventIds.forEach((Integer eventId) -> { + Map row = new ArrayListMap<>(); + Event event = events.get(eventId); + String eventNarrative = generateEventNarrative(c, u, event, topLevelSuperPkgs.get(eventId), true, false); row.put("EventId", eventId); row.put("HtmlNarrative", eventNarrative); row.put("Container", c); rows.add(row); - count++; - - if (logger != null && count % 1000 == 0) - { - logger.info(count + " narratives generated."); + int currentCount = count.incrementAndGet(); + if (logger != null && currentCount % 1000 == 0) { + logger.info(currentCount + " narratives generated."); } + }); + try (DbScope.Transaction tx = sndSchema.getDbSchema().getScope().ensureTransaction()) { + eventsCacheQus.insertRows(u, c, rows, errors, null, null); + tx.commit(); } - } - - if (logger != null) - { - logger.info("Inserting narratives into cache."); - } - - try (DbScope.Transaction tx = sndSchema.getDbSchema().getScope().ensureTransaction()) - { - eventsCacheQus.insertRows(u, c, rows, errors, null, null); - tx.commit(); - } - catch (QueryUpdateServiceException | BatchValidationException | SQLException | DuplicateKeyException e) - { - errors.addRowError(new ValidationException(e.getMessage())); - } + catch (QueryUpdateServiceException | BatchValidationException | SQLException | DuplicateKeyException e) { + errors.addRowError(new ValidationException(e.getMessage())); + } + }); } /** @@ -3043,25 +3082,32 @@ private String handleNarrativeDate(Container c, Event event, AttributeData attri private String generateEventDataNarrative(Container c, User u, Event event, EventData eventData, SuperPackage superPackage, int tabIndex, boolean genHtml, boolean genRedacted) { StringBuilder eventDataNarrative = new StringBuilder(); - if (superPackage.getNarrative() != null) - { - eventDataNarrative.append(superPackage.getNarrative()); - } - if (genHtml) + if (tabIndex == 0) { - eventDataNarrative.insert(0, "
"); + eventDataNarrative.append("** "); } else { - // plain text indenting - StringBuilder tabs = new StringBuilder("\n"); - for (int t = 0; t < tabIndex; t++) + eventDataNarrative.append("-- "); + } + + if (genHtml) + { + for (int i = 0; i <= tabIndex; i++) { - tabs.append("\t"); + eventDataNarrative.append(" "); } + } + + if (superPackage.getNarrative() != null) + { + eventDataNarrative.append(superPackage.getNarrative()); + } - eventDataNarrative.insert(0, tabs); + if (genHtml) + { + eventDataNarrative.insert(0, "
"); } if (superPackage.getPkg() != null) @@ -3088,7 +3134,9 @@ private String generateEventDataNarrative(Container c, User u, Event event, Even } if (pd == null) + { continue; + } if (genRedacted) { @@ -3118,27 +3166,35 @@ private String generateEventDataNarrative(Container c, User u, Event event, Even if (value != null) { if (genHtml) + { value = "" + value + ""; + } eventDataNarrative = new StringBuilder(eventDataNarrative.toString().replace("{" + pd.getName() + "}", value)); } } + if (genHtml) + { + eventDataNarrative.append(" (" + superPackage.getPkgId() + ")
"); + } + + if (genHtml) + { + eventDataNarrative.append("\n"); + } + if (eventData.getSubPackages() != null) { tabIndex++; for (EventData data : eventData.getSubPackages()) { - eventDataNarrative.append(generateEventDataNarrative(c, u, event, data, getSuperPackage(data.getSuperPkgId(), superPackage.getChildPackages()), tabIndex, genHtml, genRedacted)); + eventDataNarrative.append(generateEventDataNarrative(c, u, event, data, + getSuperPackage(data.getSuperPkgId(), superPackage.getChildPackages()), tabIndex, genHtml, genRedacted)); } } } - if (genHtml) - { - eventDataNarrative.append("
\n"); - } - return eventDataNarrative.toString(); } @@ -3155,7 +3211,8 @@ private String generateEventNarrative(Container c, User u, Event event, Map").append(dateValue).append("\n"); + narrative.append("
").append(dateValue) + .append("
\n"); } else { @@ -3167,7 +3224,8 @@ private String generateEventNarrative(Container c, User u, Event event, MapSubject Id: ").append(event.getSubjectId()).append("\n"); + narrative.append("
Subject Id: ") + .append(event.getSubjectId()).append("
\n"); } else { @@ -3175,16 +3233,31 @@ private String generateEventNarrative(Container c, User u, Event event, Map"); } - if (event.getEventData() != null) + if (event.getNote() != null) { - for (EventData eventData : event.getEventData()) + if (genHtml) + { + narrative.append("
Procedure Note:
") + .append(event.getNote()).append("
\n"); + } + else { - narrative.append(generateEventDataNarrative(c, u, event, eventData, topLevelEventDataSuperPkgs.get(eventData.getEventDataId()), 0, genHtml, genRedacted)); + narrative.append("Procedure Note: ").append(event.getNote()).append("\n"); } } @@ -3251,7 +3324,7 @@ public Map getAllCategories(Container c, User u) public List> getActiveProjects(Container c, User u, ArrayList filters, Boolean activeProjectItemsOnly) { - List> projectList = new ArrayList<>(); + List> projectList = new ArrayList<>(); UserSchema schema = getSndUserSchema(c, u); TableInfo projectsTable = getTableInfo(schema, SNDSchema.PROJECTS_TABLE_NAME); @@ -3380,5 +3453,443 @@ public List> getProjectItemsList(Container c, User u, int pr return projectItems; } - + + + /** + * Query the EventData table and create a Map of all top level SuperPackages for a set of eventIds + * + * @param c + * @param u + * @param eventIds + * @return + */ + private Map> getAllTopLevelEventDataSuperPkgs(Container c, User u, List eventIds, Map superPackages) { + + // EventData from query - SELECT * FROM EventData WHERE EventId IN {eventIds} AND ParentEventId IS NULL ORDER BY EventDataId + TableSelector eventDataSelector = getTableSelector(c, u, eventIds, SNDSchema.EVENTDATA_TABLE_NAME, "EventId", "EventDataId", "ParentEventDataId"); + List allEventData = eventDataSelector.getArrayList(EventData.class); + + // Get SuperPackages for eventData and group by EventId + Map> topLevelEventDataSuperPkgs = allEventData.stream().collect( + Collectors.groupingBy( + EventData::getEventId, + Collectors.toMap( + EventData::getEventDataId, + (EventData eventData) -> superPackages.get(eventData.getSuperPkgId()) + ) + ) + ); + + return topLevelEventDataSuperPkgs; + + } + + /** + * Query the Event table and retrieve rows for a set of eventIds and populate data/create narratives + * + * @param c + * @param u + * @param eventIds + * @param narrativeOptions + * @param topLevelSuperPkgs + * @param skipPermissionCheck + * @param errors + * @return + */ + @Nullable + public Map getAllEvents(Container c, User u, List eventIds, Set narrativeOptions, + @Nullable Map> topLevelSuperPkgs, boolean skipPermissionCheck, BatchValidationException errors, + List eventExtraFields, List eventDataExtraFields) { + + // Events from query - SELECT * FROM Events WHERE EventId IN {eventIds} + TableSelector eventSelector = getTableSelector(c, u, eventIds, SNDSchema.EVENTS_TABLE_NAME, "EventId", null, null); + List events = eventSelector.getArrayList(Event.class); + Collection> eventsExtensibleFields = eventSelector.getMapCollection(); + + //EventNotes grouped by EventId + Map eventNotes = getEventNotes(c, u, eventIds); + + //ProjectIdRev strings grouped by Event ObjectId + Map projectIdRevs = getProjectIdRevs(c, u, events.stream().map(Event::getParentObjectId).collect(Collectors.toList())); + + //EventData grouped by EventId + Map> eventData = getAllEventData(c, u, topLevelSuperPkgs, eventDataExtraFields); + + // Build events from eventData, eventNotes, and project data and group by EventId + Map eventsById = events.stream().collect(Collectors.toMap(Event::getEventId, (Event event) -> { + boolean hasPermission = skipPermissionCheck || + SNDSecurityManager.get().hasPermissionForTopLevelSuperPkgs(c, u, topLevelSuperPkgs.get(event.getEventId()), event, QCStateActionEnum.READ); + + Map extraFields = eventsExtensibleFields + .stream() + .filter((Map map) -> event.getEventId().equals(map.get("eventId"))) + .findFirst() + .orElse(Collections.emptyMap()); + + if (!event.hasErrors() && hasPermission) { + event.setNote(eventNotes.get(event.getEventId())); + event.setProjectIdRev(projectIdRevs.get(event.getParentObjectId())); + + if (eventData.containsKey(event.getEventId())) { + List sortedEventData = eventData + .get(event.getEventId()) + .stream() + .sorted(Comparator.comparing((EventData e) -> Integer.toString(e.getSuperPkgId()))).collect(Collectors.toList()); + event.setEventData(sortedEventData); + } + + addExtraFieldsToEvent(event, eventExtraFields, extraFields); + } + if (narrativeOptions != null && !narrativeOptions.isEmpty()) { + Map narratives = getNarratives(c, u, narrativeOptions, + topLevelSuperPkgs.get(event.getEventId()), event, errors); + if (narratives != null) { + event.setNarratives(narratives); + } + } + return event; + })); + + return eventsById; + } + + /** + * Query the SuperPackages table and retrieve all rows for a set of superPkgIds + * + * @param c + * @param u + * @param errors + * @return + */ + @Nullable + private Map getAllFullSuperPackages(Container c, User u, BatchValidationException errors) { + + UserSchema schema = getSndUserSchema(c, u); + + // Get All SuperPkg objects from database + SQLFragment sql = new SQLFragment( + "SELECT sp.SuperPkgId, sp.PkgId, sp.SortOrder, sp.Required, pkg.PkgId, pkg.Description, pkg.Active, pkg.Narrative, pkg.Repeatable FROM "); + sql.append(schema.getTable(SNDSchema.SUPERPKGS_TABLE_NAME), "sp"); + sql.append(" JOIN " + SNDSchema.NAME + "." + SNDSchema.PKGS_TABLE_NAME + " pkg"); + sql.append(" ON sp.PkgId = pkg.PkgId "); + sql.append(" ORDER BY sp.SuperPkgId "); + + SqlSelector superPkgSelector = new SqlSelector(schema.getDbSchema(), sql); + List superPackages = superPkgSelector.getArrayList(SuperPackage.class); + + // Group All SuperPkgs by SuperPkgId + Map superPackagesById = superPackages.stream().collect(Collectors.toMap(SuperPackage::getSuperPkgId, superPackage -> superPackage)); + + // All Child SuperPkgs grouped by TopLevelPkgId + Map> childSuperPackages = getAllFullChildSuperPkgs(c, u, true, errors); + + superPackagesById.forEach((superPkgId, superPackage) -> { + Integer pkgId = superPackage.getPkgId(); + if (pkgId != null) { + List pkgIds = Collections.singletonList(pkgId); + List packages = getPackages(c, u, pkgIds, true, true, true, errors); + if (!packages.isEmpty()) { + superPackage.setPkg(packages.get(0)); + } + superPackage.setChildPackages(childSuperPackages.get(pkgId)); + } + }); + + return superPackagesById; + + } + + /** + * Recursively get all children for the super package which corresponds to pkgId + */ + private Map> getAllFullChildSuperPkgs(Container c, User u, boolean includeFullSubpackages, BatchValidationException errors) { + + UserSchema schema = getSndUserSchema(c, u); + + SQLFragment childSql = new SQLFragment("SELECT * FROM "); + childSql.append(SNDSchema.NAME + "." + SNDSchema.ALL_SUPERPKGS_FUNCTION_NAME + "()"); + + SqlSelector selector = new SqlSelector(schema.getDbSchema(), childSql); + List allSuperPackages = selector.getArrayList(SuperPackage.class); + + // Group results of TopLevelSuperPkgs by TopLevelPkgId + Map> superPackagesByTopLevelPkgId = allSuperPackages.stream().collect(Collectors.groupingBy(SuperPackage::getTopLevelPkgId)); + + superPackagesByTopLevelPkgId.forEach((topLevelPkgId, superPackages) -> { + SuperPackage root = superPackages.stream().filter(sp -> sp.getParentSuperPkgId() == null).findFirst().orElse(null); + + List children = new ArrayList<>(); + if (root != null) { + superPackages.stream() + .filter(sp -> sp.getParentSuperPkgId() != null) + .forEach((SuperPackage superPackage) -> { + if (includeFullSubpackages) { + List pkgIds = Collections.singletonList(superPackage.getPkgId()); + superPackage.setPkg(getPackages(c, u, pkgIds, true, true, true, errors).get(0)); + } + if (superPackage.getParentSuperPkgId().intValue() == root.getSuperPkgId().intValue()) { + children.add(addChildren(superPackage, superPackages)); + } + }); + } + superPackages.clear(); + superPackages.addAll(children); + }); + + return superPackagesByTopLevelPkgId; + + } + + + /** + * Query EventData table and populate attribute data for a set of top level SuperPackages + * + * @param c + * @param u + * @param currentLevelSuperPkgs + * @return + */ + private Map> getAllEventData(Container c, User u, Map> currentLevelSuperPkgs, List eventDataExtraFields) { + + if (currentLevelSuperPkgs == null) { + return null; + } + + List eventDataIds = currentLevelSuperPkgs.values() + .stream() + .flatMap((Map map) -> map.keySet().stream()) + .collect(Collectors.toList()); + + // EventData from query - SELECT * FROM EventData WHERE EventDataId IN {eventDataIds} + TableSelector eventDataSelector = getTableSelector(c, u, eventDataIds, SNDSchema.EVENTDATA_TABLE_NAME, "EventDataId", null, null); + List allEventData = eventDataSelector.getArrayList(EventData.class); + Collection> eventDataExtensibleFields = eventDataSelector.getMapCollection(); + + // Child EventData grouped by EventId + Map> childEventData = getChildEventData(c, u, eventDataIds); + + // Build eventData from attributes and superPkgs and group by eventId + Map> eventDataByEventId = allEventData.stream().map((EventData eventData) -> { + + Map properties = OntologyManager.getPropertyObjects(c, eventData.getObjectURI()); + Map superPackagesById = currentLevelSuperPkgs.get(eventData.getEventId()); + SuperPackage superPackage = superPackagesById.get(eventData.getEventDataId()); + + if (superPackage != null) { + List attributeData = superPackage.getPkg().getAttributes() + .stream() + .map((GWTPropertyDescriptor propertyDescriptor) -> getAttributeData(propertyDescriptor, properties)) + .toList(); + + eventData.setAttributes(attributeData); + eventData.setNarrativeTemplate(superPackage.getNarrative()); + + } + + Map extraFields = eventDataExtensibleFields + .stream() + .filter((Map map) -> eventData.getEventDataId().equals(map.get("eventDataId"))) + .findFirst() + .orElse(Collections.emptyMap()); + + addExtraFieldsToEventData(eventData, eventDataExtraFields, extraFields); + + Map> nextLevelSuperPkgs = getNextLevelEventDataSuperPkgs(eventData, childEventData, currentLevelSuperPkgs); + + if (nextLevelSuperPkgs != null && !nextLevelSuperPkgs.isEmpty()) { + // Recursion for next child level of sub packages + Map> subEventData = getAllEventData(c, u, nextLevelSuperPkgs, eventDataExtraFields); + if (subEventData != null) { + List sorted = subEventData.get(eventData.getEventId()).stream().sorted(Comparator.comparing( + (EventData child) -> nextLevelSuperPkgs.get(child.getEventId()).get(child.getEventDataId()).getTreePath())) + .collect(Collectors.toList()); + eventData.setSubPackages(sorted); + } + } + + return eventData; + + }).collect(Collectors.groupingBy(EventData::getEventId)); + + return eventDataByEventId; + } + + /** + * Get EventData objects for a list of ParentEventIds + * + * @param c + * @param u + * @param parentEventDataIds + * @return + */ + private Map> getChildEventData(Container c, User u, List parentEventDataIds) { + + if (parentEventDataIds.isEmpty()) { + return Collections.emptyMap(); + } + + // EventData from query - SELECT * FROM EventData WHERE ParentEventDataId IN {parentEventDataIds} + TableSelector eventDataSelector = getTableSelector(c, u, parentEventDataIds, SNDSchema.EVENTDATA_TABLE_NAME, "ParentEventDataId", null, null); + List childEventData = eventDataSelector.getArrayList(EventData.class); + + // Group childEventData by eventId + Map> childEventDataByEventId = childEventData.stream().collect(Collectors.groupingBy(EventData::getEventId)); + + return childEventDataByEventId; + } + + /** + * Get the attribute data for a given PropertyDescriptor object + * + * @param propertyDescriptor + * @param properties + * @return + */ + private AttributeData getAttributeData(GWTPropertyDescriptor propertyDescriptor, Map properties) { + + Object property; + AttributeData attribute = new AttributeData(); + attribute.setPropertyName(propertyDescriptor.getName()); + attribute.setPropertyDescriptor(propertyDescriptor); + attribute.setPropertyId(propertyDescriptor.getPropertyId()); + if (properties.get(propertyDescriptor.getPropertyURI()) != null) { + property = properties.get(propertyDescriptor.getPropertyURI()).value(); + if (property != null) { + if (PropertyType.getFromURI(null, propertyDescriptor.getRangeURI()).equals(PropertyType.DATE)) { + property = DateUtil.formatDateTime((Date) property, AttributeData.DATE_FORMAT); + } else { + if (PropertyType.getFromURI(null, propertyDescriptor.getRangeURI()) + .equals(PropertyType.DATE_TIME)) { + property = DateUtil.formatDateTime((Date) property, AttributeData.DATE_TIME_FORMAT); + } + } + attribute.setValue(property.toString()); + } + } + return attribute; + } + + /** + * Get the next child level of SuperPackages for a given map of top level SuperPackages + * + * @param eventData + * @param childEventData + * @param currentLevelSuperPkgs + * @return + */ + private Map> getNextLevelEventDataSuperPkgs(EventData eventData, Map> childEventData, Map> currentLevelSuperPkgs) { + + if (!childEventData.containsKey(eventData.getEventId())) { + return null; + } + + // Get child packages from Current level of SuperPackages + Map childSuperPkgs = currentLevelSuperPkgs + .getOrDefault(eventData.getEventId(), Collections.emptyMap()) + .get(eventData.getEventDataId()) + .getChildPackages() + .stream() + .collect(Collectors.toMap( + SuperPackage::getSuperPkgId, + (SuperPackage superPackage) -> superPackage, + (s1, s2) -> s1 + )); + + // Get superPkg for eventData and group by eventId and then by eventId + Map> nextLevelEventDataSuperPkgs = childEventData.get(eventData.getEventId()) + .stream() + .filter((EventData child) -> childSuperPkgs.containsKey(child.getSuperPkgId())) + .collect( + Collectors.groupingBy( + EventData::getEventId, + Collectors.toMap( + EventData::getEventDataId, + (EventData child) -> childSuperPkgs.get(child.getSuperPkgId()) + ) + ) + ); + + return nextLevelEventDataSuperPkgs; + } + + /** + * Query EventNotes table for set of eventIds + * + * @param c + * @param u + * @param eventIds + * @return + */ + private Map getEventNotes(Container c, User u, List eventIds) { + + // EventNotes from query - SELECT * FROM EventNotes WHERE EventId IN {eventIds} + TableSelector eventNoteSelector = getTableSelector(c, u, eventIds, SNDSchema.EVENTNOTES_TABLE_NAME, "EventId", null, null); + List eventNotes = eventNoteSelector.getArrayList(EventNote.class); + + // Group eventNotes by eventId + Map eventNotesById = eventNotes.stream().collect( + Collectors.toMap( + EventNote::getEventId, EventNote::getNote + ) + ); + + return eventNotesById; + } + + /** + * Query the Projects table and retrieve the ID concatenated with RevisionNum for a set of objectIds + * + * @param c + * @param u + * @param objectIds + * @return + */ + private Map getProjectIdRevs(Container c, User u, List objectIds) { + + // Projects from query - SELECT * FROM Projects WHERE ObjectId IN {objectIds} + TableSelector projectSelector = getTableSelector(c, u, objectIds, SNDSchema.PROJECTS_TABLE_NAME, "ObjectId", null, null); + List projects = projectSelector.getArrayList(Project.class); + + // Concat string of ProjectId + RevisionNum and Group by ObjectId + Map projectIdRevs = projects.stream().collect( + Collectors.toMap( + Project::getObjectId, + (Project project) -> project.getProjectId() + "|" + project.getRevisionNum() + ) + ); + + return projectIdRevs; + + } + + /** + * Create a TableSelector object to query a table + * + * @param c + * @param u + * @param filterValues + * @param tableName + * @param filterColumn + * @param sortColumn + * @param isNullColumn + * @return + */ + public TableSelector getTableSelector(Container c, User u, List filterValues, String tableName, String filterColumn, String sortColumn, String isNullColumn) { + + UserSchema schema = getSndUserSchemaAdminRole(c, u); + + Sort sort = null; + if (sortColumn != null) { + sort = new Sort(); + sort.insertSortColumn(FieldKey.fromParts(sortColumn)); + } + + TableInfo tableInfo = getTableInfo(schema, tableName); + SimpleFilter filter = new SimpleFilter().addInClause(FieldKey.fromParts(filterColumn), filterValues); + if (isNullColumn != null) { + filter.addCondition(FieldKey.fromParts(isNullColumn), null, CompareType.ISBLANK); + } + return new TableSelector(tableInfo, filter, sort); + } } \ No newline at end of file diff --git a/src/org/labkey/snd/SNDServiceImpl.java b/src/org/labkey/snd/SNDServiceImpl.java index 399978d0..b5e99a95 100644 --- a/src/org/labkey/snd/SNDServiceImpl.java +++ b/src/org/labkey/snd/SNDServiceImpl.java @@ -543,7 +543,7 @@ public void deleteNarrativeCacheRows(Container c, User u, List> eventIds, Logger logger) + public void populateNarrativeCache(Container c, User u, List eventIds, Logger logger) { BatchValidationException errors = new BatchValidationException(); diff --git a/src/org/labkey/snd/table/PlainTextNarrativeDisplayColumn.java b/src/org/labkey/snd/table/PlainTextNarrativeDisplayColumn.java index b77ca700..c1fb9aea 100644 --- a/src/org/labkey/snd/table/PlainTextNarrativeDisplayColumn.java +++ b/src/org/labkey/snd/table/PlainTextNarrativeDisplayColumn.java @@ -53,6 +53,7 @@ public static String removeHtmlTagsFromNarrative(String htmlNarrative) textNarrative = htmlNarrative.replace("
", "\n"); textNarrative = textNarrative.replace("
", "\n"); textNarrative = textNarrative.replace("
", "\n"); + textNarrative = textNarrative.replace(" ", "\s\s\s\s"); // then crudely remove all other HTML open/close tags, or things that look like them textNarrative = textNarrative.replaceAll("\\<.*?\\>", ""); } diff --git a/webapp/snd/test/data.js b/webapp/snd/test/data.js index 112fe9b0..d91a6743 100644 --- a/webapp/snd/test/data.js +++ b/webapp/snd/test/data.js @@ -786,36 +786,28 @@ note: "Note for narrative generation sample JSON", projectIdRev: '61|0', qcState: 'Completed', - textNarrative: "2018-02-26 17:51\nSubject Id: 2\n\n" + - "Electrolytes\n\tPotassium: 200 mEq/L measured using Potassium Detection Kit\n\t" + - "Sodium: 100 mEq/L measured using Sodium Colorimetric Detection Kit\n\t" + - "Bicarbonate: 300 mEq/L measured using Carbon Dioxide (CO2) Colorimetric Detection Kit\n\t" + - "Chloride: 112 mEq/L measured using Chloride Blood Detection Kit", - redactedHtmlNarrative: "
2018-02-26 17:51
\n
" + - "Subject Id: 2
\n
Electrolytes
" + - "Potassium: Redacted Value mEq/L measured using " + - "Potassium Detection Kit
\n
" + - "Sodium: 100 mEq/L measured using " + - "Sodium Colorimetric Detection Kit
\n
" + - "Bicarbonate: 300 mEq/L measured using " + - "Carbon Dioxide (CO2) Colorimetric Detection Kit
\n
" + - "Chloride: 112 mEq/L measured using " + - "Chloride Blood Detection Kit
\n
\n", - redactedTextNarrative: "2018-02-26 17:51\nSubject Id: 2\n\nElectrolytes\n\t" + - "Potassium: Redacted Value mEq/L measured using Potassium Detection Kit\n\t" + - "Sodium: 100 mEq/L measured using Sodium Colorimetric Detection Kit\n\t" + - "Bicarbonate: 300 mEq/L measured using Carbon Dioxide (CO2) Colorimetric Detection Kit\n\t" + - "Chloride: 112 mEq/L measured using Chloride Blood Detection Kit", - htmlNarrative: "
2018-02-26 17:51
\n
" + - "Subject Id: 2
\n
Electrolytes
" + - "Sodium: 100 mEq/L measured using " + - "Sodium Colorimetric Detection Kit
\n
" + - "Potassium: 200 mEq/L measured using " + - "Potassium Detection Kit
\n
" + - "Bicarbonate: 300 mEq/L measured using " + - "Carbon Dioxide (CO2) Colorimetric Detection Kit
\n
" + - "Chloride: 112.67605633802818 mEq/L measured using " + - "Chloride Blood Detection Kit
\n
", + textNarrative: "2018-02-26 17:51\n" + + "Subject Id: 2\n" + + "** Electrolytes-- Potassium: 200 mEq/L measured using Potassium Detection Kit-- Sodium: 100 mEq/L measured using Sodium Colorimetric Detection Kit-- Bicarbonate: 300 mEq/L measured using Carbon Dioxide (CO2) Colorimetric Detection Kit-- Chloride: 112 mEq/L measured using Chloride Blood Detection KitProcedure Note: Note for narrative generation sample JSON\n", + redactedHtmlNarrative: "
2018-02-26 17:51
\n" + + "
Subject Id: 2
\n" + + "
**  Electrolytes (814)
\n" + + "
--   Potassium: Redacted Value mEq/L measured using Potassium Detection Kit (811)
\n" + + "
--   Sodium: 100 mEq/L measured using Sodium Colorimetric Detection Kit (810)
\n" + + "
--   Bicarbonate: 300 mEq/L measured using Carbon Dioxide (CO2) Colorimetric Detection Kit (812)
\n" + + "
--   Chloride: 112 mEq/L measured using Chloride Blood Detection Kit (813)
\n" + + "
Procedure Note:
Note for narrative generation sample JSON
\n", + redactedTextNarrative: "2018-02-26 17:51\n" + + "Subject Id: 2\n" + + "** Electrolytes-- Potassium: Redacted Value mEq/L measured using Potassium Detection Kit-- Sodium: 100 mEq/L measured using Sodium Colorimetric Detection Kit-- Bicarbonate: 300 mEq/L measured using Carbon Dioxide (CO2) Colorimetric Detection Kit-- Chloride: 112 mEq/L measured using Chloride Blood Detection KitProcedure Note: Note for narrative generation sample JSON\n", + htmlNarrative: "
2018-02-26 17:51
\n" + + "
Subject Id: 2
\n" + + "
**  Electrolytes (814)
\n" + + "
--   Sodium: 100 mEq/L measured using Sodium Colorimetric Detection Kit (810)
\n" + + "
--   Potassium: 200 mEq/L measured using Potassium Detection Kit (811)
\n" + + "
--   Bicarbonate: 300 mEq/L measured using Carbon Dioxide (CO2) Colorimetric Detection Kit (812)
\n" + + "
--   Chloride: 112.67605633802818 mEq/L measured using Chloride Blood Detection Kit (813)
\n" + + "
Procedure Note:
Note for narrative generation sample JSON
", eventData: [ { superPkgId: LABKEY.SND_PKG_CACHE['814'][0]['superPkgId'],