diff --git a/.github/workflows/comment.yaml b/.github/workflows/comment.yaml index 2811204d3..5cc02f1a3 100644 --- a/.github/workflows/comment.yaml +++ b/.github/workflows/comment.yaml @@ -21,7 +21,6 @@ jobs: template: SOP/pr_checklist.md - name: Comment uses: peter-evans/create-or-update-comment@v1 - if: ${{ github.event_name == 'pull_request' }} with: issue-number: ${{ github.event.pull_request.number }} body: ${{ steps.get-pr-checklist.outputs.result }} diff --git a/README.md b/README.md index 4cabe9eba..15fbb8de5 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,12 @@ # Emap + A monorepo for all core Emap functions -# Basic layout +# Setup + +The EMAP project follows this structure, for deploying a live instance of EMAP follow the instructions +in [docs/core.md](docs/core.md). + ``` EMAP [your root emap directory] ├── config [config files passed to docker containers, not in any repo] @@ -14,8 +19,12 @@ EMAP [your root emap directory] │ ├── [etc.] ``` -# Using IntelliJ with emap -How to [configure IntelliJ](docs/intellij.md) to build emap and run tests. +## Developer onboarding + +- How to [configure IntelliJ](docs/intellij.md) to build emap and run tests. +- [Onboarding](docs/dev/onboarding.md) gives details on how data items are processed and the test strategies used. + # Monorepo migration + How were [old repos migrated into this repo?](docs/migration.md) diff --git a/core/pom.xml b/core/pom.xml index cd6de97d5..6d2a5aefc 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -5,14 +5,14 @@ uk.ac.ucl.rits.inform core jar - 2.6 + 2.7 Emap Core Processor http://maven.apache.org org.springframework.boot spring-boot-starter-parent - 2.7.13 + 2.7.18 @@ -20,8 +20,8 @@ 17 10.3.1 3.3.0 - 2.6 - 2.6 + 2.7 + 2.7 1.2.8 diff --git a/docs/README.md b/docs/README.md index 4e586535e..d49f7e9df 100644 --- a/docs/README.md +++ b/docs/README.md @@ -12,6 +12,7 @@ This is a work in progress. * [2021-09-27](changelogs/2021-09-change_log.md) * [2021-11-30](changelogs/2021-11-change_log.md) * [2022-02-14](changelogs/2022-02-change_log.md) 💝 +* [2024-02-14](changelogs/2024-02-change_log.md) 💝 ### Data and Validation diff --git a/docs/SOP/release_procedure.md b/docs/SOP/release_procedure.md index b77e3b739..c6f21f311 100644 --- a/docs/SOP/release_procedure.md +++ b/docs/SOP/release_procedure.md @@ -41,4 +41,6 @@ At this point we create a list in the planner with vital things to be completed 1. Prepare and do demo. - 1. Shut down old instance. + 1. Shut down old instance, changing the HL7_READER_PORT in the global-configuration.yaml to something other than 8080. + + 1. Update the HL7_READER_PORT in the global-configuration.yaml in the new instance to 8080 and re-up the hl7-reader. diff --git a/docs/changelogs/2024-02-change_log.md b/docs/changelogs/2024-02-change_log.md new file mode 100644 index 000000000..cb66bf251 --- /dev/null +++ b/docs/changelogs/2024-02-change_log.md @@ -0,0 +1,86 @@ +# EMAP Release + +**Date: 2024-02-14 Changes in this release** + +--- + +### Changes/fixes + +- We have added a healthcheck endpoint for the hl7-reader, so that users can check the status of the hl7-reader without + needing to access the server directly. + - The current live version can be found at: `http://:8080/actuator/health` + - Please contact the developer team to find out the GAE hostname. +- All services now run using Java 17 (LTS) +- Open source release using mono-repo +- Improved testing to ensure database consistency + +### Tables changed + +Department specialities can change over time, we've updated the database to reflect this. +The following tables have been updated: + +| Table | Attributes added | Attributes removed | +|:----------------|:-----------------|:-------------------| +| DepartmentState | speciality | - | +| Department | - | speciality + + +```mermaid +erDiagram +bed { + varchar(255) hl7string + bigint room_id + bigint bed_id +} +department { + varchar(255) hl7string + bigint internal_id + varchar(255) name + bigint department_id +} +department_state { + timestamp-with-time-zone stored_from + timestamp-with-time-zone valid_from + timestamp-with-time-zone stored_until + timestamp-with-time-zone valid_until + varchar(255) speciality + varchar(255) status + bigint department_id + bigint department_state_id +} +location { + varchar(255) location_string + bigint bed_id + bigint department_id + bigint room_id + bigint location_id +} +room { + varchar(255) hl7string + varchar(255) name + bigint department_id + bigint room_id +} + +department ||--|{ department_state: department_id +location |{--o| department: department_id +location |{--o| room: room_id +location |{--o| bed: bed_id +department ||--o{ room: room_id +room ||--o{ bed: bed_id +``` + +--- + LabProcessor + LabCache *--> LabTestDefinitionAuditRepository + LabCache *--> LabTestDefinitionRepository + LabController *--> LabCache + LabController *--> LabOrderController + LabOrderController *--> LabOrderAuditRepository + LabOrderController *--> LabOrderRepository + LabProcessor *--> LabController + InformDbOperations ..> EmapOperationMessageProcessor diff --git a/docs/dev/img/core-message-processing.svg b/docs/dev/img/core-message-processing.svg new file mode 100644 index 000000000..56735a779 --- /dev/null +++ b/docs/dev/img/core-message-processing.svg @@ -0,0 +1 @@ +
InformDbOperations
+processMessage(AdtMessage) : void
+processMessage(LabOrderMsg) : void
+processMessage(PatientInfection) : void
LabCache
+saveEntityAndUpdateCache(RowState<LabBattery, LabBatteryAudit>) : LabBattery
+updateLabTestDefinitionCache(RowState<LabTestDefinition, LabTestDefinitionAudit>) : LabTestDefinition
+findExistingLabTestDefinition(String, String) : LabTestDefinition
+createLabBatteryElementIfNotExists(LabTestDefinition, LabBattery, Instant, Instant) : LabBatteryElement
LabController
+processLabOrder(Mrn, HospitalVisit?, LabOrderMsg, Instant) : void
+deleteLabOrdersForVisit(HospitalVisit, Instant, Instant) : void
+processLabMetadata(LabMetadataMsg, Instant) : void
«Interface»
LabOrderAuditRepository
+findAllIds(Long, Long, Instant, String) : List<Long>
+previouslyDeleted(Long, Long, InterchangeValue<Instant>, InterchangeValue<String>) : boolean
+findAllByHospitalVisitIdIn(Iterable<Long>) : List<LabOrderAudit>
LabOrderController
+processSampleAndOrderInformation(Mrn, HospitalVisit, LabBattery, LabOrderMsg, Instant, Instant) : LabOrder
+deleteLabOrder(LabOrder, Instant, Instant) : void
+findLabBatteryOrThrow(String, String) : LabBattery
+getLabOrdersForVisit(HospitalVisit) : List<LabOrder>
+processLabSampleAndDeleteLabOrder(Mrn, LabBattery, HospitalVisit, LabOrderMsg, Instant, Instant) : void
+getOrCreateLabBattery(String, String, Instant, Instant) : LabBattery
«Interface»
LabOrderRepository
+findByLabBatteryIdAndLabSampleId(LabBattery, LabSample) : Optional<LabOrder>
+findByLabBatteryIdAndLabSampleIdAndValidFromBefore(LabBattery, LabSample, Instant) : Optional<LabOrder>
+findByLabBatteryIdBatteryCodeAndLabSampleId(String, LabSample) : Optional<LabOrder>
+findByLabSampleIdExternalLabNumber(String) : Optional<LabOrder>
+findAllByHospitalVisitId(HospitalVisit) : List<LabOrder>
LabProcessor
+processMessage(LabMetadataMsg, Instant) : void
+processMessage(LabOrderMsg, Instant) : void
«Interface»
LabTestDefinitionAuditRepository
«Interface»
LabTestDefinitionRepository
+findByLabProviderAndTestLabCode(String, String) : Optional<LabTestDefinition>
+findByTestLabCode(String) : Optional<LabTestDefinition>
 \ No newline at end of file diff --git a/docs/dev/img/hl7-message-processing.mmd b/docs/dev/img/hl7-message-processing.mmd new file mode 100644 index 000000000..f16629051 --- /dev/null +++ b/docs/dev/img/hl7-message-processing.mmd @@ -0,0 +1,82 @@ +classDiagram + direction BT + class AblLabBuilder { + + build(String, ORU_R30) Collection~LabOrderMsg~ + } + class AppHl7 { + + main(String[]) void + + mainLoop(Publisher, IdsOperations) CommandLineRunner + } + class ConsultFactory { + ~ makeConsult(String, ORM_O01) ConsultRequest + } + class IdsOperations { + + getHl7DataSource() EmapDataSource + + getNextHL7IdsRecordBlocking(int) IdsMaster + + close() void + + parseAndSendNextHl7(Publisher, PipeParser) void + + getNextHL7IdsRecord(int) IdsMaster + + messageFromHl7Message(Message, int) List~EmapOperationMessage~ + + populateIDS() CommandLineRunner + + getIdsEmptyOnInit() boolean + ~ getLatestProcessedId() IdsProgress + } + class LabFunnel { + + buildMessages(String, ORU_R30, OrderCodingSystem) Collection~LabOrderMsg~ + + buildMessages(String, ORR_O02, OrderCodingSystem) Collection~LabOrderMsg~ + + buildMessages(String, ORU_R01, OrderCodingSystem) Collection~LabOrderMsg~ + + buildMessages(String, ORM_O01, OrderCodingSystem) List~LabOrderMsg~ + } + class LabOrderBuilder { + ~ getCodingSystem() OrderCodingSystem + ~ getEpicCareOrderNumberOrc() String + ~ setQuestions(Collection~NTE~, String, Pattern) void + + getMsg() LabOrderMsg + ~ getEpicCareOrderNumberObr() String + ~ setBatteryCodingSystem() void + ~ populateOrderInformation(ORC, OBR) void + ~ populateObrFields(OBR, boolean) void + ~ setSourceAndPatientIdentifiers(String, PatientInfoHl7) void + ~ populateObrFields(OBR) void + ~ setOrderTemporalInformation(Instant) void + } + class OrderAndResultService { + ~ buildMessages(String, ORR_O02) Collection~EmapOperationMessage~ + ~ buildMessages(String, ORM_O01) Collection~EmapOperationMessage~ + ~ buildMessages(String, ORU_R30) Collection~EmapOperationMessage~ + ~ buildMessages(String, ORU_R01) Collection~EmapOperationMessage~ + } + class WinPathLabBuilder { + + build(String, ORU_R01) Collection~LabOrderMsg~ + + build(String, ORR_O02) Collection~LabOrderMsg~ + + build(String, ORM_O01) List~LabOrderMsg~ + } + class ConsultRequest { + + getEpicConsultId() Long + + getMrn() String + + setQuestions(Map~String, String~) void + + getQuestions() Map~String, String~ + + setMrn(String) void + + processMessage(EmapOperationMessageProcessor) void + } + class LabOrderMsg { + + getSpecimenType() InterchangeValue~String~ + + getOrderControlId() String + + getLabResultMsgs() List~LabResultMsg~ + + setOrderDateTime(InterchangeValue~Instant~) void + + setOrderControlId(String) void + + setSpecimenType(InterchangeValue~String~) void + + processMessage(EmapOperationMessageProcessor) void + + addLabResult(LabResultMsg) void + } + + AblLabBuilder --|> LabOrderBuilder + AppHl7 ..> IdsOperations + IdsOperations *--> OrderAndResultService + LabFunnel ..> AblLabBuilder + LabFunnel ..> WinPathLabBuilder + OrderAndResultService *--> ConsultFactory + OrderAndResultService ..> LabFunnel + WinPathLabBuilder --|> LabOrderBuilder + LabOrderBuilder ..> LabOrderMsg: «create» + ConsultFactory ..> ConsultRequest: «create» diff --git a/docs/dev/img/hl7-message-processing.svg b/docs/dev/img/hl7-message-processing.svg new file mode 100644 index 000000000..56735a779 --- /dev/null +++ b/docs/dev/img/hl7-message-processing.svg @@ -0,0 +1 @@ +
InformDbOperations
+processMessage(AdtMessage) : void
+processMessage(LabOrderMsg) : void
+processMessage(PatientInfection) : void
LabCache
+saveEntityAndUpdateCache(RowState<LabBattery, LabBatteryAudit>) : LabBattery
+updateLabTestDefinitionCache(RowState<LabTestDefinition, LabTestDefinitionAudit>) : LabTestDefinition
+findExistingLabTestDefinition(String, String) : LabTestDefinition
+createLabBatteryElementIfNotExists(LabTestDefinition, LabBattery, Instant, Instant) : LabBatteryElement
LabController
+processLabOrder(Mrn, HospitalVisit?, LabOrderMsg, Instant) : void
+deleteLabOrdersForVisit(HospitalVisit, Instant, Instant) : void
+processLabMetadata(LabMetadataMsg, Instant) : void
«Interface»
LabOrderAuditRepository
+findAllIds(Long, Long, Instant, String) : List<Long>
+previouslyDeleted(Long, Long, InterchangeValue<Instant>, InterchangeValue<String>) : boolean
+findAllByHospitalVisitIdIn(Iterable<Long>) : List<LabOrderAudit>
LabOrderController
+processSampleAndOrderInformation(Mrn, HospitalVisit, LabBattery, LabOrderMsg, Instant, Instant) : LabOrder
+deleteLabOrder(LabOrder, Instant, Instant) : void
+findLabBatteryOrThrow(String, String) : LabBattery
+getLabOrdersForVisit(HospitalVisit) : List<LabOrder>
+processLabSampleAndDeleteLabOrder(Mrn, LabBattery, HospitalVisit, LabOrderMsg, Instant, Instant) : void
+getOrCreateLabBattery(String, String, Instant, Instant) : LabBattery
«Interface»
LabOrderRepository
+findByLabBatteryIdAndLabSampleId(LabBattery, LabSample) : Optional<LabOrder>
+findByLabBatteryIdAndLabSampleIdAndValidFromBefore(LabBattery, LabSample, Instant) : Optional<LabOrder>
+findByLabBatteryIdBatteryCodeAndLabSampleId(String, LabSample) : Optional<LabOrder>
+findByLabSampleIdExternalLabNumber(String) : Optional<LabOrder>
+findAllByHospitalVisitId(HospitalVisit) : List<LabOrder>
LabProcessor
+processMessage(LabMetadataMsg, Instant) : void
+processMessage(LabOrderMsg, Instant) : void
«Interface»
LabTestDefinitionAuditRepository
«Interface»
LabTestDefinitionRepository
+findByLabProviderAndTestLabCode(String, String) : Optional<LabTestDefinition>
+findByTestLabCode(String) : Optional<LabTestDefinition>
 \ No newline at end of file diff --git a/docs/dev/img/hoover-development.mmd b/docs/dev/img/hoover-development.mmd new file mode 100644 index 000000000..f8e1e30b0 --- /dev/null +++ b/docs/dev/img/hoover-development.mmd @@ -0,0 +1,63 @@ +classDiagram + direction BT + class Application { + + main(String[]) void + + locationMetadataProcessor(LocationMetadataQueryStrategy) Processor + + flowsheetProcessor(FlowsheetQueryStrategy) Processor + + runBatchProcessor(Processor, Processor, ...) BatchProcessor + } + class LocationMetadata { + + getHl7String() String + + getDepartmentId() Long + + getRoomMetadata() RoomMetadata + + getBedMetadata() BedMetadata + + setHl7String(String) void + + setDepartmentId(Long) void + + setRoomMetadata(RoomMetadata) void + + setBedMetadata(BedMetadata) void + + processMessage(EmapOperationMessageProcessor) void + } + class LocationMetadataDTO { + + getInterchangeMessage() LocationMetadata + } + class LocationMetadataQueryStrategy { + + getSqlQueryFilename() String + + getInstantCalculator() InstantCalculator + + fixInitialProgress(Instant) Instant + ~ getClarityData(Instant) List~LocationMetadataDTO~ + + getBatchOfInterchangeMessages(Instant, Instant, String) List~ImmutablePair~ EmapOperationMessage, String~~ + + getUpdatedProgress() EtlHooverProgress + + getName() String + } + class QueryStrategy { + <> + + getBatchOfInterchangeMessages(Instant, Instant, String) List~ImmutablePair~ EmapOperationMessage, String~~ + + fixInitialProgress(Instant) Instant + + getInstantCalculator() InstantCalculator + + getName() String + + getUpdatedProgress() EtlHooverProgress + + getBatchOfInterchangeMessages(DataTypeProgress) List~ImmutablePair~ EmapOperationMessage, String~~ + + getSqlQueryFilename() String + + interpretPreviousProgress(EtlHooverProgress) DataTypeProgress + + getSqlQuery() String + } + class Processor { + + getDataType() String + + getPreviousProgress() List~EtlHooverProgress~ + + execute() boolean + + executeOne(EtlHooverProgress) boolean + } + + class BatchProcessor { + + registerDataTypeProcessor(Processor) void + } + + Application ..> LocationMetadataQueryStrategy + LocationMetadataDTO ..> LocationMetadata: «create» + LocationMetadataQueryStrategy ..> LocationMetadata + LocationMetadataQueryStrategy ..> LocationMetadataDTO + LocationMetadataQueryStrategy ..|> QueryStrategy + Processor *--> QueryStrategy + BatchProcessor "1" *--> "dataTypeProcessors *" Processor + Application ..> Processor + Application ..> BatchProcessor: «create» \ No newline at end of file diff --git a/docs/dev/img/hoover-development.svg b/docs/dev/img/hoover-development.svg new file mode 100644 index 000000000..34b861a81 --- /dev/null +++ b/docs/dev/img/hoover-development.svg @@ -0,0 +1 @@ +
InformDbOperations
+processMessage(AdtMessage) : void
+processMessage(LabOrderMsg) : void
LabCache
+saveEntityAndUpdateCache(RowState<LabBattery, LabBatteryAudit>) : LabBattery
+updateLabTestDefinitionCache(RowState<LabTestDefinition, LabTestDefinitionAudit>) : LabTestDefinition
+findExistingLabTestDefinition(String, String) : LabTestDefinition
+createLabBatteryElementIfNotExists(LabTestDefinition, LabBattery, Instant, Instant) : LabBatteryElement
LabController
+processLabOrder(Mrn, HospitalVisit?, LabOrderMsg, Instant) : void
+deleteLabOrdersForVisit(HospitalVisit, Instant, Instant) : void
+processLabMetadata(LabMetadataMsg, Instant) : void
«Interface»
LabOrderAuditRepository
+findAllIds(Long, Long, Instant, String) : List<Long>
+previouslyDeleted(Long, Long, InterchangeValue<Instant>, InterchangeValue<String>) : boolean
+findAllByHospitalVisitIdIn(Iterable<Long>) : List<LabOrderAudit>
LabOrderController
+processSampleAndOrderInformation(Mrn, HospitalVisit, LabBattery, LabOrderMsg, Instant, Instant) : LabOrder
+deleteLabOrder(LabOrder, Instant, Instant) : void
+findLabBatteryOrThrow(String, String) : LabBattery
+getLabOrdersForVisit(HospitalVisit) : List<LabOrder>
+processLabSampleAndDeleteLabOrder(Mrn, LabBattery, HospitalVisit, LabOrderMsg, Instant, Instant) : void
+getOrCreateLabBattery(String, String, Instant, Instant) : LabBattery
«Interface»
LabOrderRepository
+findByLabBatteryIdAndLabSampleId(LabBattery, LabSample) : Optional<LabOrder>
+findByLabBatteryIdAndLabSampleIdAndValidFromBefore(LabBattery, LabSample, Instant) : Optional<LabOrder>
+findByLabBatteryIdBatteryCodeAndLabSampleId(String, LabSample) : Optional<LabOrder>
+findByLabSampleIdExternalLabNumber(String) : Optional<LabOrder>
+findAllByHospitalVisitId(HospitalVisit) : List<LabOrder>
LabProcessor
+processMessage(LabMetadataMsg, Instant) : void
+processMessage(LabOrderMsg, Instant) : void
«Interface»
LabTestDefinitionAuditRepository
«Interface»
LabTestDefinitionRepository
+findByLabProviderAndTestLabCode(String, String) : Optional<LabTestDefinition>
+findByTestLabCode(String) : Optional<LabTestDefinition>
«Interface»
EmapOperationMessageProcessor
+processMessage(LabMetadataMsg) : void
+processMessage(AdtMessage) : void
 \ No newline at end of file diff --git a/docs/dev/onboarding.md b/docs/dev/onboarding.md new file mode 100644 index 000000000..af9523283 --- /dev/null +++ b/docs/dev/onboarding.md @@ -0,0 +1,238 @@ +# Experimental Medical Application Platform (EMAP) + +## Introduction + +A technical overview of EMAP can be found in +the [inform-health-informatics/emap_documentation repository](https://github.com/inform-health-informatics/emap_documentation/blob/main/technical_overview/Technical_overview_of_EMAP.md) + +There are currently two data sources for EMAP: + +- HL7 data + - Persisted in + the [Immutable Data Store](https://github.com/inform-health-informatics/emap_documentation/blob/main/technical_overview/Technical_overview_of_EMAP.md#immutable-data-store-ids) + (IDS), from a copy of specific HL7 message streams + - The IDS is read by + the [HL7 reader](https://github.com/inform-health-informatics/emap_documentation/blob/main/technical_overview/Technical_overview_of_EMAP.md#hl7-reader), + (defined in the [hl7-reader](https://github.com/UCLH-DHCT/emap/tree/main/hl7-reader) module) + converting the HL7 message into a source-agnostic format (interchange message, defined in + the [emap-interchange](https://github.com/UCLH-DHCT/emap/tree/main/emap-interchange) module) + and published to a rabbitMQ queue for processing by the core processor. +- Hospital database polling + - + The [Hoover](https://github.com/inform-health-informatics/emap_documentation/blob/main/technical_overview/Technical_overview_of_EMAP.md#hoover) + (defined in the [hoover](https://github.com/UCLH-DHCT/hoover) repository) + service polls hospital databases (Clarity and Caboodle) for data that has changed since the last poll. + It converts the query outputs into the interchange message and publishes these to a rabbitMQ queue for processing by + the core processor. + We can't make the Hoover repository public because the SQL queries contain the intellectual property of the hospital + patient record system, EPIC. + +The [core processor](https://github.com/inform-health-informatics/emap_documentation/blob/main/technical_overview/Technical_overview_of_EMAP.md#the-eventprocessor) +(defined in the [core](https://github.com/UCLH-DHCT/emap/tree/main/core) module) is responsible for processing the +interchange messages and +updating +the [emap database](https://github.com/inform-health-informatics/emap_documentation/blob/main/technical_overview/Technical_overview_of_EMAP.md#star-schema) +(defined in the [emap-star](https://github.com/UCLH-DHCT/emap/tree/main/emap-star) module). + +The core processor compares what is already known in the EMAP database, with the data in the interchange message and +updates the EMAP database accordingly. +We can receive HL7 messages out of order so the processor must be able to handle this. + +## Development guide + +All the EMAP services use the Spring-Boot framework and are written in Java. Setup instructions are found in +the [emap repo](https://github.com/UCLH-DHCT/emap/blob/main/docs/intellij.md) +with additional information for hoover in the [hoover repo](https://github.com/UCLH-DHCT/hoover). + +A decision log for technical choices for a module can be found in +its [dev/design_choices.md](https://github.com/UCLH-DHCT/emap/blob/main/core/dev/design_choices.md) file. + +### Hl7-reader + +#### Development + +Each HL7 message can produce one or more interchange messages, depending on the type of the message there are different +patterns used in the codebase to process the HL7 message. + +As an example, the following diagram shows the processing of an `ORM^O01` HL7 message type which can either result in +a single `ConsultRequest` interchange message or a list of `LabOrderMsg` interchange messages +(these have been simplified in the diagram). + +![HL7 message processing class diagram](img/hl7-message-processing.svg) + +Flow of the processing: + +- All HL7 messages are processed by the `mainLoop` method of the `AppHl7` class, + which delegates reading and processing of HL7 messages into interchange messages to the `IdsOperations` class, and + then publishes to the queue using the `Publisher` class. +- The `IdsOperations` class is responsible for reading the HL7 messages from the IDS and delegates the processing of + HL7. + In this case `ORM` messages are an Order Message, so the message is routed to the `OrderAndResultService`. +- The `OrderAndResultService` can determine the source and type of the message, which can delegate to + the `ConsultFactory` for a consultation request, or `LabFunnel` for a lab order. + - If this is a consultation request, the `ConsultFactory` will create a `ConsultRequest` interchange message and + return this up the call stack for publishing. +- The `LabFunnel` will use the `OrderCodingSystem` to route the HL7 message type to the correct `LabOrderBuilder` + subclass. + - Each builder extracts common elements from the HL7 message, using its parent class' methods to create one or + more `LabOrderMsg` interchange message. + +#### Testing + +- For service testing, fake HL7 messages are manually created for each message type and stored in + the `src/test/resources` directory. +- To reduce repetition of configuration and annotations, the `TestHl7MessageStream` class is extended in each test + class. + - This contains a `processSingleAdtMessage` method which takes the path of the fake HL7 message and processes it + into + an interchange message for assertions. + - This method tests at the `IdsOperations` level, so the `Publisher` does not need to be mocked. +- Unless there is very tricky areas of logic, we don't unit test message processing, instead setting up test cases of + HL7 -> interchange messages and checking that this is processed as expected +- To have certainty that our end-to-end testing from hl7-reader -> core -> emap-star database works correctly, + test methods are added to the `TestHl7ParsingMatchesInterchangeFactoryOutput` test class, which takes in HL7 messages + as an input + and serialised interchange messages in yaml format as an expected output and asserts that they match. + - These serialised interchange messages are then used in emap core testing + - To ensure that all serialised interchange messages from the hl7-reader have an HL7 message that produces them, + the `InterchangeMessageFactory` + is created with Monitored Files, an exception will be thrown if there are any interchange messages which have not + been read while running the test class. + +### Hoover + +#### Development + +The hoover service requires native queries to be written for Clarity and Caboodle, so local development uses docker +containers running sqlserver +with fake data to test the queries. These are defined in `test-files/clarity` and `test-files/caboodle`. +Be sure to follow +the [local setup instructions](https://github.com/UCLH-DHCT/hoover#local-setup-instructions-using-intellij-idea) +before starting work on the service. + +Each data type processed by Hoover is represented by its own class that implements the `QueryStrategy` interface, and +has a SQL file in the `src/main/resources/sql` directory. + +An example is shown in the following diagram (simplified): + +![Hoover class diagram](img/hoover-development.svg) + +- The `Application` class creates a Spring Component that is an instance of the `Processor` class, initialised with + a `QueryStrategy` instance, in this case `LocationMetadataQueryStrategy`. + - This component is taken as an argument to the `runBatchProcessor` method, which delegates the scheduling of the + database polling to the `BatchProcessor` class. +- The `Processor` class uses the `QueryStrategy` interface to get the previous progress for the data type, and query for + any + new data since the most recent progress. +- Defining the SQL query and how this data is processed is implemented by the `LocationMetadataQueryStrategy` class. + - To allow for sqlfluff linting of SQL queries, the SQL is persisted to the `src/main/resources/sql` directory, and + linked to by the `getQueryFilename` method. + - the `getBatchOfInterchangeMessages` method queries the database, returns a list of Data Transfer Objects (DTOs), + in + this case `LocationMetadataDTO`. + - In this case, the `LocationMetadataDTO` can build a `LocationMetadata` interchange message and these are returned + up + the call stack for publishing using the `Publisher` class. + +#### Testing + +- For each data type, as a minimum you should carry out testing from a query within a time window and assert that the + expected data is returned. + - The expected data should include the serialised interchange messages in yaml format that will be read by the core + processor during testing. + - As the databases are static, creating specific test conditions within a time window is the easiest way to have + specific tests. +- Each test class should implement the `TestQueryStrategy` interface, which adds in default tests once you have + implemented the required methods for metadata about the test. + - This also gives helper methods to be able to test a time window of data and assert that the batch of interchange + messages matches the yaml files. + +### Core + +#### Development + +Message processing within core follows a general pattern of: + +- Read message from queue and delegate to processor class +- Processor class uses one or more controllers to update or create the relevant entities from the hl7 message +- Controllers use repositories carry out business logic for what exists in the database, and what should be updated or + created. +- Repositories use Spring Data JPA to interact with the database tables directly. + +An example is shown in the following diagram (simplified) + +![Core message processing](img/core-message-processing.svg) + +- Each interchange message uses double dispatch so that the class of the interchange message can be known at runtime + without checks. + The `InformDbOperations` class implements the `EmapOperationMessageProcessor` interface to enable the double + dispatch. + The `processMessage` method delegates to a `Processor` class for each family of messages, in this case + the `LabProcessor` is used. +- Each processor class has one or more `Controller` classes, which allow for the business logic of comparing the current + and previous state from the database and making a decision on what the correct outcome is. + The processor class uses these delegated classes to update or create entities which are used by other controllers. +- Each controller class can use other controllers (for complex data types which span 6+ tables), interact with tables + using a `Cache` component or directly interact with the database using a `Repository` interface. + - In this case, the `LabController` uses the `LabCache` component, + this is because Spring Boot caching annotations are ignored when a method call is made by the same class. + Breaking this out into a separate component allows for data type metadata to be cached to reduce the number of + queries to the database and improve performance. + The `LabCache` component itself can interact with `Repositories` because it is a Spring Component. + - The `LabController` also uses the `LabOrderController`, which then uses specific `Repositories` to interact with + the + SQL tables in emap star. +- Most tables in emap star have an `Audit` equivalent, this allows for the history of each entity to be tracked. + - We have defined an `@AuditTable` to generate Java classes for these entities that are then represented as tables + in + emap star. + This can be found in + the [emap-star/emap-star-annotations](https://github.com/UCLH-DHCT/emap/tree/main/emap-star/emap-star-annotations) + maven module. + - An Audit table must extend the `TemporalCore` class, generics are used to link the entity class and its audit + entity + class. + - The `RowState` class acts as a wrapper around an entity to help with determining if differences should be + persisted + to the database (and if it already exists, audited). + +#### Testing + +- All testing is carried out from interchange messages persisted in yaml format, as used by the source services. +- Test classes should extend the `MessageProcessingBase`, which provides class fields and configuration for testing. + - the `messageFactory` field is used to create interchange messages from yaml files + - there are `processSingleMessage` and `processMessages` methods which take interchange messages and process the + message(s) into an in memory database for testing. +- For complex message flows such as Lab Orders (where there are several messages with different data to and from the lab + system and EPIC), + a test class has been created, which uses a class that extends the `OrderPermutationBase` class. + - In this case the test class is `TestLabsProcessingUnorderedMessages` and the permutation class is + the `LabsPermutationTestProducer`, this has test methods which take in a set of yaml filenames, which are + processed + in every possible non-repeating order permutation. + - This ensures that the processing of the messages is not dependent on the order of the messages, and that the + correct + outcome is reached regardless of the order of the messages. + - Admissions, discharges and transfers are also checked using this method. + +## Validation and Deployment + +The EMAP services are deployed using Docker containers, which can interact with each-other using docker compose. +To simplify the configuration and deployment of the containers, we use +the [emap-setup](https://github.com/UCLH-DHCT/emap/tree/main/emap-setup) python package. +This also has functionality to deploy a validation run of EMAP, setting a specific start and end date for the data to be +processed from all sources. + +### Validation + +As all input data during development is created by the developer and this is clinical data, a validation run is always +required before changes should impact the running codebase. +If this is an entirely new data type with no effect on existing data, then feature flags can be used to disable the +processing of the messages in production. +For a change to an existing data type or to release into production, then follow +the [validation SOP](https://github.com/UCLH-DHCT/internal_emap_documentation/blob/main/SOP/validation_run.md). + +### Deployment + +Deployment is carried out using the emap-setup tool, follow +the [release procedure SOP](https://github.com/UCLH-DHCT/internal_emap_documentation/blob/main/SOP/release_procedure.md) diff --git a/emap-interchange/pom.xml b/emap-interchange/pom.xml index b38337a04..5df2cdb0d 100644 --- a/emap-interchange/pom.xml +++ b/emap-interchange/pom.xml @@ -4,14 +4,14 @@ 4.0.0 uk.ac.ucl.rits.inform emap-interchange - 2.6 + 2.7 Emap Interchange Emap interchange format for sending data to the core processor org.springframework.boot spring-boot-starter-parent - 2.7.13 + 2.7.18 diff --git a/emap-setup/emap_runner/docker/docker_runner.py b/emap-setup/emap_runner/docker/docker_runner.py index e0a333c7e..9c21352ca 100644 --- a/emap-setup/emap_runner/docker/docker_runner.py +++ b/emap-setup/emap_runner/docker/docker_runner.py @@ -94,21 +94,6 @@ def docker_compose_paths(self) -> List[Path]: def core_docker_compose_path(self) -> Path: return Path(self.emap_dir, "core", "docker-compose.yml") - def inject_ports(self) -> None: - """Inject the required ports into the docker-compose yamls""" - - file = File(self.core_docker_compose_path) - file.replace("${RABBITMQ_PORT}", self.config["global"]["RABBITMQ_PORT"]) - file.replace( - "${RABBITMQ_ADMIN_PORT}", self.config["global"]["RABBITMQ_ADMIN_PORT"] - ) - file.replace( - "${GLOWROOT_ADMIN_PORT}", self.config["glowroot"]["GLOWROOT_ADMIN_PORT"] - ) - - file.write() - return None - def setup_glowroot_password(self) -> None: """Run the required command to password protect glowroot""" @@ -159,10 +144,9 @@ def _all_global_environment_variables() -> dict: env_vars = os.environ.copy() for item in config_dir_path.iterdir(): - # only necessary to read the global config variables; rest will be # pulled through containers directly - if item.is_file() and str(item) == "global-config-envs": + if item.is_file() and item.stem == "global-config-envs": env_vars.update(EnvironmentFile(item).environment_variables) return env_vars diff --git a/emap-setup/emap_runner/runner.py b/emap-setup/emap_runner/runner.py index d3d142180..a843c6db0 100644 --- a/emap-setup/emap_runner/runner.py +++ b/emap-setup/emap_runner/runner.py @@ -184,7 +184,6 @@ def docker(self) -> None: if "up" in self.args.docker_compose_args and not runner.glowroot_is_up: runner.setup_glowroot_password() - runner.inject_ports() runner.run(*self.args.docker_compose_args) return None diff --git a/emap-setup/emap_runner/validation/validation_runner.py b/emap-setup/emap_runner/validation/validation_runner.py index e8c928c55..a19e291c6 100644 --- a/emap-setup/emap_runner/validation/validation_runner.py +++ b/emap-setup/emap_runner/validation/validation_runner.py @@ -120,7 +120,6 @@ def _run_emap(self) -> None: ) self._create_logs_directory() - self.docker.inject_ports() self.docker.run("down") if self.should_build: diff --git a/emap-setup/global-configuration-EXAMPLE.yaml b/emap-setup/global-configuration-EXAMPLE.yaml index 28d3dfe60..b07740bea 100644 --- a/emap-setup/global-configuration-EXAMPLE.yaml +++ b/emap-setup/global-configuration-EXAMPLE.yaml @@ -50,7 +50,8 @@ global: CABOODLE_JDBC_URL: jdbc:postgresql://host.docker.internal:5432/caboodle CABOODLE_USERNAME: caboodle_username CABOODLE_PASSWORD: caboodle_readaccess - + ACTUATOR_ALLOWED_ORIGINS: "" + HL7_READER_PORT: 9999 # in global so that the ports are set # Configuration for the IDS ids: @@ -79,3 +80,4 @@ glowroot: GLOWROOT_USERNAME: glowrootuser GLOWROOT_PASSWORD: glowrootpw GLOWROOT_ADMIN_PORT: 4000 + diff --git a/emap-setup/setup.py b/emap-setup/setup.py index bf06ba6c1..68ec4ba7e 100644 --- a/emap-setup/setup.py +++ b/emap-setup/setup.py @@ -3,7 +3,7 @@ setup( name="emap_runner", - version="0.0.2", + version="0.1.0", packages=find_packages("."), author="Sarah Keating, Tom Young", url="https://github.com/inform-health-informatics/emap-setup", diff --git a/emap-star/emap-star-annotations/pom.xml b/emap-star/emap-star-annotations/pom.xml index 378a2d951..4a2a2c46c 100644 --- a/emap-star/emap-star-annotations/pom.xml +++ b/emap-star/emap-star-annotations/pom.xml @@ -3,7 +3,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 emap-star-annotations - 2.6 + 2.7 Emap Star Annotations jar Inform-DB Spring JPA Annotation Helpers @@ -11,7 +11,7 @@ uk.ac.ucl.rits.inform emap-star-parent - 2.6 + 2.7 diff --git a/emap-star/emap-star/pom.xml b/emap-star/emap-star/pom.xml index 52b0609ff..d6458057a 100644 --- a/emap-star/emap-star/pom.xml +++ b/emap-star/emap-star/pom.xml @@ -3,7 +3,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 emap-star - 2.6 + 2.7 Emap Star Database Definition jar Emap Star JPA entity classes @@ -11,12 +11,12 @@ uk.ac.ucl.rits.inform emap-star-parent - 2.6 + 2.7 UTF-8 - 2.6 + 2.7 0.10.2 3.2.1 10.3.1 diff --git a/emap-star/emap-star/src/main/java/uk/ac/ucl/rits/inform/informdb/labs/LabOrder.java b/emap-star/emap-star/src/main/java/uk/ac/ucl/rits/inform/informdb/labs/LabOrder.java index e90a55d6e..46c474f8c 100644 --- a/emap-star/emap-star/src/main/java/uk/ac/ucl/rits/inform/informdb/labs/LabOrder.java +++ b/emap-star/emap-star/src/main/java/uk/ac/ucl/rits/inform/informdb/labs/LabOrder.java @@ -33,6 +33,7 @@ @AuditTable(indexes = {@Index(name = "loa_lab_sample_id", columnList = "labSampleId")}) @Table(indexes = {@Index(name = "lo_lab_battery_id", columnList = "labBatteryId"), @Index(name = "lo_lab_sample_id", columnList = "labSampleId"), + @Index(name = "lo_hospital_visit_id", columnList = "hospitalVisitId"), @Index(name = "lo_order_datetime", columnList = "orderDatetime")}) public class LabOrder extends TemporalCore { diff --git a/emap-star/pom.xml b/emap-star/pom.xml index 902707fb6..3ef3a892b 100644 --- a/emap-star/pom.xml +++ b/emap-star/pom.xml @@ -6,7 +6,7 @@ uk.ac.ucl.rits.inform emap-star-parent pom - 2.6 + 2.7 Emap Star Schema diff --git a/global-config-envs.EXAMPLE b/global-config-envs.EXAMPLE index 2350b93c3..e96471e55 100644 --- a/global-config-envs.EXAMPLE +++ b/global-config-envs.EXAMPLE @@ -3,3 +3,4 @@ RABBITMQ_PORT=5672 RABBITMQ_ADMIN_PORT=5674 GLOWROOT_ADMIN_PORT=4000 FAKEUDS_PORT=5433 +HL7_READER_PORT=9999 diff --git a/hl7-reader/docker-compose.yml b/hl7-reader/docker-compose.yml index 14ef16d2d..7a0aaaec9 100644 --- a/hl7-reader/docker-compose.yml +++ b/hl7-reader/docker-compose.yml @@ -18,3 +18,5 @@ services: # Uses services from core, orchestrate using the EMAP setup package - glowroot-central - rabbitmq + ports: + - "${HL7_READER_PORT}:8080" diff --git a/hl7-reader/hl7-reader-config-envs.EXAMPLE b/hl7-reader/hl7-reader-config-envs.EXAMPLE index 35c141981..8bb2599ff 100644 --- a/hl7-reader/hl7-reader-config-envs.EXAMPLE +++ b/hl7-reader/hl7-reader-config-envs.EXAMPLE @@ -14,5 +14,7 @@ SPRING_RABBITMQ_HOST=rabbitmq SPRING_RABBITMQ_PORT=5672 SPRING_RABBITMQ_USERNAME=emap SPRING_RABBITMQ_PASSWORD=yourstrongpassword +# Web application origins that are allowed to use the actuator endpoints +ACTUATOR_ALLOWED_ORIGINS= LOGGING_LEVEL_UK_AC_UCL_RITS_INFORM=INFO TZ=Europe/London diff --git a/hl7-reader/pom.xml b/hl7-reader/pom.xml index 72564c68c..fd3b9c29e 100644 --- a/hl7-reader/pom.xml +++ b/hl7-reader/pom.xml @@ -5,7 +5,7 @@ uk.ac.ucl.rits.inform hl7-reader jar - 2.6 + 2.7 HL7 Reader http://maven.apache.org @@ -13,7 +13,7 @@ org.springframework.boot spring-boot-starter-parent - 2.7.13 + 2.7.18 @@ -22,7 +22,7 @@ 10.3.1 3.3.0 2.3 - 2.6 + 2.7 1.2.8 @@ -54,6 +54,16 @@ spring-boot-starter-data-jpa + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-actuator + + org.springframework.boot spring-boot-starter-test diff --git a/hl7-reader/src/main/java/uk/ac/ucl/rits/inform/datasources/ids/HttpTraceActuatorConfiguration.java b/hl7-reader/src/main/java/uk/ac/ucl/rits/inform/datasources/ids/HttpTraceActuatorConfiguration.java new file mode 100644 index 000000000..0b5712bef --- /dev/null +++ b/hl7-reader/src/main/java/uk/ac/ucl/rits/inform/datasources/ids/HttpTraceActuatorConfiguration.java @@ -0,0 +1,16 @@ +package uk.ac.ucl.rits.inform.datasources.ids; + +import org.springframework.boot.actuate.trace.http.HttpTraceRepository; +import org.springframework.boot.actuate.trace.http.InMemoryHttpTraceRepository; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class HttpTraceActuatorConfiguration { + + @Bean + public HttpTraceRepository httpTraceRepository() { + return new InMemoryHttpTraceRepository(); + } + +} diff --git a/hl7-reader/src/main/resources/application.properties b/hl7-reader/src/main/resources/application.properties index d13020932..019469560 100644 --- a/hl7-reader/src/main/resources/application.properties +++ b/hl7-reader/src/main/resources/application.properties @@ -21,6 +21,10 @@ spring.datasource.hikari.maximum-pool-size=2 #spring.jpa.properties.javax.persistence.schema-generation.scripts.action=create #spring.jpa.properties.javax.persistence.schema-generation.scripts.create-target=create.sql +# spring actuator endpoints +management.endpoints.web.cors.allowed-origins=${ACTUATOR_ALLOWED_ORIGINS} + + rabbitmq.queue.length=100000 rabbitmq.max.batches=5 rabbitmq.max.intransit=1