diff --git a/.github/workflows/test-rio.yml b/.github/workflows/test-rio.yml new file mode 100644 index 00000000..a7d66e17 --- /dev/null +++ b/.github/workflows/test-rio.yml @@ -0,0 +1,51 @@ +name: Check if RIO queue is up or down + +on: + push: + branches: + - 'main' + - 'test-rio' +# - cron: '6 9-17 * * 1-5' # every hour on weekdays during working hours + +jobs: + deps: + runs-on: ubuntu-latest + + steps: + - name: Cache lein project dependencies + uses: actions/cache@v4 + with: + path: "~/.m2/repository" + key: "${{ runner.os }}-clojure-${{ hashFiles('**/project.clj') }}" + + - name: Checkout code + uses: actions/checkout@v4 + + - name: Check rio queue status + env: + CLIENTS_INFO_PATH: ${{ secrets.CLIENTS_INFO_PATH }} + CLIENT_ID: ${{ secrets.CLIENT_ID }} + CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }} + GATEWAY_PASSWORD: ${{ secrets.GATEWAY_PASSWORD }} + GATEWAY_ROOT_URL: ${{ secrets.GATEWAY_ROOT_URL }} + GATEWAY_USER: ${{ secrets.GATEWAY_USER }} + KEYSTORE_ALIAS: ${{ secrets.KEYSTORE_ALIAS }} + KEYSTORE_PASSWORD: ${{ secrets.KEYSTORE_PASSWORD }} + OS_AUTH_URL: ${{ secrets.OS_AUTH_URL }} + OS_CONTAINER_NAME: ${{ secrets.OS_CONTAINER_NAME }} + OS_PASSWORD: ${{ secrets.OS_PASSWORD }} + OS_PROJECT_NAME: ${{ secrets.OS_PROJECT_NAME }} + OS_USERNAME: ${{ secrets.OS_USERNAME }} + RIO_READ_URL: ${{ secrets.RIO_READ_URL }} + RIO_RECIPIENT_OIN: ${{ secrets.RIO_RECIPIENT_OIN }} + RIO_SENDER_OIN: ${{ secrets.RIO_SENDER_OIN }} + RIO_UPDATE_URL: ${{ secrets.RIO_UPDATE_URL }} + STORE_HTTP_REQUESTS: true + SURF_CONEXT_CLIENT_ID: ${{ secrets.SURF_CONEXT_CLIENT_ID }} + SURF_CONEXT_CLIENT_SECRET: ${{ secrets.SURF_CONEXT_CLIENT_SECRET }} + SURF_CONEXT_INTROSPECTION_ENDPOINT: ${{ secrets.SURF_CONEXT_INTROSPECTION_ENDPOINT }} + TOKEN_ENDPOINT: ${{ secrets.TOKEN_ENDPOINT }} + TRUSTSTORE: truststore.jks + TRUSTSTORE_PASSWORD: ${{ secrets.TRUSTSTORE_PASSWORD }} + KEYSTORE: test/keystore.jks + run: lein mapper test-rio rio-mapper-dev.jomco.nl diff --git a/src/nl/surf/eduhub_rio_mapper/cli_commands.clj b/src/nl/surf/eduhub_rio_mapper/cli_commands.clj index 09e7da86..bccfbbfc 100644 --- a/src/nl/surf/eduhub_rio_mapper/cli_commands.clj +++ b/src/nl/surf/eduhub_rio_mapper/cli_commands.clj @@ -18,6 +18,7 @@ (ns nl.surf.eduhub-rio-mapper.cli-commands (:require [clojure.data.json :as json] + [clojure.java.io :as io] [clojure.string :as str] [nl.surf.eduhub-rio-mapper.clients-info :as clients-info] [nl.surf.eduhub-rio-mapper.endpoints.api :as api] @@ -26,9 +27,10 @@ [nl.surf.eduhub-rio-mapper.rio.loader :as rio.loader] [nl.surf.eduhub-rio-mapper.specs.ooapi :as ooapi] [nl.surf.eduhub-rio-mapper.specs.rio :as rio] - [nl.surf.eduhub-rio-mapper.worker :as worker])) + [nl.surf.eduhub-rio-mapper.worker :as worker]) + (:import [java.util UUID])) -(defn parse-getter-args [[type id & [pagina]]] +(defn- parse-getter-args [[type id & [pagina]]] {:pre [type id (string? type)]} (let [[type response-type] (reverse (str/split type #":" 2)) response-type (and response-type (keyword response-type)) @@ -56,7 +58,7 @@ (System/exit 1)) [client-info rest-args])) -(defn process-command [command args {{:keys [getter resolver ooapi-loader dry-run! link!] :as handlers} :handlers {:keys [clients] :as config} :config}] +(defn process-command [command args {{:keys [getter resolver ooapi-loader dry-run! link! insert!] :as handlers} :handlers {:keys [clients] :as config} :config}] {:pre [getter]} (case command "serve-api" @@ -70,6 +72,30 @@ (worker/wait-worker (worker/start-worker! config))) + "test-rio" + (let [[client-info _] (parse-client-info-args args clients) + uuid (UUID/randomUUID) + eduspec (-> "../test/fixtures/ooapi/education-specification-template.json" + io/resource + slurp + (json/read-str :key-fn keyword) + (assoc :educationSpecificationId uuid))] + + (try + (insert! {:institution-oin (:institution-oin client-info) + :institution-schac-home (:institution-schac-home client-info) + ::ooapi/type "education-specification" + ::ooapi/id uuid + ::ooapi/entity eduspec}) + (println "The RIO Queue is UP") + (catch Exception ex + (when-let [ex-data (ex-data ex)] + (when (= :down (:rio-queue-status ex-data)) + (println "The RIO Queue is DOWN") + (System/exit -1))) + (println "An unexpected exception has occurred: " ex) + (System/exit -2)))) + "get" (let [[client-info rest-args] (parse-client-info-args args clients)] (getter (assoc (parse-getter-args rest-args) diff --git a/src/nl/surf/eduhub_rio_mapper/commands/processing.clj b/src/nl/surf/eduhub_rio_mapper/commands/processing.clj index 1341e51a..f05c2019 100644 --- a/src/nl/surf/eduhub_rio_mapper/commands/processing.clj +++ b/src/nl/surf/eduhub_rio_mapper/commands/processing.clj @@ -128,7 +128,7 @@ :eduspec eduspec :mutate-result mutate-result}) (throw (ex-info (str "Processing this job takes longer than expected. Our developers have been informed and will contact DUO. Please try again in a few hours." - ": " type " " id) {}))))) + ": " type " " id) {:rio-queue-status :down}))))) (defn- make-updater-sync-relations-phase [handlers] (fn sync-relations-phase [{:keys [job eduspec] :as request}] @@ -145,6 +145,18 @@ (assoc (ex-data ex) :phase phase) ex)))))) +(defn- make-insert [handlers rio-config] + (let [fs [[:preparing (make-updater-soap-phase)] + [:upserting (make-updater-mutate-rio-phase handlers)] + [:confirming (make-updater-confirm-rio-phase handlers rio-config)]] + wrapped-fs (map wrap-phase fs)] + (fn [request] + {:pre [(:institution-oin request) + (::ooapi/entity request)]} + (as-> request $ + (reduce (fn [req f] (f req)) $ wrapped-fs) + (:mutate-result $))))) + (defn- make-update [handlers rio-config] (let [fs [[:fetching-ooapi (make-updater-load-ooapi-phase handlers)] [:resolving (make-updater-resolve-phase handlers)] @@ -237,6 +249,7 @@ :resolver resolver} update! (make-update handlers rio-config) delete! (make-deleter handlers) + insert! (make-insert handlers rio-config) dry-run! (make-dry-runner handlers) link! (link/make-linker rio-config getter)] - (assoc handlers :update! update!, :delete! delete!, :dry-run! dry-run!, :link! link!))) + (assoc handlers :update! update!, :delete! delete!, :insert! insert!, :dry-run! dry-run!, :link! link!))) diff --git a/src/nl/surf/eduhub_rio_mapper/main.clj b/src/nl/surf/eduhub_rio_mapper/main.clj index 200132dd..aeac8a97 100644 --- a/src/nl/surf/eduhub_rio_mapper/main.clj +++ b/src/nl/surf/eduhub_rio_mapper/main.clj @@ -26,7 +26,7 @@ (:gen-class)) (def commands - #{"upsert" "delete" "delete-by-code" "get" "show" "resolve" "serve-api" "worker" "help" "dry-run-upsert" "link"}) + #{"upsert" "delete" "delete-by-code" "get" "show" "resolve" "serve-api" "worker" "help" "dry-run-upsert" "link" "test-rio"}) (defn -main [command & args] diff --git a/src/nl/surf/eduhub_rio_mapper/rio/loader.clj b/src/nl/surf/eduhub_rio_mapper/rio/loader.clj index 6d73eca5..b0c4cae8 100644 --- a/src/nl/surf/eduhub_rio_mapper/rio/loader.clj +++ b/src/nl/surf/eduhub_rio_mapper/rio/loader.clj @@ -151,7 +151,6 @@ (fn resolver [type id institution-oin] {:pre [institution-oin]} - (when id (logging/with-mdc {:soap-action "opvragen_rioIdentificatiecode" :ooapi-id id} diff --git a/test/fixtures/ooapi/education-specification-template.json b/test/fixtures/ooapi/education-specification-template.json new file mode 100644 index 00000000..1cf203b3 --- /dev/null +++ b/test/fixtures/ooapi/education-specification-template.json @@ -0,0 +1,63 @@ +{ + "educationSpecificationId": "123e4567-e89b-12d3-a456-426614174000", + "primaryCode": { + "codeType": "crohoCreboCode", + "code": "string" + }, + "otherCodes": [ + { + "codeType": "crohoCreboCode", + "code": "1234123" + } + ], + "educationSpecificationType": "program", + "name": [ + { + "language": "en-GB", + "value": "Bachelor Chemical technology" + }, + { + "language": "nl-BE", + "value": "Bachelor Chemische technologie" + } + ], + "abbreviation": "B Scheikundige Technologie", + "description": [ + { + "language": "en-GB", + "value": "program that is a place holder for all courses that are made available for student mobility, program that is a place holder for all courses that are made available for student mobilityprogram that is a place holder for all courses that are made available for student mobilityprogram that is a place holder for all courses that are made available for student mobilityprogram that is a place holder for all courses that are made available for student mobilityprogram that is a place holder for all courses that are made available for student mobilityprogram that is a place holder for all courses that are made available for student mobilityprogram that is a place holder for all courses that are made available for student mobilityprogram that is a place holder for all courses that are made available for student mobilityprogram that is a place holder for all courses that are made available for student mobilityprogram that is a place holder for all courses that are made available for student mobilityprogram that is a place holder for all courses that are made available for student mobility" + } + ], + "formalDocument": "diploma", + "level": "master", + "sector": "university education", + "levelOfQualification": "6", + "fieldsOfStudy": "0732", + "studyLoad": { + "studyLoadUnit": "ects", + "value": 3 + }, + "learningOutcomes": [ + [ + { + "language": "en-GB", + "value": "Executable knowledge of Chemical technology, including: Acquire knowledge of research paradigms." + } + ] + ], + "link": "https://bijvak.nl", + "parent": "42e2bc1b-6741-4e2e-b138-97b4a342c999", + "children": [ + "497f6eca-6276-4993-bfeb-53cbbbba6f08" + ], + "organization": "452c1a86-a0af-475b-b03f-724878b0f387", + "consumers": [ + { + "consumerKey": "rio", + "educationSpecificationSubType": "variant" + } + ], + "ext": {}, + "validFrom": "2019-08-24", + "validTo": "2019-08-25" +}