From 7749eae35ed9b93c2bd1206ef1953ae6322c8ec9 Mon Sep 17 00:00:00 2001 From: "@s.roertgen" Date: Fri, 18 Aug 2023 13:12:21 +0200 Subject: [PATCH 01/37] WIP: PoC successfull Need to change to Document indexing instead of "normal" Index. Then add all the respective labels there. PoC was kind of successfull, now needs to be polished. --- gatsby-node.js | 39 ++++++++++++++++++++++++----------- src/components/nestedList.jsx | 18 +++++++++------- src/templates/App.jsx | 33 ++++++++++++++++++++++++----- 3 files changed, 66 insertions(+), 24 deletions(-) diff --git a/gatsby-node.js b/gatsby-node.js index fa79f6c..76e3ab6 100644 --- a/gatsby-node.js +++ b/gatsby-node.js @@ -10,7 +10,7 @@ const { DataFactory } = n3 const { namedNode } = DataFactory const path = require("path") const fs = require("fs-extra") -const { Index } = require("flexsearch") +const { Index, Document } = require("flexsearch") const omitEmpty = require("omit-empty") const { i18n, getFilePath, parseLanguages } = require("./src/common") const context = require("./src/context") @@ -56,11 +56,11 @@ jsonld.registerRDFParser("text/turtle", (ttlString) => { const createData = ({ path, data }) => fs.outputFile(`public${path}`, data, (err) => err && console.error(err)) -const getTurtleFiles = function (dirPath, arrayOfFiles) { +const getTurtleFiles = function(dirPath, arrayOfFiles) { const files = fs.readdirSync(dirPath) arrayOfFiles = arrayOfFiles || [] - files.forEach(function (file) { + files.forEach(function(file) { if (fs.statSync(dirPath + "/" + file).isDirectory()) { arrayOfFiles = getTurtleFiles(dirPath + "/" + file, arrayOfFiles) } else { @@ -79,7 +79,7 @@ const getTurtleFiles = function (dirPath, arrayOfFiles) { * **/ const exportIndex = (index, conceptScheme, language) => { - index.export(function (key, data) { + index.export(function(key, data) { const path = getFilePath( conceptScheme.id + `/search/${language}/${key}`, `json` @@ -138,10 +138,10 @@ exports.onPreBootstrap = async ({ createContentDigest, actions, getNode }) => { } = graph const type = Array.isArray(properties.type) ? properties.type.find((t) => [ - "Concept", - "ConceptScheme", - "Collection", - ]) + "Concept", + "ConceptScheme", + "Collection", + ]) : properties.type const inSchemeNodes = [...(inScheme || []), ...(topConceptOf || [])] @@ -272,9 +272,14 @@ exports.createPages = async ({ graphql, actions: { createPage } }) => { const languagesOfCS = languagesByCS[conceptScheme.id] const indexes = Object.fromEntries( [...languagesOfCS].map((l) => { - const index = new Index({ + const index = new Document({ tokenize: tokenizer, charset: "latin", + id: "id", + index: [ + "prefLabel", + "altLabel" + ] }) return [l, index] }) @@ -322,10 +327,20 @@ exports.createPages = async ({ graphql, actions: { createPage } }) => { data: JSON.stringify(jsonld, null, 2), }) } - languagesOfCS.forEach((language) => - indexes[language].add(concept.id, i18n(language)(concept.prefLabel)) - ) + // add labels to index + languagesOfCS.forEach((language) => { + const document = { + id: concept.id, + prefLabel: i18n(language)(concept.prefLabel), + ...(concept.altLabel && Object.hasOwn(concept.altLabel, language) && { altLabel: i18n(language)(concept.altLabel) }), + } + console.log(document) + indexes[language].add(document) + }) }) + console.log(indexes["de"].search("Konzept")) + console.log(indexes["de"].search("Konzept", ["prefLabel"])) + console.log(indexes["de"].search("Alterna")) languagesOfCS.forEach((language) => createPage({ diff --git a/src/components/nestedList.jsx b/src/components/nestedList.jsx index d3dc7a0..a8896d7 100644 --- a/src/components/nestedList.jsx +++ b/src/components/nestedList.jsx @@ -25,6 +25,9 @@ const getNestedItems = (item) => { */ const NestedList = ({ items, current, filter, highlight, language }) => { + console.log(filter && filter.length && filter.map(f => f.result)) + // TODO put this in App.jsx and pass here just the ids as before + const filteredIds = filter && filter.length && filter.map(f => f.result)[0] const { config } = getConfigAndConceptSchemes() const style = css` list-style-type: none; @@ -115,11 +118,12 @@ const NestedList = ({ items, current, filter, highlight, language }) => { ` const filteredItems = filter ? items.filter( - (item) => - !filter || - filter.some((filter) => getNestedItems(item).includes(filter)) - ) + (item) => + !filter || + filteredIds.some((filter) => getNestedItems(item).includes(filter)) + ) : items + console.log(filteredItems) const t = i18n(language) const isExpanded = (item, truthy, falsy) => { @@ -142,9 +146,9 @@ const NestedList = ({ items, current, filter, highlight, language }) => { dangerouslySetInnerHTML={{ __html: highlight ? t(item.prefLabel).replace( - highlight, - (str) => `${str}` - ) + highlight, + (str) => `${str}` + ) : t(item.prefLabel), }} /> diff --git a/src/templates/App.jsx b/src/templates/App.jsx index 46c0a94..04f2f2d 100644 --- a/src/templates/App.jsx +++ b/src/templates/App.jsx @@ -1,5 +1,5 @@ import React, { useEffect, useState } from "react" -import Index from "flexsearch/dist/module/index.js" +import Document from "flexsearch/dist/module/document.js" import escapeRegExp from "lodash.escaperegexp" import { i18n, getFilePath } from "../common" import NestedList from "../components/nestedList" @@ -19,7 +19,7 @@ const App = ({ pageContext, children }) => { const [conceptSchemeId, setConceptSchemeId] = useState( data?.currentScheme?.id ) - const [index, setIndex] = useState(new Index()) + const [index, setIndex] = useState({}) const [query, setQuery] = useState(null) const [tree, setTree] = useState( pageContext.node.type === "ConceptScheme" ? pageContext.node : null @@ -36,9 +36,30 @@ const App = ({ pageContext, children }) => { } const importIndex = async () => { - const idx = new Index() - const keys = ["cfg", "ctx", "map", "reg"] - + /** FIXME the document options need to be imported from somewhere + * maybe store them in a separate file and create it in gatsby-node.js ? + * maybe along with all the necessary keys + **/ + const idx = new Document({ + tokenize: "full", + charset: "latin", + id: "id", + index: [ + "prefLabel", + "altLabel" + ] + }) + const keys = [ + "altLabel.cfg", + "altLabel.ctx", + "altLabel.map", + // "altLabel.store", + "prefLabel.cfg", + "prefLabel.ctx", + "prefLabel.map", + "reg" + ] + console.log(conceptSchemeId) for (let i = 0, key; i < keys.length; i += 1) { key = keys[i] const data = await fetch( @@ -56,6 +77,8 @@ const App = ({ pageContext, children }) => { } setIndex(idx) + console.log(idx.search("Konzept")) + console.log(idx.search("Alternativ")) } // get concept scheme id from context From a350920c59b19ab4a3a5bb800086f721d2d3519d Mon Sep 17 00:00:00 2001 From: "@s.roertgen" Date: Tue, 29 Aug 2023 15:37:44 +0200 Subject: [PATCH 02/37] WIP add checkboxes and load index based on selection --- src/common.js | 6 +-- src/components/LabelFilter.jsx | 21 +++++++++++ src/components/nestedList.jsx | 52 +++++++++++++++++--------- src/templates/App.jsx | 67 +++++++++++++++++++++------------- 4 files changed, 100 insertions(+), 46 deletions(-) create mode 100644 src/components/LabelFilter.jsx diff --git a/src/common.js b/src/common.js index 70fd7c4..94ed892 100644 --- a/src/common.js +++ b/src/common.js @@ -53,10 +53,10 @@ const getDomId = (url) => { /** * Parses languages from a json ld graph (Concept or Concept Scheme) - * @param {array} json + * @param {array} graph * @returns {array} languages - found languages */ -const parseLanguages = (json) => { +const parseLanguages = (graph) => { const languages = new Set() const parse = (arrayOfObj) => { for (let obj of arrayOfObj) { @@ -80,7 +80,7 @@ const parseLanguages = (json) => { obj?.narrower && parse(obj.narrower) } } - parse(json) + parse(graph) return languages } diff --git a/src/components/LabelFilter.jsx b/src/components/LabelFilter.jsx new file mode 100644 index 0000000..67c5ce0 --- /dev/null +++ b/src/components/LabelFilter.jsx @@ -0,0 +1,21 @@ +const LabelFilter = ({ labels, toggleClick }) => { + console.log(labels) + const handleClick = (e) => { + toggleClick(e) + } + const labelBoxes = Object.entries(labels).map((label) => ( + + )) + + + return ( +
+ {labelBoxes} +
+ ) +} + +export default LabelFilter diff --git a/src/components/nestedList.jsx b/src/components/nestedList.jsx index a8896d7..d1fc321 100644 --- a/src/components/nestedList.jsx +++ b/src/components/nestedList.jsx @@ -16,18 +16,17 @@ const getNestedItems = (item) => { } /** - * @param {array} items list of concepts - * @param {string} current current concept id - * @param {[string]|null} filter + * Building the tree view for the vocabs + * @param {Array} items - list of concepts + * @param {string} current - current concept id + * @param {Object|null} queryFilter + * @param {string} queryFilter.field + * @param {Array} queryFilter.result * @param {RegExp|null} highlight * @param {string} language * @returns */ - -const NestedList = ({ items, current, filter, highlight, language }) => { - console.log(filter && filter.length && filter.map(f => f.result)) - // TODO put this in App.jsx and pass here just the ids as before - const filteredIds = filter && filter.length && filter.map(f => f.result)[0] +const NestedList = ({ items, current, queryFilter, highlight, language, topLevel = false }) => { const { config } = getConfigAndConceptSchemes() const style = css` list-style-type: none; @@ -116,18 +115,33 @@ const NestedList = ({ items, current, filter, highlight, language }) => { } } ` - const filteredItems = filter - ? items.filter( - (item) => - !filter || - filteredIds.some((filter) => getNestedItems(item).includes(filter)) - ) - : items + console.log("highlight", highlight) + console.log("filter", queryFilter) + const filteredIds = queryFilter && queryFilter.length ? queryFilter.flatMap(f => f.result) : [] + console.log("filteredIds", filteredIds) + console.log("items", items) + + const getFilteredItems = () => { + if (!queryFilter) { + console.log("all items") + return items + } else if (filteredIds.length) { + return items.filter( + (item) => + !queryFilter || + filteredIds.some((filter) => getNestedItems(item).includes(filter)) + ) + } else { + return [] + } + } + + const filteredItems = getFilteredItems() console.log(filteredItems) const t = i18n(language) const isExpanded = (item, truthy, falsy) => { - return filter || getNestedItems(item).some((id) => id === current) + return queryFilter || getNestedItems(item).some((id) => id === current) ? truthy : falsy } @@ -173,6 +187,10 @@ const NestedList = ({ items, current, filter, highlight, language }) => { return Link } + // only return nothing found for topLevel nestedList + if (!filteredItems.length && topLevel) + return (

Nothing found!

) + return (