add all prefLabels to flexSearch index

The problem was that through multiple calls of `index.add(concept.id, i18n(language)(concept.prefLabel))` the content for the respective id got overwritten.
This way only the last added label was present in the flexSearch index.
To overcome this I added two functions in a new file `searchIndex.js`. `buildIndexAndScheme` creates the index scheme and the index dependent on how many languages are present.
In `addToIndex` labels are concatenated to their language.
With the array labelsToAdd, we have the possibility to later add on more labels per language, e.g. `skos:example` or `skos:altLabel`.
We then save it just like the `.index` file, to later load it in the `App.js`template.
With the adjustemnt in line 79 it is possible to choose whether to search across all languages or just the current chosen one from pageContext.

Might resolve: #127, #128

gatsby-node.js

@@ -9,13 +9,13 @@ const { DataFactory } = n3
 const { namedNode } = DataFactory
 const path = require('path')
 const fs = require('fs-extra')
-const flexsearch = require('flexsearch')
 const omitEmpty = require('omit-empty')
 const urlTemplate = require('url-template')
 const { i18n, getPath, getFilePath } = require('./src/common')
 const context = require('./src/context')
 const queries = require('./src/queries')
 const types = require('./src/types')
+const { buildIndexAndScheme, addToIndex } = require('./src/searchIndex')
 
 require('dotenv').config()
 require('graceful-fs').gracefulify(require('fs'))
@@ -123,8 +123,7 @@ exports.createPages = async ({ graphql, actions: { createPage } }) => {
   conceptSchemes.errors && console.error(conceptSchemes.errors)
 
   conceptSchemes.data.allConceptScheme.edges.forEach(async ({ node: conceptScheme }) => {
-    const index = flexsearch.create()
-
+    let {index, indexScheme} = buildIndexAndScheme(languages)
     const conceptsInScheme = await graphql(queries.allConcept(conceptScheme.id, languages))
     const embeddedConcepts = []
     conceptsInScheme.data.allConcept.edges.forEach(({ node: concept }) => {
@@ -174,7 +173,7 @@ exports.createPages = async ({ graphql, actions: { createPage } }) => {
           data: JSON.stringify(jsonas, null, 2)
         })
       }
-      languages.forEach(language => index.add(concept.id, i18n(language)(concept.prefLabel)))
+      index = addToIndex(languages, concept, index)
     })
 
     console.log("Built index", index.info())
@@ -202,6 +201,10 @@ exports.createPages = async ({ graphql, actions: { createPage } }) => {
       path: getFilePath(conceptScheme.id, 'index'),
       data: JSON.stringify(index.export(), null, 2)
     })
+    createData({
+      path: getFilePath(conceptScheme.id, 'index.scheme'),
+      data: JSON.stringify(indexScheme, null, 2)
+    })
   })
 
   // Build index pages

src/components/nestedList.js

@@ -103,8 +103,9 @@ const getNestedItems = item => {
 }
 
 const NestedList = ({ items, current, filter, highlight, language }) => {
-  const filteredItems = filter
-    ? items.filter(item => !filter || filter.some(filter => getNestedItems(item).includes(filter)))
+  const filterIds = filter ? filter.map(f => f.id) : null
+  const filteredItems = filterIds
+    ? items.filter(item => !filterIds || filterIds.some(filter => getNestedItems(item).includes(filter)))
     : items
   const t = i18n(language)
 
@@ -116,7 +117,7 @@ const NestedList = ({ items, current, filter, highlight, language }) => {
         >
           {(item.narrower && item.narrower.length > 0) && (
             <button
-              className={`treeItemIcon inputStyle${(filter || getNestedItems(item).some( id => id === current))
+              className={`treeItemIcon inputStyle${(filterIds || getNestedItems(item).some( id => id === current))
                 ?  '' : ' collapsed'}`}
               onClick={(e) => {
                 e.target.classList.toggle("collapsed")
@@ -163,7 +164,7 @@ const NestedList = ({ items, current, filter, highlight, language }) => {
               <NestedList
                 items={item.narrower}
                 current={current}
-                filter={filter}
+                filter={filterIds}
                 highlight={highlight}
                 language={language}
               />

src/searchIndex.js

@@ -0,0 +1,39 @@
+const { i18n } = require('./common')
+const flexsearch = require('flexsearch')
+
+const buildIndexAndScheme = (languages) => {
+  const indexScheme = {
+    doc: {
+      id: 'id',
+      field: [...languages],
+    },
+  }
+  const index = flexsearch.create(indexScheme)
+  return { index, indexScheme }
+}
+
+const addToIndex = (languages, concept, index) => {
+  // object that holds languages as key and we add labels to them in an array
+  const fields = Object.fromEntries(
+    [...languages].map((language) => [[language], ''])
+  )
+  // which labels should be added to the search index
+  const labelsToAdd = ['prefLabel', 'altLabel']
+
+  labelsToAdd.forEach((label) => {
+    [...languages].forEach((language) => {
+        const i18nLabels =
+          Array.isArray(concept[label]) && concept[label][language] ? i18n(language)(concept[label])
+            : concept[label] ? [i18n(language)(concept[label])]
+            : ['']
+        fields[language] = fields[language].concat(' ' + i18nLabels.join(' '))
+    })
+  })
+  index.add({ id: concept.id, ...fields })
+  return index
+}
+
+module.exports = {
+  buildIndexAndScheme,
+  addToIndex,
+}

src/templates/App.js

@@ -16,19 +16,26 @@ const App = ({pageContext, children}) => {
   const conceptSchemeId = pageContext.node.type === 'ConceptScheme'
     ? pageContext.node.id
     : pageContext.node.inScheme.id
-  const [index, setIndex] = useState(FlexSearch.create('speed'))
+  const [index, setIndex] = useState(null)
   const [query, setQuery] = useState(null)
   const [tree, setTree] = useState(pageContext.node.type === 'ConceptScheme' ? pageContext.node : null)
 
-  // Fetch and load the serialized index
+  // Set index
   useEffect(() => {
-    fetch(pageContext.baseURL +  getFilePath(conceptSchemeId, 'index'))
-      .then(response => response.json())
-      .then(serialized => {
-        const idx = FlexSearch.create()
-        idx.import(serialized)
-        setIndex(idx)
-        console.log("index loaded", idx.info())
+    // fetch and load the index scheme
+    fetch(pageContext.baseURL + getFilePath(conceptSchemeId, "index.scheme"))
+      .then((response) => response.json())
+      .then((serialized) => {
+        const idx = FlexSearch.create(serialized)
+        console.log("index created", idx.info())
+        // fetch and load the serialized index
+        fetch(pageContext.baseURL + getFilePath(conceptSchemeId, "index"))
+          .then((response) => response.json())
+          .then((serialized) => {
+            idx.import(serialized)
+            setIndex(idx)
+            console.log("index loaded", idx.info())
+          })
       })
   }, [])
 
@@ -69,7 +76,7 @@ const App = ({pageContext, children}) => {
             <NestedList
               items={tree.hasTopConcept}
               current={pageContext.node.id}
-              filter={query ? index.search(query) : null}
+              filter={query ? index.search({query: query, field: pageContext.language}) : null}
               highlight={query ? RegExp(escapeRegExp(query), 'gi'): null}
               language={pageContext.language}
             />