make opensearchandingest (containers)

rohberg · Jun 16, 2024 · 39a547c · 39a547c
1 parent 32d4f26
commit 39a547c
Show file tree

Hide file tree

Showing 17 changed files with 71 additions and 60 deletions.
diff --git a/Makefile b/Makefile
@@ -24,9 +24,10 @@ DEV_COMPOSE=dockerfiles/docker-compose.yml
 ACCEPTANCE_COMPOSE=acceptance/docker-compose.yml
 CMD_ENVS=CURRENT_DIR=${CURRENT_DIR} ADDON_NAME=${ADDON_NAME} ADDON_PATH=${ADDON_PATH} VOLTO_VERSION=${VOLTO_VERSION} PLONE_VERSION=${PLONE_VERSION} BACKEND_ADDONS=${BACKEND_ADDONS}
 CMD=${CMD_ENVS} docker compose
-DOCKER_COMPOSE=${CMD} -p ${ADDON_PATH} -f ${DEV_COMPOSE}
-ACCEPTANCE=${CMD} -p ${ADDON_PATH}-acceptance -f ${ACCEPTANCE_COMPOSE}
-ACCEPTANCE_MULTILINGUAL=${CMD} -p ${ADDON_PATH}-acceptance-multilingual -f ${ACCEPTANCE_COMPOSE}
+PROJECT_NAME=${ADDON_PATH}
+DOCKER_COMPOSE=${CMD} -p ${PROJECT_NAME} -f ${DEV_COMPOSE}
+ACCEPTANCE=${CMD} -p ${PROJECT_NAME}-acceptance -f ${ACCEPTANCE_COMPOSE}
+ACCEPTANCE_MULTILINGUAL=${CMD} -p ${PROJECT_NAME}-acceptance-multilingual -f ${ACCEPTANCE_COMPOSE}
 
 .PHONY: all
 all: help
@@ -74,6 +75,20 @@ dev-start: ## Start development/demo environment without rebuilding images
 	${DOCKER_COMPOSE} --profile dev up
 
 
+# Opensearch and ingest containers (everything but backend and frontend)
+.PHONY: opensearchandingest-build
+opensearchandingest-build: ## Build containers for opensearch and ingest `make opensearchandingest-build PROJECT_NAME=foo`
+	@echo "$(GREEN)==> Build containers for opensearch and ingest $(RESET)"
+	${DOCKER_COMPOSE} --profile opensearchandingest build
+	@echo "$(GREEN)==> Successfully build containers for opensearch and ingest $(RESET)"
+	${DOCKER_COMPOSE} --profile opensearchandingest config
+
+.PHONY: opensearchandingest-up
+opensearchandingest-up: ## Start containers for opensearch and ingest. `make opensearchandingest-up PROJECT_NAME=foo`
+	@echo "$(GREEN)==> Start containers for opensearch and ingest $(RESET)"
+	${DOCKER_COMPOSE} --profile opensearchandingest up
+
+
 # Dev Helpers
 .PHONY: i18n
 i18n: ## Sync i18n

diff --git a/README.md b/README.md
@@ -37,7 +37,7 @@ Install Plone backend add-on [`collective.elastic.plone 2.x`](https://github.com
 Install Plone backend add-on [`collective.elastic.ingest 2.x`](https://github.com/collective/collective.elastic.ingest) to index Plone content.
 
 Setting up OpenSearch/ElasticSearch instructions can be found on [`collective.elastic.plone 2.x`](https://github.com/collective/collective.elastic.plone).
-See the [example](docker-opensearch) configuration of collective.elastic of a mapping, attachment handling and last but not least analysis.
+See the [example](dockerfiles/opensearch) configuration of collective.elastic of a mapping, attachment handling and last but not least analysis.
 
 
 # Configuration

diff --git a/acceptance/ci.yml b/acceptance/ci.yml
@@ -99,7 +99,7 @@ services:
   ingest:
     build:
       context: ../
-      dockerfile: ./docker-opensearch/Dockerfile.ingest
+      dockerfile: .//Dockerfile.ingest
     environment:
       MAPPINGS_FILE: "/configuration/mappings.json"
       ANALYSIS_FILE: "/configuration/analysis.json"
@@ -120,7 +120,7 @@ services:
   ingest-multilingual:
     build:
       context: ../
-      dockerfile: ./docker-opensearch/Dockerfile.ingest
+      dockerfile: .//Dockerfile.ingest
     environment:
       # Different INDEX_NAME
       MAPPINGS_FILE: "/configuration/mappings.json"
@@ -148,7 +148,7 @@ services:
   opensearch:
     build:
       context: ../
-      dockerfile: ./docker-opensearch/Dockerfile.opensearch
+      dockerfile: .//Dockerfile.opensearch
     environment:
       - plugins.security.disabled=true
       - cluster.name=opensearch-cluster

diff --git a/acceptance/docker-compose.yml b/acceptance/docker-compose.yml
@@ -1,14 +1,13 @@
-version: "3"
+version: '3'
 
 services:
-
   addon-acceptance:
     build:
       context: ../
       dockerfile: ./dockerfiles/frontend/Dockerfile.acceptance
       args:
-        ADDON_NAME: "${ADDON_NAME}"
-        ADDON_PATH: "${ADDON_PATH}"
+        ADDON_NAME: '${ADDON_NAME}'
+        ADDON_PATH: '${ADDON_PATH}'
         VOLTO_VERSION: ${VOLTO_VERSION:-17}
     volumes:
       - ${CURRENT_DIR}:/app/src/addons/${ADDON_PATH}/
@@ -30,8 +29,8 @@ services:
       context: ../
       dockerfile: ./dockerfiles/frontend/Dockerfile.acceptance.multilingual
       args:
-        ADDON_NAME: "${ADDON_NAME}"
-        ADDON_PATH: "${ADDON_PATH}"
+        ADDON_NAME: '${ADDON_NAME}'
+        ADDON_PATH: '${ADDON_PATH}'
         VOLTO_VERSION: ${VOLTO_VERSION:-17}
     volumes:
       - ${CURRENT_DIR}:/app/src/addons/${ADDON_PATH}/
@@ -55,18 +54,18 @@ services:
       args:
         PLONE_VERSION: ${PLONE_VERSION:-6.0}
     environment:
-      ZSERVER_HOST: "0.0.0.0"
-      ZSERVER_PORT: "55001"
+      ZSERVER_HOST: '0.0.0.0'
+      ZSERVER_PORT: '55001'
       CELERY_BROKER: ${CELERY_BROKER?unset}
       INDEX_SERVER: ${INDEX_SERVER?unset}
       INDEX_NAME: ${INDEX_NAME?unset}
       INDEX_OPENSEARCH: ${INDEX_OPENSEARCH?unset}
       INDEX_USE_SSL: ${INDEX_USE_SSL?unset}
       INDEX_LOGIN: ${INDEX_LOGIN?unset}
       INDEX_PASSWORD: ${INDEX_PASSWORD?unset}
-      CONFIGURE_PACKAGES: "plone.restapi,plone.volto,plone.volto.cors,collective.elastic.plone"
+      CONFIGURE_PACKAGES: 'plone.restapi,plone.volto,plone.volto.cors,collective.elastic.plone'
       # Run initialize of collective.elastic.plone to register ElasticSearchProxyIndex
-      INSTALL_PRODUCTS: "collective.elastic.plone"
+      INSTALL_PRODUCTS: 'collective.elastic.plone'
     volumes:
       - backend-data:/data
     ports:
@@ -82,18 +81,18 @@ services:
       args:
         PLONE_VERSION: ${PLONE_VERSION:-6.0}
     environment:
-      ZSERVER_HOST: "0.0.0.0"
-      ZSERVER_PORT: "55001"
+      ZSERVER_HOST: '0.0.0.0'
+      ZSERVER_PORT: '55001'
       CELERY_BROKER: ${CELERY_BROKER?unset}
       INDEX_SERVER: ${INDEX_SERVER?unset}
       INDEX_NAME: multilingual
       INDEX_OPENSEARCH: ${INDEX_OPENSEARCH?unset}
       INDEX_USE_SSL: ${INDEX_USE_SSL?unset}
       INDEX_LOGIN: ${INDEX_LOGIN?unset}
       INDEX_PASSWORD: ${INDEX_PASSWORD?unset}
-      CONFIGURE_PACKAGES: "plone.restapi,plone.volto,plone.volto.cors,collective.elastic.plone"
+      CONFIGURE_PACKAGES: 'plone.restapi,plone.volto,plone.volto.cors,collective.elastic.plone'
       # Run initialize of collective.elastic.plone to register ElasticSearchProxyIndex
-      INSTALL_PRODUCTS: "collective.elastic.plone"
+      INSTALL_PRODUCTS: 'collective.elastic.plone'
     volumes:
       - backend-data:/data
     ports:
@@ -161,13 +160,13 @@ services:
   opensearch:
     build:
       context: ../
-      dockerfile: ./docker-opensearch/Dockerfile.opensearch
+      dockerfile: ./dockerfiles/opensearch/Dockerfile.opensearch
     environment:
       - cluster.name=opensearch-cluster
       - node.name=opensearch
       - discovery.type=single-node
       - bootstrap.memory_lock=true
-      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx2g"
+      - 'OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx2g'
       # - plugins.security.disabled=true
       - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${INDEX_PASSWORD?unset}
     ulimits:
@@ -188,13 +187,12 @@ services:
       - multilingual
       - prod
 
-
   opensearch-dashboards:
     image: opensearchproject/opensearch-dashboards:latest
     ports:
       - 5601:5601
     expose:
-      - "5601"
+      - '5601'
     environment:
       OPENSEARCH_HOSTS: '["https://opensearch:9200"]'
     profiles:
@@ -218,5 +216,5 @@ volumes:
     driver: local
     driver_opts:
       type: none
-      device: ../docker-opensearch/ingest-configuration
-      o: bind
+      device: ../dockerfiles/opensearch/ingest-configuration
+      o: bind
diff --git a/docker-opensearch/Dockerfile.ingest b/docker-opensearch/Dockerfile.ingest
diff --git a/docker-opensearch/Dockerfile.opensearch b/docker-opensearch/Dockerfile.opensearch
diff --git a/dockerfiles/docker-compose.yml b/dockerfiles/docker-compose.yml
@@ -1,15 +1,14 @@
-version: "3"
+version: '3'
 
 services:
-
   addon-dev:
     build:
       context: ../
       dockerfile: ./dockerfiles/frontend/Dockerfile.dev
       args:
-        ADDON_NAME: "${ADDON_NAME}"
-        ADDON_PATH: "${ADDON_PATH}"
-        VOLTO_VERSION: "${VOLTO_VERSION:-17}"
+        ADDON_NAME: '${ADDON_NAME}'
+        ADDON_PATH: '${ADDON_PATH}'
+        VOLTO_VERSION: '${VOLTO_VERSION:-17}'
     volumes:
       - ${CURRENT_DIR}:/app/src/addons/${ADDON_PATH}/
     environment:
@@ -35,9 +34,9 @@ services:
       context: ../
       dockerfile: ./dockerfiles/frontend/Dockerfile.ci
       args:
-        ADDON_NAME: "${ADDON_NAME}"
-        ADDON_PATH: "${ADDON_PATH}"
-        VOLTO_VERSION: "${VOLTO_VERSION:-17}"
+        ADDON_NAME: '${ADDON_NAME}'
+        ADDON_PATH: '${ADDON_PATH}'
+        VOLTO_VERSION: '${VOLTO_VERSION:-17}'
     environment:
       RAZZLE_INTERNAL_API_PATH: http://backend:8080/Plone
       RAZZLE_API_PATH: http://localhost:8080/Plone
@@ -55,11 +54,11 @@ services:
       context: ./backend
       dockerfile: ./Dockerfile.dev
       args:
-        PLONE_VERSION: 6.0.10.1
+        PLONE_VERSION: 6.0.11.1
     environment:
-      ZSERVER_HOST: "0.0.0.0"
-      ZSERVER_PORT: "8080"
-      LANGUAGE: "de"
+      ZSERVER_HOST: '0.0.0.0'
+      ZSERVER_PORT: '8080'
+      LANGUAGE: 'de'
       CELERY_BROKER: ${CELERY_BROKER?unset}
       INDEX_SERVER: ${INDEX_SERVER?unset}
       INDEX_OPENSEARCH: ${INDEX_OPENSEARCH?unset}
@@ -68,8 +67,8 @@ services:
       INDEX_PASSWORD: ${INDEX_PASSWORD?unset}
       SITE: Plone
       CORS_: true
-      ADDONS: "collective.elastic.plone[redis,opensearch]==2.1.0"
-      PROFILES: "collective.elastic.plone:default"
+      ADDONS: 'collective.elastic.plone[redis,opensearch]==2.1.0'
+      PROFILES: 'collective.elastic.plone:default'
       DELETE_EXISTING: False
     ports:
       - 8080:8080
@@ -79,7 +78,6 @@ services:
       - dev
       - prod
 
-
   ingest:
     image: ghcr.io/collective/collective.elastic.ingest:latest
     environment:
@@ -106,6 +104,7 @@ services:
     profiles:
       - dev
       - prod
+      - opensearchandingest
 
   redis:
     image: 'redis:latest'
@@ -116,17 +115,18 @@ services:
     profiles:
       - dev
       - prod
+      - opensearchandingest
 
   opensearch:
     build:
       context: ../
-      dockerfile: ./docker-opensearch/Dockerfile.opensearch
+      dockerfile: ./dockerfiles/opensearch/Dockerfile.opensearch
     environment:
       - cluster.name=opensearch-cluster
       - node.name=opensearch
       - discovery.type=single-node
       - bootstrap.memory_lock=true
-      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx2g"
+      - 'OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx2g'
       # - plugins.security.disabled=true
       - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${INDEX_PASSWORD?unset}
     ulimits:
@@ -147,18 +147,19 @@ services:
     profiles:
       - dev
       - prod
-
+      - opensearchandingest
 
   opensearch-dashboards:
     image: opensearchproject/opensearch-dashboards:latest
     ports:
       - 5601:5601
     expose:
-      - "5601"
+      - '5601'
     environment:
       OPENSEARCH_HOSTS: '["https://opensearch:9200"]'
     profiles:
       - dev
+      - opensearchandingest
     networks:
       - opensearch-net
 
@@ -178,7 +179,7 @@ volumes:
     driver: local
     driver_opts:
       type: none
-      device: ../docker-opensearch/ingest-configuration
+      device: ../dockerfiles/opensearch/ingest-configuration
       o: bind
 
 networks:

diff --git a/dockerfiles/opensearch/Dockerfile.opensearch b/dockerfiles/opensearch/Dockerfile.opensearch
@@ -0,0 +1,8 @@
+FROM opensearchproject/opensearch:latest
+
+COPY dockerfiles/opensearch/opensearch-configuration/keywords.txt /usr/share/opensearch/config/keywords.txt
+COPY dockerfiles/opensearch/opensearch-configuration/lexicon.txt /usr/share/opensearch/config/lexicon.txt
+COPY dockerfiles/opensearch/opensearch-configuration/keywords_english.txt /usr/share/opensearch/config/keywords_english.txt
+COPY dockerfiles/opensearch/opensearch-configuration/lexicon_english.txt /usr/share/opensearch/config/lexicon_english.txt
+
+RUN /usr/share/opensearch/bin/opensearch-plugin install --batch ingest-attachment
diff --git a/docker-opensearch/README.md → dockerfiles/opensearch/README.md b/docker-opensearch/README.md → dockerfiles/opensearch/README.md
diff --git a/...search/ingest-configuration/analysis.json → ...search/ingest-configuration/analysis.json b/...search/ingest-configuration/analysis.json → ...search/ingest-configuration/analysis.json
diff --git a/...search/ingest-configuration/mappings.json → ...search/ingest-configuration/mappings.json b/...search/ingest-configuration/mappings.json → ...search/ingest-configuration/mappings.json
diff --git a/.../ingest-configuration/preprocessings.json → .../ingest-configuration/preprocessings.json b/.../ingest-configuration/preprocessings.json → .../ingest-configuration/preprocessings.json
diff --git a/...rch/opensearch-configuration/keywords.txt → ...rch/opensearch-configuration/keywords.txt b/...rch/opensearch-configuration/keywords.txt → ...rch/opensearch-configuration/keywords.txt
diff --git a/...search-configuration/keywords_english.txt → ...search-configuration/keywords_english.txt b/...search-configuration/keywords_english.txt → ...search-configuration/keywords_english.txt
diff --git a/...arch/opensearch-configuration/lexicon.txt → ...arch/opensearch-configuration/lexicon.txt b/...arch/opensearch-configuration/lexicon.txt → ...arch/opensearch-configuration/lexicon.txt
diff --git a/...nsearch-configuration/lexicon_english.txt → ...nsearch-configuration/lexicon_english.txt b/...nsearch-configuration/lexicon_english.txt → ...nsearch-configuration/lexicon_english.txt
diff --git a/variables.mk b/variables.mk
@@ -1,5 +1,5 @@
-PLONE_VERSION=6.0.10.1
-VOLTO_VERSION=17.15.2
+PLONE_VERSION=6.0.11.1
+VOLTO_VERSION=17.16.0
 
 ADDON_NAME='@rohberg/volto-searchkit-block'
 ADDON_PATH='volto-searchkit-block'