From 041aaa400a4b943317ab65101d2604b0485653a1 Mon Sep 17 00:00:00 2001 From: Ivo Branco Date: Thu, 24 Feb 2022 15:55:25 +0000 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=A7(search)=20add=20elastic=20search?= =?UTF-8?q?=20client=20custom=20settings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add settings that permit to configure elastic search client with custom configuration like custom timeouts. Closes #1613 --- CHANGELOG.md | 5 +++ sandbox/settings.py | 4 +++ src/richie/apps/search/__init__.py | 3 +- src/richie/apps/search/index_manager.py | 46 ++++++++++++++++++++----- 4 files changed, 48 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f097ff1f3b..27e0923576 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,11 @@ Versioning](https://semver.org/spec/v2.0.0.html). - Update frontend overriding system to allow to override any frontend module. +## Added + +- Add settings that permit to configure elastic search client with custom + configuration like custom timeouts. + ## [2.13.0] - 2022-02-18 ### Added diff --git a/sandbox/settings.py b/sandbox/settings.py index b9ee3c5b93..830454062c 100644 --- a/sandbox/settings.py +++ b/sandbox/settings.py @@ -315,6 +315,10 @@ class Base(StyleguideMixin, DRFMixin, RichieCoursesConfigurationMixin, Configura default="richie", environ_name="RICHIE_ES_INDICES_PREFIX", environ_prefix=None ) RICHIE_ES_STATE_WEIGHTS = values.ListValue(None) + # Example to change the timeout: RICHIE_ES_CLIENT_KWARGS={'timeout': 30} + RICHIE_ES_CLIENT_KWARGS = values.DictValue( + {}, environ_name="RICHIE_ES_CLIENT_KWARGS", environ_prefix=None + ) # LTI Content RICHIE_LTI_PROVIDERS = { diff --git a/src/richie/apps/search/__init__.py b/src/richie/apps/search/__init__.py index c0f250adb3..b966f1e906 100644 --- a/src/richie/apps/search/__init__.py +++ b/src/richie/apps/search/__init__.py @@ -10,7 +10,8 @@ default_app_config = "richie.apps.search.apps.SearchConfig" ES_CLIENT = ElasticsearchClientCompat7to6( - [getattr(settings, "RICHIE_ES_HOST", "elasticsearch")] + [getattr(settings, "RICHIE_ES_HOST", "elasticsearch")], + **getattr(settings, "RICHIE_ES_CLIENT_KWARGS", {}), ) ES_INDICES_CLIENT = ElasticsearchIndicesClientCompat7to6(ES_CLIENT) diff --git a/src/richie/apps/search/index_manager.py b/src/richie/apps/search/index_manager.py index 25008e8776..ddaf4a37eb 100644 --- a/src/richie/apps/search/index_manager.py +++ b/src/richie/apps/search/index_manager.py @@ -46,14 +46,31 @@ def perform_create_index(indexable, logger=None): # Create the new index if logger: logger.info(f'Creating a new Elasticsearch index "{new_index:s}"...') - ES_INDICES_CLIENT.create(index=new_index) + ES_INDICES_CLIENT.create( + index=new_index, + **getattr(settings, "RICHIE_ES_INDICES_CLIENT_CREATE_KWARGS", {}), + ) # The index needs to be closed before we set an analyzer - ES_INDICES_CLIENT.close(index=new_index) - ES_INDICES_CLIENT.put_settings(body=ANALYSIS_SETTINGS, index=new_index) - ES_INDICES_CLIENT.open(index=new_index) + ES_INDICES_CLIENT.close( + index=new_index, + **getattr(settings, "RICHIE_ES_INDICES_CLIENT_CLOSE_KWARGS", {}), + ) + ES_INDICES_CLIENT.put_settings( + body=ANALYSIS_SETTINGS, + index=new_index, + **getattr(settings, "RICHIE_ES_INDICES_CLIENT_PUT_SETTINGS_KWARGS", {}), + ) + ES_INDICES_CLIENT.open( + index=new_index, + **getattr(settings, "RICHIE_ES_INDICES_CLIENT_OPEN_KWARGS", {}), + ) - ES_INDICES_CLIENT.put_mapping(body=indexable.mapping, index=new_index) + ES_INDICES_CLIENT.put_mapping( + body=indexable.mapping, + index=new_index, + **getattr(settings, "RICHIE_ES_INDICES_CLIENT_PUT_MAPPING_KWARGS", {}), + ) # Populate the new index with data provided from our indexable class richie_bulk(indexable.get_es_documents(new_index)) @@ -69,7 +86,10 @@ def regenerate_indices(logger): """ # Get all existing indices once; we'll look up into this list many times try: - existing_indices = ES_INDICES_CLIENT.get_alias("*") + existing_indices = ES_INDICES_CLIENT.get_alias( + "*", + **getattr(settings, "RICHIE_ES_INDICES_CLIENT_GET_ALIAS_KWARGS", {}), + ) except NotFoundError: # Provide a fallback empty list so we don't have to check for its existence later on existing_indices = [] @@ -114,7 +134,8 @@ def regenerate_indices(logger): def perform_aliases_update(): try: ES_INDICES_CLIENT.update_aliases( - dict(actions=actions_to_create_aliases + actions_to_delete_aliases) + dict(actions=actions_to_create_aliases + actions_to_delete_aliases), + **getattr(settings, "RICHIE_ES_INDICES_CLIENT_UPDATE_ALIAS_KWARGS", {}), ) except RequestError as exception: # This operation can fail if an index exists with the same name as an alias we're @@ -131,7 +152,10 @@ def perform_aliases_update(): exception.info["error"]["reason"] ).group(1) # Delete it (it was unusable and we can recreate its data at-will) - ES_INDICES_CLIENT.delete(index=broken_index) + ES_INDICES_CLIENT.delete( + index=broken_index, + **getattr(settings, "RICHIE_ES_INDICES_CLIENT_DELETE_KWARGS", {}), + ) # Attempt to perform the operation again # We're doing this recursively in case more than one such broken indices existed # (eg. "richie_courses" and "richie_organizations") @@ -148,7 +172,11 @@ def perform_aliases_update(): # anti-pattern. # # pylint: disable=unexpected-keyword-arg - ES_INDICES_CLIENT.delete(index=useless_index, ignore=[400, 404]) + ES_INDICES_CLIENT.delete( + index=useless_index, + ignore=[400, 404], + **getattr(settings, "RICHIE_ES_INDICES_CLIENT_DELETE_KWARGS", {}), + ) def store_es_scripts(logger=None):