Skip to content

Commit

Permalink
🔧(search) add elastic search client custom settings
Browse files Browse the repository at this point in the history
Add settings that permit to configure elastic search client with custom
configuration like custom timeouts. Closes #1613
  • Loading branch information
igobranco committed Mar 11, 2022
1 parent 765a86c commit 041aaa4
Show file tree
Hide file tree
Showing 4 changed files with 48 additions and 10 deletions.
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,11 @@ Versioning](https://semver.org/spec/v2.0.0.html).

- Update frontend overriding system to allow to override any frontend module.

## Added

- Add settings that permit to configure elastic search client with custom
configuration like custom timeouts.

## [2.13.0] - 2022-02-18

### Added
Expand Down
4 changes: 4 additions & 0 deletions sandbox/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -315,6 +315,10 @@ class Base(StyleguideMixin, DRFMixin, RichieCoursesConfigurationMixin, Configura
default="richie", environ_name="RICHIE_ES_INDICES_PREFIX", environ_prefix=None
)
RICHIE_ES_STATE_WEIGHTS = values.ListValue(None)
# Example to change the timeout: RICHIE_ES_CLIENT_KWARGS={'timeout': 30}
RICHIE_ES_CLIENT_KWARGS = values.DictValue(
{}, environ_name="RICHIE_ES_CLIENT_KWARGS", environ_prefix=None
)

# LTI Content
RICHIE_LTI_PROVIDERS = {
Expand Down
3 changes: 2 additions & 1 deletion src/richie/apps/search/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@
default_app_config = "richie.apps.search.apps.SearchConfig"

ES_CLIENT = ElasticsearchClientCompat7to6(
[getattr(settings, "RICHIE_ES_HOST", "elasticsearch")]
[getattr(settings, "RICHIE_ES_HOST", "elasticsearch")],
**getattr(settings, "RICHIE_ES_CLIENT_KWARGS", {}),
)

ES_INDICES_CLIENT = ElasticsearchIndicesClientCompat7to6(ES_CLIENT)
46 changes: 37 additions & 9 deletions src/richie/apps/search/index_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,14 +46,31 @@ def perform_create_index(indexable, logger=None):
# Create the new index
if logger:
logger.info(f'Creating a new Elasticsearch index "{new_index:s}"...')
ES_INDICES_CLIENT.create(index=new_index)
ES_INDICES_CLIENT.create(
index=new_index,
**getattr(settings, "RICHIE_ES_INDICES_CLIENT_CREATE_KWARGS", {}),
)

# The index needs to be closed before we set an analyzer
ES_INDICES_CLIENT.close(index=new_index)
ES_INDICES_CLIENT.put_settings(body=ANALYSIS_SETTINGS, index=new_index)
ES_INDICES_CLIENT.open(index=new_index)
ES_INDICES_CLIENT.close(
index=new_index,
**getattr(settings, "RICHIE_ES_INDICES_CLIENT_CLOSE_KWARGS", {}),
)
ES_INDICES_CLIENT.put_settings(
body=ANALYSIS_SETTINGS,
index=new_index,
**getattr(settings, "RICHIE_ES_INDICES_CLIENT_PUT_SETTINGS_KWARGS", {}),
)
ES_INDICES_CLIENT.open(
index=new_index,
**getattr(settings, "RICHIE_ES_INDICES_CLIENT_OPEN_KWARGS", {}),
)

ES_INDICES_CLIENT.put_mapping(body=indexable.mapping, index=new_index)
ES_INDICES_CLIENT.put_mapping(
body=indexable.mapping,
index=new_index,
**getattr(settings, "RICHIE_ES_INDICES_CLIENT_PUT_MAPPING_KWARGS", {}),
)

# Populate the new index with data provided from our indexable class
richie_bulk(indexable.get_es_documents(new_index))
Expand All @@ -69,7 +86,10 @@ def regenerate_indices(logger):
"""
# Get all existing indices once; we'll look up into this list many times
try:
existing_indices = ES_INDICES_CLIENT.get_alias("*")
existing_indices = ES_INDICES_CLIENT.get_alias(
"*",
**getattr(settings, "RICHIE_ES_INDICES_CLIENT_GET_ALIAS_KWARGS", {}),
)
except NotFoundError:
# Provide a fallback empty list so we don't have to check for its existence later on
existing_indices = []
Expand Down Expand Up @@ -114,7 +134,8 @@ def regenerate_indices(logger):
def perform_aliases_update():
try:
ES_INDICES_CLIENT.update_aliases(
dict(actions=actions_to_create_aliases + actions_to_delete_aliases)
dict(actions=actions_to_create_aliases + actions_to_delete_aliases),
**getattr(settings, "RICHIE_ES_INDICES_CLIENT_UPDATE_ALIAS_KWARGS", {}),
)
except RequestError as exception:
# This operation can fail if an index exists with the same name as an alias we're
Expand All @@ -131,7 +152,10 @@ def perform_aliases_update():
exception.info["error"]["reason"]
).group(1)
# Delete it (it was unusable and we can recreate its data at-will)
ES_INDICES_CLIENT.delete(index=broken_index)
ES_INDICES_CLIENT.delete(
index=broken_index,
**getattr(settings, "RICHIE_ES_INDICES_CLIENT_DELETE_KWARGS", {}),
)
# Attempt to perform the operation again
# We're doing this recursively in case more than one such broken indices existed
# (eg. "richie_courses" and "richie_organizations")
Expand All @@ -148,7 +172,11 @@ def perform_aliases_update():
# anti-pattern.
#
# pylint: disable=unexpected-keyword-arg
ES_INDICES_CLIENT.delete(index=useless_index, ignore=[400, 404])
ES_INDICES_CLIENT.delete(
index=useless_index,
ignore=[400, 404],
**getattr(settings, "RICHIE_ES_INDICES_CLIENT_DELETE_KWARGS", {}),
)


def store_es_scripts(logger=None):
Expand Down

0 comments on commit 041aaa4

Please sign in to comment.