Skip to content

Commit

Permalink
handle search_phase_execution_exception with incompatible data types …
Browse files Browse the repository at this point in the history 
…dring search #352
  • Loading branch information
@toluaina
toluaina committed Nov 8, 2022
1 parent 39540bc commit 3fe7ffe
Show file tree
Hide file tree
Showing 5 changed files with 37 additions and 31 deletions.
10 changes: 8 additions & 2 deletions pgsync/elastichelper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

import boto3
from elasticsearch import Elasticsearch, helpers, RequestsHttpConnection
from elasticsearch.exceptions import RequestError
from elasticsearch_dsl import Q, Search
from elasticsearch_dsl.query import Bool
from requests_aws4auth import AWS4Auth
Expand Down Expand Up @@ -227,8 +228,13 @@ def _search(self, index: str, table: str, fields: Optional[dict] = None):
]
)
)
for hit in search.scan():
yield hit.meta.id
try:
for hit in search.scan():
yield hit.meta.id
except RequestError as e:
logger.warning(f"RequestError: {e}")
if "is out of range for a long" not in str(e):
raise

def search(self, index: str, body: dict):
"""
Expand Down
12 changes: 6 additions & 6 deletions pgsync/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

logger = logging.getLogger(__name__)

HIGHLIGHT_START = "\033[4m"
HIGHLIGHT_BEGIN = "\033[4m"
HIGHLIGHT_END = "\033[0m:"


Expand Down Expand Up @@ -95,20 +95,20 @@ def get_redacted_url(result: ParseResult) -> ParseResult:

def show_settings(schema: Optional[str] = None) -> None:
"""Show settings."""
logger.info(f"{HIGHLIGHT_START}Settings{HIGHLIGHT_END}")
logger.info(f"{HIGHLIGHT_BEGIN}Settings{HIGHLIGHT_END}")
logger.info(f'{"Schema":<10s}: {schema or SCHEMA}')
logger.info("-" * 65)
logger.info(f"{HIGHLIGHT_START}Checkpoint{HIGHLIGHT_END}")
logger.info(f"{HIGHLIGHT_BEGIN}Checkpoint{HIGHLIGHT_END}")
logger.info(f"Path: {CHECKPOINT_PATH}")
logger.info(f"{HIGHLIGHT_START}Postgres{HIGHLIGHT_END}")
logger.info(f"{HIGHLIGHT_BEGIN}Postgres{HIGHLIGHT_END}")
result: ParseResult = get_redacted_url(
urlparse(get_postgres_url("postgres"))
)
logger.info(f"URL: {result.geturl()}")
result = get_redacted_url(urlparse(get_elasticsearch_url()))
logger.info(f"{HIGHLIGHT_START}Elasticsearch{HIGHLIGHT_END}")
logger.info(f"{HIGHLIGHT_BEGIN}Elasticsearch{HIGHLIGHT_END}")
logger.info(f"URL: {result.geturl()}")
logger.info(f"{HIGHLIGHT_START}Redis{HIGHLIGHT_END}")
logger.info(f"{HIGHLIGHT_BEGIN}Redis{HIGHLIGHT_END}")
result = get_redacted_url(urlparse(get_redis_url()))
logger.info(f"URL: {result.geturl()}")
logger.info("-" * 65)
Expand Down
18 changes: 9 additions & 9 deletions requirements/dev.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@ attrs==22.1.0
# via pytest
black==22.10.0
# via -r requirements/base.in
boto3==1.25.4
boto3==1.26.5
# via -r requirements/base.in
botocore==1.28.4
botocore==1.29.5
# via
# boto3
# s3transfer
Expand Down Expand Up @@ -51,9 +51,9 @@ elasticsearch-dsl==7.4.0
# via -r requirements/base.in
environs==9.5.0
# via -r requirements/base.in
exceptiongroup==1.0.0
exceptiongroup==1.0.1
# via pytest
faker==15.1.1
faker==15.3.1
# via -r requirements/base.in
filelock==3.8.0
# via virtualenv
Expand All @@ -75,7 +75,7 @@ flake8-todo==0.7
# via -r requirements/test.in
freezegun==1.2.2
# via -r requirements/test.in
greenlet==1.1.3.post0
greenlet==2.0.1
# via sqlalchemy
identify==2.5.8
# via pre-commit
Expand Down Expand Up @@ -112,7 +112,7 @@ pep517==0.13.0
# via build
pip-tools==6.9.0
# via -r requirements/dev.in
platformdirs==2.5.2
platformdirs==2.5.3
# via
# black
# virtualenv
Expand Down Expand Up @@ -146,7 +146,7 @@ pytest-mock==3.10.0
# via -r requirements/test.in
pytest-runner==6.0.0
# via -r requirements/test.in
pytest-sugar==0.9.5
pytest-sugar==0.9.6
# via -r requirements/test.in
python-dateutil==2.8.2
# via
Expand All @@ -173,7 +173,7 @@ six==1.16.0
# requests-aws4auth
snowballstemmer==2.2.0
# via pydocstyle
sqlalchemy==1.4.42
sqlalchemy==1.4.43
# via -r requirements/base.in
sqlparse==0.4.3
# via -r requirements/base.in
Expand All @@ -196,7 +196,7 @@ urllib3==1.26.12
# requests
virtualenv==20.16.6
# via pre-commit
wheel==0.37.1
wheel==0.38.3
# via pip-tools
wrapt==1.14.1
# via deprecated
Expand Down
12 changes: 6 additions & 6 deletions requirements/prod.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@ async-timeout==4.0.2
# via redis
black==22.10.0
# via -r requirements/base.in
boto3==1.25.4
boto3==1.26.5
# via -r requirements/base.in
botocore==1.28.4
botocore==1.29.5
# via
# boto3
# s3transfer
Expand Down Expand Up @@ -38,9 +38,9 @@ elasticsearch-dsl==7.4.0
# via -r requirements/base.in
environs==9.5.0
# via -r requirements/base.in
faker==15.1.1
faker==15.3.1
# via -r requirements/base.in
greenlet==1.1.3.post0
greenlet==2.0.1
# via sqlalchemy
idna==3.4
# via requests
Expand All @@ -60,7 +60,7 @@ packaging==21.3
# redis
pathspec==0.10.1
# via black
platformdirs==2.5.2
platformdirs==2.5.3
# via black
psycopg2-binary==2.9.5
# via -r requirements/base.in
Expand All @@ -86,7 +86,7 @@ six==1.16.0
# elasticsearch-dsl
# python-dateutil
# requests-aws4auth
sqlalchemy==1.4.42
sqlalchemy==1.4.43
# via -r requirements/base.in
sqlparse==0.4.3
# via -r requirements/base.in
Expand Down
16 changes: 8 additions & 8 deletions requirements/test.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@ attrs==22.1.0
# via pytest
black==22.10.0
# via -r requirements/base.in
boto3==1.25.4
boto3==1.26.5
# via -r requirements/base.in
botocore==1.28.4
botocore==1.29.5
# via
# boto3
# s3transfer
Expand Down Expand Up @@ -42,9 +42,9 @@ elasticsearch-dsl==7.4.0
# via -r requirements/base.in
environs==9.5.0
# via -r requirements/base.in
exceptiongroup==1.0.0
exceptiongroup==1.0.1
# via pytest
faker==15.1.1
faker==15.3.1
# via -r requirements/base.in
flake8==5.0.4
# via
Expand All @@ -64,7 +64,7 @@ flake8-todo==0.7
# via -r requirements/test.in
freezegun==1.2.2
# via -r requirements/test.in
greenlet==1.1.3.post0
greenlet==2.0.1
# via sqlalchemy
idna==3.4
# via requests
Expand Down Expand Up @@ -92,7 +92,7 @@ packaging==21.3
# redis
pathspec==0.10.1
# via black
platformdirs==2.5.2
platformdirs==2.5.3
# via black
pluggy==1.0.0
# via pytest
Expand Down Expand Up @@ -122,7 +122,7 @@ pytest-mock==3.10.0
# via -r requirements/test.in
pytest-runner==6.0.0
# via -r requirements/test.in
pytest-sugar==0.9.5
pytest-sugar==0.9.6
# via -r requirements/test.in
python-dateutil==2.8.2
# via
Expand All @@ -147,7 +147,7 @@ six==1.16.0
# requests-aws4auth
snowballstemmer==2.2.0
# via pydocstyle
sqlalchemy==1.4.42
sqlalchemy==1.4.43
# via -r requirements/base.in
sqlparse==0.4.3
# via -r requirements/base.in
Expand Down

0 comments on commit 3fe7ffe

Please sign in to comment.