From 23eb45c0ae27948de276c07c327d17f50777687f Mon Sep 17 00:00:00 2001 From: Tiago Santana <54704492+SantanaTiago@users.noreply.github.com> Date: Thu, 27 Jul 2023 09:23:35 +0100 Subject: [PATCH 1/4] set coordinates payload to dict Signed-off-by: Tiago Santana <54704492+SantanaTiago@users.noreply.github.com> --- deepsearch/cps/client/components/data_indices.py | 2 +- deepsearch/cps/data_indices/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deepsearch/cps/client/components/data_indices.py b/deepsearch/cps/client/components/data_indices.py index fe5c351b..a5f04bd1 100644 --- a/deepsearch/cps/client/components/data_indices.py +++ b/deepsearch/cps/client/components/data_indices.py @@ -102,7 +102,7 @@ def delete( def upload_file( self, coords: ElasticProjectDataCollectionSource, - body: Union[Dict[str, List[str]], Dict[str, Dict[str, S3Coordinates]]], + body: Union[Dict[str, List[str]], Dict[str, Dict[str, Dict]]], ) -> str: """ Call api for converting and uploading file to a project's data index. diff --git a/deepsearch/cps/data_indices/utils.py b/deepsearch/cps/data_indices/utils.py index a8ca44ce..53058035 100644 --- a/deepsearch/cps/data_indices/utils.py +++ b/deepsearch/cps/data_indices/utils.py @@ -183,7 +183,7 @@ def process_external_cos( bar_format=progressbar.bar_format, ) as progress: # upload using coordinates - payload = {"s3_source": {"coordinates": s3_coordinates}} + payload = {"s3_source": {"coordinates": s3_coordinates.dict()}} task_id = api.data_indices.upload_file( coords=coords, body=payload, From d4d39bb9ed9edc5ca34bb28965ea43ca4a39caaf Mon Sep 17 00:00:00 2001 From: Tiago Santana <54704492+SantanaTiago@users.noreply.github.com> Date: Thu, 27 Jul 2023 09:55:59 +0100 Subject: [PATCH 2/4] safe convert S3Coordinates to dict Signed-off-by: Tiago Santana <54704492+SantanaTiago@users.noreply.github.com> --- deepsearch/cps/data_indices/utils.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/deepsearch/cps/data_indices/utils.py b/deepsearch/cps/data_indices/utils.py index 53058035..706d19ad 100644 --- a/deepsearch/cps/data_indices/utils.py +++ b/deepsearch/cps/data_indices/utils.py @@ -175,6 +175,13 @@ def process_external_cos( # container for task_ids task_ids = [] + # coordinates to dict + s3_coordinates_dict: dict = {} + if isinstance(s3_coordinates, S3Coordinates): + s3_coordinates_dict = s3_coordinates.dict() + else: + s3_coordinates_dict = s3_coordinates + with tqdm( total=1, desc=f"{'Submitting input:': <{progressbar.padding}}", @@ -183,7 +190,7 @@ def process_external_cos( bar_format=progressbar.bar_format, ) as progress: # upload using coordinates - payload = {"s3_source": {"coordinates": s3_coordinates.dict()}} + payload = {"s3_source": {"coordinates": s3_coordinates_dict}} task_id = api.data_indices.upload_file( coords=coords, body=payload, From 85ec5e7bd65e702bd0f641e48ace2f797756a8c1 Mon Sep 17 00:00:00 2001 From: Tiago Santana <54704492+SantanaTiago@users.noreply.github.com> Date: Thu, 27 Jul 2023 12:20:21 +0100 Subject: [PATCH 3/4] fix parsing cos file. updated doc guide Signed-off-by: Tiago Santana <54704492+SantanaTiago@users.noreply.github.com> --- deepsearch/cps/cli/data_indices_typer.py | 2 +- deepsearch/cps/data_indices/utils.py | 9 +-------- docs/guide/data_indices.md | 13 ++++++++++++- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/deepsearch/cps/cli/data_indices_typer.py b/deepsearch/cps/cli/data_indices_typer.py index 3c735a21..ade0f569 100644 --- a/deepsearch/cps/cli/data_indices_typer.py +++ b/deepsearch/cps/cli/data_indices_typer.py @@ -150,7 +150,7 @@ def upload_files( cos_coordinates: Optional[S3Coordinates] = None if s3_coordinates is not None: try: - cos_coordinates = json.load(open(s3_coordinates)) + cos_coordinates = S3Coordinates.parse_file(s3_coordinates) except Exception as e: typer.echo(f"Error occurred: {e}") typer.echo(ERROR_MSG) diff --git a/deepsearch/cps/data_indices/utils.py b/deepsearch/cps/data_indices/utils.py index 706d19ad..53058035 100644 --- a/deepsearch/cps/data_indices/utils.py +++ b/deepsearch/cps/data_indices/utils.py @@ -175,13 +175,6 @@ def process_external_cos( # container for task_ids task_ids = [] - # coordinates to dict - s3_coordinates_dict: dict = {} - if isinstance(s3_coordinates, S3Coordinates): - s3_coordinates_dict = s3_coordinates.dict() - else: - s3_coordinates_dict = s3_coordinates - with tqdm( total=1, desc=f"{'Submitting input:': <{progressbar.padding}}", @@ -190,7 +183,7 @@ def process_external_cos( bar_format=progressbar.bar_format, ) as progress: # upload using coordinates - payload = {"s3_source": {"coordinates": s3_coordinates_dict}} + payload = {"s3_source": {"coordinates": s3_coordinates.dict()}} task_id = api.data_indices.upload_file( coords=coords, body=payload, diff --git a/docs/guide/data_indices.md b/docs/guide/data_indices.md index 12394ecc..0aa55ae9 100644 --- a/docs/guide/data_indices.md +++ b/docs/guide/data_indices.md @@ -102,6 +102,9 @@ Documents can be converted and added, directly, to a data index in a project. Br // for online documents $ deepsearch cps data-indices upload -p PROJ_KEY -x INDEX_KEY -u PATH_URL + + // for COS documents + $ deepsearch cps data-indices upload -p PROJ_KEY -x INDEX_KEY -c PATH_COS_COORDINATES ``` @@ -123,7 +126,15 @@ Documents can be converted and added, directly, to a data index in a project. Br # or, define a list directly #input_urls = ["https:///URL1", "https://URL2", "https://URL3"] - data_indices_utils.upload_files(api=api, coords=coords, url=input_urls) + # For COS documents + cos_coordinates = S3Coordinates.parse_file(s3_coordinates) + + data_indices_utils.upload_files( + api=api, + coords=coords, + url=input_urls, + s3_coordinates=cos_coordinates, + ) ``` --- From 836eef92d741d542425b97805bbd4d4b87ba91cb Mon Sep 17 00:00:00 2001 From: Tiago Santana <54704492+SantanaTiago@users.noreply.github.com> Date: Thu, 27 Jul 2023 12:55:30 +0100 Subject: [PATCH 4/4] fixed data-indices guide Signed-off-by: Tiago Santana <54704492+SantanaTiago@users.noreply.github.com> --- docs/guide/data_indices.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/docs/guide/data_indices.md b/docs/guide/data_indices.md index 0aa55ae9..3a81a591 100644 --- a/docs/guide/data_indices.md +++ b/docs/guide/data_indices.md @@ -126,15 +126,12 @@ Documents can be converted and added, directly, to a data index in a project. Br # or, define a list directly #input_urls = ["https:///URL1", "https://URL2", "https://URL3"] + data_indices_utils.upload_files(api=api, coords=coords, url=input_urls) + # For COS documents cos_coordinates = S3Coordinates.parse_file(s3_coordinates) - data_indices_utils.upload_files( - api=api, - coords=coords, - url=input_urls, - s3_coordinates=cos_coordinates, - ) + data_indices_utils.upload_files(api=api, coords=coords, s3_coordinates=cos_coordinates) ``` ---