From 5446d47b74ca7c387d0275330254ea2725d4df41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Brand=C3=A3o?= <555migalves555@gmail.com> Date: Mon, 31 Jul 2023 14:45:46 +0100 Subject: [PATCH 1/5] #3827 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miguel Brandão <555migalves555@gmail.com> --- deepsearch/cps/cli/data_indices_typer.py | 5 ++++- deepsearch/cps/cli/projects.py | 4 ++-- deepsearch/cps/data_indices/utils.py | 7 +++---- deepsearch/documents/core/common_routines.py | 3 ++- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/deepsearch/cps/cli/data_indices_typer.py b/deepsearch/cps/cli/data_indices_typer.py index ade0f569..2f8e5b1f 100644 --- a/deepsearch/cps/cli/data_indices_typer.py +++ b/deepsearch/cps/cli/data_indices_typer.py @@ -55,7 +55,7 @@ def list( for index in indices ] except ValueError as e: - print(f"Error occurred: {e}") + typer.echo(f"Error occurred: {e}") cli_output(results, output, headers="keys") return @@ -165,6 +165,9 @@ def upload_files( s3_coordinates=cos_coordinates, ) + # TODO since statuses are not returned from utils this is not very useful! + typer.echo("File upload queued successfully") + @app.command( name="add-attachment", help="Add attachment to a index item", no_args_is_help=True diff --git a/deepsearch/cps/cli/projects.py b/deepsearch/cps/cli/projects.py index 7b0e6eb7..e82c3903 100644 --- a/deepsearch/cps/cli/projects.py +++ b/deepsearch/cps/cli/projects.py @@ -49,7 +49,7 @@ def assign_user( role=role, ) else: - print("Project not found") + typer.echo("Project not found") raise typer.Exit(code=1) @@ -63,7 +63,7 @@ def remove( if project is not None: api.projects.remove(project=project) else: - print("Project not found") + typer.echo("Project not found") raise typer.Exit(code=1) diff --git a/deepsearch/cps/data_indices/utils.py b/deepsearch/cps/data_indices/utils.py index 53058035..15fcee3e 100644 --- a/deepsearch/cps/data_indices/utils.py +++ b/deepsearch/cps/data_indices/utils.py @@ -13,7 +13,7 @@ from deepsearch.cps.client.components.data_indices import S3Coordinates from deepsearch.cps.client.components.elastic import ElasticProjectDataCollectionSource from deepsearch.documents.core import convert, input_process -from deepsearch.documents.core.common_routines import progressbar, success_message +from deepsearch.documents.core.common_routines import progressbar from deepsearch.documents.core.utils import cleanup, create_root_dir logger = logging.getLogger(__name__) @@ -92,7 +92,6 @@ def process_url_input( statuses = convert.check_cps_status_running_tasks( api=api, cps_proj_key=coords.proj_key, task_ids=task_ids ) - print(success_message) return @@ -158,7 +157,7 @@ def process_local_file( statuses = convert.check_cps_status_running_tasks( api=api, cps_proj_key=coords.proj_key, task_ids=task_ids ) - print(success_message) + cleanup(root_dir=root_dir) return @@ -196,5 +195,5 @@ def process_external_cos( statuses = convert.check_cps_status_running_tasks( api=api, cps_proj_key=coords.proj_key, task_ids=task_ids ) - print(success_message) + return diff --git a/deepsearch/documents/core/common_routines.py b/deepsearch/documents/core/common_routines.py index 80783190..a22935a2 100644 --- a/deepsearch/documents/core/common_routines.py +++ b/deepsearch/documents/core/common_routines.py @@ -15,4 +15,5 @@ class ProgressBarParameters: progressbar = ProgressBarParameters() -success_message = "Completed successfully" +# TODO possibly deprecated? confirm with @pva +# success_message = "Completed successfully" From 164f40b003f5cb41ace9f5b4f090c466227aa1ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Brand=C3=A3o?= <555migalves555@gmail.com> Date: Wed, 2 Aug 2023 11:16:07 +0100 Subject: [PATCH 2/5] statuses are now returned for file uploading and success message only printed if they are all successful, some TODOs still for #3820 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miguel Brandão <555migalves555@gmail.com> --- deepsearch/cps/cli/data_indices_typer.py | 11 ++++++++--- deepsearch/cps/data_indices/utils.py | 6 +++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/deepsearch/cps/cli/data_indices_typer.py b/deepsearch/cps/cli/data_indices_typer.py index 2f8e5b1f..069359b5 100644 --- a/deepsearch/cps/cli/data_indices_typer.py +++ b/deepsearch/cps/cli/data_indices_typer.py @@ -157,7 +157,7 @@ def upload_files( raise typer.Abort() coords = ElasticProjectDataCollectionSource(proj_key=proj_key, index_key=index_key) - utils.upload_files( + statuses = utils.upload_files( api=api, coords=coords, url=urls, @@ -165,8 +165,13 @@ def upload_files( s3_coordinates=cos_coordinates, ) - # TODO since statuses are not returned from utils this is not very useful! - typer.echo("File upload queued successfully") + if all([status=="SUCCESS" for status in statuses]): + typer.echo("File upload completed successfully") + else: + # TODO + typer.echo("Some files did not upload successfully") + + @app.command( diff --git a/deepsearch/cps/data_indices/utils.py b/deepsearch/cps/data_indices/utils.py index 15fcee3e..938d72b0 100644 --- a/deepsearch/cps/data_indices/utils.py +++ b/deepsearch/cps/data_indices/utils.py @@ -93,7 +93,7 @@ def process_url_input( api=api, cps_proj_key=coords.proj_key, task_ids=task_ids ) - return + return statuses def process_local_file( @@ -159,7 +159,7 @@ def process_local_file( ) cleanup(root_dir=root_dir) - return + return statuses def process_external_cos( @@ -196,4 +196,4 @@ def process_external_cos( api=api, cps_proj_key=coords.proj_key, task_ids=task_ids ) - return + return statuses From 5efa09560e11d0cd67f538b873b29702690e6d11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Brand=C3=A3o?= <555migalves555@gmail.com> Date: Wed, 2 Aug 2023 11:16:52 +0100 Subject: [PATCH 3/5] pre-commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miguel Brandão <555migalves555@gmail.com> --- deepsearch/cps/cli/data_indices_typer.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/deepsearch/cps/cli/data_indices_typer.py b/deepsearch/cps/cli/data_indices_typer.py index 069359b5..eebb0e33 100644 --- a/deepsearch/cps/cli/data_indices_typer.py +++ b/deepsearch/cps/cli/data_indices_typer.py @@ -165,15 +165,13 @@ def upload_files( s3_coordinates=cos_coordinates, ) - if all([status=="SUCCESS" for status in statuses]): + if all([status == "SUCCESS" for status in statuses]): typer.echo("File upload completed successfully") else: # TODO typer.echo("Some files did not upload successfully") - - @app.command( name="add-attachment", help="Add attachment to a index item", no_args_is_help=True ) From c58ae6c8054a933b1ac7745a567544347fddbdc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Brand=C3=A3o?= <555migalves555@gmail.com> Date: Fri, 4 Aug 2023 11:16:59 +0100 Subject: [PATCH 4/5] typer abort correcly being invoked in data-indices list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miguel Brandão <555migalves555@gmail.com> --- deepsearch/cps/cli/data_indices_typer.py | 2 ++ deepsearch/documents/core/common_routines.py | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deepsearch/cps/cli/data_indices_typer.py b/deepsearch/cps/cli/data_indices_typer.py index eebb0e33..b32ad1ab 100644 --- a/deepsearch/cps/cli/data_indices_typer.py +++ b/deepsearch/cps/cli/data_indices_typer.py @@ -44,6 +44,7 @@ def list( try: indices = api.data_indices.list(proj_key=proj_key) + print(indices) results = [ { "Index key": index.source.index_key, @@ -56,6 +57,7 @@ def list( ] except ValueError as e: typer.echo(f"Error occurred: {e}") + raise typer.Abort() cli_output(results, output, headers="keys") return diff --git a/deepsearch/documents/core/common_routines.py b/deepsearch/documents/core/common_routines.py index a22935a2..b54b3ce3 100644 --- a/deepsearch/documents/core/common_routines.py +++ b/deepsearch/documents/core/common_routines.py @@ -15,5 +15,3 @@ class ProgressBarParameters: progressbar = ProgressBarParameters() -# TODO possibly deprecated? confirm with @pva -# success_message = "Completed successfully" From 92974e9dcf422f18c4474e8405b05a200c27ac88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Brand=C3=A3o?= <555migalves555@gmail.com> Date: Mon, 7 Aug 2023 10:36:00 +0100 Subject: [PATCH 5/5] removed redundant print MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miguel Brandão <555migalves555@gmail.com> --- deepsearch/cps/cli/data_indices_typer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/deepsearch/cps/cli/data_indices_typer.py b/deepsearch/cps/cli/data_indices_typer.py index b32ad1ab..fd689fb6 100644 --- a/deepsearch/cps/cli/data_indices_typer.py +++ b/deepsearch/cps/cli/data_indices_typer.py @@ -44,7 +44,6 @@ def list( try: indices = api.data_indices.list(proj_key=proj_key) - print(indices) results = [ { "Index key": index.source.index_key,