From 18d4ae2b523a0d287a13d8834e8c3b8b0683bf7e Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Thu, 22 Jun 2023 17:54:48 +0000 Subject: [PATCH 01/70] initial zstandard compression for parameters and results; also remove embedded liveaction doc in executions --- .../action_chain_runner.py | 4 +- .../tests/unit/test_actionchain_cancel.py | 12 +-- .../unit/test_actionchain_notifications.py | 8 +- .../unit/test_actionchain_pause_resume.py | 14 +-- .../inquirer_runner/inquirer_runner.py | 2 +- .../orquesta_runner/orquesta_runner.py | 13 +-- .../orquesta_runner/tests/unit/test_basic.py | 20 ++--- .../orquesta_runner/tests/unit/test_cancel.py | 8 +- .../tests/unit/test_data_flow.py | 8 +- .../tests/unit/test_error_handling.py | 51 +++++++---- .../tests/unit/test_functions_common.py | 2 +- .../tests/unit/test_functions_task.py | 2 +- .../tests/unit/test_inquiries.py | 38 ++++---- .../orquesta_runner/tests/unit/test_notify.py | 17 ++-- .../tests/unit/test_pause_and_resume.py | 54 ++++++------ .../orquesta_runner/tests/unit/test_rerun.py | 24 ++--- .../tests/unit/test_with_items.py | 8 +- st2actions/st2actions/container/base.py | 4 +- st2actions/st2actions/notifier/notifier.py | 2 +- .../policies/concurrency_by_attr.py | 56 +++++++----- st2actions/st2actions/scheduler/entrypoint.py | 2 +- st2actions/st2actions/scheduler/handler.py | 2 +- st2actions/st2actions/worker.py | 6 +- st2actions/st2actions/workflows/workflows.py | 5 +- .../tests/unit/policies/test_concurrency.py | 2 +- .../tests/unit/policies/test_retry_policy.py | 6 +- st2actions/tests/unit/test_executions.py | 15 ++-- st2actions/tests/unit/test_notifier.py | 14 +-- .../st2api/controllers/v1/actionexecutions.py | 13 +-- .../st2api/controllers/v1/aliasexecution.py | 12 ++- .../controllers/v1/test_alias_execution.py | 1 + .../unit/controllers/v1/test_executions.py | 4 +- st2common/bin/st2-track-result | 4 +- st2common/st2common/fields.py | 88 ++++--------------- .../garbage_collection/executions.py | 2 +- .../st2common/garbage_collection/inquiries.py | 2 +- st2common/st2common/models/api/action.py | 17 +++- st2common/st2common/models/api/base.py | 40 +++++---- st2common/st2common/models/api/execution.py | 19 +++- st2common/st2common/models/db/execution.py | 57 ++++-------- st2common/st2common/models/db/liveaction.py | 24 ++--- st2common/st2common/openapi.yaml | 2 +- st2common/st2common/openapi.yaml.j2 | 2 +- st2common/st2common/services/action.py | 16 ++-- st2common/st2common/services/executions.py | 9 +- st2common/st2common/services/inquiry.py | 2 +- st2common/st2common/services/policies.py | 9 +- st2common/st2common/services/trace.py | 2 +- st2common/st2common/services/workflows.py | 23 +++-- st2common/st2common/util/param.py | 2 + .../test_v35_migrate_db_dict_field_values.py | 10 +++ st2common/tests/unit/services/test_trace.py | 2 +- .../test_workflow_identify_orphans.py | 4 +- .../services/test_workflow_service_retries.py | 8 +- st2common/tests/unit/test_db_execution.py | 22 ++--- st2common/tests/unit/test_db_fields.py | 70 ++------------- st2common/tests/unit/test_executions.py | 33 +++---- st2common/tests/unit/test_executions_util.py | 14 +-- st2common/tests/unit/test_purge_executions.py | 2 +- .../v1/test_stream_execution_output.py | 4 +- st2tests/st2tests/api.py | 2 +- .../descendants/executions/child1_level1.yaml | 3 +- .../descendants/executions/child1_level2.yaml | 3 +- .../descendants/executions/child1_level3.yaml | 3 +- .../descendants/executions/child2_level1.yaml | 3 +- .../descendants/executions/child2_level2.yaml | 3 +- .../descendants/executions/child2_level3.yaml | 3 +- .../descendants/executions/child3_level2.yaml | 3 +- .../descendants/executions/child3_level3.yaml | 3 +- .../executions/root_execution.yaml | 3 +- .../generic/executions/execution1.yaml | 12 +-- .../generic/liveactions/parentliveaction.yaml | 2 +- .../actions/workflows/__init__.py | 0 .../packs/executions/liveactions.yaml | 3 + .../executions/execution_with_parent.yaml | 12 +-- .../executions/rule_fired_execution.yaml | 12 +-- .../executions/traceable_execution.yaml | 12 +-- 77 files changed, 447 insertions(+), 553 deletions(-) delete mode 100644 st2tests/st2tests/fixtures/packs/dummy_pack_23/actions/workflows/__init__.py diff --git a/contrib/runners/action_chain_runner/action_chain_runner/action_chain_runner.py b/contrib/runners/action_chain_runner/action_chain_runner/action_chain_runner.py index 57c015adcf..1e690f5640 100644 --- a/contrib/runners/action_chain_runner/action_chain_runner/action_chain_runner.py +++ b/contrib/runners/action_chain_runner/action_chain_runner/action_chain_runner.py @@ -333,7 +333,7 @@ def cancel(self): and child_exec.status in action_constants.LIVEACTION_CANCELABLE_STATES ): action_service.request_cancellation( - LiveAction.get(id=child_exec.liveaction["id"]), + LiveAction.get(id=child_exec.liveaction), self.context.get("user", None), ) @@ -353,7 +353,7 @@ def pause(self): and child_exec.status == action_constants.LIVEACTION_STATUS_RUNNING ): action_service.request_pause( - LiveAction.get(id=child_exec.liveaction["id"]), + LiveAction.get(id=child_exec.liveaction), self.context.get("user", None), ) diff --git a/contrib/runners/action_chain_runner/tests/unit/test_actionchain_cancel.py b/contrib/runners/action_chain_runner/tests/unit/test_actionchain_cancel.py index bf6ffdd47a..e72240e8f4 100644 --- a/contrib/runners/action_chain_runner/tests/unit/test_actionchain_cancel.py +++ b/contrib/runners/action_chain_runner/tests/unit/test_actionchain_cancel.py @@ -171,7 +171,7 @@ def test_chain_cancel_cascade_to_subworkflow(self): # Wait until the subworkflow is running. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"]) + task1_live = LiveAction.get_by_id(task1_exec.liveaction) task1_live = self._wait_on_status( task1_live, action_constants.LIVEACTION_STATUS_RUNNING ) @@ -189,7 +189,7 @@ def test_chain_cancel_cascade_to_subworkflow(self): # Wait until the subworkflow is canceling. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"]) + task1_live = LiveAction.get_by_id(task1_exec.liveaction) task1_live = self._wait_on_status( task1_live, action_constants.LIVEACTION_STATUS_CANCELING ) @@ -206,7 +206,7 @@ def test_chain_cancel_cascade_to_subworkflow(self): # Wait until the subworkflow is canceled. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"]) + task1_live = LiveAction.get_by_id(task1_exec.liveaction) task1_live = self._wait_on_status( task1_live, action_constants.LIVEACTION_STATUS_CANCELED ) @@ -248,7 +248,7 @@ def test_chain_cancel_cascade_to_parent_workflow(self): # Wait until the subworkflow is running. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"]) + task1_live = LiveAction.get_by_id(task1_exec.liveaction) task1_live = self._wait_on_status( task1_live, action_constants.LIVEACTION_STATUS_RUNNING ) @@ -260,7 +260,7 @@ def test_chain_cancel_cascade_to_parent_workflow(self): # Wait until the subworkflow is canceling. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"]) + task1_live = LiveAction.get_by_id(task1_exec.liveaction) task1_live = self._wait_on_status( task1_live, action_constants.LIVEACTION_STATUS_CANCELING ) @@ -271,7 +271,7 @@ def test_chain_cancel_cascade_to_parent_workflow(self): # Wait until the subworkflow is canceled. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"]) + task1_live = LiveAction.get_by_id(task1_exec.liveaction) task1_live = self._wait_on_status( task1_live, action_constants.LIVEACTION_STATUS_CANCELED ) diff --git a/contrib/runners/action_chain_runner/tests/unit/test_actionchain_notifications.py b/contrib/runners/action_chain_runner/tests/unit/test_actionchain_notifications.py index f2f2a680c8..d7eaf8c56c 100644 --- a/contrib/runners/action_chain_runner/tests/unit/test_actionchain_notifications.py +++ b/contrib/runners/action_chain_runner/tests/unit/test_actionchain_notifications.py @@ -151,7 +151,7 @@ def test_skip_notify_for_task_with_notify(self): # Assert task1 notify is skipped task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"]) + task1_live = LiveAction.get_by_id(task1_exec.liveaction) task1_live = self._wait_on_status( task1_live, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -162,7 +162,7 @@ def test_skip_notify_for_task_with_notify(self): # Assert task2 notify is not skipped task2_exec = ActionExecution.get_by_id(execution.children[1]) - task2_live = LiveAction.get_by_id(task2_exec.liveaction["id"]) + task2_live = LiveAction.get_by_id(task2_exec.liveaction) notify = notify_api_models.NotificationsHelper.from_model( notify_model=task2_live.notify ) @@ -186,7 +186,7 @@ def test_skip_notify_default_for_task_with_notify(self): # Assert task1 notify is set. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"]) + task1_live = LiveAction.get_by_id(task1_exec.liveaction) task1_live = self._wait_on_status( task1_live, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -200,7 +200,7 @@ def test_skip_notify_default_for_task_with_notify(self): # Assert task2 notify is not skipped by default. task2_exec = ActionExecution.get_by_id(execution.children[1]) - task2_live = LiveAction.get_by_id(task2_exec.liveaction["id"]) + task2_live = LiveAction.get_by_id(task2_exec.liveaction) self.assertIsNone(task2_live.notify) MockLiveActionPublisherNonBlocking.wait_all() diff --git a/contrib/runners/action_chain_runner/tests/unit/test_actionchain_pause_resume.py b/contrib/runners/action_chain_runner/tests/unit/test_actionchain_pause_resume.py index e0dfecc4d6..acacbeb0cd 100644 --- a/contrib/runners/action_chain_runner/tests/unit/test_actionchain_pause_resume.py +++ b/contrib/runners/action_chain_runner/tests/unit/test_actionchain_pause_resume.py @@ -431,7 +431,7 @@ def test_chain_pause_resume_cascade_to_subworkflow(self): # Wait until the subworkflow is running. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"]) + task1_live = LiveAction.get_by_id(task1_exec.liveaction) task1_live = self._wait_for_status( task1_live, action_constants.LIVEACTION_STATUS_RUNNING ) @@ -452,7 +452,7 @@ def test_chain_pause_resume_cascade_to_subworkflow(self): # Wait until the subworkflow is pausing. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"]) + task1_live = LiveAction.get_by_id(task1_exec.liveaction) task1_live = self._wait_for_status( task1_live, action_constants.LIVEACTION_STATUS_PAUSING ) @@ -477,7 +477,7 @@ def test_chain_pause_resume_cascade_to_subworkflow(self): # Wait until the subworkflow is paused. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"]) + task1_live = LiveAction.get_by_id(task1_exec.liveaction) task1_live = self._wait_for_status( task1_live, action_constants.LIVEACTION_STATUS_PAUSED ) @@ -548,7 +548,7 @@ def test_chain_pause_resume_cascade_to_parent_workflow(self): # Wait until the subworkflow is running. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"]) + task1_live = LiveAction.get_by_id(task1_exec.liveaction) task1_live = self._wait_for_status( task1_live, action_constants.LIVEACTION_STATUS_RUNNING ) @@ -559,7 +559,7 @@ def test_chain_pause_resume_cascade_to_parent_workflow(self): # Wait until the subworkflow is pausing. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"]) + task1_live = LiveAction.get_by_id(task1_exec.liveaction) task1_live = self._wait_for_status( task1_live, action_constants.LIVEACTION_STATUS_PAUSING ) @@ -574,7 +574,7 @@ def test_chain_pause_resume_cascade_to_parent_workflow(self): # Wait until the subworkflow is paused. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"]) + task1_live = LiveAction.get_by_id(task1_exec.liveaction) task1_live = self._wait_for_status( task1_live, action_constants.LIVEACTION_STATUS_PAUSED ) @@ -611,7 +611,7 @@ def test_chain_pause_resume_cascade_to_parent_workflow(self): # Wait until the subworkflow is paused. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"]) + task1_live = LiveAction.get_by_id(task1_exec.liveaction) task1_live = self._wait_for_status( task1_live, action_constants.LIVEACTION_STATUS_SUCCEEDED ) diff --git a/contrib/runners/inquirer_runner/inquirer_runner/inquirer_runner.py b/contrib/runners/inquirer_runner/inquirer_runner/inquirer_runner.py index af0f0c6f34..b33bd95761 100644 --- a/contrib/runners/inquirer_runner/inquirer_runner/inquirer_runner.py +++ b/contrib/runners/inquirer_runner/inquirer_runner/inquirer_runner.py @@ -74,7 +74,7 @@ def pre_run(self): def run(self, action_parameters): liveaction_db = action_utils.get_liveaction_by_id(self.liveaction_id) - exc = ex_db_access.ActionExecution.get(liveaction__id=str(liveaction_db.id)) + exc = ex_db_access.ActionExecution.get(liveaction=str(liveaction_db.id)) # Assemble and dispatch trigger trigger_ref = sys_db_models.ResourceReference.to_string_reference( diff --git a/contrib/runners/orquesta_runner/orquesta_runner/orquesta_runner.py b/contrib/runners/orquesta_runner/orquesta_runner/orquesta_runner.py index 586a9d0cc9..3657218e05 100644 --- a/contrib/runners/orquesta_runner/orquesta_runner/orquesta_runner.py +++ b/contrib/runners/orquesta_runner/orquesta_runner/orquesta_runner.py @@ -136,12 +136,15 @@ def start_workflow(self, action_parameters): wf_def, self.execution, st2_ctx, notify_cfg=notify_cfg ) except wf_exc.WorkflowInspectionError as e: + _, ex, tb = sys.exc_info() status = ac_const.LIVEACTION_STATUS_FAILED - result = {"errors": e.args[1], "output": None} + result = {"errors": e.args[1], "output": None, "traceback": "".join(traceback.format_tb(tb, 20))} return (status, result, self.context) except Exception as e: + _, ex, tb = sys.exc_info() status = ac_const.LIVEACTION_STATUS_FAILED - result = {"errors": [{"message": six.text_type(e)}], "output": None} + result = {"errors": [{"message": six.text_type(e)}], "output": None, + "traceback": "".join(traceback.format_tb(tb, 20))} return (status, result, self.context) return self._handle_workflow_return_value(wf_ex_db) @@ -178,7 +181,7 @@ def pause(self): child_ex = ex_db_access.ActionExecution.get(id=child_ex_id) if self.task_pauseable(child_ex): ac_svc.request_pause( - lv_db_access.LiveAction.get(id=child_ex.liveaction["id"]), + lv_db_access.LiveAction.get(id=child_ex.liveaction), self.context.get("user", None), ) @@ -209,7 +212,7 @@ def resume(self): child_ex = ex_db_access.ActionExecution.get(id=child_ex_id) if self.task_resumeable(child_ex): ac_svc.request_resume( - lv_db_access.LiveAction.get(id=child_ex.liveaction["id"]), + lv_db_access.LiveAction.get(id=child_ex.liveaction), self.context.get("user", None), ) @@ -270,7 +273,7 @@ def cancel(self): child_ex = ex_db_access.ActionExecution.get(id=child_ex_id) if self.task_cancelable(child_ex): ac_svc.request_cancellation( - lv_db_access.LiveAction.get(id=child_ex.liveaction["id"]), + lv_db_access.LiveAction.get(id=child_ex.liveaction), self.context.get("user", None), ) diff --git a/contrib/runners/orquesta_runner/tests/unit/test_basic.py b/contrib/runners/orquesta_runner/tests/unit/test_basic.py index 7c9351423a..b6da3e38e5 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_basic.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_basic.py @@ -184,7 +184,7 @@ def test_run_workflow(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) self.assertEqual(tk1_lv_ac_db.context.get("user"), username) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) self.assertTrue(wf_svc.is_action_execution_under_workflow_context(tk1_ac_ex_db)) @@ -204,7 +204,7 @@ def test_run_workflow(self): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction["id"]) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction) self.assertEqual(tk2_lv_ac_db.context.get("user"), username) self.assertEqual(tk2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) self.assertTrue(wf_svc.is_action_execution_under_workflow_context(tk2_ac_ex_db)) @@ -224,7 +224,7 @@ def test_run_workflow(self): tk3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk3_ex_db.id) )[0] - tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction["id"]) + tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction) self.assertEqual(tk3_lv_ac_db.context.get("user"), username) self.assertEqual(tk3_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) self.assertTrue(wf_svc.is_action_execution_under_workflow_context(tk3_ac_ex_db)) @@ -274,7 +274,7 @@ def test_run_workflow_with_unicode_input(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) wf_svc.handle_action_execution_completion(tk1_ac_ex_db) tk1_ex_db = wf_db_access.TaskExecution.get_by_id(tk1_ex_db.id) @@ -286,7 +286,7 @@ def test_run_workflow_with_unicode_input(self): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction["id"]) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction) self.assertEqual(tk2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) wf_svc.handle_action_execution_completion(tk2_ac_ex_db) tk2_ex_db = wf_db_access.TaskExecution.get_by_id(tk2_ex_db.id) @@ -298,7 +298,7 @@ def test_run_workflow_with_unicode_input(self): tk3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk3_ex_db.id) )[0] - tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction["id"]) + tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction) self.assertEqual(tk3_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) wf_svc.handle_action_execution_completion(tk3_ac_ex_db) tk3_ex_db = wf_db_access.TaskExecution.get_by_id(tk3_ex_db.id) @@ -347,7 +347,7 @@ def test_run_workflow_action_config_context(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) self.assertTrue(wf_svc.is_action_execution_under_workflow_context(tk1_ac_ex_db)) @@ -400,7 +400,7 @@ def test_run_workflow_with_action_less_tasks(self): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction["id"]) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction) self.assertEqual(tk2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion. @@ -412,7 +412,7 @@ def test_run_workflow_with_action_less_tasks(self): tk3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk3_ex_db.id) )[0] - tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction["id"]) + tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction) self.assertEqual(tk3_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion. @@ -433,7 +433,7 @@ def test_run_workflow_with_action_less_tasks(self): tk5_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk5_ex_db.id) )[0] - tk5_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk5_ac_ex_db.liveaction["id"]) + tk5_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk5_ac_ex_db.liveaction) self.assertEqual(tk5_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion. diff --git a/contrib/runners/orquesta_runner/tests/unit/test_cancel.py b/contrib/runners/orquesta_runner/tests/unit/test_cancel.py index cdffd6949d..0ec03aa15e 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_cancel.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_cancel.py @@ -140,7 +140,7 @@ def test_cancel_workflow_cascade_down_to_subworkflow(self): self.assertEqual(len(tk_ac_ex_dbs), 1) tk_lv_ac_db = lv_db_access.LiveAction.get_by_id( - tk_ac_ex_dbs[0].liveaction["id"] + tk_ac_ex_dbs[0].liveaction ) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) @@ -183,7 +183,7 @@ def test_cancel_subworkflow_cascade_up_to_workflow(self): self.assertEqual(len(tk_ac_ex_dbs), 1) tk_lv_ac_db = lv_db_access.LiveAction.get_by_id( - tk_ac_ex_dbs[0].liveaction["id"] + tk_ac_ex_dbs[0].liveaction ) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) @@ -231,7 +231,7 @@ def test_cancel_subworkflow_cascade_up_to_workflow_with_other_subworkflows(self) self.assertEqual(len(tk1_ac_ex_dbs), 1) tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id( - tk1_ac_ex_dbs[0].liveaction["id"] + tk1_ac_ex_dbs[0].liveaction ) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) @@ -241,7 +241,7 @@ def test_cancel_subworkflow_cascade_up_to_workflow_with_other_subworkflows(self) self.assertEqual(len(tk2_ac_ex_dbs), 1) tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id( - tk2_ac_ex_dbs[0].liveaction["id"] + tk2_ac_ex_dbs[0].liveaction ) self.assertEqual(tk2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) diff --git a/contrib/runners/orquesta_runner/tests/unit/test_data_flow.py b/contrib/runners/orquesta_runner/tests/unit/test_data_flow.py index eadadbf469..27871474ac 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_data_flow.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_data_flow.py @@ -143,7 +143,7 @@ def assert_data_flow(self, data): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion. @@ -161,7 +161,7 @@ def assert_data_flow(self, data): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction["id"]) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction) self.assertEqual(tk2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion. @@ -179,7 +179,7 @@ def assert_data_flow(self, data): tk3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk3_ex_db.id) )[0] - tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction["id"]) + tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction) self.assertEqual(tk3_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion. @@ -197,7 +197,7 @@ def assert_data_flow(self, data): tk4_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk4_ex_db.id) )[0] - tk4_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk4_ac_ex_db.liveaction["id"]) + tk4_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk4_ac_ex_db.liveaction) self.assertEqual(tk4_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion. diff --git a/contrib/runners/orquesta_runner/tests/unit/test_error_handling.py b/contrib/runners/orquesta_runner/tests/unit/test_error_handling.py index 9a4dd1cd5b..359d16bd1e 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_error_handling.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_error_handling.py @@ -363,7 +363,10 @@ def test_fail_start_task_input_value_type(self): workflow_execution=str(wf_ex_db.id) )[0] self.assertEqual(tk_ex_db.status, wf_statuses.FAILED) - self.assertDictEqual(tk_ex_db.result, {"errors": expected_errors}) + self.assertEqual(tk_ex_db.result["errors"][0]["type"], expected_errors[0]["type"]) + self.assertEqual(tk_ex_db.result["errors"][0]["message"], expected_errors[0]["message"]) + self.assertEqual(tk_ex_db.result["errors"][0]["task_id"], expected_errors[0]["task_id"]) + self.assertEqual(tk_ex_db.result["errors"][0]["route"], expected_errors[0]["route"]) lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id)) self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED) @@ -404,7 +407,7 @@ def test_fail_next_task_action(self): tk_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_db.id) )[0] - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction["id"]) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion for task1 which has an error in publish. @@ -460,7 +463,7 @@ def test_fail_next_task_input_expr_eval(self): tk_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_db.id) )[0] - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction["id"]) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion for task1 which has an error in publish. @@ -512,7 +515,7 @@ def test_fail_next_task_input_value_type(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) self.assertEqual(wf_ex_db.status, wf_statuses.RUNNING) @@ -522,13 +525,25 @@ def test_fail_next_task_input_value_type(self): # Assert workflow execution and task2 execution failed. wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(str(wf_ex_db.id)) self.assertEqual(wf_ex_db.status, wf_statuses.FAILED) - self.assertListEqual( - self.sort_workflow_errors(wf_ex_db.errors), expected_errors + self.assertEqual( + self.sort_workflow_errors(wf_ex_db.errors)[0]["type"], expected_errors[0]["type"] + ) + self.assertEqual( + self.sort_workflow_errors(wf_ex_db.errors)[0]["message"], expected_errors[0]["message"] + ) + self.assertEqual( + self.sort_workflow_errors(wf_ex_db.errors)[0]["task_id"], expected_errors[0]["task_id"] + ) + self.assertEqual( + self.sort_workflow_errors(wf_ex_db.errors)[0]["route"], expected_errors[0]["route"] ) tk2_ex_db = wf_db_access.TaskExecution.query(task_id="task2")[0] self.assertEqual(tk2_ex_db.status, wf_statuses.FAILED) - self.assertDictEqual(tk2_ex_db.result, {"errors": expected_errors}) + self.assertEqual(tk2_ex_db.result["errors"][0]["type"], expected_errors[0]["type"]) + self.assertEqual(tk2_ex_db.result["errors"][0]["message"], expected_errors[0]["message"]) + self.assertEqual(tk2_ex_db.result["errors"][0]["task_id"], expected_errors[0]["task_id"]) + self.assertEqual(tk2_ex_db.result["errors"][0]["route"], expected_errors[0]["route"]) lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id)) self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED) @@ -572,7 +587,7 @@ def test_fail_task_execution(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED) wf_svc.handle_action_execution_completion(tk1_ac_ex_db) @@ -623,7 +638,7 @@ def test_fail_task_transition(self): tk_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_db.id) )[0] - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction["id"]) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion for task1 which has an error in publish. @@ -679,7 +694,7 @@ def test_fail_task_publish(self): tk_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_db.id) )[0] - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction["id"]) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion for task1 which has an error in publish. @@ -732,7 +747,7 @@ def test_fail_output_rendering(self): tk_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_db.id) )[0] - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction["id"]) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion for task1 which has an error in publish. @@ -788,7 +803,7 @@ def test_output_on_error(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) wf_svc.handle_action_execution_completion(tk1_ac_ex_db) wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id) @@ -800,7 +815,7 @@ def test_output_on_error(self): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction["id"]) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction) self.assertEqual(tk2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED) wf_svc.handle_action_execution_completion(tk2_ac_ex_db) @@ -831,7 +846,7 @@ def test_fail_manually(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED) wf_svc.handle_action_execution_completion(tk1_ac_ex_db) wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id) @@ -843,7 +858,7 @@ def test_fail_manually(self): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction["id"]) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction) self.assertEqual(tk2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) wf_svc.handle_action_execution_completion(tk2_ac_ex_db) wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id) @@ -889,7 +904,7 @@ def test_fail_manually_with_recovery_failure(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED) wf_svc.handle_action_execution_completion(tk1_ac_ex_db) wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id) @@ -902,7 +917,7 @@ def test_fail_manually_with_recovery_failure(self): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction["id"]) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction) self.assertEqual(tk2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED) wf_svc.handle_action_execution_completion(tk2_ac_ex_db) wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id) @@ -978,7 +993,7 @@ def test_include_result_to_error_log(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) self.assertEqual(tk1_lv_ac_db.context.get("user"), username) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED) diff --git a/contrib/runners/orquesta_runner/tests/unit/test_functions_common.py b/contrib/runners/orquesta_runner/tests/unit/test_functions_common.py index 4019f9a890..04b79b8be7 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_functions_common.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_functions_common.py @@ -115,7 +115,7 @@ def _execute_workflow(self, wf_name, expected_output): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) self.assertTrue(wf_svc.is_action_execution_under_workflow_context(tk1_ac_ex_db)) diff --git a/contrib/runners/orquesta_runner/tests/unit/test_functions_task.py b/contrib/runners/orquesta_runner/tests/unit/test_functions_task.py index b325839c9d..8aaabc61ab 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_functions_task.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_functions_task.py @@ -130,7 +130,7 @@ def _execute_workflow( task_execution=str(tk_ex_db.id) )[0] tk_lv_ac_db = lv_db_access.LiveAction.get_by_id( - tk_ac_ex_db.liveaction["id"] + tk_ac_ex_db.liveaction ) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) diff --git a/contrib/runners/orquesta_runner/tests/unit/test_inquiries.py b/contrib/runners/orquesta_runner/tests/unit/test_inquiries.py index f60f9415e8..2b27c88d05 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_inquiries.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_inquiries.py @@ -113,7 +113,7 @@ def test_inquiry(self): t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t1_ex_db.id) )[0] - t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction["id"]) + t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction) self.assertEqual( t1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -134,7 +134,7 @@ def test_inquiry(self): t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t2_ex_db.id) )[0] - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) self.assertEqual(t2_lv_ac_db.status, action_constants.LIVEACTION_STATUS_PENDING) workflows.get_engine().process(t2_ac_ex_db) t2_ex_db = wf_db_access.TaskExecution.get_by_id(t2_ex_db.id) @@ -170,7 +170,7 @@ def test_inquiry(self): t3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t3_ex_db.id) )[0] - t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction["id"]) + t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction) self.assertEqual( t3_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -203,7 +203,7 @@ def test_consecutive_inquiries(self): t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t1_ex_db.id) )[0] - t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction["id"]) + t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction) self.assertEqual( t1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -224,7 +224,7 @@ def test_consecutive_inquiries(self): t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t2_ex_db.id) )[0] - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) self.assertEqual(t2_lv_ac_db.status, action_constants.LIVEACTION_STATUS_PENDING) workflows.get_engine().process(t2_ac_ex_db) t2_ex_db = wf_db_access.TaskExecution.get_by_id(t2_ex_db.id) @@ -263,7 +263,7 @@ def test_consecutive_inquiries(self): t3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t3_ex_db.id) )[0] - t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction["id"]) + t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction) self.assertEqual(t3_lv_ac_db.status, action_constants.LIVEACTION_STATUS_PENDING) workflows.get_engine().process(t3_ac_ex_db) t3_ex_db = wf_db_access.TaskExecution.get_by_id(t3_ex_db.id) @@ -299,7 +299,7 @@ def test_consecutive_inquiries(self): t4_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t4_ex_db.id) )[0] - t4_lv_ac_db = lv_db_access.LiveAction.get_by_id(t4_ac_ex_db.liveaction["id"]) + t4_lv_ac_db = lv_db_access.LiveAction.get_by_id(t4_ac_ex_db.liveaction) self.assertEqual( t4_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -332,7 +332,7 @@ def test_parallel_inquiries(self): t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t1_ex_db.id) )[0] - t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction["id"]) + t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction) self.assertEqual( t1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -350,7 +350,7 @@ def test_parallel_inquiries(self): t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t2_ex_db.id) )[0] - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) self.assertEqual(t2_lv_ac_db.status, action_constants.LIVEACTION_STATUS_PENDING) workflows.get_engine().process(t2_ac_ex_db) t2_ex_db = wf_db_access.TaskExecution.get_by_id(t2_ex_db.id) @@ -366,7 +366,7 @@ def test_parallel_inquiries(self): t3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t3_ex_db.id) )[0] - t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction["id"]) + t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction) self.assertEqual(t3_lv_ac_db.status, action_constants.LIVEACTION_STATUS_PENDING) workflows.get_engine().process(t3_ac_ex_db) t3_ex_db = wf_db_access.TaskExecution.get_by_id(t3_ex_db.id) @@ -423,7 +423,7 @@ def test_parallel_inquiries(self): t4_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t4_ex_db.id) )[0] - t4_lv_ac_db = lv_db_access.LiveAction.get_by_id(t4_ac_ex_db.liveaction["id"]) + t4_lv_ac_db = lv_db_access.LiveAction.get_by_id(t4_ac_ex_db.liveaction) self.assertEqual( t4_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -456,7 +456,7 @@ def test_nested_inquiry(self): t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t1_ex_db.id) )[0] - t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction["id"]) + t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction) self.assertEqual( t1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -477,7 +477,7 @@ def test_nested_inquiry(self): t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t2_ex_db.id) )[0] - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) self.assertEqual(t2_lv_ac_db.status, action_constants.LIVEACTION_STATUS_RUNNING) workflows.get_engine().process(t2_ac_ex_db) t2_ex_db = wf_db_access.TaskExecution.get_by_id(t2_ex_db.id) @@ -494,7 +494,7 @@ def test_nested_inquiry(self): task_execution=str(t2_t1_ex_db.id) )[0] t2_t1_lv_ac_db = lv_db_access.LiveAction.get_by_id( - t2_t1_ac_ex_db.liveaction["id"] + t2_t1_ac_ex_db.liveaction ) self.assertEqual( t2_t1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED @@ -515,7 +515,7 @@ def test_nested_inquiry(self): task_execution=str(t2_t2_ex_db.id) )[0] t2_t2_lv_ac_db = lv_db_access.LiveAction.get_by_id( - t2_t2_ac_ex_db.liveaction["id"] + t2_t2_ac_ex_db.liveaction ) self.assertEqual( t2_t2_lv_ac_db.status, action_constants.LIVEACTION_STATUS_PENDING @@ -530,7 +530,7 @@ def test_nested_inquiry(self): t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t2_ex_db.id) )[0] - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) self.assertEqual(t2_lv_ac_db.status, action_constants.LIVEACTION_STATUS_PAUSED) workflows.get_engine().process(t2_ac_ex_db) t2_ex_db = wf_db_access.TaskExecution.get_by_id(t2_ex_db.id) @@ -569,7 +569,7 @@ def test_nested_inquiry(self): task_execution=str(t2_t3_ex_db.id) )[0] t2_t3_lv_ac_db = lv_db_access.LiveAction.get_by_id( - t2_t3_ac_ex_db.liveaction["id"] + t2_t3_ac_ex_db.liveaction ) self.assertEqual( t2_t3_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED @@ -582,7 +582,7 @@ def test_nested_inquiry(self): t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t2_ex_db.id) )[0] - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) self.assertEqual( t2_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -598,7 +598,7 @@ def test_nested_inquiry(self): t3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t3_ex_db.id) )[0] - t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction["id"]) + t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction) self.assertEqual( t3_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) diff --git a/contrib/runners/orquesta_runner/tests/unit/test_notify.py b/contrib/runners/orquesta_runner/tests/unit/test_notify.py index ff7114a318..3c287c5c26 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_notify.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_notify.py @@ -235,7 +235,8 @@ def test_notify_task_list_nonexistent_task(self): } self.assertEqual(lv_ac_db.status, action_constants.LIVEACTION_STATUS_FAILED) - self.assertDictEqual(lv_ac_db.result, expected_result) + self.assertEqual(lv_ac_db.result["errors"][0]["message"], expected_result["errors"][0]["message"]) + self.assertIsNone(lv_ac_db.result["output"], expected_result["output"]) def test_notify_task_list_item_value(self): wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, "sequential.yaml") @@ -275,7 +276,7 @@ def test_cascade_notify_to_tasks(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) self.assertIsNone(tk1_lv_ac_db.notify) self.assertEqual( tk1_ac_ex_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED @@ -296,7 +297,7 @@ def test_cascade_notify_to_tasks(self): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction["id"]) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction) notify = notify_api_models.NotificationsHelper.from_model( notify_model=tk2_lv_ac_db.notify ) @@ -320,7 +321,7 @@ def test_cascade_notify_to_tasks(self): tk3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk3_ex_db.id) )[0] - tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction["id"]) + tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction) self.assertIsNone(tk3_lv_ac_db.notify) self.assertEqual( tk3_ac_ex_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED @@ -367,7 +368,7 @@ def test_notify_task_list_for_task_with_notify(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) self.assertIsNone(tk1_lv_ac_db.notify) # Assert task2 notify is set. query_filters = {"workflow_execution": str(wf_ex_db.id), "task_id": "task2"} @@ -375,7 +376,7 @@ def test_notify_task_list_for_task_with_notify(self): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction["id"]) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction) notify = notify_api_models.NotificationsHelper.from_model( notify_model=tk2_lv_ac_db.notify ) @@ -402,7 +403,7 @@ def test_no_notify_for_task_with_notify(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) self.assertIsNone(tk1_lv_ac_db.notify) # Assert task2 notify is not set. @@ -411,5 +412,5 @@ def test_no_notify_for_task_with_notify(self): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction["id"]) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction) self.assertIsNone(tk2_lv_ac_db.notify) diff --git a/contrib/runners/orquesta_runner/tests/unit/test_pause_and_resume.py b/contrib/runners/orquesta_runner/tests/unit/test_pause_and_resume.py index 7473d9db8e..3ba91d2972 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_pause_and_resume.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_pause_and_resume.py @@ -154,7 +154,7 @@ def test_pause_subworkflow_not_cascade_up_to_workflow(self): self.assertEqual(len(tk_ac_ex_dbs), 1) tk_lv_ac_db = lv_db_access.LiveAction.get_by_id( - tk_ac_ex_dbs[0].liveaction["id"] + tk_ac_ex_dbs[0].liveaction ) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) @@ -196,7 +196,7 @@ def test_pause_workflow_cascade_down_to_subworkflow(self): self.assertEqual(len(tk_ac_ex_dbs), 1) tk_ac_ex_db = tk_ac_ex_dbs[0] - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction["id"]) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Identify the records for the subworkflow. @@ -263,7 +263,7 @@ def test_pause_subworkflow_while_another_subworkflow_running(self): t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_dbs[0].id) )[0] - t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction["id"]) + t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction) t1_wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(t1_ac_ex_db.id) )[0] @@ -273,7 +273,7 @@ def test_pause_subworkflow_while_another_subworkflow_running(self): t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_dbs[1].id) )[0] - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) t2_wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(t2_ac_ex_db.id) )[0] @@ -291,7 +291,7 @@ def test_pause_subworkflow_while_another_subworkflow_running(self): self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Assert the other subworkflow is still running. - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Manually notify action execution completion for the task in the subworkflow. @@ -316,7 +316,7 @@ def test_pause_subworkflow_while_another_subworkflow_running(self): self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Assert the other subworkflow is still running. - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Manually notify action execution completion for the tasks in the other subworkflow. @@ -375,7 +375,7 @@ def test_pause_subworkflow_while_another_subworkflow_completed(self): t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_dbs[0].id) )[0] - t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction["id"]) + t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction) t1_wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(t1_ac_ex_db.id) )[0] @@ -385,7 +385,7 @@ def test_pause_subworkflow_while_another_subworkflow_completed(self): t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_dbs[1].id) )[0] - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) t2_wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(t2_ac_ex_db.id) )[0] @@ -403,7 +403,7 @@ def test_pause_subworkflow_while_another_subworkflow_completed(self): self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Assert the other subworkflow is still running. - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Manually notify action execution completion for the tasks in the other subworkflow. @@ -441,7 +441,7 @@ def test_pause_subworkflow_while_another_subworkflow_completed(self): self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Assert the target subworkflow is still pausing. - t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction["id"]) + t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction) self.assertEqual(t1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_PAUSING) # Manually notify action execution completion for the task in the subworkflow. @@ -492,7 +492,7 @@ def test_resume(self): task_execution=str(tk_ex_dbs[0].id) ) tk_lv_ac_db = lv_db_access.LiveAction.get_by_id( - tk_ac_ex_dbs[0].liveaction["id"] + tk_ac_ex_dbs[0].liveaction ) self.assertEqual(tk_ac_ex_dbs[0].status, ac_const.LIVEACTION_STATUS_SUCCEEDED) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) @@ -550,7 +550,7 @@ def test_resume_cascade_to_subworkflow(self): self.assertEqual(len(tk_ac_ex_dbs), 1) tk_ac_ex_db = tk_ac_ex_dbs[0] - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction["id"]) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Identify the records for the subworkflow. @@ -626,7 +626,7 @@ def test_resume_from_each_subworkflow_when_parent_is_paused(self): t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_dbs[0].id) )[0] - t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction["id"]) + t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction) t1_wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(t1_ac_ex_db.id) )[0] @@ -636,7 +636,7 @@ def test_resume_from_each_subworkflow_when_parent_is_paused(self): t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_dbs[1].id) )[0] - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) t2_wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(t2_ac_ex_db.id) )[0] @@ -654,7 +654,7 @@ def test_resume_from_each_subworkflow_when_parent_is_paused(self): self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Assert the other subworkflow is still running. - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Manually notify action execution completion for the task in the subworkflow. @@ -679,7 +679,7 @@ def test_resume_from_each_subworkflow_when_parent_is_paused(self): self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Assert the other subworkflow is still running. - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Pause the other subworkflow. @@ -773,7 +773,7 @@ def test_resume_from_subworkflow_when_parent_is_paused(self): t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_dbs[0].id) )[0] - t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction["id"]) + t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction) t1_wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(t1_ac_ex_db.id) )[0] @@ -783,7 +783,7 @@ def test_resume_from_subworkflow_when_parent_is_paused(self): t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_dbs[1].id) )[0] - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) t2_wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(t2_ac_ex_db.id) )[0] @@ -801,7 +801,7 @@ def test_resume_from_subworkflow_when_parent_is_paused(self): self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Assert the other subworkflow is still running. - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Manually notify action execution completion for the task in the subworkflow. @@ -826,7 +826,7 @@ def test_resume_from_subworkflow_when_parent_is_paused(self): self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Assert the other subworkflow is still running. - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Manually notify action execution completion for the tasks in the other subworkflow. @@ -907,7 +907,7 @@ def test_resume_from_subworkflow_when_parent_is_paused(self): t3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t3_ex_db.id) )[0] - t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction["id"]) + t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction) self.assertEqual(t3_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) wf_svc.handle_action_execution_completion(t3_ac_ex_db) @@ -937,7 +937,7 @@ def test_resume_from_subworkflow_when_parent_is_running(self): t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_dbs[0].id) )[0] - t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction["id"]) + t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction) t1_wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(t1_ac_ex_db.id) )[0] @@ -947,7 +947,7 @@ def test_resume_from_subworkflow_when_parent_is_running(self): t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_dbs[1].id) )[0] - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) t2_wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(t2_ac_ex_db.id) )[0] @@ -965,7 +965,7 @@ def test_resume_from_subworkflow_when_parent_is_running(self): self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Assert the other subworkflow is still running. - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Manually notify action execution completion for the task in the subworkflow. @@ -990,7 +990,7 @@ def test_resume_from_subworkflow_when_parent_is_running(self): self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Assert the other subworkflow is still running. - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Resume the subworkflow and assert it is running. @@ -1005,7 +1005,7 @@ def test_resume_from_subworkflow_when_parent_is_running(self): self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Assert the other subworkflow is still running. - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Manually notify action execution completion for the tasks in the subworkflow. @@ -1071,7 +1071,7 @@ def test_resume_from_subworkflow_when_parent_is_running(self): t3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t3_ex_db.id) )[0] - t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction["id"]) + t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction) self.assertEqual(t3_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) wf_svc.handle_action_execution_completion(t3_ac_ex_db) diff --git a/contrib/runners/orquesta_runner/tests/unit/test_rerun.py b/contrib/runners/orquesta_runner/tests/unit/test_rerun.py index 420b909e27..7981b8f42c 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_rerun.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_rerun.py @@ -127,7 +127,7 @@ def test_rerun_workflow(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) self.assertEqual(tk1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_FAILED) workflow_service.handle_action_execution_completion(tk1_ac_ex_db) tk1_ex_db = wf_db_access.TaskExecution.get_by_id(tk1_ex_db.id) @@ -166,7 +166,7 @@ def test_rerun_workflow(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) self.assertEqual( tk1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -196,7 +196,7 @@ def test_rerun_with_missing_workflow_execution_id(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) self.assertEqual(tk1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_FAILED) workflow_service.handle_action_execution_completion(tk1_ac_ex_db) tk1_ex_db = wf_db_access.TaskExecution.get_by_id(tk1_ex_db.id) @@ -264,7 +264,7 @@ def test_rerun_with_invalid_workflow_execution(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) self.assertEqual(tk1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_FAILED) workflow_service.handle_action_execution_completion(tk1_ac_ex_db) tk1_ex_db = wf_db_access.TaskExecution.get_by_id(tk1_ex_db.id) @@ -322,7 +322,7 @@ def test_rerun_workflow_still_running(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) self.assertEqual( tk1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_RUNNING ) @@ -381,7 +381,7 @@ def test_rerun_with_unexpected_error(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) self.assertEqual(tk1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_FAILED) workflow_service.handle_action_execution_completion(tk1_ac_ex_db) tk1_ex_db = wf_db_access.TaskExecution.get_by_id(tk1_ex_db.id) @@ -436,7 +436,7 @@ def test_rerun_workflow_already_succeeded(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) self.assertEqual( tk1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -450,7 +450,7 @@ def test_rerun_workflow_already_succeeded(self): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction["id"]) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction) self.assertEqual( tk2_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -464,7 +464,7 @@ def test_rerun_workflow_already_succeeded(self): tk3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk3_ex_db.id) )[0] - tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction["id"]) + tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction) self.assertEqual( tk3_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -505,7 +505,7 @@ def test_rerun_workflow_already_succeeded(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) self.assertEqual( tk1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -522,7 +522,7 @@ def test_rerun_workflow_already_succeeded(self): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction["id"]) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction) self.assertEqual( tk2_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -539,7 +539,7 @@ def test_rerun_workflow_already_succeeded(self): tk3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk3_ex_db.id) )[0] - tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction["id"]) + tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction) self.assertEqual( tk3_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) diff --git a/contrib/runners/orquesta_runner/tests/unit/test_with_items.py b/contrib/runners/orquesta_runner/tests/unit/test_with_items.py index 8e8b67bd94..44909fe831 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_with_items.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_with_items.py @@ -368,7 +368,7 @@ def test_with_items_cancellation(self): # Manually succeed the action executions and process completion. for ac_ex in t1_ac_ex_dbs: self.set_execution_status( - ac_ex.liveaction["id"], action_constants.LIVEACTION_STATUS_SUCCEEDED + ac_ex.liveaction, action_constants.LIVEACTION_STATUS_SUCCEEDED ) t1_ac_ex_dbs = ex_db_access.ActionExecution.query( @@ -440,7 +440,7 @@ def test_with_items_concurrency_cancellation(self): # Manually succeed the action executions and process completion. for ac_ex in t1_ac_ex_dbs: self.set_execution_status( - ac_ex.liveaction["id"], action_constants.LIVEACTION_STATUS_SUCCEEDED + ac_ex.liveaction, action_constants.LIVEACTION_STATUS_SUCCEEDED ) t1_ac_ex_dbs = ex_db_access.ActionExecution.query( @@ -509,7 +509,7 @@ def test_with_items_pause_and_resume(self): # Manually succeed the action executions and process completion. for ac_ex in t1_ac_ex_dbs: self.set_execution_status( - ac_ex.liveaction["id"], action_constants.LIVEACTION_STATUS_SUCCEEDED + ac_ex.liveaction, action_constants.LIVEACTION_STATUS_SUCCEEDED ) t1_ac_ex_dbs = ex_db_access.ActionExecution.query( @@ -599,7 +599,7 @@ def test_with_items_concurrency_pause_and_resume(self): # Manually succeed the action executions and process completion. for ac_ex in t1_ac_ex_dbs: self.set_execution_status( - ac_ex.liveaction["id"], action_constants.LIVEACTION_STATUS_SUCCEEDED + ac_ex.liveaction, action_constants.LIVEACTION_STATUS_SUCCEEDED ) t1_ac_ex_dbs = ex_db_access.ActionExecution.query( diff --git a/st2actions/st2actions/container/base.py b/st2actions/st2actions/container/base.py index 71fe218292..2f98d29e28 100644 --- a/st2actions/st2actions/container/base.py +++ b/st2actions/st2actions/container/base.py @@ -141,11 +141,11 @@ def _do_run(self, runner): ): queries.setup_query(runner.liveaction.id, runner.runner_type, context) except: - LOG.exception("Failed to run action.") _, ex, tb = sys.exc_info() # mark execution as failed. status = action_constants.LIVEACTION_STATUS_FAILED # include the error message and traceback to try and provide some hints. + LOG.exception("Failed to run action. traceback: %s".format("".join(traceback.format_tb(tb, 20)))) result = { "error": str(ex), "traceback": "".join(traceback.format_tb(tb, 20)), @@ -456,7 +456,7 @@ def _get_runner(self, runner_type_db, action_db, liveaction_db): runner.action_name = action_db.name runner.liveaction = liveaction_db runner.liveaction_id = str(liveaction_db.id) - runner.execution = ActionExecution.get(liveaction__id=runner.liveaction_id) + runner.execution = ActionExecution.get(liveaction=str(runner.liveaction_id)) runner.execution_id = str(runner.execution.id) runner.entry_point = resolved_entry_point runner.context = context diff --git a/st2actions/st2actions/notifier/notifier.py b/st2actions/st2actions/notifier/notifier.py index ea1a537733..e7680d4648 100644 --- a/st2actions/st2actions/notifier/notifier.py +++ b/st2actions/st2actions/notifier/notifier.py @@ -83,7 +83,7 @@ def process(self, execution_db): LOG.debug('Processing action execution "%s".', execution_id, extra=extra) # Get the corresponding liveaction record. - liveaction_db = LiveAction.get_by_id(execution_db.liveaction["id"]) + liveaction_db = LiveAction.get_by_id(execution_db.liveaction) if execution_db.status in LIVEACTION_COMPLETED_STATES: # If the action execution is executed under an orquesta workflow, policies for the diff --git a/st2actions/st2actions/policies/concurrency_by_attr.py b/st2actions/st2actions/policies/concurrency_by_attr.py index 9f503bf18a..9d4555b671 100644 --- a/st2actions/st2actions/policies/concurrency_by_attr.py +++ b/st2actions/st2actions/policies/concurrency_by_attr.py @@ -15,10 +15,9 @@ from __future__ import absolute_import -import six - from st2common.constants import action as action_constants from st2common import log as logging +from st2common.fields import JSONDictEscapedFieldCompatibilityField from st2common.persistence import action as action_access from st2common.services import action as action_service from st2common.policies.concurrency import BaseConcurrencyApplicator @@ -41,31 +40,40 @@ def __init__( ) self.attributes = attributes or [] - def _get_filters(self, target): - filters = { - ("parameters__%s" % k): v - for k, v in six.iteritems(target.parameters) - if k in self.attributes - } - - filters["action"] = target.action - filters["status"] = None - - return filters - def _apply_before(self, target): - # Get the count of scheduled and running instances of the action. - filters = self._get_filters(target) - - # Get the count of scheduled instances of the action. - filters["status"] = action_constants.LIVEACTION_STATUS_SCHEDULED - scheduled = action_access.LiveAction.count(**filters) - # Get the count of running instances of the action. - filters["status"] = action_constants.LIVEACTION_STATUS_RUNNING - running = action_access.LiveAction.count(**filters) + scheduled_filters = { + "status": action_constants.LIVEACTION_STATUS_SCHEDULED, + "action": target.action + } + scheduled = [i for i in + action_access.LiveAction.query(**scheduled_filters)] - count = scheduled + running + running_filters = { + "status": action_constants.LIVEACTION_STATUS_RUNNING, + "action": target.action + } + running = [i for i in + action_access.LiveAction.query(**running_filters)] + running.extend(scheduled) + count = 0 + target_parameters = JSONDictEscapedFieldCompatibilityField( + ).parse_field_value(target.parameters) + target_key_value_policy_attributes = { + k: v for k, v in + target_parameters.items() if k in self.attributes} + + for i in running: + running_event_parameters = \ + JSONDictEscapedFieldCompatibilityField( + ).parse_field_value(i.parameters) + # list of event parameter values that are also in policy + running_event_policy_item_key_value_attributes = { + k: v for k, v in + running_event_parameters.items() if k in self.attributes} + if running_event_policy_item_key_value_attributes == \ + target_key_value_policy_attributes: + count += 1 # Mark the execution as scheduled if threshold is not reached or delayed otherwise. if count < self.threshold: diff --git a/st2actions/st2actions/scheduler/entrypoint.py b/st2actions/st2actions/scheduler/entrypoint.py index 14d816ded3..5782a436a6 100644 --- a/st2actions/st2actions/scheduler/entrypoint.py +++ b/st2actions/st2actions/scheduler/entrypoint.py @@ -97,7 +97,7 @@ def _create_execution_queue_item_db_from_liveaction(self, liveaction, delay=None """ Create ActionExecutionSchedulingQueueItemDB from live action. """ - execution = ActionExecution.get(liveaction__id=str(liveaction.id)) + execution = ActionExecution.get(liveaction=str(liveaction.id)) execution_queue_item_db = ActionExecutionSchedulingQueueItemDB() execution_queue_item_db.action_execution_id = str(execution.id) diff --git a/st2actions/st2actions/scheduler/handler.py b/st2actions/st2actions/scheduler/handler.py index 2e2598f5da..35e2e57a86 100644 --- a/st2actions/st2actions/scheduler/handler.py +++ b/st2actions/st2actions/scheduler/handler.py @@ -136,7 +136,7 @@ def _fix_missing_action_execution_id(self): for entry in ActionExecutionSchedulingQueue.query( action_execution_id__in=["", None] ): - execution_db = ActionExecution.get(liveaction__id=entry.liveaction_id) + execution_db = ActionExecution.get(liveaction=entry.liveaction_id) if not execution_db: continue diff --git a/st2actions/st2actions/worker.py b/st2actions/st2actions/worker.py index 30af0d56a7..9537050fc0 100644 --- a/st2actions/st2actions/worker.py +++ b/st2actions/st2actions/worker.py @@ -235,7 +235,7 @@ def _run_action(self, liveaction_db): return result def _cancel_action(self, liveaction_db): - action_execution_db = ActionExecution.get(liveaction__id=str(liveaction_db.id)) + action_execution_db = ActionExecution.get(liveaction=str(liveaction_db.id)) extra = { "action_execution_db": action_execution_db, "liveaction_db": liveaction_db, @@ -265,7 +265,7 @@ def _cancel_action(self, liveaction_db): return result def _pause_action(self, liveaction_db): - action_execution_db = ActionExecution.get(liveaction__id=str(liveaction_db.id)) + action_execution_db = ActionExecution.get(liveaction=str(liveaction_db.id)) extra = { "action_execution_db": action_execution_db, "liveaction_db": liveaction_db, @@ -294,7 +294,7 @@ def _pause_action(self, liveaction_db): return result def _resume_action(self, liveaction_db): - action_execution_db = ActionExecution.get(liveaction__id=str(liveaction_db.id)) + action_execution_db = ActionExecution.get(liveaction=str(liveaction_db.id)) extra = { "action_execution_db": action_execution_db, "liveaction_db": liveaction_db, diff --git a/st2actions/st2actions/workflows/workflows.py b/st2actions/st2actions/workflows/workflows.py index 700244d058..f38c0515a3 100644 --- a/st2actions/st2actions/workflows/workflows.py +++ b/st2actions/st2actions/workflows/workflows.py @@ -100,6 +100,7 @@ def process(self, message): # error handling routine will fail as well because it will try to update # the database and fail the workflow execution gracefully. In this case, # the garbage collector will find and cancel these workflow executions. + LOG.error(e, exc_info=True) self.fail_workflow_execution(message, e) finally: with self._semaphore: @@ -132,7 +133,7 @@ def shutdown(self): if cfg.CONF.coordination.service_registry and not member_ids: ac_ex_dbs = self._get_running_workflows() for ac_ex_db in ac_ex_dbs: - lv_ac = action_utils.get_liveaction_by_id(ac_ex_db.liveaction["id"]) + lv_ac = action_utils.get_liveaction_by_id(ac_ex_db.liveaction) ac_svc.request_pause(lv_ac, WORKFLOW_ENGINE_START_STOP_SEQ) def _get_running_workflows(self): @@ -251,7 +252,7 @@ def handle_action_execution(self, ac_ex_db): return # Apply post run policies. - lv_ac_db = lv_db_access.LiveAction.get_by_id(ac_ex_db.liveaction["id"]) + lv_ac_db = lv_db_access.LiveAction.get_by_id(ac_ex_db.liveaction) pc_svc.apply_post_run_policies(lv_ac_db) # Process completion of the action execution. diff --git a/st2actions/tests/unit/policies/test_concurrency.py b/st2actions/tests/unit/policies/test_concurrency.py index 1be4b86da3..c1a75a4f33 100644 --- a/st2actions/tests/unit/policies/test_concurrency.py +++ b/st2actions/tests/unit/policies/test_concurrency.py @@ -218,7 +218,7 @@ def test_over_threshold_delay_executions(self): self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Check the status changes. - execution = ActionExecution.get(liveaction__id=str(liveaction.id)) + execution = ActionExecution.get(liveaction=str(liveaction.id)) expected_status_changes = [ "requested", "delayed", diff --git a/st2actions/tests/unit/policies/test_retry_policy.py b/st2actions/tests/unit/policies/test_retry_policy.py index 8e16a7029c..86feb96d4f 100644 --- a/st2actions/tests/unit/policies/test_retry_policy.py +++ b/st2actions/tests/unit/policies/test_retry_policy.py @@ -128,7 +128,7 @@ def test_retry_on_timeout_first_retry_is_successful(self): self.assertEqual(action_execution_dbs[1].status, LIVEACTION_STATUS_REQUESTED) # Verify retried execution contains policy related context - original_liveaction_id = action_execution_dbs[0].liveaction["id"] + original_liveaction_id = action_execution_dbs[0].liveaction context = action_execution_dbs[1].context self.assertIn("policies", context) @@ -183,7 +183,7 @@ def test_retry_on_timeout_policy_is_retried_twice(self): self.assertEqual(action_execution_dbs[1].status, LIVEACTION_STATUS_REQUESTED) # Verify retried execution contains policy related context - original_liveaction_id = action_execution_dbs[0].liveaction["id"] + original_liveaction_id = action_execution_dbs[0].liveaction context = action_execution_dbs[1].context self.assertIn("policies", context) @@ -216,7 +216,7 @@ def test_retry_on_timeout_policy_is_retried_twice(self): self.assertEqual(action_execution_dbs[2].status, LIVEACTION_STATUS_REQUESTED) # Verify retried execution contains policy related context - original_liveaction_id = action_execution_dbs[1].liveaction["id"] + original_liveaction_id = action_execution_dbs[1].liveaction context = action_execution_dbs[2].context self.assertIn("policies", context) diff --git a/st2actions/tests/unit/test_executions.py b/st2actions/tests/unit/test_executions.py index 1c95a51061..d436ffbbda 100644 --- a/st2actions/tests/unit/test_executions.py +++ b/st2actions/tests/unit/test_executions.py @@ -99,7 +99,7 @@ def test_basic_execution(self): ) execution = self._get_action_execution( - liveaction__id=str(liveaction.id), raise_exception=True + liveaction=str(liveaction.id), raise_exception=True ) self.assertDictEqual(execution.trigger, {}) @@ -121,8 +121,7 @@ def test_basic_execution(self): self.assertEqual(execution.result, liveaction.result) self.assertEqual(execution.status, liveaction.status) self.assertEqual(execution.context, liveaction.context) - self.assertEqual(execution.liveaction["callback"], liveaction.callback) - self.assertEqual(execution.liveaction["action"], liveaction.action) + self.assertEqual(execution.liveaction, str(liveaction.id)) def test_basic_execution_history_create_failed(self): MOCK_FAIL_EXECUTION_CREATE = True # noqa @@ -136,7 +135,7 @@ def test_chained_executions(self): ) execution = self._get_action_execution( - liveaction__id=str(liveaction.id), raise_exception=True + liveaction=str(liveaction.id), raise_exception=True ) action = action_utils.get_action_by_ref("executions.chain") @@ -154,8 +153,7 @@ def test_chained_executions(self): self.assertEqual(execution.result, liveaction.result) self.assertEqual(execution.status, liveaction.status) self.assertEqual(execution.context, liveaction.context) - self.assertEqual(execution.liveaction["callback"], liveaction.callback) - self.assertEqual(execution.liveaction["action"], liveaction.action) + self.assertEqual(execution.liveaction, str(liveaction.id)) self.assertGreater(len(execution.children), 0) for child in execution.children: @@ -202,7 +200,7 @@ def test_triggered_execution(self): ) execution = self._get_action_execution( - liveaction__id=str(liveaction.id), raise_exception=True + liveaction=str(liveaction.id), raise_exception=True ) self.assertDictEqual(execution.trigger, vars(TriggerAPI.from_model(trigger))) @@ -229,8 +227,7 @@ def test_triggered_execution(self): self.assertEqual(execution.result, liveaction.result) self.assertEqual(execution.status, liveaction.status) self.assertEqual(execution.context, liveaction.context) - self.assertEqual(execution.liveaction["callback"], liveaction.callback) - self.assertEqual(execution.liveaction["action"], liveaction.action) + self.assertEqual(execution.liveaction, str(liveaction.id)) def _get_action_execution(self, **kwargs): return ActionExecution.get(**kwargs) diff --git a/st2actions/tests/unit/test_notifier.py b/st2actions/tests/unit/test_notifier.py index b648d7fad3..f1609664b1 100644 --- a/st2actions/tests/unit/test_notifier.py +++ b/st2actions/tests/unit/test_notifier.py @@ -135,7 +135,7 @@ def test_notify_triggers(self): LiveAction.add_or_update(liveaction_db) execution = MOCK_EXECUTION - execution.liveaction = vars(LiveActionAPI.from_model(liveaction_db)) + execution.liveaction = str(liveaction_db.id) execution.status = liveaction_db.status dispatcher = NotifierTestCase.MockDispatcher(self) @@ -185,7 +185,7 @@ def test_notify_triggers_end_timestamp_none(self): LiveAction.add_or_update(liveaction_db) execution = MOCK_EXECUTION - execution.liveaction = vars(LiveActionAPI.from_model(liveaction_db)) + execution.liveaction = str(liveaction_db.id) execution.status = liveaction_db.status dispatcher = NotifierTestCase.MockDispatcher(self) @@ -238,7 +238,7 @@ def test_notify_triggers_jinja_patterns(self, dispatch): LiveAction.add_or_update(liveaction_db) execution = MOCK_EXECUTION - execution.liveaction = vars(LiveActionAPI.from_model(liveaction_db)) + execution.liveaction = str(liveaction_db.id) execution.status = liveaction_db.status notifier = Notifier(connection=None, queues=[]) @@ -270,7 +270,7 @@ def test_post_generic_trigger_emit_when_default_value_is_used(self, dispatch): liveaction_db = LiveActionDB(action="core.local") liveaction_db.status = status execution = MOCK_EXECUTION - execution.liveaction = vars(LiveActionAPI.from_model(liveaction_db)) + execution.liveaction = str(liveaction_db.id) execution.status = liveaction_db.status notifier = Notifier(connection=None, queues=[]) @@ -307,7 +307,7 @@ def test_post_generic_trigger_with_emit_condition(self, dispatch): liveaction_db = LiveActionDB(action="core.local") liveaction_db.status = status execution = MOCK_EXECUTION - execution.liveaction = vars(LiveActionAPI.from_model(liveaction_db)) + execution.liveaction = str(liveaction_db.id) execution.status = liveaction_db.status notifier = Notifier(connection=None, queues=[]) @@ -354,7 +354,7 @@ def test_process_post_generic_notify_trigger_on_completed_state_default( liveaction_db = LiveActionDB(id=bson.ObjectId(), action="core.local") liveaction_db.status = status execution = MOCK_EXECUTION - execution.liveaction = vars(LiveActionAPI.from_model(liveaction_db)) + execution.liveaction = str(liveaction_db.id) execution.status = liveaction_db.status mock_LiveAction.get_by_id.return_value = liveaction_db @@ -404,7 +404,7 @@ def test_process_post_generic_notify_trigger_on_custom_emit_when_states( liveaction_db = LiveActionDB(id=bson.ObjectId(), action="core.local") liveaction_db.status = status execution = MOCK_EXECUTION - execution.liveaction = vars(LiveActionAPI.from_model(liveaction_db)) + execution.liveaction = str(liveaction_db.id) execution.status = liveaction_db.status mock_LiveAction.get_by_id.return_value = liveaction_db diff --git a/st2api/st2api/controllers/v1/actionexecutions.py b/st2api/st2api/controllers/v1/actionexecutions.py index 70d709192e..020894b3e9 100644 --- a/st2api/st2api/controllers/v1/actionexecutions.py +++ b/st2api/st2api/controllers/v1/actionexecutions.py @@ -39,6 +39,7 @@ from st2common.exceptions import apivalidation as validation_exc from st2common.exceptions import param as param_exc from st2common.exceptions import trace as trace_exc +from st2common.fields import JSONDictEscapedFieldCompatibilityField from st2common.models.api.action import LiveActionAPI from st2common.models.api.action import LiveActionCreateAPI from st2common.models.api.base import cast_argument_value @@ -205,7 +206,6 @@ def _schedule_execution( runnertype_db = action_utils.get_runnertype_by_name( action_db.runner_type["name"] ) - try: liveaction_db.parameters = param_utils.render_live_params( runnertype_db.runner_parameters, @@ -241,7 +241,6 @@ def _schedule_execution( liveaction_db, actionexecution_db = action_service.create_request( liveaction=liveaction_db, action_db=action_db, runnertype_db=runnertype_db ) - _, actionexecution_db = action_service.publish_request( liveaction_db, actionexecution_db ) @@ -634,8 +633,10 @@ def post(self, spec_api, id, requester_user, no_merge=False, show_secrets=False) # Merge in any parameters provided by the user new_parameters = {} + original_parameters = getattr(existing_execution, "parameters", b"{}") + original_params_decoded = JSONDictEscapedFieldCompatibilityField().parse_field_value(original_parameters) if not no_merge: - new_parameters.update(getattr(existing_execution, "parameters", {})) + new_parameters.update(original_params_decoded) new_parameters.update(spec_api.parameters) # Create object for the new execution @@ -842,7 +843,7 @@ def put(self, id, liveaction_api, requester_user, show_secrets=False): if not execution_api: abort(http_client.NOT_FOUND, "Execution with id %s not found." % id) - liveaction_id = execution_api.liveaction["id"] + liveaction_id = execution_api.liveaction if not liveaction_id: abort( http_client.INTERNAL_SERVER_ERROR, @@ -867,7 +868,7 @@ def update_status(liveaction_api, liveaction_db): liveaction_db, status, result, set_result_size=True ) actionexecution_db = ActionExecution.get( - liveaction__id=str(liveaction_db.id) + liveaction=str(liveaction_db.id) ) return (liveaction_db, actionexecution_db) @@ -971,7 +972,7 @@ def delete(self, id, requester_user, show_secrets=False): if not execution_api: abort(http_client.NOT_FOUND, "Execution with id %s not found." % id) - liveaction_id = execution_api.liveaction["id"] + liveaction_id = execution_api.liveaction if not liveaction_id: abort( http_client.INTERNAL_SERVER_ERROR, diff --git a/st2api/st2api/controllers/v1/aliasexecution.py b/st2api/st2api/controllers/v1/aliasexecution.py index 4e0f780896..48d2514fb5 100644 --- a/st2api/st2api/controllers/v1/aliasexecution.py +++ b/st2api/st2api/controllers/v1/aliasexecution.py @@ -25,7 +25,7 @@ from st2common.models.api.action import ActionAliasAPI from st2common.models.api.action import AliasMatchAndExecuteInputAPI from st2common.models.api.auth import get_system_username -from st2common.models.api.execution import ActionExecutionAPI +from st2common.models.api.execution import ActionExecutionAPI, LiveActionAPI from st2common.models.db.auth import UserDB from st2common.models.db.liveaction import LiveActionDB from st2common.models.db.notification import NotificationSchema, NotificationSubSchema @@ -35,6 +35,7 @@ ) from st2common.models.utils.action_alias_utils import inject_immutable_parameters from st2common.persistence.actionalias import ActionAlias +from st2common.persistence.liveaction import LiveAction from st2common.services import action as action_service from st2common.util import action_db as action_utils from st2common.util import reference @@ -182,7 +183,14 @@ def _post(self, payload, requester_user, show_secrets=False, match_multiple=Fals show_secrets=show_secrets, requester_user=requester_user, ) - + if hasattr(execution, "liveaction"): + liveaction = LiveAction.get_by_id(execution.liveaction) + mask_secrets = self._get_mask_secrets( + requester_user, show_secrets=show_secrets + ) + liveaction = LiveActionAPI.from_model(liveaction, + mask_secrets=mask_secrets) + execution.liveaction = liveaction result = { "execution": execution, "actionalias": ActionAliasAPI.from_model(action_alias_db), diff --git a/st2api/tests/unit/controllers/v1/test_alias_execution.py b/st2api/tests/unit/controllers/v1/test_alias_execution.py index 44261fde3f..0ca78758a6 100644 --- a/st2api/tests/unit/controllers/v1/test_alias_execution.py +++ b/st2api/tests/unit/controllers/v1/test_alias_execution.py @@ -159,6 +159,7 @@ def test_execution_secret_parameter(self, request): self.assertEqual(post_resp.status_int, 201) expected_parameters = {"param1": "value1", "param4": SUPER_SECRET_PARAMETER} self.assertEqual(request.call_args[0][0].parameters, expected_parameters) + #above working post_resp = self._do_post( alias_execution=self.alias4, command=command, diff --git a/st2api/tests/unit/controllers/v1/test_executions.py b/st2api/tests/unit/controllers/v1/test_executions.py index eb3face2ba..9ef28155fa 100644 --- a/st2api/tests/unit/controllers/v1/test_executions.py +++ b/st2api/tests/unit/controllers/v1/test_executions.py @@ -2000,7 +2000,7 @@ def test_get_output_running_execution(self): status=status, action={"ref": "core.local"}, runner={"name": "local-shell-cmd"}, - liveaction={"ref": "foo"}, + liveaction="ref", ) action_execution_db = ActionExecution.add_or_update(action_execution_db) @@ -2081,7 +2081,7 @@ def test_get_output_finished_execution(self): status=status, action={"ref": "core.local"}, runner={"name": "local-shell-cmd"}, - liveaction={"ref": "foo"}, + liveaction="ref", ) action_execution_db = ActionExecution.add_or_update(action_execution_db) diff --git a/st2common/bin/st2-track-result b/st2common/bin/st2-track-result index 773421bf82..e461f40668 100755 --- a/st2common/bin/st2-track-result +++ b/st2common/bin/st2-track-result @@ -68,7 +68,7 @@ def add_result_tracker(exec_id): LOG.info("Retrieving runner type and liveaction records...") runnertype_db = action_db.get_runnertype_by_name(exec_db.action.get("runner_type")) - liveaction_db = action_db.get_liveaction_by_id(exec_db.liveaction["id"]) + liveaction_db = action_db.get_liveaction_by_id(exec_db.liveaction) # Skip if liveaction is completed. if liveaction_db.status in action_constants.LIVEACTION_COMPLETED_STATES: @@ -100,7 +100,7 @@ def del_result_tracker(exec_id): LOG.info('Found action execution record for "%s".', exec_id) LOG.info("Retrieving runner type and liveaction records...") - liveaction_db = action_db.get_liveaction_by_id(exec_db.liveaction["id"]) + liveaction_db = action_db.get_liveaction_by_id(exec_db.liveaction) LOG.info("Removing result tracker entry...") removed = queries.remove_query(liveaction_db.id) diff --git a/st2common/st2common/fields.py b/st2common/st2common/fields.py index 0e94f11f85..bf523ee869 100644 --- a/st2common/st2common/fields.py +++ b/st2common/st2common/fields.py @@ -31,6 +31,7 @@ import weakref import orjson +import zstandard from mongoengine import LongField from mongoengine import BinaryField @@ -331,13 +332,14 @@ def _mark_as_changed(self, key=None): class JSONDictField(BinaryField): """ - Custom field types which stores dictionary as JSON serialized strings. + Custom field types which stores dictionary as zstandard compressed JSON serialized strings. - This is done because storing large objects as JSON serialized strings is much more fficient + This is done because storing large objects as compressed JSON serialized + strings is much more efficient on the serialize and unserialize paths compared to used EscapedDictField which needs to escape all the special values ($, .). - Only downside is that to MongoDB those values are plain raw strings which means you can't query + Only downside is that to MongoDB those values are compressed plain raw strings which means you can't query on actual dictionary field values. That's not an issue for us, because in places where we use it, those values are already treated as plain binary blobs to the database layer and we never directly query on those field values. @@ -358,25 +360,12 @@ class JSONDictField(BinaryField): IMPLEMENTATION DETAILS: - If header is used, values are stored in the following format: - ::. - For example: - n:o:... - No compression, (or)json serialization - z:o:... - Zstandard compression, (or)json serialization - - If header is not used, value is stored as a serialized JSON string of the input dictionary. """ def __init__(self, *args, **kwargs): - # True if we should use field header which is more future proof approach and also allows - # us to support optional per-field compression, etc. - # This option is only exposed so we can benchmark different approaches and how much overhead - # using a header adds. - self.use_header = kwargs.pop("use_header", False) - self.compression_algorithm = kwargs.pop("compression_algorithm", "none") - + self.compression_algorithm = JSONDictFieldCompressionAlgorithmEnum.ZSTANDARD.value super(JSONDictField, self).__init__(*args, **kwargs) def to_mongo(self, value): @@ -406,8 +395,6 @@ def parse_field_value(self, value: Optional[Union[bytes, dict]]) -> dict: For example: - - (n, o, ...) - no compression, data is serialized using orjson - - (z, o, ...) - zstandard compression, data is serialized using orjson """ if not value: return self.default @@ -415,41 +402,12 @@ def parse_field_value(self, value: Optional[Union[bytes, dict]]) -> dict: if isinstance(value, dict): # Already deserializaed return value - - if not self.use_header: - return orjson.loads(value) - - split = value.split(JSON_DICT_FIELD_DELIMITER, 2) - - if len(split) != 3: - raise ValueError( - "Expected 3 values when splitting field value, got %s" % (len(split)) - ) - - compression_algorithm = split[0] - serialization_format = split[1] - data = split[2] - - if compression_algorithm not in VALID_JSON_DICT_COMPRESSION_ALGORITHMS: - raise ValueError( - "Invalid or unsupported value for compression algorithm header " - "value: %s" % (compression_algorithm) - ) - - if serialization_format not in VALID_JSON_DICT_SERIALIZATION_FORMATS: - raise ValueError( - "Invalid or unsupported value for serialization format header " - "value: %s" % (serialization_format) - ) - - if ( - compression_algorithm - == JSONDictFieldCompressionAlgorithmEnum.ZSTANDARD.value - ): - # NOTE: At this point zstandard is only test dependency - import zstandard - - data = zstandard.ZstdDecompressor().decompress(data) + data = value + try: + data = zstandard.ZstdDecompressor().decompress(value) + # skip if already a byte string and not compressed + except zstandard.ZstdError: + pass data = orjson.loads(data) return data @@ -474,21 +432,10 @@ def default(obj): return list(obj) raise TypeError - if not self.use_header: - return orjson.dumps(value, default=default) + value = orjson.dumps(value, default=default) + data = zstandard.ZstdCompressor().compress(value) - data = orjson.dumps(value, default=default) - - if self.compression_algorithm == "zstandard": - # NOTE: At this point zstandard is only test dependency - import zstandard - - compression_header = JSONDictFieldCompressionAlgorithmEnum.ZSTANDARD - data = zstandard.ZstdCompressor().compress(data) - else: - compression_header = JSONDictFieldCompressionAlgorithmEnum.NONE - - return compression_header.value + b":" + b"o:" + data + return data def __get__(self, instance, owner): """ @@ -522,11 +469,6 @@ class JSONDictEscapedFieldCompatibilityField(JSONDictField): def to_mongo(self, value): if isinstance(value, bytes): # Already serialized - if value[0] == b"{" and self.use_header: - # Serialized, but doesn't contain header prefix, add it (assume migration from - # format without a header) - return "n:o:" + value - return value if not isinstance(value, dict): diff --git a/st2common/st2common/garbage_collection/executions.py b/st2common/st2common/garbage_collection/executions.py index ae0f3296f4..aae36c9cde 100644 --- a/st2common/st2common/garbage_collection/executions.py +++ b/st2common/st2common/garbage_collection/executions.py @@ -223,5 +223,5 @@ def purge_orphaned_workflow_executions(logger): # as a result of the original failure, the garbage collection routine here cancels # the workflow execution so it cannot be rerun from failed task(s). for ac_ex_db in workflow_service.identify_orphaned_workflows(): - lv_ac_db = LiveAction.get(id=ac_ex_db.liveaction["id"]) + lv_ac_db = LiveAction.get(id=ac_ex_db.liveaction) action_service.request_cancellation(lv_ac_db, None) diff --git a/st2common/st2common/garbage_collection/inquiries.py b/st2common/st2common/garbage_collection/inquiries.py index 2381472a73..6182d2b491 100644 --- a/st2common/st2common/garbage_collection/inquiries.py +++ b/st2common/st2common/garbage_collection/inquiries.py @@ -78,7 +78,7 @@ def purge_inquiries(logger): liveaction_db = action_utils.update_liveaction_status( status=action_constants.LIVEACTION_STATUS_TIMED_OUT, result=inquiry.result, - liveaction_id=inquiry.liveaction.get("id"), + liveaction_id=inquiry.liveaction, ) executions.update_execution(liveaction_db) diff --git a/st2common/st2common/models/api/action.py b/st2common/st2common/models/api/action.py index dc18ba02cd..75c1574999 100644 --- a/st2common/st2common/models/api/action.py +++ b/st2common/st2common/models/api/action.py @@ -34,6 +34,7 @@ from st2common.models.db.runner import RunnerTypeDB from st2common.constants.action import LIVEACTION_STATUSES from st2common.models.system.common import ResourceReference +from st2common.fields import JSONDictEscapedFieldCompatibilityField __all__ = [ @@ -442,8 +443,23 @@ class LiveActionAPI(BaseAPI): } skip_unescape_field_names = [ "result", + "parameters" ] + @classmethod + def convert_raw(cls, doc, raw_values): + """ + override this class to + convert any raw byte values into dict + + :param doc: dict + :param raw_values: dict[field]:bytestring + """ + + for field_name, field_value in raw_values.items(): + doc[field_name] = JSONDictEscapedFieldCompatibilityField().parse_field_value(field_value) + return doc + @classmethod def from_model(cls, model, mask_secrets=False): doc = super(cls, cls)._from_model(model, mask_secrets=mask_secrets) @@ -451,7 +467,6 @@ def from_model(cls, model, mask_secrets=False): doc["start_timestamp"] = isotime.format(model.start_timestamp, offset=False) if model.end_timestamp: doc["end_timestamp"] = isotime.format(model.end_timestamp, offset=False) - if getattr(model, "notify", None): doc["notify"] = NotificationsHelper.from_model(model.notify) diff --git a/st2common/st2common/models/api/base.py b/st2common/st2common/models/api/base.py index 6cdb16feef..996bc6c8ee 100644 --- a/st2common/st2common/models/api/base.py +++ b/st2common/st2common/models/api/base.py @@ -22,6 +22,7 @@ from st2common.util import mongoescape as util_mongodb from st2common import log as logging +from st2common.models.db.stormbase import EscapedDynamicField, EscapedDictField __all__ = ["BaseAPI", "APIUIDMixin"] @@ -86,6 +87,9 @@ def validate(self): @classmethod def _from_model(cls, model, mask_secrets=False): + unescape_fields = [k for k, v in model._fields.items() if type(v) in + [EscapedDynamicField, EscapedDictField]] + unescape_fields = set(unescape_fields) - set(cls.skip_unescape_field_names) doc = model.to_mongo() if "_id" in doc: @@ -94,32 +98,36 @@ def _from_model(cls, model, mask_secrets=False): # Special case for models which utilize JSONDictField - there is no need to escape those # fields since it contains a JSON string and not a dictionary which doesn't need to be # mongo escaped. Skipping this step here substantially speeds things up for that field. - - # Right now we do this here manually for all those fields types but eventually we should - # refactor the code to just call unescape chars on escaped fields - more generic and - # faster. raw_values = {} - for field_name in cls.skip_unescape_field_names: if isinstance(doc.get(field_name, None), bytes): raw_values[field_name] = doc.pop(field_name) - - # TODO (Tomaz): In general we really shouldn't need to call unescape chars on the whole doc, - # but just on the EscapedDict and EscapedDynamicField fields - doing it on the whole doc - # level is slow and not necessary! - doc = util_mongodb.unescape_chars(doc) - - # Now add the JSON string field value which shouldn't be escaped back. - # We don't JSON parse the field value here because that happens inside the model specific - # "from_model()" method where we also parse and convert all the other field values. - for field_name, field_value in raw_values.items(): - doc[field_name] = field_value + for key in unescape_fields: + if key in doc.keys(): + doc[key] = util_mongodb.unescape_chars(doc[key]) + # convert raw fields and add back ; no need to unescape + doc = cls.convert_raw(doc, raw_values) if mask_secrets and cfg.CONF.log.mask_secrets: doc = model.mask_secrets(value=doc) return doc + @classmethod + def convert_raw(cls, doc, raw_values): + """ + override this class to + convert any raw byte values into dict + you can also use this to fix any other fields that need 'fixing' + + :param doc: dict + :param raw_values: dict[field]:bytestring + """ + + for field_name, field_value in raw_values.items(): + doc[field_name] = field_value + return doc + @classmethod def from_model(cls, model, mask_secrets=False): """ diff --git a/st2common/st2common/models/api/execution.py b/st2common/st2common/models/api/execution.py index 76aa9fbdf8..4df1c4c75b 100644 --- a/st2common/st2common/models/api/execution.py +++ b/st2common/st2common/models/api/execution.py @@ -29,6 +29,7 @@ from st2common.models.api.action import RunnerTypeAPI, ActionAPI, LiveActionAPI from st2common import log as logging from st2common.util.deep_copy import fast_deepcopy_dict +from st2common.fields import JSONDictEscapedFieldCompatibilityField __all__ = ["ActionExecutionAPI", "ActionExecutionOutputAPI"] @@ -147,13 +148,13 @@ class ActionExecutionAPI(BaseAPI): } skip_unescape_field_names = [ "result", + "parameters" ] @classmethod def from_model(cls, model, mask_secrets=False): - doc = cls._from_model(model, mask_secrets=mask_secrets) - doc["result"] = ActionExecutionDB.result.parse_field_value(doc["result"]) + doc = cls._from_model(model, mask_secrets=mask_secrets) start_timestamp = model.start_timestamp start_timestamp_iso = isotime.format(start_timestamp, offset=False) @@ -171,6 +172,20 @@ def from_model(cls, model, mask_secrets=False): attrs = {attr: value for attr, value in six.iteritems(doc) if value} return cls(**attrs) + @classmethod + def convert_raw(cls, doc, raw_values): + """ + override this class to + convert any raw byte values into dict + + :param doc: dict + :param raw_values: dict[field]:bytestring + """ + + for field_name, field_value in raw_values.items(): + doc[field_name] = JSONDictEscapedFieldCompatibilityField().parse_field_value(field_value) + return doc + @classmethod def to_model(cls, instance): values = {} diff --git a/st2common/st2common/models/db/execution.py b/st2common/st2common/models/db/execution.py index 0de35a5c31..1c5d817828 100644 --- a/st2common/st2common/models/db/execution.py +++ b/st2common/st2common/models/db/execution.py @@ -29,7 +29,6 @@ from st2common.util.secrets import mask_inquiry_response from st2common.util.secrets import mask_secret_parameters from st2common.constants.types import ResourceType - __all__ = ["ActionExecutionDB", "ActionExecutionOutputDB"] @@ -39,16 +38,7 @@ class ActionExecutionDB(stormbase.StormFoundationDB): RESOURCE_TYPE = ResourceType.EXECUTION UID_FIELDS = ["id"] - - trigger = stormbase.EscapedDictField() - trigger_type = stormbase.EscapedDictField() - trigger_instance = stormbase.EscapedDictField() - rule = stormbase.EscapedDictField() - action = stormbase.EscapedDictField(required=True) - runner = stormbase.EscapedDictField(required=True) - # Only the diff between the liveaction type and what is replicated - # in the ActionExecutionDB object. - liveaction = stormbase.EscapedDictField(required=True) + # SAME as liveaction workflow_execution = me.StringField() task_execution = me.StringField() status = me.StringField( @@ -61,29 +51,39 @@ class ActionExecutionDB(stormbase.StormFoundationDB): end_timestamp = ComplexDateTimeField( help_text="The timestamp when the liveaction has finished." ) - parameters = stormbase.EscapedDynamicField( + action = stormbase.EscapedDictField(required=True) + parameters = JSONDictEscapedFieldCompatibilityField( default={}, help_text="The key-value pairs passed as to the action runner & action.", ) result = JSONDictEscapedFieldCompatibilityField( default={}, help_text="Action defined result." ) - result_size = me.IntField(default=0, help_text="Serialized result size in bytes") context = me.DictField( default={}, help_text="Contextual information on the action execution." ) + delay = me.IntField(min_value=0) + + # diff from liveaction + runner = stormbase.EscapedDictField(required=True) + trigger = stormbase.EscapedDictField() + trigger_type = stormbase.EscapedDictField() + trigger_instance = stormbase.EscapedDictField() + rule = stormbase.EscapedDictField() + result_size = me.IntField(default=0, help_text="Serialized result size in bytes") parent = me.StringField() children = me.ListField(field=me.StringField()) log = me.ListField(field=me.DictField()) - delay = me.IntField(min_value=0) # Do not use URLField for web_url. If host doesn't have FQDN set, URLField validation blows. web_url = me.StringField(required=False) + # liveaction id + liveaction = me.StringField() + meta = { "indexes": [ {"fields": ["rule.ref"]}, {"fields": ["action.ref"]}, - {"fields": ["liveaction.id"]}, {"fields": ["start_timestamp"]}, {"fields": ["end_timestamp"]}, {"fields": ["status"]}, @@ -115,10 +115,8 @@ def mask_secrets(self, value): :return: result: action execution object with masked secret paramters in input and output schema. :rtype: result: ``dict`` """ - result = copy.deepcopy(value) - liveaction = result["liveaction"] parameters = {} # pylint: disable=no-member parameters.update(value.get("action", {}).get("parameters", {})) @@ -128,31 +126,6 @@ def mask_secrets(self, value): result["parameters"] = mask_secret_parameters( parameters=result.get("parameters", {}), secret_parameters=secret_parameters ) - - if "parameters" in liveaction: - liveaction["parameters"] = mask_secret_parameters( - parameters=liveaction["parameters"], secret_parameters=secret_parameters - ) - - if liveaction.get("action", "") == "st2.inquiry.respond": - # Special case to mask parameters for `st2.inquiry.respond` action - # In this case, this execution is just a plain python action, not - # an inquiry, so we don't natively have a handle on the response - # schema. - # - # To prevent leakage, we can just mask all response fields. - # - # Note: The 'string' type in secret_parameters doesn't matter, - # it's just a placeholder to tell mask_secret_parameters() - # that this parameter is indeed a secret parameter and to - # mask it. - result["parameters"]["response"] = mask_secret_parameters( - parameters=liveaction["parameters"]["response"], - secret_parameters={ - p: "string" for p in liveaction["parameters"]["response"] - }, - ) - output_value = ActionExecutionDB.result.parse_field_value(result["result"]) masked_output_value = output_schema.mask_secret_output(result, output_value) result["result"] = masked_output_value diff --git a/st2common/st2common/models/db/liveaction.py b/st2common/st2common/models/db/liveaction.py index aef52462a6..73be661d05 100644 --- a/st2common/st2common/models/db/liveaction.py +++ b/st2common/st2common/models/db/liveaction.py @@ -38,6 +38,7 @@ class LiveActionDB(stormbase.StormFoundationDB): + # same as action execution workflow_execution = me.StringField() task_execution = me.StringField() # TODO: Can status be an enum at the Mongo layer? @@ -54,11 +55,7 @@ class LiveActionDB(stormbase.StormFoundationDB): action = me.StringField( required=True, help_text="Reference to the action that has to be executed." ) - action_is_workflow = me.BooleanField( - default=False, - help_text="A flag indicating whether the referenced action is a workflow.", - ) - parameters = stormbase.EscapedDynamicField( + parameters = JSONDictEscapedFieldCompatibilityField( default={}, help_text="The key-value pairs passed as to the action runner & execution.", ) @@ -68,20 +65,25 @@ class LiveActionDB(stormbase.StormFoundationDB): context = me.DictField( default={}, help_text="Contextual information on the action execution." ) + delay = me.IntField( + min_value=0, + help_text="How long (in milliseconds) to delay the execution before scheduling.", + ) + + # diff from action execution + action_is_workflow = me.BooleanField( + default=False, + help_text="A flag indicating whether the referenced action is a workflow.", + ) callback = me.DictField( default={}, help_text="Callback information for the on completion of action execution.", ) + notify = me.EmbeddedDocumentField(NotificationSchema) runner_info = me.DictField( default={}, help_text="Information about the runner which executed this live action (hostname, pid).", ) - notify = me.EmbeddedDocumentField(NotificationSchema) - delay = me.IntField( - min_value=0, - help_text="How long (in milliseconds) to delay the execution before scheduling.", - ) - meta = { "indexes": [ {"fields": ["-start_timestamp", "action"]}, diff --git a/st2common/st2common/openapi.yaml b/st2common/st2common/openapi.yaml index 93d4bcdec6..832d70ff35 100644 --- a/st2common/st2common/openapi.yaml +++ b/st2common/st2common/openapi.yaml @@ -4935,7 +4935,7 @@ definitions: runner: $ref: '#/definitions/RunnerType' liveaction: - $ref: '#/definitions/LiveAction' + type: string task_execution: type: string workflow_execution: diff --git a/st2common/st2common/openapi.yaml.j2 b/st2common/st2common/openapi.yaml.j2 index 6b10362640..1e0bb75701 100644 --- a/st2common/st2common/openapi.yaml.j2 +++ b/st2common/st2common/openapi.yaml.j2 @@ -4931,7 +4931,7 @@ definitions: runner: $ref: '#/definitions/RunnerType' liveaction: - $ref: '#/definitions/LiveAction' + type: string task_execution: type: string workflow_execution: diff --git a/st2common/st2common/services/action.py b/st2common/st2common/services/action.py index 9c026f5507..ef3806461e 100644 --- a/st2common/st2common/services/action.py +++ b/st2common/st2common/services/action.py @@ -61,6 +61,7 @@ def create_request( ): """ Create an action execution. + :param liveaction: LiveActionDB :param action_db: Action model to operate one. If not provided, one is retrieved from the database using values from "liveaction". @@ -167,7 +168,6 @@ def create_request( runnertype_db=runnertype_db, publish=False, ) - if trace_db: trace_service.add_or_update_given_trace_db( trace_db=trace_db, @@ -316,7 +316,7 @@ def request_cancellation(liveaction, requester): liveaction, status, result=result, context=liveaction.context ) - execution = ActionExecution.get(liveaction__id=str(liveaction.id)) + execution = ActionExecution.get(liveaction=str(liveaction.id)) return (liveaction, execution) @@ -347,7 +347,7 @@ def request_pause(liveaction, requester): liveaction.status == action_constants.LIVEACTION_STATUS_PAUSING or liveaction.status == action_constants.LIVEACTION_STATUS_PAUSED ): - execution = ActionExecution.get(liveaction__id=str(liveaction.id)) + execution = ActionExecution.get(liveaction=str(liveaction.id)) return (liveaction, execution) if liveaction.status != action_constants.LIVEACTION_STATUS_RUNNING: @@ -363,7 +363,7 @@ def request_pause(liveaction, requester): context=liveaction.context, ) - execution = ActionExecution.get(liveaction__id=str(liveaction.id)) + execution = ActionExecution.get(liveaction=str(liveaction.id)) return (liveaction, execution) @@ -396,7 +396,7 @@ def request_resume(liveaction, requester): ] if liveaction.status in running_states: - execution = ActionExecution.get(liveaction__id=str(liveaction.id)) + execution = ActionExecution.get(liveaction=str(liveaction.id)) return (liveaction, execution) if liveaction.status != action_constants.LIVEACTION_STATUS_PAUSED: @@ -412,7 +412,7 @@ def request_resume(liveaction, requester): context=liveaction.context, ) - execution = ActionExecution.get(liveaction__id=str(liveaction.id)) + execution = ActionExecution.get(liveaction=str(liveaction.id)) return (liveaction, execution) @@ -433,7 +433,7 @@ def get_parent_liveaction(liveaction_db): return None parent_execution_db = ActionExecution.get(id=parent["execution_id"]) - parent_liveaction_db = LiveAction.get(id=parent_execution_db.liveaction["id"]) + parent_liveaction_db = LiveAction.get(id=parent_execution_db.liveaction) return parent_liveaction_db @@ -541,7 +541,7 @@ def store_execution_output_data_ex( def is_children_active(liveaction_id): - execution_db = ActionExecution.get(liveaction__id=str(liveaction_id)) + execution_db = ActionExecution.get(liveaction=str(liveaction_id)) if execution_db.runner["name"] not in action_constants.WORKFLOW_RUNNER_TYPES: return False diff --git a/st2common/st2common/services/executions.py b/st2common/st2common/services/executions.py index 80706e8f79..4c190b69fb 100644 --- a/st2common/st2common/services/executions.py +++ b/st2common/st2common/services/executions.py @@ -82,12 +82,10 @@ def _decompose_liveaction(liveaction_db): """ Splits the liveaction into an ActionExecution compatible dict. """ - decomposed = {"liveaction": {}} + decomposed = {"liveaction": str(liveaction_db.id)} liveaction_api = vars(LiveActionAPI.from_model(liveaction_db)) for k in liveaction_api.keys(): - if k in LIVEACTION_ATTRIBUTES: - decomposed["liveaction"][k] = liveaction_api[k] - else: + if k not in LIVEACTION_ATTRIBUTES: decomposed[k] = getattr(liveaction_db, k) return decomposed @@ -155,6 +153,7 @@ def create_execution_object( # NOTE: User input data is already validate as part of the API request, # other data is set by us. Skipping validation here makes operation 10%-30% faster + execution.liveaction = str(liveaction.id) execution = ActionExecution.add_or_update( execution, publish=publish, validate=False ) @@ -194,7 +193,7 @@ def update_execution(liveaction_db, publish=True, set_result_size=False): :param set_result_size: True to calculate size of the serialized result field value and set it on the "result_size" database field. """ - execution = ActionExecution.get(liveaction__id=str(liveaction_db.id)) + execution = ActionExecution.get(liveaction=str(liveaction_db.id)) with coordination.get_coordinator().get_lock(str(liveaction_db.id).encode()): # Skip execution object update when action is already in completed state. diff --git a/st2common/st2common/services/inquiry.py b/st2common/st2common/services/inquiry.py index 1301a62c4b..c52d182d7a 100644 --- a/st2common/st2common/services/inquiry.py +++ b/st2common/st2common/services/inquiry.py @@ -126,7 +126,7 @@ def respond(inquiry, response, requester=None): requester = cfg.CONF.system_user.user # Retrieve the liveaction from the database. - liveaction_db = lv_db_access.LiveAction.get_by_id(inquiry.liveaction.get("id")) + liveaction_db = lv_db_access.LiveAction.get_by_id(inquiry.liveaction) # Resume the parent workflow first. If the action execution for the inquiry is updated first, # it triggers handling of the action execution completion which will interact with the paused diff --git a/st2common/st2common/services/policies.py b/st2common/st2common/services/policies.py index 46e24ce290..771078e5d7 100644 --- a/st2common/st2common/services/policies.py +++ b/st2common/st2common/services/policies.py @@ -15,6 +15,9 @@ from __future__ import absolute_import +import sys +import traceback + from st2common.constants import action as ac_const from st2common import log as logging from st2common.persistence import policy as pc_db_access @@ -58,9 +61,11 @@ def apply_pre_run_policies(lv_ac_db): LOG.info(message % (policy_db.ref, policy_db.policy_type, str(lv_ac_db.id))) lv_ac_db = driver.apply_before(lv_ac_db) except: - message = 'An exception occurred while applying policy "%s" (%s) for liveaction "%s".' + _, ex, tb = sys.exc_info() + traceback_var = "".join(traceback.format_tb(tb, 20)) + message = 'An exception occurred while applying policy "%s" (%s) for liveaction "%s". traceback "%s"' LOG.exception( - message % (policy_db.ref, policy_db.policy_type, str(lv_ac_db.id)) + message % (policy_db.ref, policy_db.policy_type, str(lv_ac_db.id), traceback_var) ) if lv_ac_db.status == ac_const.LIVEACTION_STATUS_DELAYED: diff --git a/st2common/st2common/services/trace.py b/st2common/st2common/services/trace.py index 2d51161838..67035411c0 100644 --- a/st2common/st2common/services/trace.py +++ b/st2common/st2common/services/trace.py @@ -197,7 +197,7 @@ def get_trace_db_by_live_action(liveaction): ) return (created, trace_db) # 3. Check if the action_execution associated with liveaction leads to a trace_db - execution = ActionExecution.get(liveaction__id=str(liveaction.id)) + execution = ActionExecution.get(liveaction=str(liveaction.id)) if execution: trace_db = get_trace_db_by_action_execution(action_execution=execution) # 4. No trace_db found, therefore create one. This typically happens diff --git a/st2common/st2common/services/workflows.py b/st2common/st2common/services/workflows.py index b84671f8b1..f10963ddb6 100644 --- a/st2common/st2common/services/workflows.py +++ b/st2common/st2common/services/workflows.py @@ -19,6 +19,8 @@ import datetime import retrying import six +import sys +import traceback from orquesta import conducting from orquesta import events @@ -248,6 +250,7 @@ def request(wf_def, ac_ex_db, st2_ctx, notify_cfg=None): ) # Instantiate the workflow conductor. + LOG.info("action_params: " + str(action_params)) conductor_params = {"inputs": action_params, "context": st2_ctx} conductor = conducting.WorkflowConductor(wf_spec, **conductor_params) @@ -469,7 +472,7 @@ def request_cancellation(ac_ex_db): and root_ac_ex_db.status not in ac_const.LIVEACTION_CANCEL_STATES ): LOG.info("[%s] Cascading cancelation request to parent workflow.", wf_ac_ex_id) - root_lv_ac_db = lv_db_access.LiveAction.get(id=root_ac_ex_db.liveaction["id"]) + root_lv_ac_db = lv_db_access.LiveAction.get(id=root_ac_ex_db.liveaction) ac_svc.request_cancellation(root_lv_ac_db, None) LOG.debug("[%s] %s", wf_ac_ex_id, conductor.serialize()) @@ -666,7 +669,7 @@ def request_task_execution(wf_ex_db, st2_ctx, task_ex_req): except Exception as e: msg = 'Failed action execution(s) for task "%s", route "%s".' msg = msg % (task_id, str(task_route)) - LOG.exception(msg) + LOG.exception(msg, exc_info=True) msg = "%s %s: %s" % (msg, type(e).__name__, six.text_type(e)) update_progress(wf_ex_db, msg, severity="error", log=False) msg = "%s: %s" % (type(e).__name__, six.text_type(e)) @@ -676,7 +679,10 @@ def request_task_execution(wf_ex_db, st2_ctx, task_ex_req): "task_id": task_id, "route": task_route, } - update_task_execution(str(task_ex_db.id), statuses.FAILED, {"errors": [error]}) + exc_type, exc_value, exc_traceback = sys.exc_info() + traceback_in_var = traceback.format_tb(exc_traceback) + update_task_execution(str(task_ex_db.id), statuses.FAILED, {"errors": + [error], "traceback": traceback_in_var}) raise e return task_ex_db @@ -906,7 +912,7 @@ def handle_action_execution_resume(ac_ex_db): if parent_ac_ex_db.status == ac_const.LIVEACTION_STATUS_PAUSED: action_utils.update_liveaction_status( - liveaction_id=parent_ac_ex_db.liveaction["id"], + liveaction_id=parent_ac_ex_db.liveaction, status=ac_const.LIVEACTION_STATUS_RUNNING, publish=False, ) @@ -1184,12 +1190,19 @@ def request_next_tasks(wf_ex_db, task_ex_id=None): # Request the task execution. request_task_execution(wf_ex_db, st2_ctx, task) except Exception as e: + import sys + import traceback + + exc_type, exc_value, exc_traceback = sys.exc_info() + traceback_in_var = traceback.format_tb(exc_traceback) msg = 'Failed task execution for task "%s", route "%s".' msg = msg % (task["id"], str(task["route"])) update_progress( wf_ex_db, "%s %s" % (msg, str(e)), severity="error", log=False ) + LOG.error(e, exc_info=True) LOG.exception(msg) + fail_workflow_execution(str(wf_ex_db.id), e, task=task) return @@ -1435,7 +1448,7 @@ def update_execution_records( # Update the corresponding liveaction and action execution for the workflow. wf_ac_ex_db = ex_db_access.ActionExecution.get_by_id(wf_ex_db.action_execution) - wf_lv_ac_db = action_utils.get_liveaction_by_id(wf_ac_ex_db.liveaction["id"]) + wf_lv_ac_db = action_utils.get_liveaction_by_id(wf_ac_ex_db.liveaction) # Gather result for liveaction and action execution. result = {"output": wf_ex_db.output or None} diff --git a/st2common/st2common/util/param.py b/st2common/st2common/util/param.py index 67fb83e9ac..b8bb038369 100644 --- a/st2common/st2common/util/param.py +++ b/st2common/st2common/util/param.py @@ -310,9 +310,11 @@ def render_live_params( additional_contexts=None, ): """ + :param params BaseDict Renders list of parameters. Ensures that there's no cyclic or missing dependencies. Returns a dict of plain rendered parameters. """ + params = params additional_contexts = additional_contexts or {} pack = action_context.get("pack") diff --git a/st2common/tests/unit/migrations/test_v35_migrate_db_dict_field_values.py b/st2common/tests/unit/migrations/test_v35_migrate_db_dict_field_values.py index 84347fcbab..df6f316465 100644 --- a/st2common/tests/unit/migrations/test_v35_migrate_db_dict_field_values.py +++ b/st2common/tests/unit/migrations/test_v35_migrate_db_dict_field_values.py @@ -80,10 +80,20 @@ def test_migrate_executions(self): class ActionExecutionDB_OldFieldType(ActionExecutionDB): result = stormbase.EscapedDynamicField(default={}) + liveaction = stormbase.EscapedDictField(required=True) + parameters = stormbase.EscapedDynamicField(default={}) class LiveActionDB_OldFieldType(LiveActionDB): result = stormbase.EscapedDynamicField(default={}) + #todo(aj) need ActionExecutionDB_NewFieldType to be used for update + #here. the model field type has changed to string in current models + + class ActionExecutionDB_NewFieldType(ActionExecutionDB): + liveaction = stormbase.EscapedDictField(required=True) + parameters = stormbase.EscapedDynamicField(default={}) + + execution_dbs = ActionExecution.query( __raw__={ "result": { diff --git a/st2common/tests/unit/services/test_trace.py b/st2common/tests/unit/services/test_trace.py index 39db4c7ade..5933cdc4ef 100644 --- a/st2common/tests/unit/services/test_trace.py +++ b/st2common/tests/unit/services/test_trace.py @@ -254,7 +254,7 @@ def test_get_trace_db_by_live_action_from_execution(self): traceable_liveaction = copy.copy(self.traceable_liveaction) # fixtures id value in liveaction is not persisted in DB. traceable_liveaction.id = bson.ObjectId( - self.traceable_execution.liveaction["id"] + self.traceable_execution.liveaction ) created, trace_db = trace_service.get_trace_db_by_live_action( traceable_liveaction diff --git a/st2common/tests/unit/services/test_workflow_identify_orphans.py b/st2common/tests/unit/services/test_workflow_identify_orphans.py index 7110b509c9..c94892e98b 100644 --- a/st2common/tests/unit/services/test_workflow_identify_orphans.py +++ b/st2common/tests/unit/services/test_workflow_identify_orphans.py @@ -175,7 +175,7 @@ def mock_workflow_records(self, completed=False, expired=True, log=True): workflow_execution=str(wf_ex_db.id), action={"runner_type": runner, "ref": action_ref}, runner={"name": runner}, - liveaction={"id": str(lv_ac_db.id)}, + liveaction=str(lv_ac_db.id), context={"user": user, "workflow_execution": str(wf_ex_db.id)}, status=status, start_timestamp=start_timestamp, @@ -269,7 +269,7 @@ def mock_task_records( task_execution=str(tk_ex_db.id), action={"runner_type": runner, "ref": action_ref}, runner={"name": runner}, - liveaction={"id": str(lv_ac_db.id)}, + liveaction=str(lv_ac_db.id), context=context, status=status, start_timestamp=tk_ex_db.start_timestamp, diff --git a/st2common/tests/unit/services/test_workflow_service_retries.py b/st2common/tests/unit/services/test_workflow_service_retries.py index 0e322fe573..45257f8236 100644 --- a/st2common/tests/unit/services/test_workflow_service_retries.py +++ b/st2common/tests/unit/services/test_workflow_service_retries.py @@ -144,7 +144,7 @@ def test_recover_from_coordinator_connection_error(self, mock_get_lock): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) mock_get_lock.side_effect = [ coordination.ToozConnectionError("foobar"), @@ -178,7 +178,7 @@ def test_retries_exhausted_from_coordinator_connection_error(self, mock_get_lock tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) mock_get_lock.side_effect = [ @@ -220,7 +220,7 @@ def test_recover_from_database_connection_error(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) wf_svc.handle_action_execution_completion(tk1_ac_ex_db) @@ -247,7 +247,7 @@ def test_retries_exhausted_from_database_connection_error(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # The connection error should raise if retries are exhaused. diff --git a/st2common/tests/unit/test_db_execution.py b/st2common/tests/unit/test_db_execution.py index ca322c0f1b..cfcf2bf981 100644 --- a/st2common/tests/unit/test_db_execution.py +++ b/st2common/tests/unit/test_db_execution.py @@ -60,6 +60,7 @@ }, }, }, + "id": "liveaction_inquiry", "action": "core.ask", } @@ -70,6 +71,7 @@ } }, "action": "st2.inquiry.respond", + "id": "liveaction_respond" } OUTPUT_SCHEMA_RESULT = { @@ -82,6 +84,7 @@ } OUTPUT_SCHEMA_LIVEACTION = { + "id": "output_schema", "action": "core.ask", "parameters": {}, } @@ -91,14 +94,14 @@ "action": {"uid": "action:core:ask", "output_schema": {}}, "status": "succeeded", "runner": {"name": "inquirer"}, - "liveaction": INQUIRY_LIVEACTION, + "liveaction": INQUIRY_LIVEACTION["id"], "result": INQUIRY_RESULT, }, "execution_2": { "action": {"uid": "action:st2:inquiry.respond", "output_schema": {}}, "status": "succeeded", "runner": {"name": "python-script"}, - "liveaction": RESPOND_LIVEACTION, + "liveaction": RESPOND_LIVEACTION["id"], "result": {"exit_code": 0, "result": None, "stderr": "", "stdout": ""}, }, "execution_3": { @@ -119,7 +122,7 @@ }, "status": "succeeded", "runner": {"name": "inquirer", "output_key": "result"}, - "liveaction": OUTPUT_SCHEMA_LIVEACTION, + "liveaction": OUTPUT_SCHEMA_LIVEACTION["id"], "result": OUTPUT_SCHEMA_RESULT, }, } @@ -188,19 +191,6 @@ def test_execution_inquiry_secrets(self): "supersecretvalue", ) - def test_execution_inquiry_response_action(self): - """Test that the response parameters for any `st2.inquiry.respond` executions are masked - - We aren't bothering to get the inquiry schema in the `st2.inquiry.respond` action, - so we mask all response values. This test ensures this happens. - """ - - masked = self.executions["execution_2"].mask_secrets( - self.executions["execution_2"].to_serializable_dict() - ) - for value in masked["parameters"]["response"].values(): - self.assertEqual(value, MASKED_ATTRIBUTE_VALUE) - def test_output_schema_secret_param_masking(self): """Test that the output marked as secret in the output schema is masked in the output result diff --git a/st2common/tests/unit/test_db_fields.py b/st2common/tests/unit/test_db_fields.py index 9abd587fb5..bc4a25632a 100644 --- a/st2common/tests/unit/test_db_fields.py +++ b/st2common/tests/unit/test_db_fields.py @@ -94,6 +94,7 @@ def test_to_mongo(self): result = field.to_mongo(MOCK_DATA_DICT) self.assertTrue(isinstance(result, bytes)) + result = zstandard.ZstdDecompressor().decompress(result) self.assertEqual(result, orjson.dumps(MOCK_DATA_DICT)) def test_to_python(self): @@ -147,78 +148,17 @@ def test_parse_field_value(self): self.assertEqual(result, {"c": "d"}) -class JSONDictFieldTestCaseWithHeader(unittest2.TestCase): - def test_to_mongo_no_compression(self): - field = JSONDictField(use_header=True) - - result = field.to_mongo(MOCK_DATA_DICT) - self.assertTrue(isinstance(result, bytes)) - - split = result.split(b":", 2) - self.assertEqual(split[0], JSONDictFieldCompressionAlgorithmEnum.NONE.value) - self.assertEqual(split[1], JSONDictFieldSerializationFormatEnum.ORJSON.value) - self.assertEqual(orjson.loads(split[2]), MOCK_DATA_DICT) - - parsed_value = field.parse_field_value(result) - self.assertEqual(parsed_value, MOCK_DATA_DICT) - - def test_to_mongo_zstandard_compression(self): - field = JSONDictField(use_header=True, compression_algorithm="zstandard") - - result = field.to_mongo(MOCK_DATA_DICT) - self.assertTrue(isinstance(result, bytes)) - - split = result.split(b":", 2) - self.assertEqual( - split[0], JSONDictFieldCompressionAlgorithmEnum.ZSTANDARD.value - ) - self.assertEqual(split[1], JSONDictFieldSerializationFormatEnum.ORJSON.value) - self.assertEqual( - orjson.loads(zstandard.ZstdDecompressor().decompress(split[2])), - MOCK_DATA_DICT, - ) - - parsed_value = field.parse_field_value(result) - self.assertEqual(parsed_value, MOCK_DATA_DICT) - - def test_to_python_no_compression(self): - field = JSONDictField(use_header=True) - - serialized_data = field.to_mongo(MOCK_DATA_DICT) - - self.assertTrue(isinstance(serialized_data, bytes)) - split = serialized_data.split(b":", 2) - self.assertEqual(split[0], JSONDictFieldCompressionAlgorithmEnum.NONE.value) - self.assertEqual(split[1], JSONDictFieldSerializationFormatEnum.ORJSON.value) - - desserialized_data = field.to_python(serialized_data) - self.assertEqual(desserialized_data, MOCK_DATA_DICT) - - def test_to_python_zstandard_compression(self): - field = JSONDictField(use_header=True, compression_algorithm="zstandard") - - serialized_data = field.to_mongo(MOCK_DATA_DICT) - self.assertTrue(isinstance(serialized_data, bytes)) - - split = serialized_data.split(b":", 2) - self.assertEqual( - split[0], JSONDictFieldCompressionAlgorithmEnum.ZSTANDARD.value - ) - self.assertEqual(split[1], JSONDictFieldSerializationFormatEnum.ORJSON.value) - - desserialized_data = field.to_python(serialized_data) - self.assertEqual(desserialized_data, MOCK_DATA_DICT) - - class JSONDictEscapedFieldCompatibilityFieldTestCase(DbTestCase): def test_to_mongo(self): field = JSONDictEscapedFieldCompatibilityField(use_header=False) result_to_mongo_1 = field.to_mongo(MOCK_DATA_DICT) + result_to_mongo_1 = zstandard.ZstdDecompressor().decompress(result_to_mongo_1) self.assertEqual(result_to_mongo_1, orjson.dumps(MOCK_DATA_DICT)) # Already serialized result_to_mongo_2 = field.to_mongo(MOCK_DATA_DICT) + result_to_mongo_2 = zstandard.ZstdDecompressor().decompress(result_to_mongo_2) self.assertEqual(result_to_mongo_2, result_to_mongo_1) def test_existing_db_value_is_using_escaped_dict_field_compatibility(self): @@ -275,7 +215,9 @@ def test_existing_db_value_is_using_escaped_dict_field_compatibility(self): self.assertEqual(len(pymongo_result), 1) self.assertEqual(pymongo_result[0]["_id"], inserted_model_db.id) self.assertTrue(isinstance(pymongo_result[0]["result"], bytes)) - self.assertEqual(orjson.loads(pymongo_result[0]["result"]), expected_data) + + result = zstandard.ZstdDecompressor().decompress(pymongo_result[0]["result"]) + self.assertEqual(orjson.loads(result), expected_data) self.assertEqual(pymongo_result[0]["counter"], 1) def test_field_state_changes_are_correctly_detected_add_or_update_method(self): diff --git a/st2common/tests/unit/test_executions.py b/st2common/tests/unit/test_executions.py index 0be1ca7c9d..aa5efa3311 100644 --- a/st2common/tests/unit/test_executions.py +++ b/st2common/tests/unit/test_executions.py @@ -38,7 +38,7 @@ def setUp(self): "id": str(bson.ObjectId()), "action": copy.deepcopy(fixture.ARTIFACTS["actions"]["local"]), "runner": copy.deepcopy(fixture.ARTIFACTS["runners"]["run-local"]), - "liveaction": copy.deepcopy(fixture.ARTIFACTS["liveactions"]["task1"]), + "liveaction": copy.deepcopy(fixture.ARTIFACTS["liveactions"]["task1"]["id"]), "status": fixture.ARTIFACTS["liveactions"]["task1"]["status"], "start_timestamp": fixture.ARTIFACTS["liveactions"]["task1"][ "start_timestamp" @@ -71,7 +71,7 @@ def setUp(self): "rule": copy.deepcopy(fixture.ARTIFACTS["rule"]), "action": copy.deepcopy(fixture.ARTIFACTS["actions"]["chain"]), "runner": copy.deepcopy(fixture.ARTIFACTS["runners"]["action-chain"]), - "liveaction": copy.deepcopy(fixture.ARTIFACTS["liveactions"]["workflow"]), + "liveaction": copy.deepcopy(fixture.ARTIFACTS["liveactions"]["workflow"]["id"]), "children": [task["id"] for task in self.fake_history_subtasks], "status": fixture.ARTIFACTS["liveactions"]["workflow"]["status"], "start_timestamp": fixture.ARTIFACTS["liveactions"]["workflow"][ @@ -117,10 +117,7 @@ def test_model_complete(self): self.assertDictEqual(model.rule, self.fake_history_workflow["rule"]) self.assertDictEqual(model.action, self.fake_history_workflow["action"]) self.assertDictEqual(model.runner, self.fake_history_workflow["runner"]) - doc = copy.deepcopy(self.fake_history_workflow["liveaction"]) - doc["start_timestamp"] = doc["start_timestamp"] - doc["end_timestamp"] = doc["end_timestamp"] - self.assertDictEqual(model.liveaction, doc) + self.assertEqual(model.liveaction, self.fake_history_workflow["liveaction"]) self.assertIsNone(getattr(model, "parent", None)) self.assertListEqual(model.children, self.fake_history_workflow["children"]) @@ -137,7 +134,7 @@ def test_model_complete(self): self.assertDictEqual(obj.rule, self.fake_history_workflow["rule"]) self.assertDictEqual(obj.action, self.fake_history_workflow["action"]) self.assertDictEqual(obj.runner, self.fake_history_workflow["runner"]) - self.assertDictEqual(obj.liveaction, self.fake_history_workflow["liveaction"]) + self.assertEqual(obj.liveaction, self.fake_history_workflow["liveaction"]) self.assertIsNone(getattr(obj, "parent", None)) self.assertListEqual(obj.children, self.fake_history_workflow["children"]) @@ -157,10 +154,7 @@ def test_crud_complete(self): self.assertDictEqual(model.rule, self.fake_history_workflow["rule"]) self.assertDictEqual(model.action, self.fake_history_workflow["action"]) self.assertDictEqual(model.runner, self.fake_history_workflow["runner"]) - doc = copy.deepcopy(self.fake_history_workflow["liveaction"]) - doc["start_timestamp"] = doc["start_timestamp"] - doc["end_timestamp"] = doc["end_timestamp"] - self.assertDictEqual(model.liveaction, doc) + self.assertEqual(model.liveaction, self.fake_history_workflow["liveaction"]) self.assertIsNone(getattr(model, "parent", None)) self.assertListEqual(model.children, self.fake_history_workflow["children"]) @@ -186,7 +180,7 @@ def test_model_partial(self): self.assertIsNone(getattr(obj, "rule", None)) self.assertDictEqual(obj.action, self.fake_history_subtasks[0]["action"]) self.assertDictEqual(obj.runner, self.fake_history_subtasks[0]["runner"]) - self.assertDictEqual( + self.assertEqual( obj.liveaction, self.fake_history_subtasks[0]["liveaction"] ) self.assertEqual(obj.parent, self.fake_history_subtasks[0]["parent"]) @@ -201,11 +195,8 @@ def test_model_partial(self): self.assertDictEqual(model.rule, {}) self.assertDictEqual(model.action, self.fake_history_subtasks[0]["action"]) self.assertDictEqual(model.runner, self.fake_history_subtasks[0]["runner"]) - doc = copy.deepcopy(self.fake_history_subtasks[0]["liveaction"]) - doc["start_timestamp"] = doc["start_timestamp"] - doc["end_timestamp"] = doc["end_timestamp"] - - self.assertDictEqual(model.liveaction, doc) + self.assertEqual(model.liveaction, + self.fake_history_subtasks[0]["liveaction"]) self.assertEqual(model.parent, self.fake_history_subtasks[0]["parent"]) self.assertListEqual(model.children, []) @@ -218,7 +209,7 @@ def test_model_partial(self): self.assertIsNone(getattr(obj, "rule", None)) self.assertDictEqual(obj.action, self.fake_history_subtasks[0]["action"]) self.assertDictEqual(obj.runner, self.fake_history_subtasks[0]["runner"]) - self.assertDictEqual( + self.assertEqual( obj.liveaction, self.fake_history_subtasks[0]["liveaction"] ) self.assertEqual(obj.parent, self.fake_history_subtasks[0]["parent"]) @@ -236,10 +227,8 @@ def test_crud_partial(self): self.assertDictEqual(model.rule, {}) self.assertDictEqual(model.action, self.fake_history_subtasks[0]["action"]) self.assertDictEqual(model.runner, self.fake_history_subtasks[0]["runner"]) - doc = copy.deepcopy(self.fake_history_subtasks[0]["liveaction"]) - doc["start_timestamp"] = doc["start_timestamp"] - doc["end_timestamp"] = doc["end_timestamp"] - self.assertDictEqual(model.liveaction, doc) + self.assertEqual(model.liveaction, + self.fake_history_subtasks[0]["liveaction"]) self.assertEqual(model.parent, self.fake_history_subtasks[0]["parent"]) self.assertListEqual(model.children, []) diff --git a/st2common/tests/unit/test_executions_util.py b/st2common/tests/unit/test_executions_util.py index 4c2530155a..3ae45169a7 100644 --- a/st2common/tests/unit/test_executions_util.py +++ b/st2common/tests/unit/test_executions_util.py @@ -79,7 +79,7 @@ def test_execution_creation_manual_action_run(self): executions_util.create_execution_object(liveaction) post_creation_timestamp = date_utils.get_datetime_utc_now() execution = self._get_action_execution( - liveaction__id=str(liveaction.id), raise_exception=True + liveaction=str(liveaction.id), raise_exception=True ) self.assertDictEqual(execution.trigger, {}) self.assertDictEqual(execution.trigger_type, {}) @@ -90,7 +90,7 @@ def test_execution_creation_manual_action_run(self): runner = RunnerType.get_by_name(action.runner_type["name"]) self.assertDictEqual(execution.runner, vars(RunnerTypeAPI.from_model(runner))) liveaction = LiveAction.get_by_id(str(liveaction.id)) - self.assertEqual(execution.liveaction["id"], str(liveaction.id)) + self.assertEqual(execution.liveaction, str(liveaction.id)) self.assertEqual(len(execution.log), 1) self.assertEqual(execution.log[0]["status"], liveaction.status) self.assertGreater(execution.log[0]["timestamp"], pre_creation_timestamp) @@ -120,7 +120,7 @@ def test_execution_creation_action_triggered_by_rule(self): ) executions_util.create_execution_object(liveaction) execution = self._get_action_execution( - liveaction__id=str(liveaction.id), raise_exception=True + liveaction=str(liveaction.id), raise_exception=True ) self.assertDictEqual(execution.trigger, vars(TriggerAPI.from_model(trigger))) self.assertDictEqual( @@ -136,13 +136,13 @@ def test_execution_creation_action_triggered_by_rule(self): runner = RunnerType.get_by_name(action.runner_type["name"]) self.assertDictEqual(execution.runner, vars(RunnerTypeAPI.from_model(runner))) liveaction = LiveAction.get_by_id(str(liveaction.id)) - self.assertEqual(execution.liveaction["id"], str(liveaction.id)) + self.assertEqual(execution.liveaction, str(liveaction.id)) def test_execution_creation_with_web_url(self): liveaction = self.MODELS["liveactions"]["liveaction1.yaml"] executions_util.create_execution_object(liveaction) execution = self._get_action_execution( - liveaction__id=str(liveaction.id), raise_exception=True + liveaction=str(liveaction.id), raise_exception=True ) self.assertIsNotNone(execution.web_url) execution_id = str(execution.id) @@ -164,7 +164,7 @@ def test_execution_update(self): executions_util.update_execution(liveaction) post_update_timestamp = date_utils.get_datetime_utc_now() execution = self._get_action_execution( - liveaction__id=str(liveaction.id), raise_exception=True + liveaction=str(liveaction.id), raise_exception=True ) self.assertEqual(len(execution.log), 2) self.assertEqual(execution.log[1]["status"], liveaction.status) @@ -178,7 +178,7 @@ def test_skip_execution_update(self): liveaction.status = "running" executions_util.update_execution(liveaction) execution = self._get_action_execution( - liveaction__id=str(liveaction.id), raise_exception=True + liveaction=str(liveaction.id), raise_exception=True ) self.assertEqual(len(execution.log), 1) # Check status is not updated if it's already in completed state. diff --git a/st2common/tests/unit/test_purge_executions.py b/st2common/tests/unit/test_purge_executions.py index f43266a121..80fa4d2dec 100644 --- a/st2common/tests/unit/test_purge_executions.py +++ b/st2common/tests/unit/test_purge_executions.py @@ -194,7 +194,7 @@ def test_liveaction_gets_deleted(self): exec_model["end_timestamp"] = end_ts exec_model["status"] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model["id"] = bson.ObjectId() - exec_model["liveaction"]["id"] = str(liveaction.id) + exec_model["liveaction"] = str(liveaction.id) ActionExecution.add_or_update(exec_model) liveactions = LiveAction.get_all() diff --git a/st2stream/tests/unit/controllers/v1/test_stream_execution_output.py b/st2stream/tests/unit/controllers/v1/test_stream_execution_output.py index 9a135d1789..8ddd983ea8 100644 --- a/st2stream/tests/unit/controllers/v1/test_stream_execution_output.py +++ b/st2stream/tests/unit/controllers/v1/test_stream_execution_output.py @@ -59,7 +59,7 @@ def test_get_output_running_execution(self): status=status, action={"ref": "core.local"}, runner={"name": "local-shell-cmd"}, - liveaction={"ref": "foo"}, + liveaction="ref", ) action_execution_db = ActionExecution.add_or_update(action_execution_db) @@ -141,7 +141,7 @@ def test_get_output_finished_execution(self): status=status, action={"ref": "core.local"}, runner={"name": "local-shell-cmd"}, - liveaction={"ref": "foo"}, + liveaction="ref", ) action_execution_db = ActionExecution.add_or_update(action_execution_db) diff --git a/st2tests/st2tests/api.py b/st2tests/st2tests/api.py index dae8eb7831..c4625ec5de 100644 --- a/st2tests/st2tests/api.py +++ b/st2tests/st2tests/api.py @@ -395,7 +395,7 @@ def _get_actionexecution_id(resp): @staticmethod def _get_liveaction_id(resp): - return resp.json["liveaction"]["id"] + return resp.json["liveaction"] def _do_get_one(self, actionexecution_id, *args, **kwargs): return self.app.get("/v1/executions/%s" % actionexecution_id, *args, **kwargs) diff --git a/st2tests/st2tests/fixtures/descendants/executions/child1_level1.yaml b/st2tests/st2tests/fixtures/descendants/executions/child1_level1.yaml index 30467cfb18..0b7faddb17 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/child1_level1.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/child1_level1.yaml @@ -7,8 +7,7 @@ children: - 54e6583d0640fd16887d685b end_timestamp: '2014-09-01T00:00:57.000001Z' id: 54e657f20640fd16887d6857 -liveaction: - action: pointlessaction +liveaction: pointlessaction parent: 54e657d60640fd16887d6855 runner: name: pointlessrunner diff --git a/st2tests/st2tests/fixtures/descendants/executions/child1_level2.yaml b/st2tests/st2tests/fixtures/descendants/executions/child1_level2.yaml index 2ee548ccb2..1ebf3d5f39 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/child1_level2.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/child1_level2.yaml @@ -5,8 +5,7 @@ action: children: [] end_timestamp: '2014-09-01T00:00:56.000002Z' id: 54e657fa0640fd16887d6858 -liveaction: - action: pointlessaction +liveaction: pointlessaction parent: 54e657f20640fd16887d6857 runner: name: pointlessrunner diff --git a/st2tests/st2tests/fixtures/descendants/executions/child1_level3.yaml b/st2tests/st2tests/fixtures/descendants/executions/child1_level3.yaml index 276c5aa0b7..9443d4e0cb 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/child1_level3.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/child1_level3.yaml @@ -5,8 +5,7 @@ action: children: [] end_timestamp: '2014-09-01T00:00:55.100000Z' id: 54e6581b0640fd16887d6859 -liveaction: - action: pointlessaction +liveaction: pointlessaction parent: 54e6583d0640fd16887d685b runner: name: pointlessrunner diff --git a/st2tests/st2tests/fixtures/descendants/executions/child2_level1.yaml b/st2tests/st2tests/fixtures/descendants/executions/child2_level1.yaml index 35050c15cd..65a6dfc803 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/child2_level1.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/child2_level1.yaml @@ -6,8 +6,7 @@ children: - 54e658570640fd16887d685d end_timestamp: '2014-09-01T00:00:55.000000Z' id: 54e658290640fd16887d685a -liveaction: - action: pointlessaction +liveaction: pointlessaction parent: 54e657d60640fd16887d6855 runner: name: pointlessrunner diff --git a/st2tests/st2tests/fixtures/descendants/executions/child2_level2.yaml b/st2tests/st2tests/fixtures/descendants/executions/child2_level2.yaml index 7d57ceb171..fe96706346 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/child2_level2.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/child2_level2.yaml @@ -6,8 +6,7 @@ children: - 54e6581b0640fd16887d6859 end_timestamp: '2014-09-01T00:00:55.000000Z' id: 54e6583d0640fd16887d685b -liveaction: - action: pointlessaction +liveaction: pointlessaction parent: 54e657f20640fd16887d6857 runner: name: pointlessrunner diff --git a/st2tests/st2tests/fixtures/descendants/executions/child2_level3.yaml b/st2tests/st2tests/fixtures/descendants/executions/child2_level3.yaml index a10bcd016b..7e0cebd8ab 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/child2_level3.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/child2_level3.yaml @@ -5,8 +5,7 @@ action: children: [] end_timestamp: '2014-09-01T00:00:59.000010Z' id: 54e6584a0640fd16887d685c -liveaction: - action: pointlessaction +liveaction: pointlessaction parent: 54e658570640fd16887d685d runner: name: pointlessrunner diff --git a/st2tests/st2tests/fixtures/descendants/executions/child3_level2.yaml b/st2tests/st2tests/fixtures/descendants/executions/child3_level2.yaml index d803654d6b..e80356860b 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/child3_level2.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/child3_level2.yaml @@ -7,8 +7,7 @@ children: - 54e6585f0640fd16887d685e end_timestamp: '2014-09-01T00:00:55.000000Z' id: 54e658570640fd16887d685d -liveaction: - action: pointlessaction +liveaction: pointlessaction parent: 54e658290640fd16887d685a runner: name: pointlessrunner diff --git a/st2tests/st2tests/fixtures/descendants/executions/child3_level3.yaml b/st2tests/st2tests/fixtures/descendants/executions/child3_level3.yaml index ad1aae1bad..754f29831a 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/child3_level3.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/child3_level3.yaml @@ -5,8 +5,7 @@ action: children: [] end_timestamp: '2014-09-01T00:00:55.000000Z' id: 54e6585f0640fd16887d685e -liveaction: - action: pointlessaction +liveaction: pointlessaction parent: 54e658570640fd16887d685d runner: name: pointlessrunner diff --git a/st2tests/st2tests/fixtures/descendants/executions/root_execution.yaml b/st2tests/st2tests/fixtures/descendants/executions/root_execution.yaml index d993f9c140..37a5b3f221 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/root_execution.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/root_execution.yaml @@ -7,8 +7,7 @@ children: - 54e658290640fd16887d685a end_timestamp: '2014-09-01T00:00:59.000000Z' id: 54e657d60640fd16887d6855 -liveaction: - action: pointlessaction +liveaction: pointlessaction runner: name: pointlessrunner runner_module: no.module diff --git a/st2tests/st2tests/fixtures/generic/executions/execution1.yaml b/st2tests/st2tests/fixtures/generic/executions/execution1.yaml index 8d519fad7a..d7a7329dad 100644 --- a/st2tests/st2tests/fixtures/generic/executions/execution1.yaml +++ b/st2tests/st2tests/fixtures/generic/executions/execution1.yaml @@ -13,17 +13,7 @@ action: runner_type: run-local end_timestamp: '2014-09-01T00:00:05.000000Z' id: 54c6bb640640fd5211edef0d -liveaction: - action: core.someworkflow - callback: {} - context: - user: system - end_timestamp: '2014-09-01T00:00:05.000000Z' - id: 54c6b6d60640fd4f5354e74a - parameters: {} - result: {} - start_timestamp: '2014-09-01T00:00:01.000000Z' - status: scheduled +liveaction: 54c6b6d60640fd4f5354e74a parameters: {} result: {} runner: diff --git a/st2tests/st2tests/fixtures/generic/liveactions/parentliveaction.yaml b/st2tests/st2tests/fixtures/generic/liveactions/parentliveaction.yaml index ed56d4c449..087be792b1 100644 --- a/st2tests/st2tests/fixtures/generic/liveactions/parentliveaction.yaml +++ b/st2tests/st2tests/fixtures/generic/liveactions/parentliveaction.yaml @@ -1,10 +1,10 @@ --- action: core.someworkflow +id: 54c6b6d60640fd4f5354e74a callback: {} context: user: system end_timestamp: '2014-09-01T00:00:05.000000Z' -id: 54c6b6d60640fd4f5354e74a parameters: {} result: {} start_timestamp: '2014-09-01T00:00:01.000000Z' diff --git a/st2tests/st2tests/fixtures/packs/dummy_pack_23/actions/workflows/__init__.py b/st2tests/st2tests/fixtures/packs/dummy_pack_23/actions/workflows/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/st2tests/st2tests/fixtures/packs/executions/liveactions.yaml b/st2tests/st2tests/fixtures/packs/executions/liveactions.yaml index 6f87578650..2113d0ea99 100644 --- a/st2tests/st2tests/fixtures/packs/executions/liveactions.yaml +++ b/st2tests/st2tests/fixtures/packs/executions/liveactions.yaml @@ -1,6 +1,7 @@ --- task1: action: executions.local + id: "liveaction1" callback: {} end_timestamp: '2014-09-01T00:00:05.000000Z' parameters: @@ -19,6 +20,7 @@ task1: status: succeeded task2: action: executions.local + id: "liveaction2" callback: {} end_timestamp: '2014-09-01T00:00:05.000000Z' parameters: @@ -36,6 +38,7 @@ task2: start_timestamp: '2014-09-01T00:00:03.000000Z' status: succeeded workflow: + id: "workflow1" action: executions.chain callback: {} end_timestamp: '2014-09-01T00:00:05.000000Z' diff --git a/st2tests/st2tests/fixtures/traces/executions/execution_with_parent.yaml b/st2tests/st2tests/fixtures/traces/executions/execution_with_parent.yaml index d627ef3777..a3e680145a 100644 --- a/st2tests/st2tests/fixtures/traces/executions/execution_with_parent.yaml +++ b/st2tests/st2tests/fixtures/traces/executions/execution_with_parent.yaml @@ -17,17 +17,7 @@ action: runner_type: action-chain end_timestamp: '2014-09-01T00:00:05.000000Z' id: 54c6bb640640fd5211edef3d -liveaction: - action: traces.someworkflow - callback: {} - context: - user: system - end_timestamp: '2014-09-01T00:00:05.000000Z' - id: 54c6b6d60640fd4f5354e75a - parameters: {} - result: {} - start_timestamp: '2014-09-01T00:00:01.000000Z' - status: scheduled +liveaction: 54c6b6d60640fd4f5354e75a parameters: {} result: {} runner: diff --git a/st2tests/st2tests/fixtures/traces/executions/rule_fired_execution.yaml b/st2tests/st2tests/fixtures/traces/executions/rule_fired_execution.yaml index 9e5f3af967..18bb047a82 100644 --- a/st2tests/st2tests/fixtures/traces/executions/rule_fired_execution.yaml +++ b/st2tests/st2tests/fixtures/traces/executions/rule_fired_execution.yaml @@ -17,17 +17,7 @@ action: runner_type: action-chain end_timestamp: '2014-09-01T00:00:05.000000Z' id: 54c6bb640640fd5211edef0d -liveaction: - action: traces.someworkflow - callback: {} - context: - user: system - end_timestamp: '2014-09-01T00:00:05.000000Z' - id: 54c6b6d60640fd4f5354e74a - parameters: {} - result: {} - start_timestamp: '2014-09-01T00:00:01.000000Z' - status: scheduled +liveaction: 54c6b6d60640fd4f5354e74a parameters: {} result: {} runner: diff --git a/st2tests/st2tests/fixtures/traces/executions/traceable_execution.yaml b/st2tests/st2tests/fixtures/traces/executions/traceable_execution.yaml index 55d7d404fa..cfded0e2da 100644 --- a/st2tests/st2tests/fixtures/traces/executions/traceable_execution.yaml +++ b/st2tests/st2tests/fixtures/traces/executions/traceable_execution.yaml @@ -17,17 +17,7 @@ action: runner_type: action-chain end_timestamp: '2014-09-01T00:00:05.000000Z' id: 54c6bb640640fd5211edef0d -liveaction: - action: traces.someworkflow - callback: {} - context: - user: system - end_timestamp: '2014-09-01T00:00:05.000000Z' - id: 54c6b6d60640fd4f5354e74a - parameters: {} - result: {} - start_timestamp: '2014-09-01T00:00:01.000000Z' - status: scheduled +liveaction: 54c6b6d60640fd4f5354e74a parameters: {} result: {} runner: From cf051bbbe90ccfe3dcf1b51ad8af4d0da3138d95 Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Fri, 23 Jun 2023 13:42:09 +0000 Subject: [PATCH 02/70] fix test_executions_fixtures.py tests --- .../controllers/v1/test_executions_filters.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/st2api/tests/unit/controllers/v1/test_executions_filters.py b/st2api/tests/unit/controllers/v1/test_executions_filters.py index af451ca519..470b810088 100644 --- a/st2api/tests/unit/controllers/v1/test_executions_filters.py +++ b/st2api/tests/unit/controllers/v1/test_executions_filters.py @@ -60,16 +60,19 @@ def setUpClass(cls): "rule": copy.deepcopy(fixture.ARTIFACTS["rule"]), "action": copy.deepcopy(fixture.ARTIFACTS["actions"]["chain"]), "runner": copy.deepcopy(fixture.ARTIFACTS["runners"]["action-chain"]), - "liveaction": copy.deepcopy( - fixture.ARTIFACTS["liveactions"]["workflow"] - ), + "liveaction": fixture.ARTIFACTS["liveactions"]["workflow"]["id"], + "status": fixture.ARTIFACTS["liveactions"]["workflow"]["status"], + "result": copy.deepcopy(fixture.ARTIFACTS["liveactions"]["workflow"]["result"]), "context": copy.deepcopy(fixture.ARTIFACTS["context"]), "children": [], }, { "action": copy.deepcopy(fixture.ARTIFACTS["actions"]["local"]), "runner": copy.deepcopy(fixture.ARTIFACTS["runners"]["run-local"]), - "liveaction": copy.deepcopy(fixture.ARTIFACTS["liveactions"]["task1"]), + "liveaction": fixture.ARTIFACTS["liveactions"]["task1"]["id"], + "status": fixture.ARTIFACTS["liveactions"]["task1"]["status"], + "result": copy.deepcopy(fixture.ARTIFACTS["liveactions"]["task1"]["result"]), + }, ] @@ -89,8 +92,8 @@ def assign_parent(child): data["id"] = obj_id data["start_timestamp"] = isotime.format(timestamp, offset=False) data["end_timestamp"] = isotime.format(timestamp, offset=False) - data["status"] = data["liveaction"]["status"] - data["result"] = data["liveaction"]["result"] + data["status"] = data["status"] + data["result"] = data["result"] if fake_type["action"]["name"] == "local" and random.choice([True, False]): assign_parent(data) wb_obj = ActionExecutionAPI(**data) @@ -135,7 +138,7 @@ def test_get_one(self): self.assertEqual(record["id"], obj_id) self.assertDictEqual(record["action"], fake_record.action) self.assertDictEqual(record["runner"], fake_record.runner) - self.assertDictEqual(record["liveaction"], fake_record.liveaction) + self.assertEqual(record["liveaction"], fake_record.liveaction) def test_get_one_failed(self): response = self.app.get( From c02b350ab6b544e59b99fada32c20690180f2386 Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Fri, 23 Jun 2023 13:42:23 +0000 Subject: [PATCH 03/70] index liveaction --- st2common/st2common/models/db/execution.py | 1 + 1 file changed, 1 insertion(+) diff --git a/st2common/st2common/models/db/execution.py b/st2common/st2common/models/db/execution.py index 1c5d817828..7c320ad3ab 100644 --- a/st2common/st2common/models/db/execution.py +++ b/st2common/st2common/models/db/execution.py @@ -84,6 +84,7 @@ class ActionExecutionDB(stormbase.StormFoundationDB): "indexes": [ {"fields": ["rule.ref"]}, {"fields": ["action.ref"]}, + {"fields": ["liveaction"]}, {"fields": ["start_timestamp"]}, {"fields": ["end_timestamp"]}, {"fields": ["status"]}, From d38a95543a8646c3be2f165c4b516682def116d3 Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Fri, 23 Jun 2023 14:10:50 +0000 Subject: [PATCH 04/70] fix action_chain unit testing --- .../action_chain_runner/action_chain_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/runners/action_chain_runner/action_chain_runner/action_chain_runner.py b/contrib/runners/action_chain_runner/action_chain_runner/action_chain_runner.py index 1e690f5640..e8b95cb9b1 100644 --- a/contrib/runners/action_chain_runner/action_chain_runner/action_chain_runner.py +++ b/contrib/runners/action_chain_runner/action_chain_runner/action_chain_runner.py @@ -966,7 +966,7 @@ def _format_action_exec_result( execution_db = None if liveaction_db: - execution_db = ActionExecution.get(liveaction__id=str(liveaction_db.id)) + execution_db = ActionExecution.get(liveaction=str(liveaction_db.id)) result["id"] = action_node.name result["name"] = action_node.name From cced8e3036f2117084961dffa3891826adc27128 Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Fri, 23 Jun 2023 14:26:22 +0000 Subject: [PATCH 05/70] fix execution1.yaml fixture for rule enforcement testing --- .../rule_enforcements/executions/execution1.yaml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/st2tests/st2tests/fixtures/rule_enforcements/executions/execution1.yaml b/st2tests/st2tests/fixtures/rule_enforcements/executions/execution1.yaml index b76bd99d57..90c9e9e09b 100644 --- a/st2tests/st2tests/fixtures/rule_enforcements/executions/execution1.yaml +++ b/st2tests/st2tests/fixtures/rule_enforcements/executions/execution1.yaml @@ -13,17 +13,7 @@ action: runner_type: run-local end_timestamp: '2014-09-01T00:00:05.000000Z' id: 565e15ce32ed350857dfa626 -liveaction: - action: core.someworkflow - callback: {} - context: - user: system - end_timestamp: '2014-09-01T00:00:05.000000Z' - id: 54c6b6d60640fd4f5354e74a - parameters: {} - result: {} - start_timestamp: '2014-09-01T00:00:01.000000Z' - status: scheduled +liveaction: 54c6b6d60640fd4f5354e74a parameters: cmd: echo bar result: {} From 2532784b86540216460be1e84725d048e2f86de4 Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Fri, 23 Jun 2023 14:41:59 +0000 Subject: [PATCH 06/70] black reformat, fix log format string --- st2actions/st2actions/container/base.py | 5 ++- .../policies/concurrency_by_attr.py | 37 ++++++++++--------- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/st2actions/st2actions/container/base.py b/st2actions/st2actions/container/base.py index 2f98d29e28..e7e38d9451 100644 --- a/st2actions/st2actions/container/base.py +++ b/st2actions/st2actions/container/base.py @@ -145,7 +145,10 @@ def _do_run(self, runner): # mark execution as failed. status = action_constants.LIVEACTION_STATUS_FAILED # include the error message and traceback to try and provide some hints. - LOG.exception("Failed to run action. traceback: %s".format("".join(traceback.format_tb(tb, 20)))) + LOG.exception( + "Failed to run action. traceback: %s" + % "".join(traceback.format_tb(tb, 20)) + ) result = { "error": str(ex), "traceback": "".join(traceback.format_tb(tb, 20)), diff --git a/st2actions/st2actions/policies/concurrency_by_attr.py b/st2actions/st2actions/policies/concurrency_by_attr.py index 9d4555b671..6ba064f7e5 100644 --- a/st2actions/st2actions/policies/concurrency_by_attr.py +++ b/st2actions/st2actions/policies/concurrency_by_attr.py @@ -44,35 +44,38 @@ def _apply_before(self, target): scheduled_filters = { "status": action_constants.LIVEACTION_STATUS_SCHEDULED, - "action": target.action + "action": target.action, } - scheduled = [i for i in - action_access.LiveAction.query(**scheduled_filters)] + scheduled = [i for i in action_access.LiveAction.query(**scheduled_filters)] running_filters = { "status": action_constants.LIVEACTION_STATUS_RUNNING, - "action": target.action + "action": target.action, } - running = [i for i in - action_access.LiveAction.query(**running_filters)] + running = [i for i in action_access.LiveAction.query(**running_filters)] running.extend(scheduled) count = 0 - target_parameters = JSONDictEscapedFieldCompatibilityField( - ).parse_field_value(target.parameters) + target_parameters = JSONDictEscapedFieldCompatibilityField().parse_field_value( + target.parameters + ) target_key_value_policy_attributes = { - k: v for k, v in - target_parameters.items() if k in self.attributes} + k: v for k, v in target_parameters.items() if k in self.attributes + } for i in running: - running_event_parameters = \ - JSONDictEscapedFieldCompatibilityField( - ).parse_field_value(i.parameters) + running_event_parameters = ( + JSONDictEscapedFieldCompatibilityField().parse_field_value(i.parameters) + ) # list of event parameter values that are also in policy running_event_policy_item_key_value_attributes = { - k: v for k, v in - running_event_parameters.items() if k in self.attributes} - if running_event_policy_item_key_value_attributes == \ - target_key_value_policy_attributes: + k: v + for k, v in running_event_parameters.items() + if k in self.attributes + } + if ( + running_event_policy_item_key_value_attributes + == target_key_value_policy_attributes + ): count += 1 # Mark the execution as scheduled if threshold is not reached or delayed otherwise. From 71a1a530dc4e909f43dc0e713ee6faea2183cfe6 Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Fri, 23 Jun 2023 14:47:31 +0000 Subject: [PATCH 07/70] fix test_garbage_collector.py integration test --- st2reactor/tests/integration/test_garbage_collector.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/st2reactor/tests/integration/test_garbage_collector.py b/st2reactor/tests/integration/test_garbage_collector.py index 649d09e534..2af839d669 100644 --- a/st2reactor/tests/integration/test_garbage_collector.py +++ b/st2reactor/tests/integration/test_garbage_collector.py @@ -88,7 +88,7 @@ def test_garbage_collection(self): status=status, action={"ref": "core.local"}, runner={"name": "local-shell-cmd"}, - liveaction={"ref": "foo"}, + liveaction="ref", ) ActionExecution.add_or_update(action_execution_db) @@ -124,7 +124,7 @@ def test_garbage_collection(self): status=status, action={"ref": "core.local"}, runner={"name": "local-shell-cmd"}, - liveaction={"ref": "foo"}, + liveaction="ref", ) ActionExecution.add_or_update(action_execution_db) @@ -159,7 +159,7 @@ def test_garbage_collection(self): status=status, action={"ref": "core.local"}, runner={"name": "local-shell-cmd"}, - liveaction={"ref": "foo"}, + liveaction="ref", ) ActionExecution.add_or_update(action_execution_db) From 9e0c5a559c0fdf7550b24386a1f39ad639d38e1d Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Fri, 23 Jun 2023 15:08:14 +0000 Subject: [PATCH 08/70] black formatting --- .../st2api/controllers/v1/actionexecutions.py | 10 +++++---- .../st2api/controllers/v1/aliasexecution.py | 5 +++-- .../controllers/v1/test_alias_execution.py | 2 +- .../controllers/v1/test_executions_filters.py | 9 +++++--- st2common/st2common/fields.py | 4 +++- st2common/st2common/models/api/action.py | 11 +++++----- st2common/st2common/models/api/base.py | 9 +++++--- st2common/st2common/models/api/execution.py | 11 +++++----- st2common/st2common/models/db/execution.py | 1 + st2common/st2common/services/policies.py | 8 ++++++- st2common/st2common/services/workflows.py | 9 +++++--- .../test_v35_migrate_db_dict_field_values.py | 5 ++--- st2common/tests/unit/services/test_trace.py | 4 +--- st2common/tests/unit/test_db_execution.py | 2 +- st2common/tests/unit/test_executions.py | 22 +++++++++---------- 15 files changed, 63 insertions(+), 49 deletions(-) diff --git a/st2api/st2api/controllers/v1/actionexecutions.py b/st2api/st2api/controllers/v1/actionexecutions.py index 020894b3e9..714f1977f1 100644 --- a/st2api/st2api/controllers/v1/actionexecutions.py +++ b/st2api/st2api/controllers/v1/actionexecutions.py @@ -634,7 +634,11 @@ def post(self, spec_api, id, requester_user, no_merge=False, show_secrets=False) # Merge in any parameters provided by the user new_parameters = {} original_parameters = getattr(existing_execution, "parameters", b"{}") - original_params_decoded = JSONDictEscapedFieldCompatibilityField().parse_field_value(original_parameters) + original_params_decoded = ( + JSONDictEscapedFieldCompatibilityField().parse_field_value( + original_parameters + ) + ) if not no_merge: new_parameters.update(original_params_decoded) new_parameters.update(spec_api.parameters) @@ -867,9 +871,7 @@ def update_status(liveaction_api, liveaction_db): liveaction_db = action_service.update_status( liveaction_db, status, result, set_result_size=True ) - actionexecution_db = ActionExecution.get( - liveaction=str(liveaction_db.id) - ) + actionexecution_db = ActionExecution.get(liveaction=str(liveaction_db.id)) return (liveaction_db, actionexecution_db) try: diff --git a/st2api/st2api/controllers/v1/aliasexecution.py b/st2api/st2api/controllers/v1/aliasexecution.py index 48d2514fb5..3ba706e1b9 100644 --- a/st2api/st2api/controllers/v1/aliasexecution.py +++ b/st2api/st2api/controllers/v1/aliasexecution.py @@ -188,8 +188,9 @@ def _post(self, payload, requester_user, show_secrets=False, match_multiple=Fals mask_secrets = self._get_mask_secrets( requester_user, show_secrets=show_secrets ) - liveaction = LiveActionAPI.from_model(liveaction, - mask_secrets=mask_secrets) + liveaction = LiveActionAPI.from_model( + liveaction, mask_secrets=mask_secrets + ) execution.liveaction = liveaction result = { "execution": execution, diff --git a/st2api/tests/unit/controllers/v1/test_alias_execution.py b/st2api/tests/unit/controllers/v1/test_alias_execution.py index 0ca78758a6..a530268622 100644 --- a/st2api/tests/unit/controllers/v1/test_alias_execution.py +++ b/st2api/tests/unit/controllers/v1/test_alias_execution.py @@ -159,7 +159,7 @@ def test_execution_secret_parameter(self, request): self.assertEqual(post_resp.status_int, 201) expected_parameters = {"param1": "value1", "param4": SUPER_SECRET_PARAMETER} self.assertEqual(request.call_args[0][0].parameters, expected_parameters) - #above working + # above working post_resp = self._do_post( alias_execution=self.alias4, command=command, diff --git a/st2api/tests/unit/controllers/v1/test_executions_filters.py b/st2api/tests/unit/controllers/v1/test_executions_filters.py index 470b810088..d688d5a76d 100644 --- a/st2api/tests/unit/controllers/v1/test_executions_filters.py +++ b/st2api/tests/unit/controllers/v1/test_executions_filters.py @@ -62,7 +62,9 @@ def setUpClass(cls): "runner": copy.deepcopy(fixture.ARTIFACTS["runners"]["action-chain"]), "liveaction": fixture.ARTIFACTS["liveactions"]["workflow"]["id"], "status": fixture.ARTIFACTS["liveactions"]["workflow"]["status"], - "result": copy.deepcopy(fixture.ARTIFACTS["liveactions"]["workflow"]["result"]), + "result": copy.deepcopy( + fixture.ARTIFACTS["liveactions"]["workflow"]["result"] + ), "context": copy.deepcopy(fixture.ARTIFACTS["context"]), "children": [], }, @@ -71,8 +73,9 @@ def setUpClass(cls): "runner": copy.deepcopy(fixture.ARTIFACTS["runners"]["run-local"]), "liveaction": fixture.ARTIFACTS["liveactions"]["task1"]["id"], "status": fixture.ARTIFACTS["liveactions"]["task1"]["status"], - "result": copy.deepcopy(fixture.ARTIFACTS["liveactions"]["task1"]["result"]), - + "result": copy.deepcopy( + fixture.ARTIFACTS["liveactions"]["task1"]["result"] + ), }, ] diff --git a/st2common/st2common/fields.py b/st2common/st2common/fields.py index bf523ee869..8ee1a0f603 100644 --- a/st2common/st2common/fields.py +++ b/st2common/st2common/fields.py @@ -365,7 +365,9 @@ class JSONDictField(BinaryField): """ def __init__(self, *args, **kwargs): - self.compression_algorithm = JSONDictFieldCompressionAlgorithmEnum.ZSTANDARD.value + self.compression_algorithm = ( + JSONDictFieldCompressionAlgorithmEnum.ZSTANDARD.value + ) super(JSONDictField, self).__init__(*args, **kwargs) def to_mongo(self, value): diff --git a/st2common/st2common/models/api/action.py b/st2common/st2common/models/api/action.py index 75c1574999..f82220950b 100644 --- a/st2common/st2common/models/api/action.py +++ b/st2common/st2common/models/api/action.py @@ -441,23 +441,22 @@ class LiveActionAPI(BaseAPI): }, "additionalProperties": False, } - skip_unescape_field_names = [ - "result", - "parameters" - ] + skip_unescape_field_names = ["result", "parameters"] @classmethod def convert_raw(cls, doc, raw_values): """ override this class to - convert any raw byte values into dict + convert any raw byte values into dict :param doc: dict :param raw_values: dict[field]:bytestring """ for field_name, field_value in raw_values.items(): - doc[field_name] = JSONDictEscapedFieldCompatibilityField().parse_field_value(field_value) + doc[ + field_name + ] = JSONDictEscapedFieldCompatibilityField().parse_field_value(field_value) return doc @classmethod diff --git a/st2common/st2common/models/api/base.py b/st2common/st2common/models/api/base.py index 996bc6c8ee..3be5b2d69a 100644 --- a/st2common/st2common/models/api/base.py +++ b/st2common/st2common/models/api/base.py @@ -87,8 +87,11 @@ def validate(self): @classmethod def _from_model(cls, model, mask_secrets=False): - unescape_fields = [k for k, v in model._fields.items() if type(v) in - [EscapedDynamicField, EscapedDictField]] + unescape_fields = [ + k + for k, v in model._fields.items() + if type(v) in [EscapedDynamicField, EscapedDictField] + ] unescape_fields = set(unescape_fields) - set(cls.skip_unescape_field_names) doc = model.to_mongo() @@ -117,7 +120,7 @@ def _from_model(cls, model, mask_secrets=False): def convert_raw(cls, doc, raw_values): """ override this class to - convert any raw byte values into dict + convert any raw byte values into dict you can also use this to fix any other fields that need 'fixing' :param doc: dict diff --git a/st2common/st2common/models/api/execution.py b/st2common/st2common/models/api/execution.py index 4df1c4c75b..17b9dcf2ad 100644 --- a/st2common/st2common/models/api/execution.py +++ b/st2common/st2common/models/api/execution.py @@ -146,10 +146,7 @@ class ActionExecutionAPI(BaseAPI): }, "additionalProperties": False, } - skip_unescape_field_names = [ - "result", - "parameters" - ] + skip_unescape_field_names = ["result", "parameters"] @classmethod def from_model(cls, model, mask_secrets=False): @@ -176,14 +173,16 @@ def from_model(cls, model, mask_secrets=False): def convert_raw(cls, doc, raw_values): """ override this class to - convert any raw byte values into dict + convert any raw byte values into dict :param doc: dict :param raw_values: dict[field]:bytestring """ for field_name, field_value in raw_values.items(): - doc[field_name] = JSONDictEscapedFieldCompatibilityField().parse_field_value(field_value) + doc[ + field_name + ] = JSONDictEscapedFieldCompatibilityField().parse_field_value(field_value) return doc @classmethod diff --git a/st2common/st2common/models/db/execution.py b/st2common/st2common/models/db/execution.py index 7c320ad3ab..05b903d81b 100644 --- a/st2common/st2common/models/db/execution.py +++ b/st2common/st2common/models/db/execution.py @@ -29,6 +29,7 @@ from st2common.util.secrets import mask_inquiry_response from st2common.util.secrets import mask_secret_parameters from st2common.constants.types import ResourceType + __all__ = ["ActionExecutionDB", "ActionExecutionOutputDB"] diff --git a/st2common/st2common/services/policies.py b/st2common/st2common/services/policies.py index 771078e5d7..c4aa1ac87e 100644 --- a/st2common/st2common/services/policies.py +++ b/st2common/st2common/services/policies.py @@ -65,7 +65,13 @@ def apply_pre_run_policies(lv_ac_db): traceback_var = "".join(traceback.format_tb(tb, 20)) message = 'An exception occurred while applying policy "%s" (%s) for liveaction "%s". traceback "%s"' LOG.exception( - message % (policy_db.ref, policy_db.policy_type, str(lv_ac_db.id), traceback_var) + message + % ( + policy_db.ref, + policy_db.policy_type, + str(lv_ac_db.id), + traceback_var, + ) ) if lv_ac_db.status == ac_const.LIVEACTION_STATUS_DELAYED: diff --git a/st2common/st2common/services/workflows.py b/st2common/st2common/services/workflows.py index f10963ddb6..b313130764 100644 --- a/st2common/st2common/services/workflows.py +++ b/st2common/st2common/services/workflows.py @@ -681,8 +681,11 @@ def request_task_execution(wf_ex_db, st2_ctx, task_ex_req): } exc_type, exc_value, exc_traceback = sys.exc_info() traceback_in_var = traceback.format_tb(exc_traceback) - update_task_execution(str(task_ex_db.id), statuses.FAILED, {"errors": - [error], "traceback": traceback_in_var}) + update_task_execution( + str(task_ex_db.id), + statuses.FAILED, + {"errors": [error], "traceback": traceback_in_var}, + ) raise e return task_ex_db @@ -1202,7 +1205,7 @@ def request_next_tasks(wf_ex_db, task_ex_id=None): ) LOG.error(e, exc_info=True) LOG.exception(msg) - + fail_workflow_execution(str(wf_ex_db.id), e, task=task) return diff --git a/st2common/tests/unit/migrations/test_v35_migrate_db_dict_field_values.py b/st2common/tests/unit/migrations/test_v35_migrate_db_dict_field_values.py index df6f316465..a56937e25d 100644 --- a/st2common/tests/unit/migrations/test_v35_migrate_db_dict_field_values.py +++ b/st2common/tests/unit/migrations/test_v35_migrate_db_dict_field_values.py @@ -86,14 +86,13 @@ class ActionExecutionDB_OldFieldType(ActionExecutionDB): class LiveActionDB_OldFieldType(LiveActionDB): result = stormbase.EscapedDynamicField(default={}) - #todo(aj) need ActionExecutionDB_NewFieldType to be used for update - #here. the model field type has changed to string in current models + # todo(aj) need ActionExecutionDB_NewFieldType to be used for update + # here. the model field type has changed to string in current models class ActionExecutionDB_NewFieldType(ActionExecutionDB): liveaction = stormbase.EscapedDictField(required=True) parameters = stormbase.EscapedDynamicField(default={}) - execution_dbs = ActionExecution.query( __raw__={ "result": { diff --git a/st2common/tests/unit/services/test_trace.py b/st2common/tests/unit/services/test_trace.py index 5933cdc4ef..83a39a53ef 100644 --- a/st2common/tests/unit/services/test_trace.py +++ b/st2common/tests/unit/services/test_trace.py @@ -253,9 +253,7 @@ def test_get_trace_db_by_live_action_parent_fail(self): def test_get_trace_db_by_live_action_from_execution(self): traceable_liveaction = copy.copy(self.traceable_liveaction) # fixtures id value in liveaction is not persisted in DB. - traceable_liveaction.id = bson.ObjectId( - self.traceable_execution.liveaction - ) + traceable_liveaction.id = bson.ObjectId(self.traceable_execution.liveaction) created, trace_db = trace_service.get_trace_db_by_live_action( traceable_liveaction ) diff --git a/st2common/tests/unit/test_db_execution.py b/st2common/tests/unit/test_db_execution.py index cfcf2bf981..8a9e2b95b9 100644 --- a/st2common/tests/unit/test_db_execution.py +++ b/st2common/tests/unit/test_db_execution.py @@ -71,7 +71,7 @@ } }, "action": "st2.inquiry.respond", - "id": "liveaction_respond" + "id": "liveaction_respond", } OUTPUT_SCHEMA_RESULT = { diff --git a/st2common/tests/unit/test_executions.py b/st2common/tests/unit/test_executions.py index aa5efa3311..6a89e1d9fe 100644 --- a/st2common/tests/unit/test_executions.py +++ b/st2common/tests/unit/test_executions.py @@ -38,7 +38,9 @@ def setUp(self): "id": str(bson.ObjectId()), "action": copy.deepcopy(fixture.ARTIFACTS["actions"]["local"]), "runner": copy.deepcopy(fixture.ARTIFACTS["runners"]["run-local"]), - "liveaction": copy.deepcopy(fixture.ARTIFACTS["liveactions"]["task1"]["id"]), + "liveaction": copy.deepcopy( + fixture.ARTIFACTS["liveactions"]["task1"]["id"] + ), "status": fixture.ARTIFACTS["liveactions"]["task1"]["status"], "start_timestamp": fixture.ARTIFACTS["liveactions"]["task1"][ "start_timestamp" @@ -71,7 +73,9 @@ def setUp(self): "rule": copy.deepcopy(fixture.ARTIFACTS["rule"]), "action": copy.deepcopy(fixture.ARTIFACTS["actions"]["chain"]), "runner": copy.deepcopy(fixture.ARTIFACTS["runners"]["action-chain"]), - "liveaction": copy.deepcopy(fixture.ARTIFACTS["liveactions"]["workflow"]["id"]), + "liveaction": copy.deepcopy( + fixture.ARTIFACTS["liveactions"]["workflow"]["id"] + ), "children": [task["id"] for task in self.fake_history_subtasks], "status": fixture.ARTIFACTS["liveactions"]["workflow"]["status"], "start_timestamp": fixture.ARTIFACTS["liveactions"]["workflow"][ @@ -180,9 +184,7 @@ def test_model_partial(self): self.assertIsNone(getattr(obj, "rule", None)) self.assertDictEqual(obj.action, self.fake_history_subtasks[0]["action"]) self.assertDictEqual(obj.runner, self.fake_history_subtasks[0]["runner"]) - self.assertEqual( - obj.liveaction, self.fake_history_subtasks[0]["liveaction"] - ) + self.assertEqual(obj.liveaction, self.fake_history_subtasks[0]["liveaction"]) self.assertEqual(obj.parent, self.fake_history_subtasks[0]["parent"]) self.assertIsNone(getattr(obj, "children", None)) @@ -195,8 +197,7 @@ def test_model_partial(self): self.assertDictEqual(model.rule, {}) self.assertDictEqual(model.action, self.fake_history_subtasks[0]["action"]) self.assertDictEqual(model.runner, self.fake_history_subtasks[0]["runner"]) - self.assertEqual(model.liveaction, - self.fake_history_subtasks[0]["liveaction"]) + self.assertEqual(model.liveaction, self.fake_history_subtasks[0]["liveaction"]) self.assertEqual(model.parent, self.fake_history_subtasks[0]["parent"]) self.assertListEqual(model.children, []) @@ -209,9 +210,7 @@ def test_model_partial(self): self.assertIsNone(getattr(obj, "rule", None)) self.assertDictEqual(obj.action, self.fake_history_subtasks[0]["action"]) self.assertDictEqual(obj.runner, self.fake_history_subtasks[0]["runner"]) - self.assertEqual( - obj.liveaction, self.fake_history_subtasks[0]["liveaction"] - ) + self.assertEqual(obj.liveaction, self.fake_history_subtasks[0]["liveaction"]) self.assertEqual(obj.parent, self.fake_history_subtasks[0]["parent"]) self.assertIsNone(getattr(obj, "children", None)) @@ -227,8 +226,7 @@ def test_crud_partial(self): self.assertDictEqual(model.rule, {}) self.assertDictEqual(model.action, self.fake_history_subtasks[0]["action"]) self.assertDictEqual(model.runner, self.fake_history_subtasks[0]["runner"]) - self.assertEqual(model.liveaction, - self.fake_history_subtasks[0]["liveaction"]) + self.assertEqual(model.liveaction, self.fake_history_subtasks[0]["liveaction"]) self.assertEqual(model.parent, self.fake_history_subtasks[0]["parent"]) self.assertListEqual(model.children, []) From d06a14d0ba7c00b837694a727b295e95eaac932b Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Fri, 23 Jun 2023 15:12:58 +0000 Subject: [PATCH 09/70] black fixes for contrib --- .../orquesta_runner/orquesta_runner.py | 13 ++++-- .../orquesta_runner/tests/unit/test_cancel.py | 16 ++----- .../tests/unit/test_error_handling.py | 44 ++++++++++++++----- .../tests/unit/test_functions_task.py | 4 +- .../tests/unit/test_inquiries.py | 12 ++--- .../orquesta_runner/tests/unit/test_notify.py | 5 ++- .../tests/unit/test_pause_and_resume.py | 8 +--- 7 files changed, 56 insertions(+), 46 deletions(-) diff --git a/contrib/runners/orquesta_runner/orquesta_runner/orquesta_runner.py b/contrib/runners/orquesta_runner/orquesta_runner/orquesta_runner.py index 3657218e05..d7a2d4c3a9 100644 --- a/contrib/runners/orquesta_runner/orquesta_runner/orquesta_runner.py +++ b/contrib/runners/orquesta_runner/orquesta_runner/orquesta_runner.py @@ -138,13 +138,20 @@ def start_workflow(self, action_parameters): except wf_exc.WorkflowInspectionError as e: _, ex, tb = sys.exc_info() status = ac_const.LIVEACTION_STATUS_FAILED - result = {"errors": e.args[1], "output": None, "traceback": "".join(traceback.format_tb(tb, 20))} + result = { + "errors": e.args[1], + "output": None, + "traceback": "".join(traceback.format_tb(tb, 20)), + } return (status, result, self.context) except Exception as e: _, ex, tb = sys.exc_info() status = ac_const.LIVEACTION_STATUS_FAILED - result = {"errors": [{"message": six.text_type(e)}], "output": None, - "traceback": "".join(traceback.format_tb(tb, 20))} + result = { + "errors": [{"message": six.text_type(e)}], + "output": None, + "traceback": "".join(traceback.format_tb(tb, 20)), + } return (status, result, self.context) return self._handle_workflow_return_value(wf_ex_db) diff --git a/contrib/runners/orquesta_runner/tests/unit/test_cancel.py b/contrib/runners/orquesta_runner/tests/unit/test_cancel.py index 0ec03aa15e..602951be00 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_cancel.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_cancel.py @@ -139,9 +139,7 @@ def test_cancel_workflow_cascade_down_to_subworkflow(self): ) self.assertEqual(len(tk_ac_ex_dbs), 1) - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id( - tk_ac_ex_dbs[0].liveaction - ) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_dbs[0].liveaction) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Cancel the main workflow. @@ -182,9 +180,7 @@ def test_cancel_subworkflow_cascade_up_to_workflow(self): ) self.assertEqual(len(tk_ac_ex_dbs), 1) - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id( - tk_ac_ex_dbs[0].liveaction - ) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_dbs[0].liveaction) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Cancel the subworkflow. @@ -230,9 +226,7 @@ def test_cancel_subworkflow_cascade_up_to_workflow_with_other_subworkflows(self) ) self.assertEqual(len(tk1_ac_ex_dbs), 1) - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id( - tk1_ac_ex_dbs[0].liveaction - ) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_dbs[0].liveaction) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) tk2_ac_ex_dbs = ex_db_access.ActionExecution.query( @@ -240,9 +234,7 @@ def test_cancel_subworkflow_cascade_up_to_workflow_with_other_subworkflows(self) ) self.assertEqual(len(tk2_ac_ex_dbs), 1) - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id( - tk2_ac_ex_dbs[0].liveaction - ) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_dbs[0].liveaction) self.assertEqual(tk2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Cancel the subworkflow which should cascade up to the root. diff --git a/contrib/runners/orquesta_runner/tests/unit/test_error_handling.py b/contrib/runners/orquesta_runner/tests/unit/test_error_handling.py index 359d16bd1e..ace3278882 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_error_handling.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_error_handling.py @@ -363,10 +363,18 @@ def test_fail_start_task_input_value_type(self): workflow_execution=str(wf_ex_db.id) )[0] self.assertEqual(tk_ex_db.status, wf_statuses.FAILED) - self.assertEqual(tk_ex_db.result["errors"][0]["type"], expected_errors[0]["type"]) - self.assertEqual(tk_ex_db.result["errors"][0]["message"], expected_errors[0]["message"]) - self.assertEqual(tk_ex_db.result["errors"][0]["task_id"], expected_errors[0]["task_id"]) - self.assertEqual(tk_ex_db.result["errors"][0]["route"], expected_errors[0]["route"]) + self.assertEqual( + tk_ex_db.result["errors"][0]["type"], expected_errors[0]["type"] + ) + self.assertEqual( + tk_ex_db.result["errors"][0]["message"], expected_errors[0]["message"] + ) + self.assertEqual( + tk_ex_db.result["errors"][0]["task_id"], expected_errors[0]["task_id"] + ) + self.assertEqual( + tk_ex_db.result["errors"][0]["route"], expected_errors[0]["route"] + ) lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id)) self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED) @@ -526,24 +534,36 @@ def test_fail_next_task_input_value_type(self): wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(str(wf_ex_db.id)) self.assertEqual(wf_ex_db.status, wf_statuses.FAILED) self.assertEqual( - self.sort_workflow_errors(wf_ex_db.errors)[0]["type"], expected_errors[0]["type"] + self.sort_workflow_errors(wf_ex_db.errors)[0]["type"], + expected_errors[0]["type"], ) self.assertEqual( - self.sort_workflow_errors(wf_ex_db.errors)[0]["message"], expected_errors[0]["message"] + self.sort_workflow_errors(wf_ex_db.errors)[0]["message"], + expected_errors[0]["message"], ) self.assertEqual( - self.sort_workflow_errors(wf_ex_db.errors)[0]["task_id"], expected_errors[0]["task_id"] + self.sort_workflow_errors(wf_ex_db.errors)[0]["task_id"], + expected_errors[0]["task_id"], ) self.assertEqual( - self.sort_workflow_errors(wf_ex_db.errors)[0]["route"], expected_errors[0]["route"] + self.sort_workflow_errors(wf_ex_db.errors)[0]["route"], + expected_errors[0]["route"], ) tk2_ex_db = wf_db_access.TaskExecution.query(task_id="task2")[0] self.assertEqual(tk2_ex_db.status, wf_statuses.FAILED) - self.assertEqual(tk2_ex_db.result["errors"][0]["type"], expected_errors[0]["type"]) - self.assertEqual(tk2_ex_db.result["errors"][0]["message"], expected_errors[0]["message"]) - self.assertEqual(tk2_ex_db.result["errors"][0]["task_id"], expected_errors[0]["task_id"]) - self.assertEqual(tk2_ex_db.result["errors"][0]["route"], expected_errors[0]["route"]) + self.assertEqual( + tk2_ex_db.result["errors"][0]["type"], expected_errors[0]["type"] + ) + self.assertEqual( + tk2_ex_db.result["errors"][0]["message"], expected_errors[0]["message"] + ) + self.assertEqual( + tk2_ex_db.result["errors"][0]["task_id"], expected_errors[0]["task_id"] + ) + self.assertEqual( + tk2_ex_db.result["errors"][0]["route"], expected_errors[0]["route"] + ) lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id)) self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED) diff --git a/contrib/runners/orquesta_runner/tests/unit/test_functions_task.py b/contrib/runners/orquesta_runner/tests/unit/test_functions_task.py index 8aaabc61ab..c6b83f3bcc 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_functions_task.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_functions_task.py @@ -129,9 +129,7 @@ def _execute_workflow( tk_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_db.id) )[0] - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id( - tk_ac_ex_db.liveaction - ) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) self.assertTrue( diff --git a/contrib/runners/orquesta_runner/tests/unit/test_inquiries.py b/contrib/runners/orquesta_runner/tests/unit/test_inquiries.py index 2b27c88d05..342a97420e 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_inquiries.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_inquiries.py @@ -493,9 +493,7 @@ def test_nested_inquiry(self): t2_t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t2_t1_ex_db.id) )[0] - t2_t1_lv_ac_db = lv_db_access.LiveAction.get_by_id( - t2_t1_ac_ex_db.liveaction - ) + t2_t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_t1_ac_ex_db.liveaction) self.assertEqual( t2_t1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -514,9 +512,7 @@ def test_nested_inquiry(self): t2_t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t2_t2_ex_db.id) )[0] - t2_t2_lv_ac_db = lv_db_access.LiveAction.get_by_id( - t2_t2_ac_ex_db.liveaction - ) + t2_t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_t2_ac_ex_db.liveaction) self.assertEqual( t2_t2_lv_ac_db.status, action_constants.LIVEACTION_STATUS_PENDING ) @@ -568,9 +564,7 @@ def test_nested_inquiry(self): t2_t3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t2_t3_ex_db.id) )[0] - t2_t3_lv_ac_db = lv_db_access.LiveAction.get_by_id( - t2_t3_ac_ex_db.liveaction - ) + t2_t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_t3_ac_ex_db.liveaction) self.assertEqual( t2_t3_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) diff --git a/contrib/runners/orquesta_runner/tests/unit/test_notify.py b/contrib/runners/orquesta_runner/tests/unit/test_notify.py index 3c287c5c26..546809ead5 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_notify.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_notify.py @@ -235,7 +235,10 @@ def test_notify_task_list_nonexistent_task(self): } self.assertEqual(lv_ac_db.status, action_constants.LIVEACTION_STATUS_FAILED) - self.assertEqual(lv_ac_db.result["errors"][0]["message"], expected_result["errors"][0]["message"]) + self.assertEqual( + lv_ac_db.result["errors"][0]["message"], + expected_result["errors"][0]["message"], + ) self.assertIsNone(lv_ac_db.result["output"], expected_result["output"]) def test_notify_task_list_item_value(self): diff --git a/contrib/runners/orquesta_runner/tests/unit/test_pause_and_resume.py b/contrib/runners/orquesta_runner/tests/unit/test_pause_and_resume.py index 3ba91d2972..a5e6c05091 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_pause_and_resume.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_pause_and_resume.py @@ -153,9 +153,7 @@ def test_pause_subworkflow_not_cascade_up_to_workflow(self): ) self.assertEqual(len(tk_ac_ex_dbs), 1) - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id( - tk_ac_ex_dbs[0].liveaction - ) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_dbs[0].liveaction) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Pause the subworkflow. @@ -491,9 +489,7 @@ def test_resume(self): tk_ac_ex_dbs = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_dbs[0].id) ) - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id( - tk_ac_ex_dbs[0].liveaction - ) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_dbs[0].liveaction) self.assertEqual(tk_ac_ex_dbs[0].status, ac_const.LIVEACTION_STATUS_SUCCEEDED) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) wf_svc.handle_action_execution_completion(tk_ac_ex_dbs[0]) From 2245b623bc0951149cf76382bb6be3c8033b6134 Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Fri, 23 Jun 2023 19:52:31 +0000 Subject: [PATCH 10/70] fix lint errors --- st2actions/tests/unit/test_notifier.py | 1 - st2common/st2common/services/workflows.py | 6 +----- st2common/tests/unit/test_db_fields.py | 2 -- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/st2actions/tests/unit/test_notifier.py b/st2actions/tests/unit/test_notifier.py index f1609664b1..a11288805f 100644 --- a/st2actions/tests/unit/test_notifier.py +++ b/st2actions/tests/unit/test_notifier.py @@ -27,7 +27,6 @@ from st2common.constants.action import LIVEACTION_COMPLETED_STATES from st2common.constants.action import LIVEACTION_STATUSES from st2common.constants.triggers import INTERNAL_TRIGGER_TYPES -from st2common.models.api.action import LiveActionAPI from st2common.models.db.action import ActionDB from st2common.models.db.execution import ActionExecutionDB from st2common.models.db.liveaction import LiveActionDB diff --git a/st2common/st2common/services/workflows.py b/st2common/st2common/services/workflows.py index b313130764..1cd2f48cba 100644 --- a/st2common/st2common/services/workflows.py +++ b/st2common/st2common/services/workflows.py @@ -1193,18 +1193,14 @@ def request_next_tasks(wf_ex_db, task_ex_id=None): # Request the task execution. request_task_execution(wf_ex_db, st2_ctx, task) except Exception as e: - import sys - import traceback exc_type, exc_value, exc_traceback = sys.exc_info() - traceback_in_var = traceback.format_tb(exc_traceback) msg = 'Failed task execution for task "%s", route "%s".' msg = msg % (task["id"], str(task["route"])) update_progress( wf_ex_db, "%s %s" % (msg, str(e)), severity="error", log=False ) - LOG.error(e, exc_info=True) - LOG.exception(msg) + LOG.exception(msg, exc_info=True) fail_workflow_execution(str(wf_ex_db.id), e, task=task) return diff --git a/st2common/tests/unit/test_db_fields.py b/st2common/tests/unit/test_db_fields.py index bc4a25632a..c334714d6f 100644 --- a/st2common/tests/unit/test_db_fields.py +++ b/st2common/tests/unit/test_db_fields.py @@ -37,8 +37,6 @@ from st2common.models.db import MongoDBAccess from st2common.fields import JSONDictField from st2common.fields import JSONDictEscapedFieldCompatibilityField -from st2common.fields import JSONDictFieldCompressionAlgorithmEnum -from st2common.fields import JSONDictFieldSerializationFormatEnum from st2tests import DbTestCase From bdaaf4a5ba86fa217c17f39bc4062cd8c58a762e Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Mon, 3 Jul 2023 15:12:41 +0000 Subject: [PATCH 11/70] fix migration test for 3.5 --- .../test_v35_migrate_db_dict_field_values.py | 178 +++++++++++++++++- 1 file changed, 171 insertions(+), 7 deletions(-) diff --git a/st2common/tests/unit/migrations/test_v35_migrate_db_dict_field_values.py b/st2common/tests/unit/migrations/test_v35_migrate_db_dict_field_values.py index a56937e25d..8e75c0a579 100644 --- a/st2common/tests/unit/migrations/test_v35_migrate_db_dict_field_values.py +++ b/st2common/tests/unit/migrations/test_v35_migrate_db_dict_field_values.py @@ -16,11 +16,15 @@ import sys import datetime +import mongoengine as me from st2common.constants import action as action_constants +from st2common.fields import ComplexDateTimeField +from st2common.fields import JSONDictEscapedFieldCompatibilityField from st2common.models.db import stormbase from st2common.models.db.execution import ActionExecutionDB from st2common.models.db.liveaction import LiveActionDB +from st2common.models.db.notification import NotificationSchema from st2common.models.db.workflow import WorkflowExecutionDB from st2common.models.db.workflow import TaskExecutionDB from st2common.models.db.trigger import TriggerInstanceDB @@ -31,6 +35,7 @@ from st2common.persistence.trigger import TriggerInstance from st2common.constants.triggers import TRIGGER_INSTANCE_PROCESSED from st2common.constants.triggers import TRIGGER_INSTANCE_PENDING +from st2common.util import date as date_utils from st2tests import DbTestCase @@ -41,7 +46,6 @@ import st2_migrate_db_dict_field_values as migration_module - MOCK_RESULT_1 = { "foo": "bar1", "bar": 1, @@ -79,19 +83,171 @@ def test_migrate_executions(self): LiveActionDB._meta["allow_inheritance"] = True class ActionExecutionDB_OldFieldType(ActionExecutionDB): + result = stormbase.EscapedDynamicField(default={}) liveaction = stormbase.EscapedDictField(required=True) parameters = stormbase.EscapedDynamicField(default={}) + workflow_execution = me.StringField() + task_execution = me.StringField() + status = me.StringField( + required=True, help_text="The current status of the liveaction." + ) + start_timestamp = ComplexDateTimeField( + default=date_utils.get_datetime_utc_now, + help_text="The timestamp when the liveaction was created.", + ) + end_timestamp = ComplexDateTimeField( + help_text="The timestamp when the liveaction has finished." + ) + action = stormbase.EscapedDictField(required=True) + context = me.DictField( + default={}, help_text="Contextual information on the action execution." + ) + delay = me.IntField(min_value=0) + + # diff from liveaction + runner = stormbase.EscapedDictField(required=True) + trigger = stormbase.EscapedDictField() + trigger_type = stormbase.EscapedDictField() + trigger_instance = stormbase.EscapedDictField() + rule = stormbase.EscapedDictField() + result_size = me.IntField(default=0, help_text="Serialized result size in bytes") + parent = me.StringField() + children = me.ListField(field=me.StringField()) + log = me.ListField(field=me.DictField()) + # Do not use URLField for web_url. If host doesn't have FQDN set, URLField validation blows. + web_url = me.StringField(required=False) + class LiveActionDB_OldFieldType(LiveActionDB): result = stormbase.EscapedDynamicField(default={}) - - # todo(aj) need ActionExecutionDB_NewFieldType to be used for update - # here. the model field type has changed to string in current models + workflow_execution = me.StringField() + task_execution = me.StringField() + # TODO: Can status be an enum at the Mongo layer? + status = me.StringField( + required=True, help_text="The current status of the liveaction." + ) + start_timestamp = ComplexDateTimeField( + default=date_utils.get_datetime_utc_now, + help_text="The timestamp when the liveaction was created.", + ) + end_timestamp = ComplexDateTimeField( + help_text="The timestamp when the liveaction has finished." + ) + action = me.StringField( + required=True, help_text="Reference to the action that has to be executed." + ) + parameters = JSONDictEscapedFieldCompatibilityField( + default={}, + help_text="The key-value pairs passed as to the action runner & execution.", + ) + context = me.DictField( + default={}, help_text="Contextual information on the action execution." + ) + delay = me.IntField( + min_value=0, + help_text="How long (in milliseconds) to delay the execution before scheduling.", + ) + + # diff from action execution + action_is_workflow = me.BooleanField( + default=False, + help_text="A flag indicating whether the referenced action is a workflow.", + ) + callback = me.DictField( + default={}, + help_text="Callback information for the on completion of action execution.", + ) + notify = me.EmbeddedDocumentField(NotificationSchema) + runner_info = me.DictField( + default={}, + help_text="Information about the runner which executed this live action (hostname, pid).", + ) + + class LiveActionDB_NewFieldType(LiveActionDB): + result = JSONDictEscapedFieldCompatibilityField( + default={}, help_text="Action defined result." + ) + workflow_execution = me.StringField() + task_execution = me.StringField() + # TODO: Can status be an enum at the Mongo layer? + status = me.StringField( + required=True, help_text="The current status of the liveaction." + ) + start_timestamp = ComplexDateTimeField( + default=date_utils.get_datetime_utc_now, + help_text="The timestamp when the liveaction was created.", + ) + end_timestamp = ComplexDateTimeField( + help_text="The timestamp when the liveaction has finished." + ) + action = me.StringField( + required=True, help_text="Reference to the action that has to be executed." + ) + parameters = JSONDictEscapedFieldCompatibilityField( + default={}, + help_text="The key-value pairs passed as to the action runner & execution.", + ) + context = me.DictField( + default={}, help_text="Contextual information on the action execution." + ) + delay = me.IntField( + min_value=0, + help_text="How long (in milliseconds) to delay the execution before scheduling.", + ) + + # diff from action execution + action_is_workflow = me.BooleanField( + default=False, + help_text="A flag indicating whether the referenced action is a workflow.", + ) + callback = me.DictField( + default={}, + help_text="Callback information for the on completion of action execution.", + ) + notify = me.EmbeddedDocumentField(NotificationSchema) + runner_info = me.DictField( + default={}, + help_text="Information about the runner which executed this live action (hostname, pid).", + ) class ActionExecutionDB_NewFieldType(ActionExecutionDB): liveaction = stormbase.EscapedDictField(required=True) parameters = stormbase.EscapedDynamicField(default={}) + result = JSONDictEscapedFieldCompatibilityField( + default={}, help_text="Action defined result." + ) + + workflow_execution = me.StringField() + task_execution = me.StringField() + status = me.StringField( + required=True, help_text="The current status of the liveaction." + ) + start_timestamp = ComplexDateTimeField( + default=date_utils.get_datetime_utc_now, + help_text="The timestamp when the liveaction was created.", + ) + end_timestamp = ComplexDateTimeField( + help_text="The timestamp when the liveaction has finished." + ) + action = stormbase.EscapedDictField(required=True) + context = me.DictField( + default={}, help_text="Contextual information on the action execution." + ) + delay = me.IntField(min_value=0) + + # diff from liveaction + runner = stormbase.EscapedDictField(required=True) + trigger = stormbase.EscapedDictField() + trigger_type = stormbase.EscapedDictField() + trigger_instance = stormbase.EscapedDictField() + rule = stormbase.EscapedDictField() + result_size = me.IntField(default=0, help_text="Serialized result size in bytes") + parent = me.StringField() + children = me.ListField(field=me.StringField()) + log = me.ListField(field=me.DictField()) + # Do not use URLField for web_url. If host doesn't have FQDN set, URLField validation blows. + web_url = me.StringField(required=False) execution_dbs = ActionExecution.query( __raw__={ @@ -233,15 +389,23 @@ class ActionExecutionDB_NewFieldType(ActionExecutionDB): "$type": "object", }, } - ).update(set___cls="ActionExecutionDB") - + ).update(set___cls="ActionExecutionDB.ActionExecutionDB_NewFieldType") + execution_dbs = ActionExecution.query( + __raw__={ + "result": { + "$not": { + "$type": "binData", + }, + } + } + ) LiveAction.query( __raw__={ "result": { "$type": "object", }, } - ).update(set___cls="LiveActionDB") + ).update(set___cls="LiveActionDB.LiveActionDB_NewFieldType") # 2. Run migration start_dt = datetime.datetime.utcnow().replace( From 161a236ff97cf8f1347c5c539b978f78b850b260 Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Mon, 3 Jul 2023 15:17:33 +0000 Subject: [PATCH 12/70] black format v35 unit test --- .../test_v35_migrate_db_dict_field_values.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/st2common/tests/unit/migrations/test_v35_migrate_db_dict_field_values.py b/st2common/tests/unit/migrations/test_v35_migrate_db_dict_field_values.py index 8e75c0a579..fd0f0ea1ba 100644 --- a/st2common/tests/unit/migrations/test_v35_migrate_db_dict_field_values.py +++ b/st2common/tests/unit/migrations/test_v35_migrate_db_dict_field_values.py @@ -112,7 +112,9 @@ class ActionExecutionDB_OldFieldType(ActionExecutionDB): trigger_type = stormbase.EscapedDictField() trigger_instance = stormbase.EscapedDictField() rule = stormbase.EscapedDictField() - result_size = me.IntField(default=0, help_text="Serialized result size in bytes") + result_size = me.IntField( + default=0, help_text="Serialized result size in bytes" + ) parent = me.StringField() children = me.ListField(field=me.StringField()) log = me.ListField(field=me.DictField()) @@ -135,7 +137,8 @@ class LiveActionDB_OldFieldType(LiveActionDB): help_text="The timestamp when the liveaction has finished." ) action = me.StringField( - required=True, help_text="Reference to the action that has to be executed." + required=True, + help_text="Reference to the action that has to be executed.", ) parameters = JSONDictEscapedFieldCompatibilityField( default={}, @@ -182,7 +185,8 @@ class LiveActionDB_NewFieldType(LiveActionDB): help_text="The timestamp when the liveaction has finished." ) action = me.StringField( - required=True, help_text="Reference to the action that has to be executed." + required=True, + help_text="Reference to the action that has to be executed.", ) parameters = JSONDictEscapedFieldCompatibilityField( default={}, @@ -242,7 +246,9 @@ class ActionExecutionDB_NewFieldType(ActionExecutionDB): trigger_type = stormbase.EscapedDictField() trigger_instance = stormbase.EscapedDictField() rule = stormbase.EscapedDictField() - result_size = me.IntField(default=0, help_text="Serialized result size in bytes") + result_size = me.IntField( + default=0, help_text="Serialized result size in bytes" + ) parent = me.StringField() children = me.ListField(field=me.StringField()) log = me.ListField(field=me.DictField()) From 664a0fb2919291827c0121f3aff8fa1cdba6029d Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Mon, 3 Jul 2023 17:06:41 +0000 Subject: [PATCH 13/70] set actual byte size in result_size field instead of compressed size --- st2api/st2api/controllers/v1/actionexecutions.py | 9 +++++++-- st2common/st2common/fields.py | 7 ++++--- st2common/st2common/services/executions.py | 2 +- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/st2api/st2api/controllers/v1/actionexecutions.py b/st2api/st2api/controllers/v1/actionexecutions.py index 714f1977f1..4ff9f023fe 100644 --- a/st2api/st2api/controllers/v1/actionexecutions.py +++ b/st2api/st2api/controllers/v1/actionexecutions.py @@ -27,6 +27,7 @@ from oslo_config import cfg from six.moves import http_client from mongoengine.queryset.visitor import Q +import zstandard from st2api.controllers.base import BaseRestControllerMixin from st2api.controllers.resource import ResourceController @@ -438,12 +439,16 @@ def get( # For new JSON storage format we just use raw value since it's already JSON serialized # string response_body = result["result"] - + try: + response_body = zstandard.ZstdDecompressor().decompress(response_body) + # skip if already a byte string and not compressed + except zstandard.ZstdError: + pass if pretty_format: # Pretty format is not a default behavior since it adds quite some overhead (e.g. # 10-30ms for non pretty format for 4 MB json vs ~120 ms for pretty formatted) response_body = orjson.dumps( - orjson.loads(result["result"]), option=orjson.OPT_INDENT_2 + orjson.loads(response_body), option=orjson.OPT_INDENT_2 ) response = Response() diff --git a/st2common/st2common/fields.py b/st2common/st2common/fields.py index 8ee1a0f603..8ad17d7925 100644 --- a/st2common/st2common/fields.py +++ b/st2common/st2common/fields.py @@ -414,7 +414,7 @@ def parse_field_value(self, value: Optional[Union[bytes, dict]]) -> dict: data = orjson.loads(data) return data - def _serialize_field_value(self, value: dict) -> bytes: + def _serialize_field_value(self, value: dict, zstd=True) -> bytes: """ Serialize and encode the provided field value. """ @@ -434,8 +434,9 @@ def default(obj): return list(obj) raise TypeError - value = orjson.dumps(value, default=default) - data = zstandard.ZstdCompressor().compress(value) + data = orjson.dumps(value, default=default) + if zstd: + data = zstandard.ZstdCompressor().compress(data) return data diff --git a/st2common/st2common/services/executions.py b/st2common/st2common/services/executions.py index 4c190b69fb..a5d3179010 100644 --- a/st2common/st2common/services/executions.py +++ b/st2common/st2common/services/executions.py @@ -224,7 +224,7 @@ def update_execution(liveaction_db, publish=True, set_result_size=False): with Timer(key="action.executions.calculate_result_size"): result_size = len( ActionExecutionDB.result._serialize_field_value( - liveaction_db.result + value=liveaction_db.result, zstd=False ) ) kw["set__result_size"] = result_size From c7a6ce07331f9b4cf2baccb2a84544adea6c0dab Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Mon, 3 Jul 2023 18:02:10 +0000 Subject: [PATCH 14/70] setting maxDiff to see errors --- st2tests/integration/orquesta/test_wiring_error_handling.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/st2tests/integration/orquesta/test_wiring_error_handling.py b/st2tests/integration/orquesta/test_wiring_error_handling.py index 130a68c7c5..2d59169fb4 100644 --- a/st2tests/integration/orquesta/test_wiring_error_handling.py +++ b/st2tests/integration/orquesta/test_wiring_error_handling.py @@ -23,6 +23,7 @@ class ErrorHandlingTest(base.TestWorkflowExecution): def test_inspection_error(self): + self.maxDiff = None expected_errors = [ { "type": "content", @@ -194,6 +195,7 @@ def test_output_error(self): self.assertDictEqual(ex.result, {"errors": expected_errors, "output": None}) def test_task_content_errors(self): + self.maxDiff = None expected_errors = [ { "type": "content", From 02ccef147f87005b0c962d9978d429178eec572d Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Mon, 3 Jul 2023 19:07:37 +0000 Subject: [PATCH 15/70] remove traceback --- .../orquesta/test_wiring_error_handling.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/st2tests/integration/orquesta/test_wiring_error_handling.py b/st2tests/integration/orquesta/test_wiring_error_handling.py index 2d59169fb4..88465307d9 100644 --- a/st2tests/integration/orquesta/test_wiring_error_handling.py +++ b/st2tests/integration/orquesta/test_wiring_error_handling.py @@ -67,6 +67,8 @@ def test_inspection_error(self): ex = self._execute_workflow("examples.orquesta-fail-inspection") ex = self._wait_for_completion(ex) self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) + for i in ex.result.get("errors"): + i.pop("traceback", None) self.assertDictEqual(ex.result, {"errors": expected_errors, "output": None}) def test_input_error(self): @@ -83,6 +85,8 @@ def test_input_error(self): ex = self._execute_workflow("examples.orquesta-fail-input-rendering") ex = self._wait_for_completion(ex) + for i in ex.result.get("errors"): + i.pop("traceback", None) self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) self.assertDictEqual(ex.result, {"errors": expected_errors, "output": None}) @@ -100,6 +104,9 @@ def test_vars_error(self): ex = self._execute_workflow("examples.orquesta-fail-vars-rendering") ex = self._wait_for_completion(ex) + for i in ex.result.get("errors"): + i.pop("traceback", None) + self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) self.assertDictEqual(ex.result, {"errors": expected_errors, "output": None}) @@ -128,6 +135,9 @@ def test_start_task_error(self): ex = self._execute_workflow("examples.orquesta-fail-start-task") ex = self._wait_for_completion(ex) + for i in ex.result.get("errors"): + i.pop("traceback", None) + self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) self.assertDictEqual(ex.result, {"errors": expected_errors, "output": None}) @@ -149,6 +159,9 @@ def test_task_transition_error(self): ex = self._execute_workflow("examples.orquesta-fail-task-transition") ex = self._wait_for_completion(ex) + for i in ex.result.get("errors"): + i.pop("traceback", None) + self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) self.assertDictEqual( ex.result, {"errors": expected_errors, "output": expected_output} @@ -172,6 +185,9 @@ def test_task_publish_error(self): ex = self._execute_workflow("examples.orquesta-fail-task-publish") ex = self._wait_for_completion(ex) + for i in ex.result.get("errors"): + i.pop("traceback", None) + self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) self.assertDictEqual( ex.result, {"errors": expected_errors, "output": expected_output} @@ -191,6 +207,9 @@ def test_output_error(self): ex = self._execute_workflow("examples.orquesta-fail-output-rendering") ex = self._wait_for_completion(ex) + for i in ex.result.get("errors"): + i.pop("traceback", None) + self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) self.assertDictEqual(ex.result, {"errors": expected_errors, "output": None}) @@ -228,6 +247,9 @@ def test_task_content_errors(self): ex = self._execute_workflow("examples.orquesta-fail-inspection-task-contents") ex = self._wait_for_completion(ex) + for i in ex.result.get("errors"): + i.pop("traceback", None) + self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) self.assertDictEqual(ex.result, {"errors": expected_errors, "output": None}) @@ -264,6 +286,8 @@ def test_remediate_then_fail(self): self._wait_for_task(ex, "task1", ac_const.LIVEACTION_STATUS_FAILED) self._wait_for_task(ex, "log", ac_const.LIVEACTION_STATUS_SUCCEEDED) + for i in ex.result.get("errors"): + i.pop("traceback", None) # Assert workflow status and result. self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) @@ -301,6 +325,8 @@ def test_fail_manually(self): # Assert task status. self._wait_for_task(ex, "task1", ac_const.LIVEACTION_STATUS_FAILED) self._wait_for_task(ex, "task3", ac_const.LIVEACTION_STATUS_SUCCEEDED) + for i in ex.result.get("errors"): + i.pop("traceback", None) # Assert workflow status and result. self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) @@ -334,6 +360,8 @@ def test_fail_continue(self): # Assert task status. self._wait_for_task(ex, "task1", ac_const.LIVEACTION_STATUS_FAILED) + for i in ex.result.get("errors"): + i.pop("traceback", None) # Assert workflow status and result. self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) From 5e724d99289800ee3c77b671808729c5b91f904e Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Mon, 3 Jul 2023 19:13:15 +0000 Subject: [PATCH 16/70] change log entry --- CHANGELOG.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index bb04eb2e92..01acc16fe5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -3,6 +3,11 @@ Changelog in development -------------- +* implemented zstandard compression for parameters and results. #5995 + contributed by @guzzijones12 + +* removed embedded liveaction in action execution database table #5995 + contributed by @guzzijones12 Python 3.6 is no longer supported; Stackstorm requires at least Python 3.8. From c963e39da3be25d7232de62af7e4827ad06d2a30 Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Mon, 3 Jul 2023 20:02:36 +0000 Subject: [PATCH 17/70] use errors list instead of original execution.errors --- .../orquesta/test_wiring_error_handling.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/st2tests/integration/orquesta/test_wiring_error_handling.py b/st2tests/integration/orquesta/test_wiring_error_handling.py index 88465307d9..7a4f695848 100644 --- a/st2tests/integration/orquesta/test_wiring_error_handling.py +++ b/st2tests/integration/orquesta/test_wiring_error_handling.py @@ -67,9 +67,12 @@ def test_inspection_error(self): ex = self._execute_workflow("examples.orquesta-fail-inspection") ex = self._wait_for_completion(ex) self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) + errors = [] for i in ex.result.get("errors"): i.pop("traceback", None) - self.assertDictEqual(ex.result, {"errors": expected_errors, "output": None}) + errors.append(i) + self.assertDictEqual(errors, expected_errors) + self.assertIsNone(ex.result["output"]) def test_input_error(self): expected_errors = [ @@ -247,11 +250,13 @@ def test_task_content_errors(self): ex = self._execute_workflow("examples.orquesta-fail-inspection-task-contents") ex = self._wait_for_completion(ex) + errors = [] for i in ex.result.get("errors"): i.pop("traceback", None) - + errors.append(i) + self.assertDictEqual(errors, expected_errors) + self.assertIsNone(ex.result["output"]) self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) - self.assertDictEqual(ex.result, {"errors": expected_errors, "output": None}) def test_remediate_then_fail(self): expected_errors = [ From a164c59c95b1ee39c833361dd083900c41b1b4a2 Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Mon, 3 Jul 2023 21:01:25 +0000 Subject: [PATCH 18/70] test each error individually --- st2tests/integration/orquesta/test_wiring_error_handling.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/st2tests/integration/orquesta/test_wiring_error_handling.py b/st2tests/integration/orquesta/test_wiring_error_handling.py index 7a4f695848..6130f8b088 100644 --- a/st2tests/integration/orquesta/test_wiring_error_handling.py +++ b/st2tests/integration/orquesta/test_wiring_error_handling.py @@ -71,7 +71,8 @@ def test_inspection_error(self): for i in ex.result.get("errors"): i.pop("traceback", None) errors.append(i) - self.assertDictEqual(errors, expected_errors) + for index, i in errors: + self.assertDictEqual(i, expected_errors[index]) self.assertIsNone(ex.result["output"]) def test_input_error(self): @@ -141,8 +142,8 @@ def test_start_task_error(self): for i in ex.result.get("errors"): i.pop("traceback", None) - self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) self.assertDictEqual(ex.result, {"errors": expected_errors, "output": None}) + self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) def test_task_transition_error(self): expected_errors = [ From 1128e9608af4eff7b97796193b9ffc7d10e2e95d Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Wed, 5 Jul 2023 15:43:23 +0000 Subject: [PATCH 19/70] benchmarks and integration test --- .../micro/test_mongo_field_types.py | 129 ++++++++++-------- .../orquesta/test_wiring_error_handling.py | 65 +++------ 2 files changed, 88 insertions(+), 106 deletions(-) diff --git a/st2common/benchmarks/micro/test_mongo_field_types.py b/st2common/benchmarks/micro/test_mongo_field_types.py index 54e5ead509..e9b3077d3b 100644 --- a/st2common/benchmarks/micro/test_mongo_field_types.py +++ b/st2common/benchmarks/micro/test_mongo_field_types.py @@ -46,6 +46,7 @@ import pytest import mongoengine as me +import orjson from st2common.service_setup import db_setup from st2common.models.db import stormbase @@ -62,7 +63,50 @@ LiveActionDB._meta["allow_inheritance"] = True # pylint: disable=no-member -# 1. Current approach aka using EscapedDynamicField +class OldJSONDictField(JSONDictField): + def parse_field_value(self, value) -> dict: + """ + Parse provided binary field value and return parsed value (dictionary). + + For example: + + - (n, o, ...) - no compression, data is serialized using orjson + - (z, o, ...) - zstandard compression, data is serialized using orjson + """ + if not value: + return self.default + + if isinstance(value, dict): + # Already deserializaed + return value + + data = orjson.loads(value) + return data + + def _serialize_field_value(self, value: dict) -> bytes: + """ + Serialize and encode the provided field value. + """ + # Orquesta workflows support toSet() YAQL operator which returns a set which used to get + # serialized to list by mongoengine DictField. + # + # For backward compatibility reasons, we need to support serializing set to a list as + # well. + # + # Based on micro benchmarks, using default function adds very little overhead (1%) so it + # should be safe to use default for every operation. + # + # If this turns out to be not true or it adds more overhead in other scenarios, we should + # revisit this decision and only use "default" argument where needed (aka Workflow models). + def default(obj): + if isinstance(obj, set): + return list(obj) + raise TypeError + + return orjson.dumps(value, default=default) + + +# 1. old approach aka using EscapedDynamicField class LiveActionDB_EscapedDynamicField(LiveActionDB): result = stormbase.EscapedDynamicField(default={}) @@ -71,46 +115,31 @@ class LiveActionDB_EscapedDynamicField(LiveActionDB): field3 = stormbase.EscapedDynamicField(default={}) -# 2. Current approach aka using EscapedDictField +# 2. old approach aka using EscapedDictField class LiveActionDB_EscapedDictField(LiveActionDB): result = stormbase.EscapedDictField(default={}) - field1 = stormbase.EscapedDynamicField(default={}, use_header=False) - field2 = stormbase.EscapedDynamicField(default={}, use_header=False) - field3 = stormbase.EscapedDynamicField(default={}, use_header=False) - - -# 3. Approach which uses new JSONDictField where value is stored as serialized JSON string / blob -class LiveActionDB_JSONField(LiveActionDB): - result = JSONDictField(default={}, use_header=False) - - field1 = JSONDictField(default={}, use_header=False) - field2 = JSONDictField(default={}, use_header=False) - field3 = JSONDictField(default={}, use_header=False) + field1 = stormbase.EscapedDynamicField(default={}) + field2 = stormbase.EscapedDynamicField(default={}) + field3 = stormbase.EscapedDynamicField(default={}) -class LiveActionDB_JSONFieldWithHeader(LiveActionDB): - result = JSONDictField(default={}, use_header=True, compression_algorithm="none") +# 3. Old Approach which uses no compression where value is stored as serialized JSON string / blob +class LiveActionDB_OLDJSONField(LiveActionDB): + result = OldJSONDictField(default={}, use_header=False) - field1 = JSONDictField(default={}, use_header=True, compression_algorithm="none") - field2 = JSONDictField(default={}, use_header=True, compression_algorithm="none") - field3 = JSONDictField(default={}, use_header=True, compression_algorithm="none") + field1 = OldJSONDictField(default={}) + field2 = OldJSONDictField(default={}) + field3 = OldJSONDictField(default={}) -class LiveActionDB_JSONFieldWithHeaderAndZstandard(LiveActionDB): - result = JSONDictField( - default={}, use_header=True, compression_algorithm="zstandard" - ) +# 4. Current Approach which uses new JSONDictField where value is stored as zstandard compressed serialized JSON string / blob +class LiveActionDB_JSONField(LiveActionDB): + result = JSONDictField(default={}, use_header=False) - field1 = JSONDictField( - default={}, use_header=True, compression_algorithm="zstandard" - ) - field2 = JSONDictField( - default={}, use_header=True, compression_algorithm="zstandard" - ) - field3 = JSONDictField( - default={}, use_header=True, compression_algorithm="zstandard" - ) + field1 = JSONDictField(default={}) + field2 = JSONDictField(default={}) + field3 = JSONDictField(default={}) class LiveActionDB_StringField(LiveActionDB): @@ -128,10 +157,8 @@ def get_model_class_for_approach(approach: str) -> Type[LiveActionDB]: model_cls = LiveActionDB_EscapedDictField elif approach == "json_dict_field": model_cls = LiveActionDB_JSONField - elif approach == "json_dict_field_with_header": - model_cls = LiveActionDB_JSONFieldWithHeader - elif approach == "json_dict_field_with_header_and_zstd": - model_cls = LiveActionDB_JSONFieldWithHeaderAndZstandard + elif approach == "old_json_dict_field": + model_cls = LiveActionDB_OLDJSONField else: raise ValueError("Invalid approach: %s" % (approach)) @@ -142,18 +169,12 @@ def get_model_class_for_approach(approach: str) -> Type[LiveActionDB]: @pytest.mark.parametrize( "approach", [ - "escaped_dynamic_field", - "escaped_dict_field", + "old_json_dict_field", "json_dict_field", - "json_dict_field_with_header", - "json_dict_field_with_header_and_zstd", ], ids=[ - "escaped_dynamic_field", - "escaped_dict_field", + "old_json_dict_field", "json_dict_field", - "json_dict_field_w_header", - "json_dict_field_w_header_and_zstd", ], ) @pytest.mark.benchmark(group="live_action_save") @@ -187,18 +208,12 @@ def run_benchmark(): @pytest.mark.parametrize( "approach", [ - "escaped_dynamic_field", - "escaped_dict_field", + "old_json_dict_field", "json_dict_field", - "json_dict_field_with_header", - "json_dict_field_with_header_and_zstd", ], ids=[ - "escaped_dynamic_field", - "escaped_dict_field", + "old_json_dict_field", "json_dict_field", - "json_dict_field_w_header", - "json_dict_field_w_header_and_zstd", ], ) @pytest.mark.benchmark(group="live_action_save_multiple_fields") @@ -240,18 +255,12 @@ def run_benchmark(): @pytest.mark.parametrize( "approach", [ - "escaped_dynamic_field", - "escaped_dict_field", + "old_json_dict_field", "json_dict_field", - "json_dict_field_with_header", - "json_dict_field_with_header_and_zstd", ], ids=[ - "escaped_dynamic_field", - "escaped_dict_field", + "old_json_dict_field", "json_dict_field", - "json_dict_field_w_header", - "json_dict_field_w_header_and_zstd", ], ) @pytest.mark.benchmark(group="live_action_read") diff --git a/st2tests/integration/orquesta/test_wiring_error_handling.py b/st2tests/integration/orquesta/test_wiring_error_handling.py index 6130f8b088..a177beef6c 100644 --- a/st2tests/integration/orquesta/test_wiring_error_handling.py +++ b/st2tests/integration/orquesta/test_wiring_error_handling.py @@ -22,6 +22,15 @@ class ErrorHandlingTest(base.TestWorkflowExecution): + + def error_inspect(self, ex, expected_errors): + errors = [] + for i in ex.result.get("errors"): + i.pop("traceback", None) + errors.append(i) + for index, i in enumerate(errors): + self.assertDictEqual(i, expected_errors[index]) + def test_inspection_error(self): self.maxDiff = None expected_errors = [ @@ -67,12 +76,7 @@ def test_inspection_error(self): ex = self._execute_workflow("examples.orquesta-fail-inspection") ex = self._wait_for_completion(ex) self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) - errors = [] - for i in ex.result.get("errors"): - i.pop("traceback", None) - errors.append(i) - for index, i in errors: - self.assertDictEqual(i, expected_errors[index]) + self.error_inspect(ex, expected_errors) self.assertIsNone(ex.result["output"]) def test_input_error(self): @@ -89,10 +93,9 @@ def test_input_error(self): ex = self._execute_workflow("examples.orquesta-fail-input-rendering") ex = self._wait_for_completion(ex) - for i in ex.result.get("errors"): - i.pop("traceback", None) self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) - self.assertDictEqual(ex.result, {"errors": expected_errors, "output": None}) + self.error_inspect(ex, expected_errors) + def test_vars_error(self): expected_errors = [ @@ -108,11 +111,8 @@ def test_vars_error(self): ex = self._execute_workflow("examples.orquesta-fail-vars-rendering") ex = self._wait_for_completion(ex) - for i in ex.result.get("errors"): - i.pop("traceback", None) - + self.error_inspect(ex, expected_errors) self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) - self.assertDictEqual(ex.result, {"errors": expected_errors, "output": None}) def test_start_task_error(self): self.maxDiff = None @@ -139,10 +139,7 @@ def test_start_task_error(self): ex = self._execute_workflow("examples.orquesta-fail-start-task") ex = self._wait_for_completion(ex) - for i in ex.result.get("errors"): - i.pop("traceback", None) - - self.assertDictEqual(ex.result, {"errors": expected_errors, "output": None}) + self.error_inspect(ex, expected_errors) self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) def test_task_transition_error(self): @@ -163,13 +160,8 @@ def test_task_transition_error(self): ex = self._execute_workflow("examples.orquesta-fail-task-transition") ex = self._wait_for_completion(ex) - for i in ex.result.get("errors"): - i.pop("traceback", None) - self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) - self.assertDictEqual( - ex.result, {"errors": expected_errors, "output": expected_output} - ) + self.error_inspect(ex, expected_errors) def test_task_publish_error(self): expected_errors = [ @@ -189,13 +181,9 @@ def test_task_publish_error(self): ex = self._execute_workflow("examples.orquesta-fail-task-publish") ex = self._wait_for_completion(ex) - for i in ex.result.get("errors"): - i.pop("traceback", None) self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) - self.assertDictEqual( - ex.result, {"errors": expected_errors, "output": expected_output} - ) + self.error_inspect(ex, expected_errors) def test_output_error(self): expected_errors = [ @@ -211,11 +199,8 @@ def test_output_error(self): ex = self._execute_workflow("examples.orquesta-fail-output-rendering") ex = self._wait_for_completion(ex) - for i in ex.result.get("errors"): - i.pop("traceback", None) - self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) - self.assertDictEqual(ex.result, {"errors": expected_errors, "output": None}) + self.error_inspect(ex, expected_errors) def test_task_content_errors(self): self.maxDiff = None @@ -251,11 +236,7 @@ def test_task_content_errors(self): ex = self._execute_workflow("examples.orquesta-fail-inspection-task-contents") ex = self._wait_for_completion(ex) - errors = [] - for i in ex.result.get("errors"): - i.pop("traceback", None) - errors.append(i) - self.assertDictEqual(errors, expected_errors) + self.error_inspect(ex, expected_errors) self.assertIsNone(ex.result["output"]) self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) @@ -292,12 +273,9 @@ def test_remediate_then_fail(self): self._wait_for_task(ex, "task1", ac_const.LIVEACTION_STATUS_FAILED) self._wait_for_task(ex, "log", ac_const.LIVEACTION_STATUS_SUCCEEDED) - for i in ex.result.get("errors"): - i.pop("traceback", None) - # Assert workflow status and result. self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) - self.assertDictEqual(ex.result, {"errors": expected_errors, "output": None}) + self.error_inspect(ex, expected_errors) def test_fail_manually(self): expected_errors = [ @@ -331,8 +309,6 @@ def test_fail_manually(self): # Assert task status. self._wait_for_task(ex, "task1", ac_const.LIVEACTION_STATUS_FAILED) self._wait_for_task(ex, "task3", ac_const.LIVEACTION_STATUS_SUCCEEDED) - for i in ex.result.get("errors"): - i.pop("traceback", None) # Assert workflow status and result. self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) @@ -366,9 +342,6 @@ def test_fail_continue(self): # Assert task status. self._wait_for_task(ex, "task1", ac_const.LIVEACTION_STATUS_FAILED) - for i in ex.result.get("errors"): - i.pop("traceback", None) - # Assert workflow status and result. self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) self.assertDictEqual( From ea7a5a7d5a8c708792ea71e5da8f3557e587842f Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Wed, 5 Jul 2023 17:17:57 +0000 Subject: [PATCH 20/70] black fixes --- st2tests/integration/orquesta/test_wiring_error_handling.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/st2tests/integration/orquesta/test_wiring_error_handling.py b/st2tests/integration/orquesta/test_wiring_error_handling.py index a177beef6c..eec1acd9af 100644 --- a/st2tests/integration/orquesta/test_wiring_error_handling.py +++ b/st2tests/integration/orquesta/test_wiring_error_handling.py @@ -22,7 +22,6 @@ class ErrorHandlingTest(base.TestWorkflowExecution): - def error_inspect(self, ex, expected_errors): errors = [] for i in ex.result.get("errors"): @@ -96,7 +95,6 @@ def test_input_error(self): self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) self.error_inspect(ex, expected_errors) - def test_vars_error(self): expected_errors = [ { From a554d38838221d72234d2c2af679129482ab4683 Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Wed, 5 Jul 2023 17:49:32 +0000 Subject: [PATCH 21/70] fix lint error --- st2tests/integration/orquesta/test_wiring_error_handling.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/st2tests/integration/orquesta/test_wiring_error_handling.py b/st2tests/integration/orquesta/test_wiring_error_handling.py index eec1acd9af..a46727b6fb 100644 --- a/st2tests/integration/orquesta/test_wiring_error_handling.py +++ b/st2tests/integration/orquesta/test_wiring_error_handling.py @@ -159,6 +159,7 @@ def test_task_transition_error(self): ex = self._execute_workflow("examples.orquesta-fail-task-transition") ex = self._wait_for_completion(ex) self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) + self.assertDictEqual(ex.result, expected_output) self.error_inspect(ex, expected_errors) def test_task_publish_error(self): @@ -181,6 +182,7 @@ def test_task_publish_error(self): ex = self._wait_for_completion(ex) self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) + self.assertDictEqual(ex.result, expected_output) self.error_inspect(ex, expected_errors) def test_output_error(self): From 6b3b40efb2fa3a5d72d8dcc043eb55304d7d8e96 Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Wed, 5 Jul 2023 18:09:24 +0000 Subject: [PATCH 22/70] fix output in integration test --- st2tests/integration/orquesta/test_wiring_error_handling.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/st2tests/integration/orquesta/test_wiring_error_handling.py b/st2tests/integration/orquesta/test_wiring_error_handling.py index a46727b6fb..edaba668a1 100644 --- a/st2tests/integration/orquesta/test_wiring_error_handling.py +++ b/st2tests/integration/orquesta/test_wiring_error_handling.py @@ -159,7 +159,7 @@ def test_task_transition_error(self): ex = self._execute_workflow("examples.orquesta-fail-task-transition") ex = self._wait_for_completion(ex) self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) - self.assertDictEqual(ex.result, expected_output) + self.assertDictEqual(ex.result["output"], expected_output) self.error_inspect(ex, expected_errors) def test_task_publish_error(self): @@ -182,7 +182,7 @@ def test_task_publish_error(self): ex = self._wait_for_completion(ex) self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) - self.assertDictEqual(ex.result, expected_output) + self.assertDictEqual(ex.result["output"], expected_output) self.error_inspect(ex, expected_errors) def test_output_error(self): From c69f5adbb08050051483012641352e21cfc8884e Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Wed, 5 Jul 2023 19:58:51 +0000 Subject: [PATCH 23/70] add migration for liveaction --- st2common/bin/migrations/v3.9/BUILD | 3 + st2common/bin/migrations/v3.9/__init__.py | 0 .../v3.9/st2-migrate-liveaction-executiondb | 227 ++++++++++++++++++ .../st2_migrate_liveaction_executiondb.py | 1 + 4 files changed, 231 insertions(+) create mode 100644 st2common/bin/migrations/v3.9/BUILD create mode 100644 st2common/bin/migrations/v3.9/__init__.py create mode 100755 st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb create mode 120000 st2common/bin/migrations/v3.9/st2_migrate_liveaction_executiondb.py diff --git a/st2common/bin/migrations/v3.9/BUILD b/st2common/bin/migrations/v3.9/BUILD new file mode 100644 index 0000000000..255bf31004 --- /dev/null +++ b/st2common/bin/migrations/v3.9/BUILD @@ -0,0 +1,3 @@ +python_sources( + sources=["*.py", "st2*"], +) diff --git a/st2common/bin/migrations/v3.9/__init__.py b/st2common/bin/migrations/v3.9/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb new file mode 100755 index 0000000000..07c3e1ce6a --- /dev/null +++ b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb @@ -0,0 +1,227 @@ +#!/usr/bin/env python +# Copyright 2021 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Migration which which migrates data for existing objects in the database which utilize +liveaction to a string + +Migration step is idempotent and can be retried on failures / partial runs. + +Right now the script utilizes no concurrency and performs migration one object by one. That's done +for simplicity reasons and also to avoid massive CPU usage spikes when running this script with +large concurrency on large objects. + +Keep in mind that only "completed" objects are processes - this means Executions in "final" states +(succeeded, failed, timeout, etc.). + +We determine if an object should be migrating using mongodb $type query (for execution objects we +could also determine that based on the presence of result_size field). +""" + +import sys +import datetime +import time +import traceback + +from oslo_config import cfg + +from st2common import config +from st2common.service_setup import db_setup +from st2common.service_setup import db_teardown +from st2common.util import isotime +from st2common.models.db.execution import ActionExecutionDB +from st2common.persistence.execution import ActionExecution +from st2common.exceptions.db import StackStormDBObjectNotFoundError +from st2common.constants.action import LIVEACTION_COMPLETED_STATES + +# NOTE: To avoid unnecessary mongoengine object churn when retrieving only object ids (aka to avoid +# instantiating model class with a single field), we use raw pymongo value which is a dict with a +# single value + + +def migrate_executions(start_dt: datetime.datetime, end_dt: datetime.datetime) -> None: + """ + Perform migrations for execution related objects (ActionExecutionDB, LiveActionDB). + """ + print("Migrating execution objects") + + # NOTE: We first only retrieve the IDs because there could be a lot of objects in the database + # and this could result in massive ram use. Technically, mongoengine loads querysets lazily, + # but this is not always the case so it's better to first retrieve all the IDs and then retrieve + # objects one by one. + # Keep in mind we need to use ModelClass.objects and not PersistanceClass.query() so .only() + # works correctly - with PersistanceClass.query().only() all the fields will still be retrieved. + + # 1. Migrate ActionExecutionDB objects + result = ( + ActionExecutionDB.objects( + __raw__={ + "status": { + "$in": LIVEACTION_COMPLETED_STATES, + }, + }, + start_timestamp__gte=start_dt, + start_timestamp__lte=end_dt, + ) + .only("id") + .as_pymongo() + ) + execution_ids = set([str(item["_id"]) for item in result]) + objects_count = result.count() + + if not execution_ids: + print("Found no ActionExecutionDB objects to migrate.") + print("") + return None + + print("Will migrate %s ActionExecutionDB objects" % (objects_count)) + print("") + + for index, execution_id in enumerate(execution_ids, 1): + try: + execution_db = ActionExecution.get_by_id(execution_id) + except StackStormDBObjectNotFoundError: + print( + "Skipping ActionExecutionDB with id %s which is missing in the database" + % (execution_id) + ) + continue + + print( + "[%s/%s] Migrating ActionExecutionDB with id %s" + % (index, objects_count, execution_id) + ) + + # This is a bit of a "hack", but it's the easiest way to tell mongoengine that a specific + # field has been updated and should be saved. If we don't do, nothing will be re-saved on + # .save() call due to mongoengine only trying to save what has changed to make it more + # efficient instead of always re-saving the whole object. + execution_db._mark_as_changed("liveaction") + # NOTE: If you want to view changed fields, you can access execution_db._changed_fields + # will throw an exception if already a string + execution_db.liveaction = execution_db.liveaction.get("id", None) + execution_db.save() + print("ActionExecutionDB with id %s has been migrated" % (execution_db.id)) + + +def _register_cli_opts(): + cfg.CONF.register_cli_opt( + cfg.BoolOpt( + "yes", + short="y", + required=False, + default=False, + ) + ) + + # We default to past 30 days. Keep in mind that using longer period may take a long time in + # case there are many objects in the database. + now_dt = datetime.datetime.utcnow() + start_dt = now_dt - datetime.timedelta(days=30) + + cfg.CONF.register_cli_opt( + cfg.StrOpt( + "start-dt", + required=False, + help=( + "Start cut off ISO UTC iso date time string for objects which will be migrated. " + "Defaults to now - 30 days." + "Example value: 2020-03-13T19:01:27Z" + ), + default=start_dt.strftime("%Y-%m-%dT%H:%M:%SZ"), + ) + ) + cfg.CONF.register_cli_opt( + cfg.StrOpt( + "end-dt", + required=False, + help=( + "End cut off UTC ISO date time string for objects which will be migrated." + "Defaults to now." + "Example value: 2020-03-13T19:01:27Z" + ), + default=now_dt.strftime("%Y-%m-%dT%H:%M:%SZ"), + ) + ) + + +def migrate_objects( + start_dt: datetime.datetime, end_dt: datetime.datetime, display_prompt: bool = True +) -> None: + start_dt_str = start_dt.strftime("%Y-%m-%d %H:%M:%S") + end_dt_str = end_dt.strftime("%Y-%m-%d %H:%M:%S") + + print("StackStorm v3.5 database field data migration script\n") + + if display_prompt: + input( + "Will migrate objects with creation date between %s UTC and %s UTC.\n\n" + "You are strongly recommended to create database backup before proceeding.\n\n" + "Depending on the number of the objects in the database, " + "migration may take multiple hours or more. You are recommended to start the " + "script in a screen session, tmux or similar. \n\n" + "To proceed with the migration, press enter and to cancel it, press CTRL+C.\n" + % (start_dt_str, end_dt_str) + ) + print("") + + print( + "Migrating affected database objects between %s and %s" + % (start_dt_str, end_dt_str) + ) + print("") + + start_ts = int(time.time()) + migrate_executions(start_dt=start_dt, end_dt=end_dt) + end_ts = int(time.time()) + + duration = end_ts - start_ts + + print( + "SUCCESS: All database objects migrated successfully (duration: %s seconds)." + % (duration) + ) + + +def main(): + _register_cli_opts() + + config.parse_args() + db_setup() + + start_dt = isotime.parse(cfg.CONF.start_dt) + + if cfg.CONF.end_dt == "now": + end_dt = datetime.datetime.utcnow() + end_dt = end_dt.replace(tzinfo=datetime.timezone.utc) + else: + end_dt = isotime.parse(cfg.CONF.end_dt) + + try: + migrate_objects( + start_dt=start_dt, end_dt=end_dt, display_prompt=not cfg.CONF.yes + ) + exit_code = 0 + except Exception as e: + print("ABORTED: Objects migration aborted on first failure: %s" % (str(e))) + traceback.print_exc() + exit_code = 1 + + db_teardown() + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/st2common/bin/migrations/v3.9/st2_migrate_liveaction_executiondb.py b/st2common/bin/migrations/v3.9/st2_migrate_liveaction_executiondb.py new file mode 120000 index 0000000000..5bea2222da --- /dev/null +++ b/st2common/bin/migrations/v3.9/st2_migrate_liveaction_executiondb.py @@ -0,0 +1 @@ +st2-migrate-liveaction-executiondb \ No newline at end of file From 0268596df33d0345412322024b177dd6130220bd Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Wed, 5 Jul 2023 20:02:51 +0000 Subject: [PATCH 24/70] remove sym link migration --- ...action-executiondb => st2-migrate-liveaction-executiondb.py} | 2 +- .../bin/migrations/v3.9/st2_migrate_liveaction_executiondb.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) rename st2common/bin/migrations/v3.9/{st2-migrate-liveaction-executiondb => st2-migrate-liveaction-executiondb.py} (99%) delete mode 120000 st2common/bin/migrations/v3.9/st2_migrate_liveaction_executiondb.py diff --git a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb.py similarity index 99% rename from st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb rename to st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb.py index 07c3e1ce6a..184373b52a 100755 --- a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb +++ b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright 2021 The StackStorm Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/st2common/bin/migrations/v3.9/st2_migrate_liveaction_executiondb.py b/st2common/bin/migrations/v3.9/st2_migrate_liveaction_executiondb.py deleted file mode 120000 index 5bea2222da..0000000000 --- a/st2common/bin/migrations/v3.9/st2_migrate_liveaction_executiondb.py +++ /dev/null @@ -1 +0,0 @@ -st2-migrate-liveaction-executiondb \ No newline at end of file From 86243daa317e070c040952a853ab9d371ba0a663 Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Wed, 5 Jul 2023 20:39:51 +0000 Subject: [PATCH 25/70] add migration script for execution db liveaction --- st2common/BUILD | 1 + ...eaction-executiondb.py => st2-migrate-liveaction-executiondb} | 0 st2common/setup.py | 1 + 3 files changed, 2 insertions(+) rename st2common/bin/migrations/v3.9/{st2-migrate-liveaction-executiondb.py => st2-migrate-liveaction-executiondb} (100%) diff --git a/st2common/BUILD b/st2common/BUILD index 832a17e483..48a0726dfd 100644 --- a/st2common/BUILD +++ b/st2common/BUILD @@ -21,6 +21,7 @@ st2_component_python_distribution( "bin/st2-pack-setup-virtualenv", "bin/migrations/v3.5/st2-migrate-db-dict-field-values", "bin/migrations/v3.8/st2-drop-st2exporter-marker-collections", + "bin/migrations/v3.9/st2-migrate-liveaction-executiondb", "bin/st2-run-pack-tests:shell", "bin/st2ctl:shell", "bin/st2-self-check:shell", diff --git a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb.py b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb similarity index 100% rename from st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb.py rename to st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb diff --git a/st2common/setup.py b/st2common/setup.py index e67c846b90..5e2764286d 100644 --- a/st2common/setup.py +++ b/st2common/setup.py @@ -69,6 +69,7 @@ "bin/st2-pack-download", "bin/st2-pack-setup-virtualenv", "bin/migrations/v3.5/st2-migrate-db-dict-field-values", + "bin/migrations/v3.9/st2-migrate-liveaction-executiondb", ], entry_points={ "st2common.metrics.driver": [ From 7c47bca1474cf7a96b4d3c4bc9a76c449a8520b6 Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Thu, 6 Jul 2023 13:23:59 +0000 Subject: [PATCH 26/70] add config option to turn off zstandard compression --- conf/st2.conf.sample | 1 + conf/st2.dev.conf | 1 + st2common/st2common/config.py | 5 +++++ st2common/st2common/fields.py | 4 +++- 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/conf/st2.conf.sample b/conf/st2.conf.sample index 82da2ae2e1..d6acbf16a4 100644 --- a/conf/st2.conf.sample +++ b/conf/st2.conf.sample @@ -124,6 +124,7 @@ url = None [database] # Specifies database authentication mechanisms. By default, it use SCRAM-SHA-1 with MongoDB 3.0 and later, MONGODB-CR (MongoDB Challenge Response protocol) for older servers. authentication_mechanism = None +parameter_result_compression = True # Comma delimited string of compression algorithms to use for transport level compression. Actual algorithm will then be decided based on the algorithms supported by the client and the server. For example: zstd. Defaults to no compression. Keep in mind that zstd is only supported with MongoDB 4.2 and later. compressors = # Connection retry backoff max (seconds). diff --git a/conf/st2.dev.conf b/conf/st2.dev.conf index cf2b5b6596..06203213a1 100644 --- a/conf/st2.dev.conf +++ b/conf/st2.dev.conf @@ -1,6 +1,7 @@ # Config used by local development environment (tools/launch.dev.sh) [database] host = 127.0.0.1 +parameter_result_compression = True [api] # Host and port to bind the API server. diff --git a/st2common/st2common/config.py b/st2common/st2common/config.py index c88955e4bb..7eb7797e4d 100644 --- a/st2common/st2common/config.py +++ b/st2common/st2common/config.py @@ -243,6 +243,11 @@ def register_opts(ignore_errors=False): "By default, it use SCRAM-SHA-1 with MongoDB 3.0 and later, " "MONGODB-CR (MongoDB Challenge Response protocol) for older servers.", ), + cfg.BoolOpt( + "parameter_result_compression", default=True, help="use zstandard " + "compression for parameter and result storage in liveaction and " + "execution models" + ), cfg.StrOpt( "compressors", default="", diff --git a/st2common/st2common/fields.py b/st2common/st2common/fields.py index 8ad17d7925..2d16d40121 100644 --- a/st2common/st2common/fields.py +++ b/st2common/st2common/fields.py @@ -38,6 +38,7 @@ from mongoengine.base.datastructures import mark_as_changed_wrapper from mongoengine.base.datastructures import mark_key_as_changed_wrapper from mongoengine.common import _import_class +from oslo_config import cfg from st2common.util import date as date_utils from st2common.util import mongoescape @@ -435,7 +436,8 @@ def default(obj): raise TypeError data = orjson.dumps(value, default=default) - if zstd: + parameter_result_compression = cfg.CONF.database.parameter_result_compression + if zstd and parameter_result_compression: data = zstandard.ZstdCompressor().compress(data) return data From 2e7f5e5b582acf368fcf3fc8cf7681f8723c2b27 Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Thu, 6 Jul 2023 13:25:03 +0000 Subject: [PATCH 27/70] black formatting --- st2common/st2common/config.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/st2common/st2common/config.py b/st2common/st2common/config.py index 7eb7797e4d..a1226ba59c 100644 --- a/st2common/st2common/config.py +++ b/st2common/st2common/config.py @@ -244,9 +244,11 @@ def register_opts(ignore_errors=False): "MONGODB-CR (MongoDB Challenge Response protocol) for older servers.", ), cfg.BoolOpt( - "parameter_result_compression", default=True, help="use zstandard " + "parameter_result_compression", + default=True, + help="use zstandard " "compression for parameter and result storage in liveaction and " - "execution models" + "execution models", ), cfg.StrOpt( "compressors", From 92b81a9f4ff04c2664bff5fb414b05fe19e84f89 Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Thu, 6 Jul 2023 13:32:12 +0000 Subject: [PATCH 28/70] fix version on migration script --- .../bin/migrations/v3.9/st2-migrate-liveaction-executiondb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb index 184373b52a..dd0cdb2812 100755 --- a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb +++ b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb @@ -163,7 +163,7 @@ def migrate_objects( start_dt_str = start_dt.strftime("%Y-%m-%d %H:%M:%S") end_dt_str = end_dt.strftime("%Y-%m-%d %H:%M:%S") - print("StackStorm v3.5 database field data migration script\n") + print("StackStorm v3.9 database field data migration script\n") if display_prompt: input( From e4f29ee405e8f22372dabc7c09be59232be3916d Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Thu, 6 Jul 2023 13:36:19 +0000 Subject: [PATCH 29/70] update sample config --- conf/st2.conf.sample | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conf/st2.conf.sample b/conf/st2.conf.sample index d6acbf16a4..5e6851a48e 100644 --- a/conf/st2.conf.sample +++ b/conf/st2.conf.sample @@ -124,7 +124,6 @@ url = None [database] # Specifies database authentication mechanisms. By default, it use SCRAM-SHA-1 with MongoDB 3.0 and later, MONGODB-CR (MongoDB Challenge Response protocol) for older servers. authentication_mechanism = None -parameter_result_compression = True # Comma delimited string of compression algorithms to use for transport level compression. Actual algorithm will then be decided based on the algorithms supported by the client and the server. For example: zstd. Defaults to no compression. Keep in mind that zstd is only supported with MongoDB 4.2 and later. compressors = # Connection retry backoff max (seconds). @@ -139,6 +138,8 @@ connection_timeout = 3000 db_name = st2 # host of db server host = 127.0.0.1 +# use zstandard compression for parameter and result storage in liveaction and execution models +parameter_result_compression = True # password for db login password = None # port of db server From 514f77968662fb5e492824cfe19154dd28ae10bc Mon Sep 17 00:00:00 2001 From: guzzijones Date: Thu, 13 Jul 2023 19:19:50 +0000 Subject: [PATCH 30/70] initial header compression --- conf/st2.dev.conf | 2 +- st2common/st2common/config.py | 13 ++-- st2common/st2common/constants/compression.py | 64 ++++++++++++++++++++ st2common/st2common/fields.py | 47 +++++--------- st2common/st2common/services/executions.py | 2 +- st2common/st2common/services/workflows.py | 2 +- 6 files changed, 90 insertions(+), 40 deletions(-) create mode 100644 st2common/st2common/constants/compression.py diff --git a/conf/st2.dev.conf b/conf/st2.dev.conf index 06203213a1..c2276ec332 100644 --- a/conf/st2.dev.conf +++ b/conf/st2.dev.conf @@ -1,7 +1,7 @@ # Config used by local development environment (tools/launch.dev.sh) [database] host = 127.0.0.1 -parameter_result_compression = True +parameter_result_compression = zstandard [api] # Host and port to bind the API server. diff --git a/st2common/st2common/config.py b/st2common/st2common/config.py index a1226ba59c..d5ed1a96ef 100644 --- a/st2common/st2common/config.py +++ b/st2common/st2common/config.py @@ -20,6 +20,10 @@ from oslo_config import cfg +from st2common.constants.compression import ( + ZSTANDARD_COMPRESS, + VALID_COMPRESS +) from st2common.constants.system import VERSION_STRING from st2common.constants.system import DEFAULT_CONFIG_FILE_PATH from st2common.constants.runners import PYTHON_RUNNER_DEFAULT_LOG_LEVEL @@ -243,11 +247,12 @@ def register_opts(ignore_errors=False): "By default, it use SCRAM-SHA-1 with MongoDB 3.0 and later, " "MONGODB-CR (MongoDB Challenge Response protocol) for older servers.", ), - cfg.BoolOpt( + cfg.StrOpt( "parameter_result_compression", - default=True, - help="use zstandard " - "compression for parameter and result storage in liveaction and " + default=ZSTANDARD_COMPRESS, + required=True, + choices=VALID_COMPRESS, + help="compression for parameter and result storage in liveaction and " "execution models", ), cfg.StrOpt( diff --git a/st2common/st2common/constants/compression.py b/st2common/st2common/constants/compression.py new file mode 100644 index 0000000000..c2da601914 --- /dev/null +++ b/st2common/st2common/constants/compression.py @@ -0,0 +1,64 @@ +# Copyright 2020 The StackStorm Authors. +# Copyright 2019 Extreme Networks, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Mongoengine is licensed under MIT. +""" + + +import enum +import zstandard + +ZSTANDARD_COMPRESS = "zstandard" +NO_COMPRESSION = "none" + +VALID_COMPRESS = [ + ZSTANDARD_COMPRESS, + NO_COMPRESSION +] + + +class JSONDictFieldCompressionAlgorithmEnum(enum.Enum): + """ + Enum which represents compression algorithm (if any) used for a specific JSONDictField value. + """ + + ZSTANDARD = b"z" + + +VALID_JSON_DICT_COMPRESSION_ALGORITHMS = [ + JSONDictFieldCompressionAlgorithmEnum.ZSTANDARD.value, +] + + +def zstandard_compress(data): + data = JSONDictFieldCompressionAlgorithmEnum.ZSTANDARD.value \ + + zstandard.ZstdCompressor().compress(data) + return data + + +def zstandard_uncompress(data): + data = zstandard.ZstdDecompressor().decompress(data) + return data + + +MAP_COMPRESS = { + ZSTANDARD_COMPRESS: zstandard_compress, +} + + +MAP_UNCOMPRESS = { + JSONDictFieldCompressionAlgorithmEnum.ZSTANDARD.value: zstandard_uncompress, +} diff --git a/st2common/st2common/fields.py b/st2common/st2common/fields.py index 2d16d40121..0d28fc7eb5 100644 --- a/st2common/st2common/fields.py +++ b/st2common/st2common/fields.py @@ -27,7 +27,6 @@ import datetime import calendar -import enum import weakref import orjson @@ -40,6 +39,11 @@ from mongoengine.common import _import_class from oslo_config import cfg +from st2common.constants.compression import ( + JSONDictFieldCompressionAlgorithmEnum, + MAP_COMPRESS, + MAP_UNCOMPRESS, +) from st2common.util import date as date_utils from st2common.util import mongoescape @@ -51,34 +55,6 @@ JSON_DICT_FIELD_DELIMITER = b":" -class JSONDictFieldCompressionAlgorithmEnum(enum.Enum): - """ - Enum which represents compression algorithm (if any) used for a specific JSONDictField value. - """ - - NONE = b"n" - ZSTANDARD = b"z" - - -class JSONDictFieldSerializationFormatEnum(enum.Enum): - """ - Enum which represents serialization format used for a specific JSONDictField value. - """ - - ORJSON = b"o" - - -VALID_JSON_DICT_COMPRESSION_ALGORITHMS = [ - JSONDictFieldCompressionAlgorithmEnum.NONE.value, - JSONDictFieldCompressionAlgorithmEnum.ZSTANDARD.value, -] - - -VALID_JSON_DICT_SERIALIZATION_FORMATS = [ - JSONDictFieldSerializationFormatEnum.ORJSON.value, -] - - class ComplexDateTimeField(LongField): """ Date time field which handles microseconds exactly and internally stores @@ -407,7 +383,10 @@ def parse_field_value(self, value: Optional[Union[bytes, dict]]) -> dict: return value data = value try: - data = zstandard.ZstdDecompressor().decompress(value) + uncompression_header = value[0] + uncompression_method = MAP_UNCOMPRESS.get(uncompression_header, False) + if uncompression_method: + data = uncompression_method(value) # skip if already a byte string and not compressed except zstandard.ZstdError: pass @@ -415,7 +394,7 @@ def parse_field_value(self, value: Optional[Union[bytes, dict]]) -> dict: data = orjson.loads(data) return data - def _serialize_field_value(self, value: dict, zstd=True) -> bytes: + def _serialize_field_value(self, value: dict, compress=True) -> bytes: """ Serialize and encode the provided field value. """ @@ -437,8 +416,10 @@ def default(obj): data = orjson.dumps(value, default=default) parameter_result_compression = cfg.CONF.database.parameter_result_compression - if zstd and parameter_result_compression: - data = zstandard.ZstdCompressor().compress(data) + compression_method = MAP_COMPRESS.get(parameter_result_compression, False) + # none is not mapped at all so has no compression method + if compress and compression_method: + data = compression_method(data) return data diff --git a/st2common/st2common/services/executions.py b/st2common/st2common/services/executions.py index a5d3179010..4215d1e764 100644 --- a/st2common/st2common/services/executions.py +++ b/st2common/st2common/services/executions.py @@ -224,7 +224,7 @@ def update_execution(liveaction_db, publish=True, set_result_size=False): with Timer(key="action.executions.calculate_result_size"): result_size = len( ActionExecutionDB.result._serialize_field_value( - value=liveaction_db.result, zstd=False + value=liveaction_db.result, compress=False ) ) kw["set__result_size"] = result_size diff --git a/st2common/st2common/services/workflows.py b/st2common/st2common/services/workflows.py index 1cd2f48cba..f51f8f873b 100644 --- a/st2common/st2common/services/workflows.py +++ b/st2common/st2common/services/workflows.py @@ -1215,7 +1215,7 @@ def request_next_tasks(wf_ex_db, task_ex_id=None): next_tasks = conductor.get_next_tasks() if not next_tasks: - update_progress(wf_ex_db, "No tasks identified to execute next.") + update_progress(wf_ex_db, "end of while No tasks identified to execute next.") update_progress(wf_ex_db, "\n", log=False) From a18213ce0a24e5f32310ae83d3e029c9a80636e4 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Thu, 13 Jul 2023 19:23:38 +0000 Subject: [PATCH 31/70] add new conf setting for zstandard --- conf/st2.conf.sample | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/conf/st2.conf.sample b/conf/st2.conf.sample index 5e6851a48e..d812580046 100644 --- a/conf/st2.conf.sample +++ b/conf/st2.conf.sample @@ -138,8 +138,9 @@ connection_timeout = 3000 db_name = st2 # host of db server host = 127.0.0.1 -# use zstandard compression for parameter and result storage in liveaction and execution models -parameter_result_compression = True +# compression for parameter and result storage in liveaction and execution models +# Valid values: zstandard, none +parameter_result_compression = zstandard # password for db login password = None # port of db server From 8bfd3aa92c2aa2ab8c99c3965756f6a09a9b53ec Mon Sep 17 00:00:00 2001 From: guzzijones Date: Thu, 13 Jul 2023 21:19:51 +0000 Subject: [PATCH 32/70] add ability to change compression via config setting --- st2common/st2common/fields.py | 4 +-- st2common/tests/unit/test_db_fields.py | 40 ++++++++++++++++++++------ 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/st2common/st2common/fields.py b/st2common/st2common/fields.py index 0d28fc7eb5..41ecd178cf 100644 --- a/st2common/st2common/fields.py +++ b/st2common/st2common/fields.py @@ -383,10 +383,10 @@ def parse_field_value(self, value: Optional[Union[bytes, dict]]) -> dict: return value data = value try: - uncompression_header = value[0] + uncompression_header = value[0:1] uncompression_method = MAP_UNCOMPRESS.get(uncompression_header, False) if uncompression_method: - data = uncompression_method(value) + data = uncompression_method(value[1:]) # skip if already a byte string and not compressed except zstandard.ZstdError: pass diff --git a/st2common/tests/unit/test_db_fields.py b/st2common/tests/unit/test_db_fields.py index c334714d6f..baa5168696 100644 --- a/st2common/tests/unit/test_db_fields.py +++ b/st2common/tests/unit/test_db_fields.py @@ -20,9 +20,9 @@ import calendar import mock +from oslo_config import cfg import unittest2 import orjson -import zstandard # pytest: make sure monkey_patching happens before importing mongoengine from st2common.util.monkey_patch import monkey_patch @@ -77,6 +77,14 @@ class ModelWithJSONDictFieldDB(stormbase.StormFoundationDB): class JSONDictFieldTestCase(unittest2.TestCase): + def setUp(self): + # NOTE: It's important we re-establish a connection on each setUp + cfg.CONF.reset() + + def tearDown(self): + # NOTE: It's important we disconnect here otherwise tests will fail + cfg.CONF.reset() + def test_set_to_mongo(self): field = JSONDictField(use_header=False) result = field.to_mongo({"test": {1, 2}}) @@ -87,13 +95,25 @@ def test_header_set_to_mongo(self): result = field.to_mongo({"test": {1, 2}}) self.assertTrue(isinstance(result, bytes)) - def test_to_mongo(self): + def test_to_mongo_to_python_none(self): + cfg.CONF.set_override(name="parameter_result_compression", + group="database", override="none") + field = JSONDictField(use_header=False) + result = field.to_mongo(MOCK_DATA_DICT) + + self.assertTrue(isinstance(result, bytes)) + result = field.to_python(result) + self.assertEqual(result, MOCK_DATA_DICT) + + def test_to_mongo_zstandard(self): + cfg.CONF.set_override(name="parameter_result_compression", + group="database", override="zstandard") field = JSONDictField(use_header=False) result = field.to_mongo(MOCK_DATA_DICT) self.assertTrue(isinstance(result, bytes)) - result = zstandard.ZstdDecompressor().decompress(result) - self.assertEqual(result, orjson.dumps(MOCK_DATA_DICT)) + result = field.to_python(result) + self.assertEqual(result, MOCK_DATA_DICT) def test_to_python(self): field = JSONDictField(use_header=False) @@ -151,12 +171,11 @@ def test_to_mongo(self): field = JSONDictEscapedFieldCompatibilityField(use_header=False) result_to_mongo_1 = field.to_mongo(MOCK_DATA_DICT) - result_to_mongo_1 = zstandard.ZstdDecompressor().decompress(result_to_mongo_1) - self.assertEqual(result_to_mongo_1, orjson.dumps(MOCK_DATA_DICT)) + self.assertTrue(isinstance(result_to_mongo_1, bytes)) + self.assertEqual(result_to_mongo_1[0:1], b"z") # Already serialized result_to_mongo_2 = field.to_mongo(MOCK_DATA_DICT) - result_to_mongo_2 = zstandard.ZstdDecompressor().decompress(result_to_mongo_2) self.assertEqual(result_to_mongo_2, result_to_mongo_1) def test_existing_db_value_is_using_escaped_dict_field_compatibility(self): @@ -214,8 +233,11 @@ def test_existing_db_value_is_using_escaped_dict_field_compatibility(self): self.assertEqual(pymongo_result[0]["_id"], inserted_model_db.id) self.assertTrue(isinstance(pymongo_result[0]["result"], bytes)) - result = zstandard.ZstdDecompressor().decompress(pymongo_result[0]["result"]) - self.assertEqual(orjson.loads(result), expected_data) + result = pymongo_result[0]["result"] + + field = JSONDictField(use_header=False) + result = field.to_python(result) + self.assertEqual(result, expected_data) self.assertEqual(pymongo_result[0]["counter"], 1) def test_field_state_changes_are_correctly_detected_add_or_update_method(self): From 10c6a3888ad71eb8e044251c75f75f867ecd72b3 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Thu, 13 Jul 2023 21:21:23 +0000 Subject: [PATCH 33/70] black changes --- st2common/st2common/config.py | 5 +---- st2common/st2common/constants/compression.py | 9 ++++----- st2common/st2common/services/workflows.py | 4 +++- st2common/tests/unit/test_db_fields.py | 10 ++++++---- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/st2common/st2common/config.py b/st2common/st2common/config.py index d5ed1a96ef..c204fa0d93 100644 --- a/st2common/st2common/config.py +++ b/st2common/st2common/config.py @@ -20,10 +20,7 @@ from oslo_config import cfg -from st2common.constants.compression import ( - ZSTANDARD_COMPRESS, - VALID_COMPRESS -) +from st2common.constants.compression import ZSTANDARD_COMPRESS, VALID_COMPRESS from st2common.constants.system import VERSION_STRING from st2common.constants.system import DEFAULT_CONFIG_FILE_PATH from st2common.constants.runners import PYTHON_RUNNER_DEFAULT_LOG_LEVEL diff --git a/st2common/st2common/constants/compression.py b/st2common/st2common/constants/compression.py index c2da601914..91902e0af9 100644 --- a/st2common/st2common/constants/compression.py +++ b/st2common/st2common/constants/compression.py @@ -24,10 +24,7 @@ ZSTANDARD_COMPRESS = "zstandard" NO_COMPRESSION = "none" -VALID_COMPRESS = [ - ZSTANDARD_COMPRESS, - NO_COMPRESSION -] +VALID_COMPRESS = [ZSTANDARD_COMPRESS, NO_COMPRESSION] class JSONDictFieldCompressionAlgorithmEnum(enum.Enum): @@ -44,8 +41,10 @@ class JSONDictFieldCompressionAlgorithmEnum(enum.Enum): def zstandard_compress(data): - data = JSONDictFieldCompressionAlgorithmEnum.ZSTANDARD.value \ + data = ( + JSONDictFieldCompressionAlgorithmEnum.ZSTANDARD.value + zstandard.ZstdCompressor().compress(data) + ) return data diff --git a/st2common/st2common/services/workflows.py b/st2common/st2common/services/workflows.py index f51f8f873b..ac6b85f033 100644 --- a/st2common/st2common/services/workflows.py +++ b/st2common/st2common/services/workflows.py @@ -1215,7 +1215,9 @@ def request_next_tasks(wf_ex_db, task_ex_id=None): next_tasks = conductor.get_next_tasks() if not next_tasks: - update_progress(wf_ex_db, "end of while No tasks identified to execute next.") + update_progress( + wf_ex_db, "end of while No tasks identified to execute next." + ) update_progress(wf_ex_db, "\n", log=False) diff --git a/st2common/tests/unit/test_db_fields.py b/st2common/tests/unit/test_db_fields.py index baa5168696..3258aa158f 100644 --- a/st2common/tests/unit/test_db_fields.py +++ b/st2common/tests/unit/test_db_fields.py @@ -96,8 +96,9 @@ def test_header_set_to_mongo(self): self.assertTrue(isinstance(result, bytes)) def test_to_mongo_to_python_none(self): - cfg.CONF.set_override(name="parameter_result_compression", - group="database", override="none") + cfg.CONF.set_override( + name="parameter_result_compression", group="database", override="none" + ) field = JSONDictField(use_header=False) result = field.to_mongo(MOCK_DATA_DICT) @@ -106,8 +107,9 @@ def test_to_mongo_to_python_none(self): self.assertEqual(result, MOCK_DATA_DICT) def test_to_mongo_zstandard(self): - cfg.CONF.set_override(name="parameter_result_compression", - group="database", override="zstandard") + cfg.CONF.set_override( + name="parameter_result_compression", group="database", override="zstandard" + ) field = JSONDictField(use_header=False) result = field.to_mongo(MOCK_DATA_DICT) From 0083d83c3a6be0b3ba7f46ae2f14230eb259b245 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Fri, 14 Jul 2023 15:24:49 +0000 Subject: [PATCH 34/70] change liveaction to liveaction_id; also move inquiry mask code into liveaction --- .../action_chain_runner.py | 6 +-- .../tests/unit/test_actionchain_cancel.py | 10 ++-- .../unit/test_actionchain_notifications.py | 8 +-- .../unit/test_actionchain_pause_resume.py | 14 ++--- .../inquirer_runner/inquirer_runner.py | 2 +- .../orquesta_runner/orquesta_runner.py | 6 +-- .../orquesta_runner/tests/unit/test_basic.py | 20 +++---- .../orquesta_runner/tests/unit/test_cancel.py | 8 +-- .../tests/unit/test_data_flow.py | 8 +-- .../tests/unit/test_functions_common.py | 2 +- .../tests/unit/test_functions_task.py | 2 +- .../tests/unit/test_inquiries.py | 38 ++++++------- .../orquesta_runner/tests/unit/test_notify.py | 14 ++--- .../tests/unit/test_pause_and_resume.py | 54 +++++++++---------- .../orquesta_runner/tests/unit/test_rerun.py | 24 ++++----- .../tests/unit/test_with_items.py | 8 +-- st2actions/st2actions/container/base.py | 2 +- st2actions/st2actions/notifier/notifier.py | 2 +- st2actions/st2actions/scheduler/entrypoint.py | 2 +- st2actions/st2actions/scheduler/handler.py | 2 +- st2actions/st2actions/worker.py | 6 +-- st2actions/st2actions/workflows/workflows.py | 2 +- .../tests/unit/policies/test_concurrency.py | 2 +- .../tests/unit/policies/test_retry_policy.py | 6 +-- st2actions/tests/unit/test_executions.py | 10 ++-- st2actions/tests/unit/test_notifier.py | 12 ++--- .../st2api/controllers/v1/actionexecutions.py | 6 +-- .../st2api/controllers/v1/aliasexecution.py | 9 ---- .../controllers/v1/test_alias_execution.py | 1 - .../controllers/v1/test_executions_filters.py | 6 +-- .../v3.9/st2-migrate-liveaction-executiondb | 2 +- st2common/bin/st2-track-result | 4 +- st2common/st2common/fields.py | 3 -- .../garbage_collection/executions.py | 2 +- st2common/st2common/models/api/execution.py | 4 ++ st2common/st2common/models/db/execution.py | 4 +- st2common/st2common/models/db/liveaction.py | 17 ++++++ st2common/st2common/openapi.yaml | 2 +- st2common/st2common/openapi.yaml.j2 | 2 +- st2common/st2common/services/action.py | 14 ++--- st2common/st2common/services/executions.py | 6 +-- st2common/st2common/services/inquiry.py | 2 +- st2common/st2common/services/trace.py | 2 +- st2common/st2common/services/workflows.py | 6 +-- st2common/st2common/util/param.py | 3 +- .../test_v35_migrate_db_dict_field_values.py | 3 +- st2common/tests/unit/services/test_trace.py | 2 +- .../test_workflow_identify_orphans.py | 4 +- .../services/test_workflow_service_retries.py | 8 +-- st2common/tests/unit/test_db_execution.py | 6 +-- st2common/tests/unit/test_executions.py | 22 ++++---- st2common/tests/unit/test_executions_util.py | 14 ++--- st2common/tests/unit/test_purge_executions.py | 2 +- .../integration/test_garbage_collector.py | 6 +-- .../v1/test_stream_execution_output.py | 4 +- st2tests/st2tests/api.py | 2 +- .../descendants/executions/child1_level1.yaml | 2 +- .../descendants/executions/child1_level2.yaml | 2 +- .../descendants/executions/child1_level3.yaml | 2 +- .../descendants/executions/child2_level1.yaml | 2 +- .../descendants/executions/child2_level2.yaml | 2 +- .../descendants/executions/child2_level3.yaml | 2 +- .../descendants/executions/child3_level2.yaml | 2 +- .../descendants/executions/child3_level3.yaml | 2 +- .../executions/root_execution.yaml | 2 +- .../generic/executions/execution1.yaml | 2 +- .../executions/execution1.yaml | 2 +- .../executions/execution_with_parent.yaml | 2 +- .../executions/rule_fired_execution.yaml | 2 +- .../executions/traceable_execution.yaml | 2 +- 70 files changed, 237 insertions(+), 229 deletions(-) diff --git a/contrib/runners/action_chain_runner/action_chain_runner/action_chain_runner.py b/contrib/runners/action_chain_runner/action_chain_runner/action_chain_runner.py index e8b95cb9b1..b923a38c20 100644 --- a/contrib/runners/action_chain_runner/action_chain_runner/action_chain_runner.py +++ b/contrib/runners/action_chain_runner/action_chain_runner/action_chain_runner.py @@ -333,7 +333,7 @@ def cancel(self): and child_exec.status in action_constants.LIVEACTION_CANCELABLE_STATES ): action_service.request_cancellation( - LiveAction.get(id=child_exec.liveaction), + LiveAction.get(id=child_exec.liveaction_id), self.context.get("user", None), ) @@ -353,7 +353,7 @@ def pause(self): and child_exec.status == action_constants.LIVEACTION_STATUS_RUNNING ): action_service.request_pause( - LiveAction.get(id=child_exec.liveaction), + LiveAction.get(id=child_exec.liveaction_id), self.context.get("user", None), ) @@ -966,7 +966,7 @@ def _format_action_exec_result( execution_db = None if liveaction_db: - execution_db = ActionExecution.get(liveaction=str(liveaction_db.id)) + execution_db = ActionExecution.get(liveaction_id=str(liveaction_db.id)) result["id"] = action_node.name result["name"] = action_node.name diff --git a/contrib/runners/action_chain_runner/tests/unit/test_actionchain_cancel.py b/contrib/runners/action_chain_runner/tests/unit/test_actionchain_cancel.py index e72240e8f4..635747ad1c 100644 --- a/contrib/runners/action_chain_runner/tests/unit/test_actionchain_cancel.py +++ b/contrib/runners/action_chain_runner/tests/unit/test_actionchain_cancel.py @@ -171,7 +171,7 @@ def test_chain_cancel_cascade_to_subworkflow(self): # Wait until the subworkflow is running. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction) + task1_live = LiveAction.get_by_id(task1_exec.liveaction_id) task1_live = self._wait_on_status( task1_live, action_constants.LIVEACTION_STATUS_RUNNING ) @@ -189,7 +189,7 @@ def test_chain_cancel_cascade_to_subworkflow(self): # Wait until the subworkflow is canceling. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction) + task1_live = LiveAction.get_by_id(task1_exec.liveaction_id) task1_live = self._wait_on_status( task1_live, action_constants.LIVEACTION_STATUS_CANCELING ) @@ -248,7 +248,7 @@ def test_chain_cancel_cascade_to_parent_workflow(self): # Wait until the subworkflow is running. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction) + task1_live = LiveAction.get_by_id(task1_exec.liveaction_id) task1_live = self._wait_on_status( task1_live, action_constants.LIVEACTION_STATUS_RUNNING ) @@ -260,7 +260,7 @@ def test_chain_cancel_cascade_to_parent_workflow(self): # Wait until the subworkflow is canceling. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction) + task1_live = LiveAction.get_by_id(task1_exec.liveaction_id) task1_live = self._wait_on_status( task1_live, action_constants.LIVEACTION_STATUS_CANCELING ) @@ -271,7 +271,7 @@ def test_chain_cancel_cascade_to_parent_workflow(self): # Wait until the subworkflow is canceled. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction) + task1_live = LiveAction.get_by_id(task1_exec.liveaction_id) task1_live = self._wait_on_status( task1_live, action_constants.LIVEACTION_STATUS_CANCELED ) diff --git a/contrib/runners/action_chain_runner/tests/unit/test_actionchain_notifications.py b/contrib/runners/action_chain_runner/tests/unit/test_actionchain_notifications.py index d7eaf8c56c..df1c1567d9 100644 --- a/contrib/runners/action_chain_runner/tests/unit/test_actionchain_notifications.py +++ b/contrib/runners/action_chain_runner/tests/unit/test_actionchain_notifications.py @@ -151,7 +151,7 @@ def test_skip_notify_for_task_with_notify(self): # Assert task1 notify is skipped task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction) + task1_live = LiveAction.get_by_id(task1_exec.liveaction_id) task1_live = self._wait_on_status( task1_live, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -162,7 +162,7 @@ def test_skip_notify_for_task_with_notify(self): # Assert task2 notify is not skipped task2_exec = ActionExecution.get_by_id(execution.children[1]) - task2_live = LiveAction.get_by_id(task2_exec.liveaction) + task2_live = LiveAction.get_by_id(task2_exec.liveaction_id) notify = notify_api_models.NotificationsHelper.from_model( notify_model=task2_live.notify ) @@ -186,7 +186,7 @@ def test_skip_notify_default_for_task_with_notify(self): # Assert task1 notify is set. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction) + task1_live = LiveAction.get_by_id(task1_exec.liveaction_id) task1_live = self._wait_on_status( task1_live, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -200,7 +200,7 @@ def test_skip_notify_default_for_task_with_notify(self): # Assert task2 notify is not skipped by default. task2_exec = ActionExecution.get_by_id(execution.children[1]) - task2_live = LiveAction.get_by_id(task2_exec.liveaction) + task2_live = LiveAction.get_by_id(task2_exec.liveaction_id) self.assertIsNone(task2_live.notify) MockLiveActionPublisherNonBlocking.wait_all() diff --git a/contrib/runners/action_chain_runner/tests/unit/test_actionchain_pause_resume.py b/contrib/runners/action_chain_runner/tests/unit/test_actionchain_pause_resume.py index acacbeb0cd..6187522d42 100644 --- a/contrib/runners/action_chain_runner/tests/unit/test_actionchain_pause_resume.py +++ b/contrib/runners/action_chain_runner/tests/unit/test_actionchain_pause_resume.py @@ -431,7 +431,7 @@ def test_chain_pause_resume_cascade_to_subworkflow(self): # Wait until the subworkflow is running. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction) + task1_live = LiveAction.get_by_id(task1_exec.liveaction_id) task1_live = self._wait_for_status( task1_live, action_constants.LIVEACTION_STATUS_RUNNING ) @@ -452,7 +452,7 @@ def test_chain_pause_resume_cascade_to_subworkflow(self): # Wait until the subworkflow is pausing. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction) + task1_live = LiveAction.get_by_id(task1_exec.liveaction_id) task1_live = self._wait_for_status( task1_live, action_constants.LIVEACTION_STATUS_PAUSING ) @@ -477,7 +477,7 @@ def test_chain_pause_resume_cascade_to_subworkflow(self): # Wait until the subworkflow is paused. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction) + task1_live = LiveAction.get_by_id(task1_exec.liveaction_id) task1_live = self._wait_for_status( task1_live, action_constants.LIVEACTION_STATUS_PAUSED ) @@ -548,7 +548,7 @@ def test_chain_pause_resume_cascade_to_parent_workflow(self): # Wait until the subworkflow is running. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction) + task1_live = LiveAction.get_by_id(task1_exec.liveaction_id) task1_live = self._wait_for_status( task1_live, action_constants.LIVEACTION_STATUS_RUNNING ) @@ -559,7 +559,7 @@ def test_chain_pause_resume_cascade_to_parent_workflow(self): # Wait until the subworkflow is pausing. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction) + task1_live = LiveAction.get_by_id(task1_exec.liveaction_id) task1_live = self._wait_for_status( task1_live, action_constants.LIVEACTION_STATUS_PAUSING ) @@ -574,7 +574,7 @@ def test_chain_pause_resume_cascade_to_parent_workflow(self): # Wait until the subworkflow is paused. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction) + task1_live = LiveAction.get_by_id(task1_exec.liveaction_id) task1_live = self._wait_for_status( task1_live, action_constants.LIVEACTION_STATUS_PAUSED ) @@ -611,7 +611,7 @@ def test_chain_pause_resume_cascade_to_parent_workflow(self): # Wait until the subworkflow is paused. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction) + task1_live = LiveAction.get_by_id(task1_exec.liveaction_id) task1_live = self._wait_for_status( task1_live, action_constants.LIVEACTION_STATUS_SUCCEEDED ) diff --git a/contrib/runners/inquirer_runner/inquirer_runner/inquirer_runner.py b/contrib/runners/inquirer_runner/inquirer_runner/inquirer_runner.py index b33bd95761..f080c7ab30 100644 --- a/contrib/runners/inquirer_runner/inquirer_runner/inquirer_runner.py +++ b/contrib/runners/inquirer_runner/inquirer_runner/inquirer_runner.py @@ -74,7 +74,7 @@ def pre_run(self): def run(self, action_parameters): liveaction_db = action_utils.get_liveaction_by_id(self.liveaction_id) - exc = ex_db_access.ActionExecution.get(liveaction=str(liveaction_db.id)) + exc = ex_db_access.ActionExecution.get(liveaction_id=str(liveaction_db.id)) # Assemble and dispatch trigger trigger_ref = sys_db_models.ResourceReference.to_string_reference( diff --git a/contrib/runners/orquesta_runner/orquesta_runner/orquesta_runner.py b/contrib/runners/orquesta_runner/orquesta_runner/orquesta_runner.py index d7a2d4c3a9..717ec979c4 100644 --- a/contrib/runners/orquesta_runner/orquesta_runner/orquesta_runner.py +++ b/contrib/runners/orquesta_runner/orquesta_runner/orquesta_runner.py @@ -188,7 +188,7 @@ def pause(self): child_ex = ex_db_access.ActionExecution.get(id=child_ex_id) if self.task_pauseable(child_ex): ac_svc.request_pause( - lv_db_access.LiveAction.get(id=child_ex.liveaction), + lv_db_access.LiveAction.get(id=child_ex.liveaction_id), self.context.get("user", None), ) @@ -219,7 +219,7 @@ def resume(self): child_ex = ex_db_access.ActionExecution.get(id=child_ex_id) if self.task_resumeable(child_ex): ac_svc.request_resume( - lv_db_access.LiveAction.get(id=child_ex.liveaction), + lv_db_access.LiveAction.get(id=child_ex.liveaction_id), self.context.get("user", None), ) @@ -280,7 +280,7 @@ def cancel(self): child_ex = ex_db_access.ActionExecution.get(id=child_ex_id) if self.task_cancelable(child_ex): ac_svc.request_cancellation( - lv_db_access.LiveAction.get(id=child_ex.liveaction), + lv_db_access.LiveAction.get(id=child_ex.liveaction_id), self.context.get("user", None), ) diff --git a/contrib/runners/orquesta_runner/tests/unit/test_basic.py b/contrib/runners/orquesta_runner/tests/unit/test_basic.py index b6da3e38e5..dd4fdd49bf 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_basic.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_basic.py @@ -184,7 +184,7 @@ def test_run_workflow(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) self.assertEqual(tk1_lv_ac_db.context.get("user"), username) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) self.assertTrue(wf_svc.is_action_execution_under_workflow_context(tk1_ac_ex_db)) @@ -204,7 +204,7 @@ def test_run_workflow(self): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction_id) self.assertEqual(tk2_lv_ac_db.context.get("user"), username) self.assertEqual(tk2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) self.assertTrue(wf_svc.is_action_execution_under_workflow_context(tk2_ac_ex_db)) @@ -224,7 +224,7 @@ def test_run_workflow(self): tk3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk3_ex_db.id) )[0] - tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction) + tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction_id) self.assertEqual(tk3_lv_ac_db.context.get("user"), username) self.assertEqual(tk3_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) self.assertTrue(wf_svc.is_action_execution_under_workflow_context(tk3_ac_ex_db)) @@ -274,7 +274,7 @@ def test_run_workflow_with_unicode_input(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) wf_svc.handle_action_execution_completion(tk1_ac_ex_db) tk1_ex_db = wf_db_access.TaskExecution.get_by_id(tk1_ex_db.id) @@ -286,7 +286,7 @@ def test_run_workflow_with_unicode_input(self): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction_id) self.assertEqual(tk2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) wf_svc.handle_action_execution_completion(tk2_ac_ex_db) tk2_ex_db = wf_db_access.TaskExecution.get_by_id(tk2_ex_db.id) @@ -298,7 +298,7 @@ def test_run_workflow_with_unicode_input(self): tk3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk3_ex_db.id) )[0] - tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction) + tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction_id) self.assertEqual(tk3_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) wf_svc.handle_action_execution_completion(tk3_ac_ex_db) tk3_ex_db = wf_db_access.TaskExecution.get_by_id(tk3_ex_db.id) @@ -347,7 +347,7 @@ def test_run_workflow_action_config_context(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) self.assertTrue(wf_svc.is_action_execution_under_workflow_context(tk1_ac_ex_db)) @@ -400,7 +400,7 @@ def test_run_workflow_with_action_less_tasks(self): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction_id) self.assertEqual(tk2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion. @@ -412,7 +412,7 @@ def test_run_workflow_with_action_less_tasks(self): tk3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk3_ex_db.id) )[0] - tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction) + tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction_id) self.assertEqual(tk3_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion. @@ -433,7 +433,7 @@ def test_run_workflow_with_action_less_tasks(self): tk5_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk5_ex_db.id) )[0] - tk5_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk5_ac_ex_db.liveaction) + tk5_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk5_ac_ex_db.liveaction_id) self.assertEqual(tk5_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion. diff --git a/contrib/runners/orquesta_runner/tests/unit/test_cancel.py b/contrib/runners/orquesta_runner/tests/unit/test_cancel.py index 602951be00..1c6df3cf11 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_cancel.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_cancel.py @@ -139,7 +139,7 @@ def test_cancel_workflow_cascade_down_to_subworkflow(self): ) self.assertEqual(len(tk_ac_ex_dbs), 1) - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_dbs[0].liveaction) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_dbs[0].liveaction_id) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Cancel the main workflow. @@ -180,7 +180,7 @@ def test_cancel_subworkflow_cascade_up_to_workflow(self): ) self.assertEqual(len(tk_ac_ex_dbs), 1) - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_dbs[0].liveaction) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_dbs[0].liveaction_id) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Cancel the subworkflow. @@ -226,7 +226,7 @@ def test_cancel_subworkflow_cascade_up_to_workflow_with_other_subworkflows(self) ) self.assertEqual(len(tk1_ac_ex_dbs), 1) - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_dbs[0].liveaction) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_dbs[0].liveaction_id) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) tk2_ac_ex_dbs = ex_db_access.ActionExecution.query( @@ -234,7 +234,7 @@ def test_cancel_subworkflow_cascade_up_to_workflow_with_other_subworkflows(self) ) self.assertEqual(len(tk2_ac_ex_dbs), 1) - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_dbs[0].liveaction) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_dbs[0].liveaction_id) self.assertEqual(tk2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Cancel the subworkflow which should cascade up to the root. diff --git a/contrib/runners/orquesta_runner/tests/unit/test_data_flow.py b/contrib/runners/orquesta_runner/tests/unit/test_data_flow.py index 27871474ac..910ff2cf18 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_data_flow.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_data_flow.py @@ -143,7 +143,7 @@ def assert_data_flow(self, data): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion. @@ -161,7 +161,7 @@ def assert_data_flow(self, data): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction_id) self.assertEqual(tk2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion. @@ -179,7 +179,7 @@ def assert_data_flow(self, data): tk3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk3_ex_db.id) )[0] - tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction) + tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction_id) self.assertEqual(tk3_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion. @@ -197,7 +197,7 @@ def assert_data_flow(self, data): tk4_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk4_ex_db.id) )[0] - tk4_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk4_ac_ex_db.liveaction) + tk4_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk4_ac_ex_db.liveaction_id) self.assertEqual(tk4_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion. diff --git a/contrib/runners/orquesta_runner/tests/unit/test_functions_common.py b/contrib/runners/orquesta_runner/tests/unit/test_functions_common.py index 04b79b8be7..6efd2c0f8b 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_functions_common.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_functions_common.py @@ -115,7 +115,7 @@ def _execute_workflow(self, wf_name, expected_output): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) self.assertTrue(wf_svc.is_action_execution_under_workflow_context(tk1_ac_ex_db)) diff --git a/contrib/runners/orquesta_runner/tests/unit/test_functions_task.py b/contrib/runners/orquesta_runner/tests/unit/test_functions_task.py index c6b83f3bcc..721f5c5de7 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_functions_task.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_functions_task.py @@ -129,7 +129,7 @@ def _execute_workflow( tk_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_db.id) )[0] - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction_id) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) self.assertTrue( diff --git a/contrib/runners/orquesta_runner/tests/unit/test_inquiries.py b/contrib/runners/orquesta_runner/tests/unit/test_inquiries.py index 342a97420e..9e8ad560c3 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_inquiries.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_inquiries.py @@ -113,7 +113,7 @@ def test_inquiry(self): t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t1_ex_db.id) )[0] - t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction) + t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction_id) self.assertEqual( t1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -134,7 +134,7 @@ def test_inquiry(self): t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t2_ex_db.id) )[0] - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) self.assertEqual(t2_lv_ac_db.status, action_constants.LIVEACTION_STATUS_PENDING) workflows.get_engine().process(t2_ac_ex_db) t2_ex_db = wf_db_access.TaskExecution.get_by_id(t2_ex_db.id) @@ -170,7 +170,7 @@ def test_inquiry(self): t3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t3_ex_db.id) )[0] - t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction) + t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction_id) self.assertEqual( t3_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -203,7 +203,7 @@ def test_consecutive_inquiries(self): t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t1_ex_db.id) )[0] - t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction) + t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction_id) self.assertEqual( t1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -224,7 +224,7 @@ def test_consecutive_inquiries(self): t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t2_ex_db.id) )[0] - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) self.assertEqual(t2_lv_ac_db.status, action_constants.LIVEACTION_STATUS_PENDING) workflows.get_engine().process(t2_ac_ex_db) t2_ex_db = wf_db_access.TaskExecution.get_by_id(t2_ex_db.id) @@ -263,7 +263,7 @@ def test_consecutive_inquiries(self): t3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t3_ex_db.id) )[0] - t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction) + t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction_id) self.assertEqual(t3_lv_ac_db.status, action_constants.LIVEACTION_STATUS_PENDING) workflows.get_engine().process(t3_ac_ex_db) t3_ex_db = wf_db_access.TaskExecution.get_by_id(t3_ex_db.id) @@ -299,7 +299,7 @@ def test_consecutive_inquiries(self): t4_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t4_ex_db.id) )[0] - t4_lv_ac_db = lv_db_access.LiveAction.get_by_id(t4_ac_ex_db.liveaction) + t4_lv_ac_db = lv_db_access.LiveAction.get_by_id(t4_ac_ex_db.liveaction_id) self.assertEqual( t4_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -332,7 +332,7 @@ def test_parallel_inquiries(self): t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t1_ex_db.id) )[0] - t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction) + t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction_id) self.assertEqual( t1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -350,7 +350,7 @@ def test_parallel_inquiries(self): t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t2_ex_db.id) )[0] - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) self.assertEqual(t2_lv_ac_db.status, action_constants.LIVEACTION_STATUS_PENDING) workflows.get_engine().process(t2_ac_ex_db) t2_ex_db = wf_db_access.TaskExecution.get_by_id(t2_ex_db.id) @@ -366,7 +366,7 @@ def test_parallel_inquiries(self): t3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t3_ex_db.id) )[0] - t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction) + t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction_id) self.assertEqual(t3_lv_ac_db.status, action_constants.LIVEACTION_STATUS_PENDING) workflows.get_engine().process(t3_ac_ex_db) t3_ex_db = wf_db_access.TaskExecution.get_by_id(t3_ex_db.id) @@ -423,7 +423,7 @@ def test_parallel_inquiries(self): t4_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t4_ex_db.id) )[0] - t4_lv_ac_db = lv_db_access.LiveAction.get_by_id(t4_ac_ex_db.liveaction) + t4_lv_ac_db = lv_db_access.LiveAction.get_by_id(t4_ac_ex_db.liveaction_id) self.assertEqual( t4_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -456,7 +456,7 @@ def test_nested_inquiry(self): t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t1_ex_db.id) )[0] - t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction) + t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction_id) self.assertEqual( t1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -477,7 +477,7 @@ def test_nested_inquiry(self): t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t2_ex_db.id) )[0] - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) self.assertEqual(t2_lv_ac_db.status, action_constants.LIVEACTION_STATUS_RUNNING) workflows.get_engine().process(t2_ac_ex_db) t2_ex_db = wf_db_access.TaskExecution.get_by_id(t2_ex_db.id) @@ -493,7 +493,7 @@ def test_nested_inquiry(self): t2_t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t2_t1_ex_db.id) )[0] - t2_t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_t1_ac_ex_db.liveaction) + t2_t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_t1_ac_ex_db.liveaction_id) self.assertEqual( t2_t1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -512,7 +512,7 @@ def test_nested_inquiry(self): t2_t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t2_t2_ex_db.id) )[0] - t2_t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_t2_ac_ex_db.liveaction) + t2_t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_t2_ac_ex_db.liveaction_id) self.assertEqual( t2_t2_lv_ac_db.status, action_constants.LIVEACTION_STATUS_PENDING ) @@ -526,7 +526,7 @@ def test_nested_inquiry(self): t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t2_ex_db.id) )[0] - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) self.assertEqual(t2_lv_ac_db.status, action_constants.LIVEACTION_STATUS_PAUSED) workflows.get_engine().process(t2_ac_ex_db) t2_ex_db = wf_db_access.TaskExecution.get_by_id(t2_ex_db.id) @@ -564,7 +564,7 @@ def test_nested_inquiry(self): t2_t3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t2_t3_ex_db.id) )[0] - t2_t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_t3_ac_ex_db.liveaction) + t2_t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_t3_ac_ex_db.liveaction_id) self.assertEqual( t2_t3_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -576,7 +576,7 @@ def test_nested_inquiry(self): t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t2_ex_db.id) )[0] - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) self.assertEqual( t2_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -592,7 +592,7 @@ def test_nested_inquiry(self): t3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t3_ex_db.id) )[0] - t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction) + t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction_id) self.assertEqual( t3_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) diff --git a/contrib/runners/orquesta_runner/tests/unit/test_notify.py b/contrib/runners/orquesta_runner/tests/unit/test_notify.py index 546809ead5..04c786fb78 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_notify.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_notify.py @@ -279,7 +279,7 @@ def test_cascade_notify_to_tasks(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) self.assertIsNone(tk1_lv_ac_db.notify) self.assertEqual( tk1_ac_ex_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED @@ -300,7 +300,7 @@ def test_cascade_notify_to_tasks(self): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction_id) notify = notify_api_models.NotificationsHelper.from_model( notify_model=tk2_lv_ac_db.notify ) @@ -324,7 +324,7 @@ def test_cascade_notify_to_tasks(self): tk3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk3_ex_db.id) )[0] - tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction) + tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction_id) self.assertIsNone(tk3_lv_ac_db.notify) self.assertEqual( tk3_ac_ex_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED @@ -371,7 +371,7 @@ def test_notify_task_list_for_task_with_notify(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) self.assertIsNone(tk1_lv_ac_db.notify) # Assert task2 notify is set. query_filters = {"workflow_execution": str(wf_ex_db.id), "task_id": "task2"} @@ -379,7 +379,7 @@ def test_notify_task_list_for_task_with_notify(self): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction_id) notify = notify_api_models.NotificationsHelper.from_model( notify_model=tk2_lv_ac_db.notify ) @@ -406,7 +406,7 @@ def test_no_notify_for_task_with_notify(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) self.assertIsNone(tk1_lv_ac_db.notify) # Assert task2 notify is not set. @@ -415,5 +415,5 @@ def test_no_notify_for_task_with_notify(self): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction_id) self.assertIsNone(tk2_lv_ac_db.notify) diff --git a/contrib/runners/orquesta_runner/tests/unit/test_pause_and_resume.py b/contrib/runners/orquesta_runner/tests/unit/test_pause_and_resume.py index a5e6c05091..bef405cb1c 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_pause_and_resume.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_pause_and_resume.py @@ -153,7 +153,7 @@ def test_pause_subworkflow_not_cascade_up_to_workflow(self): ) self.assertEqual(len(tk_ac_ex_dbs), 1) - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_dbs[0].liveaction) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_dbs[0].liveaction_id) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Pause the subworkflow. @@ -194,7 +194,7 @@ def test_pause_workflow_cascade_down_to_subworkflow(self): self.assertEqual(len(tk_ac_ex_dbs), 1) tk_ac_ex_db = tk_ac_ex_dbs[0] - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction_id) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Identify the records for the subworkflow. @@ -261,7 +261,7 @@ def test_pause_subworkflow_while_another_subworkflow_running(self): t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_dbs[0].id) )[0] - t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction) + t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction_id) t1_wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(t1_ac_ex_db.id) )[0] @@ -271,7 +271,7 @@ def test_pause_subworkflow_while_another_subworkflow_running(self): t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_dbs[1].id) )[0] - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) t2_wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(t2_ac_ex_db.id) )[0] @@ -289,7 +289,7 @@ def test_pause_subworkflow_while_another_subworkflow_running(self): self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Assert the other subworkflow is still running. - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Manually notify action execution completion for the task in the subworkflow. @@ -314,7 +314,7 @@ def test_pause_subworkflow_while_another_subworkflow_running(self): self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Assert the other subworkflow is still running. - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Manually notify action execution completion for the tasks in the other subworkflow. @@ -373,7 +373,7 @@ def test_pause_subworkflow_while_another_subworkflow_completed(self): t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_dbs[0].id) )[0] - t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction) + t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction_id) t1_wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(t1_ac_ex_db.id) )[0] @@ -383,7 +383,7 @@ def test_pause_subworkflow_while_another_subworkflow_completed(self): t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_dbs[1].id) )[0] - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) t2_wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(t2_ac_ex_db.id) )[0] @@ -401,7 +401,7 @@ def test_pause_subworkflow_while_another_subworkflow_completed(self): self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Assert the other subworkflow is still running. - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Manually notify action execution completion for the tasks in the other subworkflow. @@ -439,7 +439,7 @@ def test_pause_subworkflow_while_another_subworkflow_completed(self): self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Assert the target subworkflow is still pausing. - t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction) + t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction_id) self.assertEqual(t1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_PAUSING) # Manually notify action execution completion for the task in the subworkflow. @@ -489,7 +489,7 @@ def test_resume(self): tk_ac_ex_dbs = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_dbs[0].id) ) - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_dbs[0].liveaction) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_dbs[0].liveaction_id) self.assertEqual(tk_ac_ex_dbs[0].status, ac_const.LIVEACTION_STATUS_SUCCEEDED) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) wf_svc.handle_action_execution_completion(tk_ac_ex_dbs[0]) @@ -546,7 +546,7 @@ def test_resume_cascade_to_subworkflow(self): self.assertEqual(len(tk_ac_ex_dbs), 1) tk_ac_ex_db = tk_ac_ex_dbs[0] - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction_id) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Identify the records for the subworkflow. @@ -622,7 +622,7 @@ def test_resume_from_each_subworkflow_when_parent_is_paused(self): t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_dbs[0].id) )[0] - t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction) + t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction_id) t1_wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(t1_ac_ex_db.id) )[0] @@ -632,7 +632,7 @@ def test_resume_from_each_subworkflow_when_parent_is_paused(self): t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_dbs[1].id) )[0] - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) t2_wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(t2_ac_ex_db.id) )[0] @@ -650,7 +650,7 @@ def test_resume_from_each_subworkflow_when_parent_is_paused(self): self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Assert the other subworkflow is still running. - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Manually notify action execution completion for the task in the subworkflow. @@ -675,7 +675,7 @@ def test_resume_from_each_subworkflow_when_parent_is_paused(self): self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Assert the other subworkflow is still running. - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Pause the other subworkflow. @@ -769,7 +769,7 @@ def test_resume_from_subworkflow_when_parent_is_paused(self): t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_dbs[0].id) )[0] - t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction) + t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction_id) t1_wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(t1_ac_ex_db.id) )[0] @@ -779,7 +779,7 @@ def test_resume_from_subworkflow_when_parent_is_paused(self): t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_dbs[1].id) )[0] - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) t2_wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(t2_ac_ex_db.id) )[0] @@ -797,7 +797,7 @@ def test_resume_from_subworkflow_when_parent_is_paused(self): self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Assert the other subworkflow is still running. - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Manually notify action execution completion for the task in the subworkflow. @@ -822,7 +822,7 @@ def test_resume_from_subworkflow_when_parent_is_paused(self): self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Assert the other subworkflow is still running. - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Manually notify action execution completion for the tasks in the other subworkflow. @@ -903,7 +903,7 @@ def test_resume_from_subworkflow_when_parent_is_paused(self): t3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t3_ex_db.id) )[0] - t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction) + t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction_id) self.assertEqual(t3_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) wf_svc.handle_action_execution_completion(t3_ac_ex_db) @@ -933,7 +933,7 @@ def test_resume_from_subworkflow_when_parent_is_running(self): t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_dbs[0].id) )[0] - t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction) + t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction_id) t1_wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(t1_ac_ex_db.id) )[0] @@ -943,7 +943,7 @@ def test_resume_from_subworkflow_when_parent_is_running(self): t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_dbs[1].id) )[0] - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) t2_wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(t2_ac_ex_db.id) )[0] @@ -961,7 +961,7 @@ def test_resume_from_subworkflow_when_parent_is_running(self): self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Assert the other subworkflow is still running. - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Manually notify action execution completion for the task in the subworkflow. @@ -986,7 +986,7 @@ def test_resume_from_subworkflow_when_parent_is_running(self): self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Assert the other subworkflow is still running. - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Resume the subworkflow and assert it is running. @@ -1001,7 +1001,7 @@ def test_resume_from_subworkflow_when_parent_is_running(self): self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Assert the other subworkflow is still running. - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Manually notify action execution completion for the tasks in the subworkflow. @@ -1067,7 +1067,7 @@ def test_resume_from_subworkflow_when_parent_is_running(self): t3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t3_ex_db.id) )[0] - t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction) + t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction_id) self.assertEqual(t3_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) wf_svc.handle_action_execution_completion(t3_ac_ex_db) diff --git a/contrib/runners/orquesta_runner/tests/unit/test_rerun.py b/contrib/runners/orquesta_runner/tests/unit/test_rerun.py index 7981b8f42c..22c40aa8c1 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_rerun.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_rerun.py @@ -127,7 +127,7 @@ def test_rerun_workflow(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) self.assertEqual(tk1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_FAILED) workflow_service.handle_action_execution_completion(tk1_ac_ex_db) tk1_ex_db = wf_db_access.TaskExecution.get_by_id(tk1_ex_db.id) @@ -166,7 +166,7 @@ def test_rerun_workflow(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) self.assertEqual( tk1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -196,7 +196,7 @@ def test_rerun_with_missing_workflow_execution_id(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) self.assertEqual(tk1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_FAILED) workflow_service.handle_action_execution_completion(tk1_ac_ex_db) tk1_ex_db = wf_db_access.TaskExecution.get_by_id(tk1_ex_db.id) @@ -264,7 +264,7 @@ def test_rerun_with_invalid_workflow_execution(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) self.assertEqual(tk1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_FAILED) workflow_service.handle_action_execution_completion(tk1_ac_ex_db) tk1_ex_db = wf_db_access.TaskExecution.get_by_id(tk1_ex_db.id) @@ -322,7 +322,7 @@ def test_rerun_workflow_still_running(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) self.assertEqual( tk1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_RUNNING ) @@ -381,7 +381,7 @@ def test_rerun_with_unexpected_error(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) self.assertEqual(tk1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_FAILED) workflow_service.handle_action_execution_completion(tk1_ac_ex_db) tk1_ex_db = wf_db_access.TaskExecution.get_by_id(tk1_ex_db.id) @@ -436,7 +436,7 @@ def test_rerun_workflow_already_succeeded(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) self.assertEqual( tk1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -450,7 +450,7 @@ def test_rerun_workflow_already_succeeded(self): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction_id) self.assertEqual( tk2_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -464,7 +464,7 @@ def test_rerun_workflow_already_succeeded(self): tk3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk3_ex_db.id) )[0] - tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction) + tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction_id) self.assertEqual( tk3_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -505,7 +505,7 @@ def test_rerun_workflow_already_succeeded(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) self.assertEqual( tk1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -522,7 +522,7 @@ def test_rerun_workflow_already_succeeded(self): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction_id) self.assertEqual( tk2_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -539,7 +539,7 @@ def test_rerun_workflow_already_succeeded(self): tk3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk3_ex_db.id) )[0] - tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction) + tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction_id) self.assertEqual( tk3_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) diff --git a/contrib/runners/orquesta_runner/tests/unit/test_with_items.py b/contrib/runners/orquesta_runner/tests/unit/test_with_items.py index 44909fe831..de9b0bea07 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_with_items.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_with_items.py @@ -368,7 +368,7 @@ def test_with_items_cancellation(self): # Manually succeed the action executions and process completion. for ac_ex in t1_ac_ex_dbs: self.set_execution_status( - ac_ex.liveaction, action_constants.LIVEACTION_STATUS_SUCCEEDED + ac_ex.liveaction_id, action_constants.LIVEACTION_STATUS_SUCCEEDED ) t1_ac_ex_dbs = ex_db_access.ActionExecution.query( @@ -440,7 +440,7 @@ def test_with_items_concurrency_cancellation(self): # Manually succeed the action executions and process completion. for ac_ex in t1_ac_ex_dbs: self.set_execution_status( - ac_ex.liveaction, action_constants.LIVEACTION_STATUS_SUCCEEDED + ac_ex.liveaction_id, action_constants.LIVEACTION_STATUS_SUCCEEDED ) t1_ac_ex_dbs = ex_db_access.ActionExecution.query( @@ -509,7 +509,7 @@ def test_with_items_pause_and_resume(self): # Manually succeed the action executions and process completion. for ac_ex in t1_ac_ex_dbs: self.set_execution_status( - ac_ex.liveaction, action_constants.LIVEACTION_STATUS_SUCCEEDED + ac_ex.liveaction_id, action_constants.LIVEACTION_STATUS_SUCCEEDED ) t1_ac_ex_dbs = ex_db_access.ActionExecution.query( @@ -599,7 +599,7 @@ def test_with_items_concurrency_pause_and_resume(self): # Manually succeed the action executions and process completion. for ac_ex in t1_ac_ex_dbs: self.set_execution_status( - ac_ex.liveaction, action_constants.LIVEACTION_STATUS_SUCCEEDED + ac_ex.liveaction_id, action_constants.LIVEACTION_STATUS_SUCCEEDED ) t1_ac_ex_dbs = ex_db_access.ActionExecution.query( diff --git a/st2actions/st2actions/container/base.py b/st2actions/st2actions/container/base.py index e7e38d9451..330fb072c3 100644 --- a/st2actions/st2actions/container/base.py +++ b/st2actions/st2actions/container/base.py @@ -459,7 +459,7 @@ def _get_runner(self, runner_type_db, action_db, liveaction_db): runner.action_name = action_db.name runner.liveaction = liveaction_db runner.liveaction_id = str(liveaction_db.id) - runner.execution = ActionExecution.get(liveaction=str(runner.liveaction_id)) + runner.execution = ActionExecution.get(liveaction_id=str(runner.liveaction_id)) runner.execution_id = str(runner.execution.id) runner.entry_point = resolved_entry_point runner.context = context diff --git a/st2actions/st2actions/notifier/notifier.py b/st2actions/st2actions/notifier/notifier.py index e7680d4648..2b8fe49059 100644 --- a/st2actions/st2actions/notifier/notifier.py +++ b/st2actions/st2actions/notifier/notifier.py @@ -83,7 +83,7 @@ def process(self, execution_db): LOG.debug('Processing action execution "%s".', execution_id, extra=extra) # Get the corresponding liveaction record. - liveaction_db = LiveAction.get_by_id(execution_db.liveaction) + liveaction_db = LiveAction.get_by_id(execution_db.liveaction_id) if execution_db.status in LIVEACTION_COMPLETED_STATES: # If the action execution is executed under an orquesta workflow, policies for the diff --git a/st2actions/st2actions/scheduler/entrypoint.py b/st2actions/st2actions/scheduler/entrypoint.py index 5782a436a6..47e3295a5d 100644 --- a/st2actions/st2actions/scheduler/entrypoint.py +++ b/st2actions/st2actions/scheduler/entrypoint.py @@ -97,7 +97,7 @@ def _create_execution_queue_item_db_from_liveaction(self, liveaction, delay=None """ Create ActionExecutionSchedulingQueueItemDB from live action. """ - execution = ActionExecution.get(liveaction=str(liveaction.id)) + execution = ActionExecution.get(liveaction_id=str(liveaction.id)) execution_queue_item_db = ActionExecutionSchedulingQueueItemDB() execution_queue_item_db.action_execution_id = str(execution.id) diff --git a/st2actions/st2actions/scheduler/handler.py b/st2actions/st2actions/scheduler/handler.py index 35e2e57a86..7e3615434b 100644 --- a/st2actions/st2actions/scheduler/handler.py +++ b/st2actions/st2actions/scheduler/handler.py @@ -136,7 +136,7 @@ def _fix_missing_action_execution_id(self): for entry in ActionExecutionSchedulingQueue.query( action_execution_id__in=["", None] ): - execution_db = ActionExecution.get(liveaction=entry.liveaction_id) + execution_db = ActionExecution.get(liveaction_id=entry.liveaction_id) if not execution_db: continue diff --git a/st2actions/st2actions/worker.py b/st2actions/st2actions/worker.py index 9537050fc0..b1d3fc790e 100644 --- a/st2actions/st2actions/worker.py +++ b/st2actions/st2actions/worker.py @@ -235,7 +235,7 @@ def _run_action(self, liveaction_db): return result def _cancel_action(self, liveaction_db): - action_execution_db = ActionExecution.get(liveaction=str(liveaction_db.id)) + action_execution_db = ActionExecution.get(liveaction_id=str(liveaction_db.id)) extra = { "action_execution_db": action_execution_db, "liveaction_db": liveaction_db, @@ -265,7 +265,7 @@ def _cancel_action(self, liveaction_db): return result def _pause_action(self, liveaction_db): - action_execution_db = ActionExecution.get(liveaction=str(liveaction_db.id)) + action_execution_db = ActionExecution.get(liveaction_id=str(liveaction_db.id)) extra = { "action_execution_db": action_execution_db, "liveaction_db": liveaction_db, @@ -294,7 +294,7 @@ def _pause_action(self, liveaction_db): return result def _resume_action(self, liveaction_db): - action_execution_db = ActionExecution.get(liveaction=str(liveaction_db.id)) + action_execution_db = ActionExecution.get(liveaction_id=str(liveaction_db.id)) extra = { "action_execution_db": action_execution_db, "liveaction_db": liveaction_db, diff --git a/st2actions/st2actions/workflows/workflows.py b/st2actions/st2actions/workflows/workflows.py index f38c0515a3..8c8968de8b 100644 --- a/st2actions/st2actions/workflows/workflows.py +++ b/st2actions/st2actions/workflows/workflows.py @@ -133,7 +133,7 @@ def shutdown(self): if cfg.CONF.coordination.service_registry and not member_ids: ac_ex_dbs = self._get_running_workflows() for ac_ex_db in ac_ex_dbs: - lv_ac = action_utils.get_liveaction_by_id(ac_ex_db.liveaction) + lv_ac = action_utils.get_liveaction_by_id(ac_ex_db.liveaction_id) ac_svc.request_pause(lv_ac, WORKFLOW_ENGINE_START_STOP_SEQ) def _get_running_workflows(self): diff --git a/st2actions/tests/unit/policies/test_concurrency.py b/st2actions/tests/unit/policies/test_concurrency.py index c1a75a4f33..a2991ad862 100644 --- a/st2actions/tests/unit/policies/test_concurrency.py +++ b/st2actions/tests/unit/policies/test_concurrency.py @@ -218,7 +218,7 @@ def test_over_threshold_delay_executions(self): self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Check the status changes. - execution = ActionExecution.get(liveaction=str(liveaction.id)) + execution = ActionExecution.get(liveaction_id=str(liveaction.id)) expected_status_changes = [ "requested", "delayed", diff --git a/st2actions/tests/unit/policies/test_retry_policy.py b/st2actions/tests/unit/policies/test_retry_policy.py index 86feb96d4f..609a607b95 100644 --- a/st2actions/tests/unit/policies/test_retry_policy.py +++ b/st2actions/tests/unit/policies/test_retry_policy.py @@ -128,7 +128,7 @@ def test_retry_on_timeout_first_retry_is_successful(self): self.assertEqual(action_execution_dbs[1].status, LIVEACTION_STATUS_REQUESTED) # Verify retried execution contains policy related context - original_liveaction_id = action_execution_dbs[0].liveaction + original_liveaction_id = action_execution_dbs[0].liveaction_id context = action_execution_dbs[1].context self.assertIn("policies", context) @@ -183,7 +183,7 @@ def test_retry_on_timeout_policy_is_retried_twice(self): self.assertEqual(action_execution_dbs[1].status, LIVEACTION_STATUS_REQUESTED) # Verify retried execution contains policy related context - original_liveaction_id = action_execution_dbs[0].liveaction + original_liveaction_id = action_execution_dbs[0].liveaction_id context = action_execution_dbs[1].context self.assertIn("policies", context) @@ -216,7 +216,7 @@ def test_retry_on_timeout_policy_is_retried_twice(self): self.assertEqual(action_execution_dbs[2].status, LIVEACTION_STATUS_REQUESTED) # Verify retried execution contains policy related context - original_liveaction_id = action_execution_dbs[1].liveaction + original_liveaction_id = action_execution_dbs[1].liveaction_id context = action_execution_dbs[2].context self.assertIn("policies", context) diff --git a/st2actions/tests/unit/test_executions.py b/st2actions/tests/unit/test_executions.py index d436ffbbda..3bd92c5034 100644 --- a/st2actions/tests/unit/test_executions.py +++ b/st2actions/tests/unit/test_executions.py @@ -99,7 +99,7 @@ def test_basic_execution(self): ) execution = self._get_action_execution( - liveaction=str(liveaction.id), raise_exception=True + liveaction_id=str(liveaction.id), raise_exception=True ) self.assertDictEqual(execution.trigger, {}) @@ -135,7 +135,7 @@ def test_chained_executions(self): ) execution = self._get_action_execution( - liveaction=str(liveaction.id), raise_exception=True + liveaction_id=str(liveaction.id), raise_exception=True ) action = action_utils.get_action_by_ref("executions.chain") @@ -153,7 +153,7 @@ def test_chained_executions(self): self.assertEqual(execution.result, liveaction.result) self.assertEqual(execution.status, liveaction.status) self.assertEqual(execution.context, liveaction.context) - self.assertEqual(execution.liveaction, str(liveaction.id)) + self.assertEqual(execution.liveaction_id, str(liveaction.id)) self.assertGreater(len(execution.children), 0) for child in execution.children: @@ -200,7 +200,7 @@ def test_triggered_execution(self): ) execution = self._get_action_execution( - liveaction=str(liveaction.id), raise_exception=True + liveaction_id=str(liveaction.id), raise_exception=True ) self.assertDictEqual(execution.trigger, vars(TriggerAPI.from_model(trigger))) @@ -227,7 +227,7 @@ def test_triggered_execution(self): self.assertEqual(execution.result, liveaction.result) self.assertEqual(execution.status, liveaction.status) self.assertEqual(execution.context, liveaction.context) - self.assertEqual(execution.liveaction, str(liveaction.id)) + self.assertEqual(execution.liveaction_id, str(liveaction.id)) def _get_action_execution(self, **kwargs): return ActionExecution.get(**kwargs) diff --git a/st2actions/tests/unit/test_notifier.py b/st2actions/tests/unit/test_notifier.py index a11288805f..9151ab0d23 100644 --- a/st2actions/tests/unit/test_notifier.py +++ b/st2actions/tests/unit/test_notifier.py @@ -184,7 +184,7 @@ def test_notify_triggers_end_timestamp_none(self): LiveAction.add_or_update(liveaction_db) execution = MOCK_EXECUTION - execution.liveaction = str(liveaction_db.id) + execution.liveaction_id = str(liveaction_db.id) execution.status = liveaction_db.status dispatcher = NotifierTestCase.MockDispatcher(self) @@ -237,7 +237,7 @@ def test_notify_triggers_jinja_patterns(self, dispatch): LiveAction.add_or_update(liveaction_db) execution = MOCK_EXECUTION - execution.liveaction = str(liveaction_db.id) + execution.liveaction_id = str(liveaction_db.id) execution.status = liveaction_db.status notifier = Notifier(connection=None, queues=[]) @@ -269,7 +269,7 @@ def test_post_generic_trigger_emit_when_default_value_is_used(self, dispatch): liveaction_db = LiveActionDB(action="core.local") liveaction_db.status = status execution = MOCK_EXECUTION - execution.liveaction = str(liveaction_db.id) + execution.liveaction_id = str(liveaction_db.id) execution.status = liveaction_db.status notifier = Notifier(connection=None, queues=[]) @@ -306,7 +306,7 @@ def test_post_generic_trigger_with_emit_condition(self, dispatch): liveaction_db = LiveActionDB(action="core.local") liveaction_db.status = status execution = MOCK_EXECUTION - execution.liveaction = str(liveaction_db.id) + execution.liveaction_id = str(liveaction_db.id) execution.status = liveaction_db.status notifier = Notifier(connection=None, queues=[]) @@ -353,7 +353,7 @@ def test_process_post_generic_notify_trigger_on_completed_state_default( liveaction_db = LiveActionDB(id=bson.ObjectId(), action="core.local") liveaction_db.status = status execution = MOCK_EXECUTION - execution.liveaction = str(liveaction_db.id) + execution.liveaction_id = str(liveaction_db.id) execution.status = liveaction_db.status mock_LiveAction.get_by_id.return_value = liveaction_db @@ -403,7 +403,7 @@ def test_process_post_generic_notify_trigger_on_custom_emit_when_states( liveaction_db = LiveActionDB(id=bson.ObjectId(), action="core.local") liveaction_db.status = status execution = MOCK_EXECUTION - execution.liveaction = str(liveaction_db.id) + execution.liveaction_id = str(liveaction_db.id) execution.status = liveaction_db.status mock_LiveAction.get_by_id.return_value = liveaction_db diff --git a/st2api/st2api/controllers/v1/actionexecutions.py b/st2api/st2api/controllers/v1/actionexecutions.py index 4ff9f023fe..38cca8d299 100644 --- a/st2api/st2api/controllers/v1/actionexecutions.py +++ b/st2api/st2api/controllers/v1/actionexecutions.py @@ -852,7 +852,7 @@ def put(self, id, liveaction_api, requester_user, show_secrets=False): if not execution_api: abort(http_client.NOT_FOUND, "Execution with id %s not found." % id) - liveaction_id = execution_api.liveaction + liveaction_id = execution_api.liveaction_id if not liveaction_id: abort( http_client.INTERNAL_SERVER_ERROR, @@ -876,7 +876,7 @@ def update_status(liveaction_api, liveaction_db): liveaction_db = action_service.update_status( liveaction_db, status, result, set_result_size=True ) - actionexecution_db = ActionExecution.get(liveaction=str(liveaction_db.id)) + actionexecution_db = ActionExecution.get(liveaction_id=str(liveaction_db.id)) return (liveaction_db, actionexecution_db) try: @@ -979,7 +979,7 @@ def delete(self, id, requester_user, show_secrets=False): if not execution_api: abort(http_client.NOT_FOUND, "Execution with id %s not found." % id) - liveaction_id = execution_api.liveaction + liveaction_id = execution_api.liveaction_id if not liveaction_id: abort( http_client.INTERNAL_SERVER_ERROR, diff --git a/st2api/st2api/controllers/v1/aliasexecution.py b/st2api/st2api/controllers/v1/aliasexecution.py index 3ba706e1b9..b46e6fd32f 100644 --- a/st2api/st2api/controllers/v1/aliasexecution.py +++ b/st2api/st2api/controllers/v1/aliasexecution.py @@ -183,15 +183,6 @@ def _post(self, payload, requester_user, show_secrets=False, match_multiple=Fals show_secrets=show_secrets, requester_user=requester_user, ) - if hasattr(execution, "liveaction"): - liveaction = LiveAction.get_by_id(execution.liveaction) - mask_secrets = self._get_mask_secrets( - requester_user, show_secrets=show_secrets - ) - liveaction = LiveActionAPI.from_model( - liveaction, mask_secrets=mask_secrets - ) - execution.liveaction = liveaction result = { "execution": execution, "actionalias": ActionAliasAPI.from_model(action_alias_db), diff --git a/st2api/tests/unit/controllers/v1/test_alias_execution.py b/st2api/tests/unit/controllers/v1/test_alias_execution.py index a530268622..44261fde3f 100644 --- a/st2api/tests/unit/controllers/v1/test_alias_execution.py +++ b/st2api/tests/unit/controllers/v1/test_alias_execution.py @@ -159,7 +159,6 @@ def test_execution_secret_parameter(self, request): self.assertEqual(post_resp.status_int, 201) expected_parameters = {"param1": "value1", "param4": SUPER_SECRET_PARAMETER} self.assertEqual(request.call_args[0][0].parameters, expected_parameters) - # above working post_resp = self._do_post( alias_execution=self.alias4, command=command, diff --git a/st2api/tests/unit/controllers/v1/test_executions_filters.py b/st2api/tests/unit/controllers/v1/test_executions_filters.py index d688d5a76d..9a1dab25fd 100644 --- a/st2api/tests/unit/controllers/v1/test_executions_filters.py +++ b/st2api/tests/unit/controllers/v1/test_executions_filters.py @@ -60,7 +60,7 @@ def setUpClass(cls): "rule": copy.deepcopy(fixture.ARTIFACTS["rule"]), "action": copy.deepcopy(fixture.ARTIFACTS["actions"]["chain"]), "runner": copy.deepcopy(fixture.ARTIFACTS["runners"]["action-chain"]), - "liveaction": fixture.ARTIFACTS["liveactions"]["workflow"]["id"], + "liveaction_id": fixture.ARTIFACTS["liveactions"]["workflow"]["id"], "status": fixture.ARTIFACTS["liveactions"]["workflow"]["status"], "result": copy.deepcopy( fixture.ARTIFACTS["liveactions"]["workflow"]["result"] @@ -71,7 +71,7 @@ def setUpClass(cls): { "action": copy.deepcopy(fixture.ARTIFACTS["actions"]["local"]), "runner": copy.deepcopy(fixture.ARTIFACTS["runners"]["run-local"]), - "liveaction": fixture.ARTIFACTS["liveactions"]["task1"]["id"], + "liveaction_id": fixture.ARTIFACTS["liveactions"]["task1"]["id"], "status": fixture.ARTIFACTS["liveactions"]["task1"]["status"], "result": copy.deepcopy( fixture.ARTIFACTS["liveactions"]["task1"]["result"] @@ -141,7 +141,7 @@ def test_get_one(self): self.assertEqual(record["id"], obj_id) self.assertDictEqual(record["action"], fake_record.action) self.assertDictEqual(record["runner"], fake_record.runner) - self.assertEqual(record["liveaction"], fake_record.liveaction) + self.assertEqual(record["liveaction_id"], fake_record.liveaction_id) def test_get_one_failed(self): response = self.app.get( diff --git a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb index dd0cdb2812..25e1e76b22 100755 --- a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb +++ b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb @@ -111,7 +111,7 @@ def migrate_executions(start_dt: datetime.datetime, end_dt: datetime.datetime) - execution_db._mark_as_changed("liveaction") # NOTE: If you want to view changed fields, you can access execution_db._changed_fields # will throw an exception if already a string - execution_db.liveaction = execution_db.liveaction.get("id", None) + execution_db.liveaction_id = execution_db.liveaction.get("id", None) execution_db.save() print("ActionExecutionDB with id %s has been migrated" % (execution_db.id)) diff --git a/st2common/bin/st2-track-result b/st2common/bin/st2-track-result index e461f40668..7f158a2453 100755 --- a/st2common/bin/st2-track-result +++ b/st2common/bin/st2-track-result @@ -68,7 +68,7 @@ def add_result_tracker(exec_id): LOG.info("Retrieving runner type and liveaction records...") runnertype_db = action_db.get_runnertype_by_name(exec_db.action.get("runner_type")) - liveaction_db = action_db.get_liveaction_by_id(exec_db.liveaction) + liveaction_db = action_db.get_liveaction_by_id(exec_db.liveaction_id) # Skip if liveaction is completed. if liveaction_db.status in action_constants.LIVEACTION_COMPLETED_STATES: @@ -100,7 +100,7 @@ def del_result_tracker(exec_id): LOG.info('Found action execution record for "%s".', exec_id) LOG.info("Retrieving runner type and liveaction records...") - liveaction_db = action_db.get_liveaction_by_id(exec_db.liveaction) + liveaction_db = action_db.get_liveaction_by_id(exec_db.liveaction_id) LOG.info("Removing result tracker entry...") removed = queries.remove_query(liveaction_db.id) diff --git a/st2common/st2common/fields.py b/st2common/st2common/fields.py index 41ecd178cf..33144962df 100644 --- a/st2common/st2common/fields.py +++ b/st2common/st2common/fields.py @@ -342,9 +342,6 @@ class JSONDictField(BinaryField): """ def __init__(self, *args, **kwargs): - self.compression_algorithm = ( - JSONDictFieldCompressionAlgorithmEnum.ZSTANDARD.value - ) super(JSONDictField, self).__init__(*args, **kwargs) def to_mongo(self, value): diff --git a/st2common/st2common/garbage_collection/executions.py b/st2common/st2common/garbage_collection/executions.py index aae36c9cde..bc0e85f1f0 100644 --- a/st2common/st2common/garbage_collection/executions.py +++ b/st2common/st2common/garbage_collection/executions.py @@ -223,5 +223,5 @@ def purge_orphaned_workflow_executions(logger): # as a result of the original failure, the garbage collection routine here cancels # the workflow execution so it cannot be rerun from failed task(s). for ac_ex_db in workflow_service.identify_orphaned_workflows(): - lv_ac_db = LiveAction.get(id=ac_ex_db.liveaction) + lv_ac_db = LiveAction.get(id=ac_ex_db.liveaction_id) action_service.request_cancellation(lv_ac_db, None) diff --git a/st2common/st2common/models/api/execution.py b/st2common/st2common/models/api/execution.py index 17b9dcf2ad..9be6cd2512 100644 --- a/st2common/st2common/models/api/execution.py +++ b/st2common/st2common/models/api/execution.py @@ -175,6 +175,10 @@ def convert_raw(cls, doc, raw_values): override this class to convert any raw byte values into dict + Now add the JSON string field value which shouldn't be escaped back. + We don't JSON parse the field value here because that happens inside the model specific + "from_model()" method where we also parse and convert all the other field values. + :param doc: dict :param raw_values: dict[field]:bytestring """ diff --git a/st2common/st2common/models/db/execution.py b/st2common/st2common/models/db/execution.py index 05b903d81b..eba2cb840e 100644 --- a/st2common/st2common/models/db/execution.py +++ b/st2common/st2common/models/db/execution.py @@ -79,13 +79,13 @@ class ActionExecutionDB(stormbase.StormFoundationDB): web_url = me.StringField(required=False) # liveaction id - liveaction = me.StringField() + liveaction_id = me.StringField() meta = { "indexes": [ {"fields": ["rule.ref"]}, {"fields": ["action.ref"]}, - {"fields": ["liveaction"]}, + {"fields": ["liveaction_id"]}, {"fields": ["start_timestamp"]}, {"fields": ["end_timestamp"]}, {"fields": ["status"]}, diff --git a/st2common/st2common/models/db/liveaction.py b/st2common/st2common/models/db/liveaction.py index 73be661d05..4f4ebcaa37 100644 --- a/st2common/st2common/models/db/liveaction.py +++ b/st2common/st2common/models/db/liveaction.py @@ -116,6 +116,23 @@ def mask_secrets(self, value): result["parameters"] = mask_secret_parameters( parameters=execution_parameters, secret_parameters=secret_parameters ) + if result.get("action", "") == "st2.inquiry.respond": + # In this case, this execution is just a plain python action, not + # an inquiry, so we don't natively have a handle on the response + # schema. + # + # To prevent leakage, we can just mask all response fields. + # + # Note: The 'string' type in secret_parameters doesn't matter, + # it's just a placeholder to tell mask_secret_parameters() + # that this parameter is indeed a secret parameter and to + # mask it. + result["parameters"]["response"] = mask_secret_parameters( + parameters=result["parameters"]["response"], + secret_parameters={ + p: "string" for p in result["parameters"]["response"] + }, + ) return result def get_masked_parameters(self): diff --git a/st2common/st2common/openapi.yaml b/st2common/st2common/openapi.yaml index 832d70ff35..c49242c627 100644 --- a/st2common/st2common/openapi.yaml +++ b/st2common/st2common/openapi.yaml @@ -4934,7 +4934,7 @@ definitions: $ref: '#/definitions/Action' runner: $ref: '#/definitions/RunnerType' - liveaction: + liveaction_id: type: string task_execution: type: string diff --git a/st2common/st2common/openapi.yaml.j2 b/st2common/st2common/openapi.yaml.j2 index 1e0bb75701..f8b08af4c2 100644 --- a/st2common/st2common/openapi.yaml.j2 +++ b/st2common/st2common/openapi.yaml.j2 @@ -4930,7 +4930,7 @@ definitions: $ref: '#/definitions/Action' runner: $ref: '#/definitions/RunnerType' - liveaction: + liveaction_id: type: string task_execution: type: string diff --git a/st2common/st2common/services/action.py b/st2common/st2common/services/action.py index ef3806461e..9aa33ee542 100644 --- a/st2common/st2common/services/action.py +++ b/st2common/st2common/services/action.py @@ -316,7 +316,7 @@ def request_cancellation(liveaction, requester): liveaction, status, result=result, context=liveaction.context ) - execution = ActionExecution.get(liveaction=str(liveaction.id)) + execution = ActionExecution.get(liveaction_id=str(liveaction.id)) return (liveaction, execution) @@ -347,7 +347,7 @@ def request_pause(liveaction, requester): liveaction.status == action_constants.LIVEACTION_STATUS_PAUSING or liveaction.status == action_constants.LIVEACTION_STATUS_PAUSED ): - execution = ActionExecution.get(liveaction=str(liveaction.id)) + execution = ActionExecution.get(liveaction_id=str(liveaction.id)) return (liveaction, execution) if liveaction.status != action_constants.LIVEACTION_STATUS_RUNNING: @@ -363,7 +363,7 @@ def request_pause(liveaction, requester): context=liveaction.context, ) - execution = ActionExecution.get(liveaction=str(liveaction.id)) + execution = ActionExecution.get(liveaction_id=str(liveaction.id)) return (liveaction, execution) @@ -396,7 +396,7 @@ def request_resume(liveaction, requester): ] if liveaction.status in running_states: - execution = ActionExecution.get(liveaction=str(liveaction.id)) + execution = ActionExecution.get(liveaction_id=str(liveaction.id)) return (liveaction, execution) if liveaction.status != action_constants.LIVEACTION_STATUS_PAUSED: @@ -412,7 +412,7 @@ def request_resume(liveaction, requester): context=liveaction.context, ) - execution = ActionExecution.get(liveaction=str(liveaction.id)) + execution = ActionExecution.get(liveaction_id=str(liveaction.id)) return (liveaction, execution) @@ -433,7 +433,7 @@ def get_parent_liveaction(liveaction_db): return None parent_execution_db = ActionExecution.get(id=parent["execution_id"]) - parent_liveaction_db = LiveAction.get(id=parent_execution_db.liveaction) + parent_liveaction_db = LiveAction.get(id=parent_execution_db.liveaction_id) return parent_liveaction_db @@ -541,7 +541,7 @@ def store_execution_output_data_ex( def is_children_active(liveaction_id): - execution_db = ActionExecution.get(liveaction=str(liveaction_id)) + execution_db = ActionExecution.get(liveaction_id=str(liveaction_id)) if execution_db.runner["name"] not in action_constants.WORKFLOW_RUNNER_TYPES: return False diff --git a/st2common/st2common/services/executions.py b/st2common/st2common/services/executions.py index 4215d1e764..926c8ac808 100644 --- a/st2common/st2common/services/executions.py +++ b/st2common/st2common/services/executions.py @@ -82,7 +82,7 @@ def _decompose_liveaction(liveaction_db): """ Splits the liveaction into an ActionExecution compatible dict. """ - decomposed = {"liveaction": str(liveaction_db.id)} + decomposed = {"liveaction_id": str(liveaction_db.id)} liveaction_api = vars(LiveActionAPI.from_model(liveaction_db)) for k in liveaction_api.keys(): if k not in LIVEACTION_ATTRIBUTES: @@ -153,7 +153,7 @@ def create_execution_object( # NOTE: User input data is already validate as part of the API request, # other data is set by us. Skipping validation here makes operation 10%-30% faster - execution.liveaction = str(liveaction.id) + execution.liveaction_id = str(liveaction.id) execution = ActionExecution.add_or_update( execution, publish=publish, validate=False ) @@ -193,7 +193,7 @@ def update_execution(liveaction_db, publish=True, set_result_size=False): :param set_result_size: True to calculate size of the serialized result field value and set it on the "result_size" database field. """ - execution = ActionExecution.get(liveaction=str(liveaction_db.id)) + execution = ActionExecution.get(liveaction_id=str(liveaction_db.id)) with coordination.get_coordinator().get_lock(str(liveaction_db.id).encode()): # Skip execution object update when action is already in completed state. diff --git a/st2common/st2common/services/inquiry.py b/st2common/st2common/services/inquiry.py index c52d182d7a..35755d2e22 100644 --- a/st2common/st2common/services/inquiry.py +++ b/st2common/st2common/services/inquiry.py @@ -126,7 +126,7 @@ def respond(inquiry, response, requester=None): requester = cfg.CONF.system_user.user # Retrieve the liveaction from the database. - liveaction_db = lv_db_access.LiveAction.get_by_id(inquiry.liveaction) + liveaction_db = lv_db_access.LiveAction.get_by_id(inquiry.liveaction_id) # Resume the parent workflow first. If the action execution for the inquiry is updated first, # it triggers handling of the action execution completion which will interact with the paused diff --git a/st2common/st2common/services/trace.py b/st2common/st2common/services/trace.py index 67035411c0..37d435f36f 100644 --- a/st2common/st2common/services/trace.py +++ b/st2common/st2common/services/trace.py @@ -197,7 +197,7 @@ def get_trace_db_by_live_action(liveaction): ) return (created, trace_db) # 3. Check if the action_execution associated with liveaction leads to a trace_db - execution = ActionExecution.get(liveaction=str(liveaction.id)) + execution = ActionExecution.get(liveaction_id=str(liveaction.id)) if execution: trace_db = get_trace_db_by_action_execution(action_execution=execution) # 4. No trace_db found, therefore create one. This typically happens diff --git a/st2common/st2common/services/workflows.py b/st2common/st2common/services/workflows.py index ac6b85f033..c99fb896b5 100644 --- a/st2common/st2common/services/workflows.py +++ b/st2common/st2common/services/workflows.py @@ -472,7 +472,7 @@ def request_cancellation(ac_ex_db): and root_ac_ex_db.status not in ac_const.LIVEACTION_CANCEL_STATES ): LOG.info("[%s] Cascading cancelation request to parent workflow.", wf_ac_ex_id) - root_lv_ac_db = lv_db_access.LiveAction.get(id=root_ac_ex_db.liveaction) + root_lv_ac_db = lv_db_access.LiveAction.get(id=root_ac_ex_db.liveaction_id) ac_svc.request_cancellation(root_lv_ac_db, None) LOG.debug("[%s] %s", wf_ac_ex_id, conductor.serialize()) @@ -915,7 +915,7 @@ def handle_action_execution_resume(ac_ex_db): if parent_ac_ex_db.status == ac_const.LIVEACTION_STATUS_PAUSED: action_utils.update_liveaction_status( - liveaction_id=parent_ac_ex_db.liveaction, + liveaction_id=parent_ac_ex_db.liveaction_id, status=ac_const.LIVEACTION_STATUS_RUNNING, publish=False, ) @@ -1449,7 +1449,7 @@ def update_execution_records( # Update the corresponding liveaction and action execution for the workflow. wf_ac_ex_db = ex_db_access.ActionExecution.get_by_id(wf_ex_db.action_execution) - wf_lv_ac_db = action_utils.get_liveaction_by_id(wf_ac_ex_db.liveaction) + wf_lv_ac_db = action_utils.get_liveaction_by_id(wf_ac_ex_db.liveaction_id) # Gather result for liveaction and action execution. result = {"output": wf_ex_db.output or None} diff --git a/st2common/st2common/util/param.py b/st2common/st2common/util/param.py index b8bb038369..104a3c5479 100644 --- a/st2common/st2common/util/param.py +++ b/st2common/st2common/util/param.py @@ -310,11 +310,10 @@ def render_live_params( additional_contexts=None, ): """ - :param params BaseDict + :param params: BaseDict Renders list of parameters. Ensures that there's no cyclic or missing dependencies. Returns a dict of plain rendered parameters. """ - params = params additional_contexts = additional_contexts or {} pack = action_context.get("pack") diff --git a/st2common/tests/unit/migrations/test_v35_migrate_db_dict_field_values.py b/st2common/tests/unit/migrations/test_v35_migrate_db_dict_field_values.py index fd0f0ea1ba..3dfe85bc86 100644 --- a/st2common/tests/unit/migrations/test_v35_migrate_db_dict_field_values.py +++ b/st2common/tests/unit/migrations/test_v35_migrate_db_dict_field_values.py @@ -83,7 +83,7 @@ def test_migrate_executions(self): LiveActionDB._meta["allow_inheritance"] = True class ActionExecutionDB_OldFieldType(ActionExecutionDB): - + liveaction_id = None result = stormbase.EscapedDynamicField(default={}) liveaction = stormbase.EscapedDictField(required=True) parameters = stormbase.EscapedDynamicField(default={}) @@ -168,6 +168,7 @@ class LiveActionDB_OldFieldType(LiveActionDB): ) class LiveActionDB_NewFieldType(LiveActionDB): + liveaction_id = None result = JSONDictEscapedFieldCompatibilityField( default={}, help_text="Action defined result." ) diff --git a/st2common/tests/unit/services/test_trace.py b/st2common/tests/unit/services/test_trace.py index 83a39a53ef..a19f8c3552 100644 --- a/st2common/tests/unit/services/test_trace.py +++ b/st2common/tests/unit/services/test_trace.py @@ -253,7 +253,7 @@ def test_get_trace_db_by_live_action_parent_fail(self): def test_get_trace_db_by_live_action_from_execution(self): traceable_liveaction = copy.copy(self.traceable_liveaction) # fixtures id value in liveaction is not persisted in DB. - traceable_liveaction.id = bson.ObjectId(self.traceable_execution.liveaction) + traceable_liveaction.id = bson.ObjectId(self.traceable_execution.liveaction_id) created, trace_db = trace_service.get_trace_db_by_live_action( traceable_liveaction ) diff --git a/st2common/tests/unit/services/test_workflow_identify_orphans.py b/st2common/tests/unit/services/test_workflow_identify_orphans.py index c94892e98b..8c3de819d3 100644 --- a/st2common/tests/unit/services/test_workflow_identify_orphans.py +++ b/st2common/tests/unit/services/test_workflow_identify_orphans.py @@ -175,7 +175,7 @@ def mock_workflow_records(self, completed=False, expired=True, log=True): workflow_execution=str(wf_ex_db.id), action={"runner_type": runner, "ref": action_ref}, runner={"name": runner}, - liveaction=str(lv_ac_db.id), + liveaction_id=str(lv_ac_db.id), context={"user": user, "workflow_execution": str(wf_ex_db.id)}, status=status, start_timestamp=start_timestamp, @@ -269,7 +269,7 @@ def mock_task_records( task_execution=str(tk_ex_db.id), action={"runner_type": runner, "ref": action_ref}, runner={"name": runner}, - liveaction=str(lv_ac_db.id), + liveaction_id=str(lv_ac_db.id), context=context, status=status, start_timestamp=tk_ex_db.start_timestamp, diff --git a/st2common/tests/unit/services/test_workflow_service_retries.py b/st2common/tests/unit/services/test_workflow_service_retries.py index 45257f8236..bfc6250e28 100644 --- a/st2common/tests/unit/services/test_workflow_service_retries.py +++ b/st2common/tests/unit/services/test_workflow_service_retries.py @@ -144,7 +144,7 @@ def test_recover_from_coordinator_connection_error(self, mock_get_lock): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) mock_get_lock.side_effect = [ coordination.ToozConnectionError("foobar"), @@ -178,7 +178,7 @@ def test_retries_exhausted_from_coordinator_connection_error(self, mock_get_lock tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) mock_get_lock.side_effect = [ @@ -220,7 +220,7 @@ def test_recover_from_database_connection_error(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) wf_svc.handle_action_execution_completion(tk1_ac_ex_db) @@ -247,7 +247,7 @@ def test_retries_exhausted_from_database_connection_error(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # The connection error should raise if retries are exhaused. diff --git a/st2common/tests/unit/test_db_execution.py b/st2common/tests/unit/test_db_execution.py index 8a9e2b95b9..d78776736f 100644 --- a/st2common/tests/unit/test_db_execution.py +++ b/st2common/tests/unit/test_db_execution.py @@ -94,14 +94,14 @@ "action": {"uid": "action:core:ask", "output_schema": {}}, "status": "succeeded", "runner": {"name": "inquirer"}, - "liveaction": INQUIRY_LIVEACTION["id"], + "liveaction_id": INQUIRY_LIVEACTION["id"], "result": INQUIRY_RESULT, }, "execution_2": { "action": {"uid": "action:st2:inquiry.respond", "output_schema": {}}, "status": "succeeded", "runner": {"name": "python-script"}, - "liveaction": RESPOND_LIVEACTION["id"], + "liveaction_id": RESPOND_LIVEACTION["id"], "result": {"exit_code": 0, "result": None, "stderr": "", "stdout": ""}, }, "execution_3": { @@ -122,7 +122,7 @@ }, "status": "succeeded", "runner": {"name": "inquirer", "output_key": "result"}, - "liveaction": OUTPUT_SCHEMA_LIVEACTION["id"], + "liveaction_id": OUTPUT_SCHEMA_LIVEACTION["id"], "result": OUTPUT_SCHEMA_RESULT, }, } diff --git a/st2common/tests/unit/test_executions.py b/st2common/tests/unit/test_executions.py index 6a89e1d9fe..aba9903cac 100644 --- a/st2common/tests/unit/test_executions.py +++ b/st2common/tests/unit/test_executions.py @@ -38,7 +38,7 @@ def setUp(self): "id": str(bson.ObjectId()), "action": copy.deepcopy(fixture.ARTIFACTS["actions"]["local"]), "runner": copy.deepcopy(fixture.ARTIFACTS["runners"]["run-local"]), - "liveaction": copy.deepcopy( + "liveaction_id": copy.deepcopy( fixture.ARTIFACTS["liveactions"]["task1"]["id"] ), "status": fixture.ARTIFACTS["liveactions"]["task1"]["status"], @@ -53,7 +53,7 @@ def setUp(self): "id": str(bson.ObjectId()), "action": copy.deepcopy(fixture.ARTIFACTS["actions"]["local"]), "runner": copy.deepcopy(fixture.ARTIFACTS["runners"]["run-local"]), - "liveaction": copy.deepcopy(fixture.ARTIFACTS["liveactions"]["task2"]), + "liveaction_id": copy.deepcopy(fixture.ARTIFACTS["liveactions"]["task2"]), "status": fixture.ARTIFACTS["liveactions"]["task2"]["status"], "start_timestamp": fixture.ARTIFACTS["liveactions"]["task2"][ "start_timestamp" @@ -73,7 +73,7 @@ def setUp(self): "rule": copy.deepcopy(fixture.ARTIFACTS["rule"]), "action": copy.deepcopy(fixture.ARTIFACTS["actions"]["chain"]), "runner": copy.deepcopy(fixture.ARTIFACTS["runners"]["action-chain"]), - "liveaction": copy.deepcopy( + "liveaction_id": copy.deepcopy( fixture.ARTIFACTS["liveactions"]["workflow"]["id"] ), "children": [task["id"] for task in self.fake_history_subtasks], @@ -104,7 +104,7 @@ def test_model_complete(self): self.assertDictEqual(obj.rule, self.fake_history_workflow["rule"]) self.assertDictEqual(obj.action, self.fake_history_workflow["action"]) self.assertDictEqual(obj.runner, self.fake_history_workflow["runner"]) - self.assertEqual(obj.liveaction, self.fake_history_workflow["liveaction"]) + self.assertEqual(obj.liveaction_id, self.fake_history_workflow["liveaction_id"]) self.assertIsNone(getattr(obj, "parent", None)) self.assertListEqual(obj.children, self.fake_history_workflow["children"]) @@ -121,7 +121,7 @@ def test_model_complete(self): self.assertDictEqual(model.rule, self.fake_history_workflow["rule"]) self.assertDictEqual(model.action, self.fake_history_workflow["action"]) self.assertDictEqual(model.runner, self.fake_history_workflow["runner"]) - self.assertEqual(model.liveaction, self.fake_history_workflow["liveaction"]) + self.assertEqual(model.liveaction_id, self.fake_history_workflow["liveaction_id"]) self.assertIsNone(getattr(model, "parent", None)) self.assertListEqual(model.children, self.fake_history_workflow["children"]) @@ -138,7 +138,7 @@ def test_model_complete(self): self.assertDictEqual(obj.rule, self.fake_history_workflow["rule"]) self.assertDictEqual(obj.action, self.fake_history_workflow["action"]) self.assertDictEqual(obj.runner, self.fake_history_workflow["runner"]) - self.assertEqual(obj.liveaction, self.fake_history_workflow["liveaction"]) + self.assertEqual(obj.liveaction_id, self.fake_history_workflow["liveaction_id"]) self.assertIsNone(getattr(obj, "parent", None)) self.assertListEqual(obj.children, self.fake_history_workflow["children"]) @@ -158,7 +158,7 @@ def test_crud_complete(self): self.assertDictEqual(model.rule, self.fake_history_workflow["rule"]) self.assertDictEqual(model.action, self.fake_history_workflow["action"]) self.assertDictEqual(model.runner, self.fake_history_workflow["runner"]) - self.assertEqual(model.liveaction, self.fake_history_workflow["liveaction"]) + self.assertEqual(model.liveaction_id, self.fake_history_workflow["liveaction_id"]) self.assertIsNone(getattr(model, "parent", None)) self.assertListEqual(model.children, self.fake_history_workflow["children"]) @@ -184,7 +184,7 @@ def test_model_partial(self): self.assertIsNone(getattr(obj, "rule", None)) self.assertDictEqual(obj.action, self.fake_history_subtasks[0]["action"]) self.assertDictEqual(obj.runner, self.fake_history_subtasks[0]["runner"]) - self.assertEqual(obj.liveaction, self.fake_history_subtasks[0]["liveaction"]) + self.assertEqual(obj.liveaction_id, self.fake_history_subtasks[0]["liveaction_id"]) self.assertEqual(obj.parent, self.fake_history_subtasks[0]["parent"]) self.assertIsNone(getattr(obj, "children", None)) @@ -197,7 +197,7 @@ def test_model_partial(self): self.assertDictEqual(model.rule, {}) self.assertDictEqual(model.action, self.fake_history_subtasks[0]["action"]) self.assertDictEqual(model.runner, self.fake_history_subtasks[0]["runner"]) - self.assertEqual(model.liveaction, self.fake_history_subtasks[0]["liveaction"]) + self.assertEqual(model.liveaction_id, self.fake_history_subtasks[0]["liveaction_id"]) self.assertEqual(model.parent, self.fake_history_subtasks[0]["parent"]) self.assertListEqual(model.children, []) @@ -210,7 +210,7 @@ def test_model_partial(self): self.assertIsNone(getattr(obj, "rule", None)) self.assertDictEqual(obj.action, self.fake_history_subtasks[0]["action"]) self.assertDictEqual(obj.runner, self.fake_history_subtasks[0]["runner"]) - self.assertEqual(obj.liveaction, self.fake_history_subtasks[0]["liveaction"]) + self.assertEqual(obj.liveaction_id, self.fake_history_subtasks[0]["liveaction_id"]) self.assertEqual(obj.parent, self.fake_history_subtasks[0]["parent"]) self.assertIsNone(getattr(obj, "children", None)) @@ -226,7 +226,7 @@ def test_crud_partial(self): self.assertDictEqual(model.rule, {}) self.assertDictEqual(model.action, self.fake_history_subtasks[0]["action"]) self.assertDictEqual(model.runner, self.fake_history_subtasks[0]["runner"]) - self.assertEqual(model.liveaction, self.fake_history_subtasks[0]["liveaction"]) + self.assertEqual(model.liveaction_id, self.fake_history_subtasks[0]["liveaction_id"]) self.assertEqual(model.parent, self.fake_history_subtasks[0]["parent"]) self.assertListEqual(model.children, []) diff --git a/st2common/tests/unit/test_executions_util.py b/st2common/tests/unit/test_executions_util.py index 3ae45169a7..6e180db1e0 100644 --- a/st2common/tests/unit/test_executions_util.py +++ b/st2common/tests/unit/test_executions_util.py @@ -79,7 +79,7 @@ def test_execution_creation_manual_action_run(self): executions_util.create_execution_object(liveaction) post_creation_timestamp = date_utils.get_datetime_utc_now() execution = self._get_action_execution( - liveaction=str(liveaction.id), raise_exception=True + liveaction_id=str(liveaction.id), raise_exception=True ) self.assertDictEqual(execution.trigger, {}) self.assertDictEqual(execution.trigger_type, {}) @@ -90,7 +90,7 @@ def test_execution_creation_manual_action_run(self): runner = RunnerType.get_by_name(action.runner_type["name"]) self.assertDictEqual(execution.runner, vars(RunnerTypeAPI.from_model(runner))) liveaction = LiveAction.get_by_id(str(liveaction.id)) - self.assertEqual(execution.liveaction, str(liveaction.id)) + self.assertEqual(execution.liveaction_id, str(liveaction.id)) self.assertEqual(len(execution.log), 1) self.assertEqual(execution.log[0]["status"], liveaction.status) self.assertGreater(execution.log[0]["timestamp"], pre_creation_timestamp) @@ -120,7 +120,7 @@ def test_execution_creation_action_triggered_by_rule(self): ) executions_util.create_execution_object(liveaction) execution = self._get_action_execution( - liveaction=str(liveaction.id), raise_exception=True + liveaction_id=str(liveaction.id), raise_exception=True ) self.assertDictEqual(execution.trigger, vars(TriggerAPI.from_model(trigger))) self.assertDictEqual( @@ -136,13 +136,13 @@ def test_execution_creation_action_triggered_by_rule(self): runner = RunnerType.get_by_name(action.runner_type["name"]) self.assertDictEqual(execution.runner, vars(RunnerTypeAPI.from_model(runner))) liveaction = LiveAction.get_by_id(str(liveaction.id)) - self.assertEqual(execution.liveaction, str(liveaction.id)) + self.assertEqual(execution.liveaction_id, str(liveaction.id)) def test_execution_creation_with_web_url(self): liveaction = self.MODELS["liveactions"]["liveaction1.yaml"] executions_util.create_execution_object(liveaction) execution = self._get_action_execution( - liveaction=str(liveaction.id), raise_exception=True + liveaction_id=str(liveaction.id), raise_exception=True ) self.assertIsNotNone(execution.web_url) execution_id = str(execution.id) @@ -164,7 +164,7 @@ def test_execution_update(self): executions_util.update_execution(liveaction) post_update_timestamp = date_utils.get_datetime_utc_now() execution = self._get_action_execution( - liveaction=str(liveaction.id), raise_exception=True + liveaction_id=str(liveaction.id), raise_exception=True ) self.assertEqual(len(execution.log), 2) self.assertEqual(execution.log[1]["status"], liveaction.status) @@ -178,7 +178,7 @@ def test_skip_execution_update(self): liveaction.status = "running" executions_util.update_execution(liveaction) execution = self._get_action_execution( - liveaction=str(liveaction.id), raise_exception=True + liveaction_id=str(liveaction.id), raise_exception=True ) self.assertEqual(len(execution.log), 1) # Check status is not updated if it's already in completed state. diff --git a/st2common/tests/unit/test_purge_executions.py b/st2common/tests/unit/test_purge_executions.py index 80fa4d2dec..559494c705 100644 --- a/st2common/tests/unit/test_purge_executions.py +++ b/st2common/tests/unit/test_purge_executions.py @@ -194,7 +194,7 @@ def test_liveaction_gets_deleted(self): exec_model["end_timestamp"] = end_ts exec_model["status"] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model["id"] = bson.ObjectId() - exec_model["liveaction"] = str(liveaction.id) + exec_model["liveaction_id"] = str(liveaction.id) ActionExecution.add_or_update(exec_model) liveactions = LiveAction.get_all() diff --git a/st2reactor/tests/integration/test_garbage_collector.py b/st2reactor/tests/integration/test_garbage_collector.py index 2af839d669..1435125e1a 100644 --- a/st2reactor/tests/integration/test_garbage_collector.py +++ b/st2reactor/tests/integration/test_garbage_collector.py @@ -88,7 +88,7 @@ def test_garbage_collection(self): status=status, action={"ref": "core.local"}, runner={"name": "local-shell-cmd"}, - liveaction="ref", + liveaction_id="ref", ) ActionExecution.add_or_update(action_execution_db) @@ -124,7 +124,7 @@ def test_garbage_collection(self): status=status, action={"ref": "core.local"}, runner={"name": "local-shell-cmd"}, - liveaction="ref", + liveaction_id="ref", ) ActionExecution.add_or_update(action_execution_db) @@ -159,7 +159,7 @@ def test_garbage_collection(self): status=status, action={"ref": "core.local"}, runner={"name": "local-shell-cmd"}, - liveaction="ref", + liveaction_id="ref", ) ActionExecution.add_or_update(action_execution_db) diff --git a/st2stream/tests/unit/controllers/v1/test_stream_execution_output.py b/st2stream/tests/unit/controllers/v1/test_stream_execution_output.py index 8ddd983ea8..ace3b11685 100644 --- a/st2stream/tests/unit/controllers/v1/test_stream_execution_output.py +++ b/st2stream/tests/unit/controllers/v1/test_stream_execution_output.py @@ -59,7 +59,7 @@ def test_get_output_running_execution(self): status=status, action={"ref": "core.local"}, runner={"name": "local-shell-cmd"}, - liveaction="ref", + liveaction_id="ref", ) action_execution_db = ActionExecution.add_or_update(action_execution_db) @@ -141,7 +141,7 @@ def test_get_output_finished_execution(self): status=status, action={"ref": "core.local"}, runner={"name": "local-shell-cmd"}, - liveaction="ref", + liveaction_id="ref", ) action_execution_db = ActionExecution.add_or_update(action_execution_db) diff --git a/st2tests/st2tests/api.py b/st2tests/st2tests/api.py index c4625ec5de..1f04d7929b 100644 --- a/st2tests/st2tests/api.py +++ b/st2tests/st2tests/api.py @@ -395,7 +395,7 @@ def _get_actionexecution_id(resp): @staticmethod def _get_liveaction_id(resp): - return resp.json["liveaction"] + return resp.json["liveaction_id"] def _do_get_one(self, actionexecution_id, *args, **kwargs): return self.app.get("/v1/executions/%s" % actionexecution_id, *args, **kwargs) diff --git a/st2tests/st2tests/fixtures/descendants/executions/child1_level1.yaml b/st2tests/st2tests/fixtures/descendants/executions/child1_level1.yaml index 0b7faddb17..49e14c3d07 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/child1_level1.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/child1_level1.yaml @@ -7,7 +7,7 @@ children: - 54e6583d0640fd16887d685b end_timestamp: '2014-09-01T00:00:57.000001Z' id: 54e657f20640fd16887d6857 -liveaction: pointlessaction +liveaction_id: pointlessaction parent: 54e657d60640fd16887d6855 runner: name: pointlessrunner diff --git a/st2tests/st2tests/fixtures/descendants/executions/child1_level2.yaml b/st2tests/st2tests/fixtures/descendants/executions/child1_level2.yaml index 1ebf3d5f39..7a7cb27470 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/child1_level2.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/child1_level2.yaml @@ -5,7 +5,7 @@ action: children: [] end_timestamp: '2014-09-01T00:00:56.000002Z' id: 54e657fa0640fd16887d6858 -liveaction: pointlessaction +liveaction_id: pointlessaction parent: 54e657f20640fd16887d6857 runner: name: pointlessrunner diff --git a/st2tests/st2tests/fixtures/descendants/executions/child1_level3.yaml b/st2tests/st2tests/fixtures/descendants/executions/child1_level3.yaml index 9443d4e0cb..c5e6ee0dad 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/child1_level3.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/child1_level3.yaml @@ -5,7 +5,7 @@ action: children: [] end_timestamp: '2014-09-01T00:00:55.100000Z' id: 54e6581b0640fd16887d6859 -liveaction: pointlessaction +liveaction_id: pointlessaction parent: 54e6583d0640fd16887d685b runner: name: pointlessrunner diff --git a/st2tests/st2tests/fixtures/descendants/executions/child2_level1.yaml b/st2tests/st2tests/fixtures/descendants/executions/child2_level1.yaml index 65a6dfc803..d7479c4b88 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/child2_level1.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/child2_level1.yaml @@ -6,7 +6,7 @@ children: - 54e658570640fd16887d685d end_timestamp: '2014-09-01T00:00:55.000000Z' id: 54e658290640fd16887d685a -liveaction: pointlessaction +liveaction_id: pointlessaction parent: 54e657d60640fd16887d6855 runner: name: pointlessrunner diff --git a/st2tests/st2tests/fixtures/descendants/executions/child2_level2.yaml b/st2tests/st2tests/fixtures/descendants/executions/child2_level2.yaml index fe96706346..ff7fcf3a2e 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/child2_level2.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/child2_level2.yaml @@ -6,7 +6,7 @@ children: - 54e6581b0640fd16887d6859 end_timestamp: '2014-09-01T00:00:55.000000Z' id: 54e6583d0640fd16887d685b -liveaction: pointlessaction +liveaction_id: pointlessaction parent: 54e657f20640fd16887d6857 runner: name: pointlessrunner diff --git a/st2tests/st2tests/fixtures/descendants/executions/child2_level3.yaml b/st2tests/st2tests/fixtures/descendants/executions/child2_level3.yaml index 7e0cebd8ab..448d4374df 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/child2_level3.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/child2_level3.yaml @@ -5,7 +5,7 @@ action: children: [] end_timestamp: '2014-09-01T00:00:59.000010Z' id: 54e6584a0640fd16887d685c -liveaction: pointlessaction +liveaction_id: pointlessaction parent: 54e658570640fd16887d685d runner: name: pointlessrunner diff --git a/st2tests/st2tests/fixtures/descendants/executions/child3_level2.yaml b/st2tests/st2tests/fixtures/descendants/executions/child3_level2.yaml index e80356860b..9076d6e41b 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/child3_level2.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/child3_level2.yaml @@ -7,7 +7,7 @@ children: - 54e6585f0640fd16887d685e end_timestamp: '2014-09-01T00:00:55.000000Z' id: 54e658570640fd16887d685d -liveaction: pointlessaction +liveaction_id: pointlessaction parent: 54e658290640fd16887d685a runner: name: pointlessrunner diff --git a/st2tests/st2tests/fixtures/descendants/executions/child3_level3.yaml b/st2tests/st2tests/fixtures/descendants/executions/child3_level3.yaml index 754f29831a..6f1bee7c45 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/child3_level3.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/child3_level3.yaml @@ -5,7 +5,7 @@ action: children: [] end_timestamp: '2014-09-01T00:00:55.000000Z' id: 54e6585f0640fd16887d685e -liveaction: pointlessaction +liveaction_id: pointlessaction parent: 54e658570640fd16887d685d runner: name: pointlessrunner diff --git a/st2tests/st2tests/fixtures/descendants/executions/root_execution.yaml b/st2tests/st2tests/fixtures/descendants/executions/root_execution.yaml index 37a5b3f221..903aa47f6d 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/root_execution.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/root_execution.yaml @@ -7,7 +7,7 @@ children: - 54e658290640fd16887d685a end_timestamp: '2014-09-01T00:00:59.000000Z' id: 54e657d60640fd16887d6855 -liveaction: pointlessaction +liveaction_id: pointlessaction runner: name: pointlessrunner runner_module: no.module diff --git a/st2tests/st2tests/fixtures/generic/executions/execution1.yaml b/st2tests/st2tests/fixtures/generic/executions/execution1.yaml index d7a7329dad..4aef0aac05 100644 --- a/st2tests/st2tests/fixtures/generic/executions/execution1.yaml +++ b/st2tests/st2tests/fixtures/generic/executions/execution1.yaml @@ -13,7 +13,7 @@ action: runner_type: run-local end_timestamp: '2014-09-01T00:00:05.000000Z' id: 54c6bb640640fd5211edef0d -liveaction: 54c6b6d60640fd4f5354e74a +liveaction_id: 54c6b6d60640fd4f5354e74a parameters: {} result: {} runner: diff --git a/st2tests/st2tests/fixtures/rule_enforcements/executions/execution1.yaml b/st2tests/st2tests/fixtures/rule_enforcements/executions/execution1.yaml index 90c9e9e09b..291eab7bcb 100644 --- a/st2tests/st2tests/fixtures/rule_enforcements/executions/execution1.yaml +++ b/st2tests/st2tests/fixtures/rule_enforcements/executions/execution1.yaml @@ -13,7 +13,7 @@ action: runner_type: run-local end_timestamp: '2014-09-01T00:00:05.000000Z' id: 565e15ce32ed350857dfa626 -liveaction: 54c6b6d60640fd4f5354e74a +liveaction_id: 54c6b6d60640fd4f5354e74a parameters: cmd: echo bar result: {} diff --git a/st2tests/st2tests/fixtures/traces/executions/execution_with_parent.yaml b/st2tests/st2tests/fixtures/traces/executions/execution_with_parent.yaml index a3e680145a..279e799b8b 100644 --- a/st2tests/st2tests/fixtures/traces/executions/execution_with_parent.yaml +++ b/st2tests/st2tests/fixtures/traces/executions/execution_with_parent.yaml @@ -17,7 +17,7 @@ action: runner_type: action-chain end_timestamp: '2014-09-01T00:00:05.000000Z' id: 54c6bb640640fd5211edef3d -liveaction: 54c6b6d60640fd4f5354e75a +liveaction_id: 54c6b6d60640fd4f5354e75a parameters: {} result: {} runner: diff --git a/st2tests/st2tests/fixtures/traces/executions/rule_fired_execution.yaml b/st2tests/st2tests/fixtures/traces/executions/rule_fired_execution.yaml index 18bb047a82..d09ee9faaa 100644 --- a/st2tests/st2tests/fixtures/traces/executions/rule_fired_execution.yaml +++ b/st2tests/st2tests/fixtures/traces/executions/rule_fired_execution.yaml @@ -17,7 +17,7 @@ action: runner_type: action-chain end_timestamp: '2014-09-01T00:00:05.000000Z' id: 54c6bb640640fd5211edef0d -liveaction: 54c6b6d60640fd4f5354e74a +liveaction_id: 54c6b6d60640fd4f5354e74a parameters: {} result: {} runner: diff --git a/st2tests/st2tests/fixtures/traces/executions/traceable_execution.yaml b/st2tests/st2tests/fixtures/traces/executions/traceable_execution.yaml index cfded0e2da..70a0667e7b 100644 --- a/st2tests/st2tests/fixtures/traces/executions/traceable_execution.yaml +++ b/st2tests/st2tests/fixtures/traces/executions/traceable_execution.yaml @@ -17,7 +17,7 @@ action: runner_type: action-chain end_timestamp: '2014-09-01T00:00:05.000000Z' id: 54c6bb640640fd5211edef0d -liveaction: 54c6b6d60640fd4f5354e74a +liveaction_id: 54c6b6d60640fd4f5354e74a parameters: {} result: {} runner: From 529604281df66cec1f3883595ba003b45144ef97 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Fri, 14 Jul 2023 18:21:45 +0000 Subject: [PATCH 35/70] remove liveaction from action execution api --- .../tests/unit/test_error_handling.py | 28 +++++++-------- st2actions/st2actions/workflows/workflows.py | 2 +- st2actions/tests/unit/test_executions.py | 2 +- st2actions/tests/unit/test_notifier.py | 2 +- .../st2api/controllers/v1/actionexecutions.py | 36 +++++-------------- .../st2api/controllers/v1/execution_views.py | 4 +-- .../controllers/v1/test_alias_execution.py | 13 ------- .../unit/controllers/v1/test_executions.py | 4 +-- st2common/st2common/models/api/execution.py | 3 +- st2common/st2common/models/api/inquiry.py | 7 ++-- st2common/st2common/services/inquiry.py | 4 +++ st2common/tests/unit/test_db_execution.py | 2 +- st2tests/st2tests/api.py | 1 - 13 files changed, 38 insertions(+), 70 deletions(-) diff --git a/contrib/runners/orquesta_runner/tests/unit/test_error_handling.py b/contrib/runners/orquesta_runner/tests/unit/test_error_handling.py index ace3278882..e1482fb3af 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_error_handling.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_error_handling.py @@ -415,7 +415,7 @@ def test_fail_next_task_action(self): tk_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_db.id) )[0] - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction_id) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion for task1 which has an error in publish. @@ -471,7 +471,7 @@ def test_fail_next_task_input_expr_eval(self): tk_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_db.id) )[0] - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction_id) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion for task1 which has an error in publish. @@ -523,7 +523,7 @@ def test_fail_next_task_input_value_type(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) self.assertEqual(wf_ex_db.status, wf_statuses.RUNNING) @@ -607,7 +607,7 @@ def test_fail_task_execution(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED) wf_svc.handle_action_execution_completion(tk1_ac_ex_db) @@ -658,7 +658,7 @@ def test_fail_task_transition(self): tk_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_db.id) )[0] - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction_id) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion for task1 which has an error in publish. @@ -714,7 +714,7 @@ def test_fail_task_publish(self): tk_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_db.id) )[0] - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction_id) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion for task1 which has an error in publish. @@ -767,7 +767,7 @@ def test_fail_output_rendering(self): tk_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_db.id) )[0] - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction_id) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion for task1 which has an error in publish. @@ -823,7 +823,7 @@ def test_output_on_error(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) wf_svc.handle_action_execution_completion(tk1_ac_ex_db) wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id) @@ -835,7 +835,7 @@ def test_output_on_error(self): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction_id) self.assertEqual(tk2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED) wf_svc.handle_action_execution_completion(tk2_ac_ex_db) @@ -866,7 +866,7 @@ def test_fail_manually(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED) wf_svc.handle_action_execution_completion(tk1_ac_ex_db) wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id) @@ -878,7 +878,7 @@ def test_fail_manually(self): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction_id) self.assertEqual(tk2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) wf_svc.handle_action_execution_completion(tk2_ac_ex_db) wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id) @@ -924,7 +924,7 @@ def test_fail_manually_with_recovery_failure(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED) wf_svc.handle_action_execution_completion(tk1_ac_ex_db) wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id) @@ -937,7 +937,7 @@ def test_fail_manually_with_recovery_failure(self): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction_id) self.assertEqual(tk2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED) wf_svc.handle_action_execution_completion(tk2_ac_ex_db) wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id) @@ -1013,7 +1013,7 @@ def test_include_result_to_error_log(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) self.assertEqual(tk1_lv_ac_db.context.get("user"), username) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED) diff --git a/st2actions/st2actions/workflows/workflows.py b/st2actions/st2actions/workflows/workflows.py index 8c8968de8b..6672069e6f 100644 --- a/st2actions/st2actions/workflows/workflows.py +++ b/st2actions/st2actions/workflows/workflows.py @@ -252,7 +252,7 @@ def handle_action_execution(self, ac_ex_db): return # Apply post run policies. - lv_ac_db = lv_db_access.LiveAction.get_by_id(ac_ex_db.liveaction) + lv_ac_db = lv_db_access.LiveAction.get_by_id(ac_ex_db.liveaction_id) pc_svc.apply_post_run_policies(lv_ac_db) # Process completion of the action execution. diff --git a/st2actions/tests/unit/test_executions.py b/st2actions/tests/unit/test_executions.py index 3bd92c5034..8814b78616 100644 --- a/st2actions/tests/unit/test_executions.py +++ b/st2actions/tests/unit/test_executions.py @@ -121,7 +121,7 @@ def test_basic_execution(self): self.assertEqual(execution.result, liveaction.result) self.assertEqual(execution.status, liveaction.status) self.assertEqual(execution.context, liveaction.context) - self.assertEqual(execution.liveaction, str(liveaction.id)) + self.assertEqual(execution.liveaction_id, str(liveaction.id)) def test_basic_execution_history_create_failed(self): MOCK_FAIL_EXECUTION_CREATE = True # noqa diff --git a/st2actions/tests/unit/test_notifier.py b/st2actions/tests/unit/test_notifier.py index 9151ab0d23..4bce9e294d 100644 --- a/st2actions/tests/unit/test_notifier.py +++ b/st2actions/tests/unit/test_notifier.py @@ -134,7 +134,7 @@ def test_notify_triggers(self): LiveAction.add_or_update(liveaction_db) execution = MOCK_EXECUTION - execution.liveaction = str(liveaction_db.id) + execution.liveaction_id = str(liveaction_db.id) execution.status = liveaction_db.status dispatcher = NotifierTestCase.MockDispatcher(self) diff --git a/st2api/st2api/controllers/v1/actionexecutions.py b/st2api/st2api/controllers/v1/actionexecutions.py index 38cca8d299..1f51a75f65 100644 --- a/st2api/st2api/controllers/v1/actionexecutions.py +++ b/st2api/st2api/controllers/v1/actionexecutions.py @@ -27,7 +27,6 @@ from oslo_config import cfg from six.moves import http_client from mongoengine.queryset.visitor import Q -import zstandard from st2api.controllers.base import BaseRestControllerMixin from st2api.controllers.resource import ResourceController @@ -416,40 +415,21 @@ def get( :rtype: ``str`` """ - # NOTE: Here we intentionally use as_pymongo() to avoid mongoengine layer even for old style - # data + # NOTE: we need to use to_python() to uncompress the data try: result = ( - self.access.impl.model.objects.filter(id=id) - .only("result") - .as_pymongo()[0] + self.access.impl.model.objects.filter(id=id).only("result")[0].result ) except IndexError: raise NotFoundException("Execution with id %s not found" % (id)) - if isinstance(result["result"], dict): - # For backward compatibility we also support old non JSON field storage format - if pretty_format: - response_body = orjson.dumps( - result["result"], option=orjson.OPT_INDENT_2 - ) - else: - response_body = orjson.dumps(result["result"]) + # For backward compatibility we also support old non JSON field storage format + if pretty_format: + response_body = orjson.dumps( + result, option=orjson.OPT_INDENT_2 + ) else: - # For new JSON storage format we just use raw value since it's already JSON serialized - # string - response_body = result["result"] - try: - response_body = zstandard.ZstdDecompressor().decompress(response_body) - # skip if already a byte string and not compressed - except zstandard.ZstdError: - pass - if pretty_format: - # Pretty format is not a default behavior since it adds quite some overhead (e.g. - # 10-30ms for non pretty format for 4 MB json vs ~120 ms for pretty formatted) - response_body = orjson.dumps( - orjson.loads(response_body), option=orjson.OPT_INDENT_2 - ) + response_body = orjson.dumps(result) response = Response() response.headers["Content-Type"] = "text/json" diff --git a/st2api/st2api/controllers/v1/execution_views.py b/st2api/st2api/controllers/v1/execution_views.py index f4240b94ab..9c44764bd1 100644 --- a/st2api/st2api/controllers/v1/execution_views.py +++ b/st2api/st2api/controllers/v1/execution_views.py @@ -31,7 +31,7 @@ SUPPORTED_FILTERS = { "action": "action.ref", "status": "status", - "liveaction": "liveaction.id", + "liveaction_id": "liveaction_id", "parent": "parent", "rule": "rule.name", "runner": "runner.name", @@ -54,7 +54,7 @@ # List of filters that are too broad to distinct by them and are very likely to represent 1 to 1 # relation between filter and particular history record. -IGNORE_FILTERS = ["parent", "timestamp", "liveaction", "trigger_instance"] +IGNORE_FILTERS = ["parent", "timestamp", "liveaction_id", "trigger_instance"] class FiltersController(object): diff --git a/st2api/tests/unit/controllers/v1/test_alias_execution.py b/st2api/tests/unit/controllers/v1/test_alias_execution.py index 44261fde3f..664d60bb92 100644 --- a/st2api/tests/unit/controllers/v1/test_alias_execution.py +++ b/st2api/tests/unit/controllers/v1/test_alias_execution.py @@ -322,15 +322,8 @@ def test_match_and_execute_list_action_param_str_cast_to_list(self): self.assertEqual(resp.status_int, 201) result = resp.json["results"][0] - live_action = result["execution"]["liveaction"] action_alias = result["actionalias"] - self.assertEqual(resp.status_int, 201) - self.assertTrue(isinstance(live_action["parameters"]["array_param"], list)) - self.assertEqual(live_action["parameters"]["array_param"][0], "one") - self.assertEqual(live_action["parameters"]["array_param"][1], "two") - self.assertEqual(live_action["parameters"]["array_param"][2], "three") - self.assertEqual(live_action["parameters"]["array_param"][3], "four") self.assertTrue( isinstance(action_alias["immutable_parameters"]["array_param"], str) ) @@ -349,15 +342,9 @@ def test_match_and_execute_list_action_param_already_a_list(self): self.assertEqual(resp.status_int, 201) result = resp.json["results"][0] - live_action = result["execution"]["liveaction"] action_alias = result["actionalias"] self.assertEqual(resp.status_int, 201) - self.assertTrue(isinstance(live_action["parameters"]["array_param"], list)) - self.assertEqual(live_action["parameters"]["array_param"][0]["key1"], "one") - self.assertEqual(live_action["parameters"]["array_param"][0]["key2"], "two") - self.assertEqual(live_action["parameters"]["array_param"][1]["key3"], "three") - self.assertEqual(live_action["parameters"]["array_param"][1]["key4"], "four") self.assertTrue( isinstance(action_alias["immutable_parameters"]["array_param"], list) ) diff --git a/st2api/tests/unit/controllers/v1/test_executions.py b/st2api/tests/unit/controllers/v1/test_executions.py index 9ef28155fa..f34e78fdbf 100644 --- a/st2api/tests/unit/controllers/v1/test_executions.py +++ b/st2api/tests/unit/controllers/v1/test_executions.py @@ -2000,7 +2000,7 @@ def test_get_output_running_execution(self): status=status, action={"ref": "core.local"}, runner={"name": "local-shell-cmd"}, - liveaction="ref", + liveaction_id="ref", ) action_execution_db = ActionExecution.add_or_update(action_execution_db) @@ -2081,7 +2081,7 @@ def test_get_output_finished_execution(self): status=status, action={"ref": "core.local"}, runner={"name": "local-shell-cmd"}, - liveaction="ref", + liveaction_id="ref", ) action_execution_db = ActionExecution.add_or_update(action_execution_db) diff --git a/st2common/st2common/models/api/execution.py b/st2common/st2common/models/api/execution.py index 9be6cd2512..fb1cdac8b3 100644 --- a/st2common/st2common/models/api/execution.py +++ b/st2common/st2common/models/api/execution.py @@ -39,7 +39,6 @@ REQUIRED_ATTR_SCHEMAS = { "action": copy.deepcopy(ActionAPI.schema), "runner": copy.deepcopy(RunnerTypeAPI.schema), - "liveaction": copy.deepcopy(LiveActionAPI.schema), } for k, v in six.iteritems(REQUIRED_ATTR_SCHEMAS): @@ -61,7 +60,7 @@ class ActionExecutionAPI(BaseAPI): "rule": RuleAPI.schema, "action": REQUIRED_ATTR_SCHEMAS["action"], "runner": REQUIRED_ATTR_SCHEMAS["runner"], - "liveaction": REQUIRED_ATTR_SCHEMAS["liveaction"], + "liveaction_id": {"type": "string", "required": True}, "status": { "description": "The current status of the action execution.", "type": "string", diff --git a/st2common/st2common/models/api/inquiry.py b/st2common/st2common/models/api/inquiry.py index d04c8efbf1..f5a3be92b8 100644 --- a/st2common/st2common/models/api/inquiry.py +++ b/st2common/st2common/models/api/inquiry.py @@ -21,7 +21,7 @@ from st2common.constants.action import LIVEACTION_STATUSES from st2common.models.api.base import BaseAPI -from st2common.models.api.action import RunnerTypeAPI, ActionAPI, LiveActionAPI +from st2common.models.api.action import RunnerTypeAPI, ActionAPI from st2common.models.db.execution import ActionExecutionDB from st2common import log as logging @@ -31,7 +31,6 @@ REQUIRED_ATTR_SCHEMAS = { "action": copy.deepcopy(ActionAPI.schema), "runner": copy.deepcopy(RunnerTypeAPI.schema), - "liveaction": copy.deepcopy(LiveActionAPI.schema), } for k, v in six.iteritems(REQUIRED_ATTR_SCHEMAS): @@ -76,7 +75,7 @@ class InquiryAPI(BaseAPI): }, "required": True, }, - "liveaction": REQUIRED_ATTR_SCHEMAS["liveaction"], + "liveaction_id": {"type": "string", "required": True}, "runner": REQUIRED_ATTR_SCHEMAS["runner"], "status": { "description": "The current status of the action execution.", @@ -112,7 +111,7 @@ def from_model(cls, model, mask_secrets=False): "id": doc["id"], "runner": doc.get("runner", None), "status": doc.get("status", None), - "liveaction": doc.get("liveaction", None), + "liveaction_id": doc.get("liveaction_id", None), "parent": doc.get("parent", None), "result": doc.get("result", None), } diff --git a/st2common/st2common/services/inquiry.py b/st2common/st2common/services/inquiry.py index 35755d2e22..e181faa5a0 100644 --- a/st2common/st2common/services/inquiry.py +++ b/st2common/st2common/services/inquiry.py @@ -121,6 +121,10 @@ def validate_response(inquiry, response): def respond(inquiry, response, requester=None): + """ + :param inquiry: InquiryAPI + :param response: dict + """ # Set requester to system user is not provided. if not requester: requester = cfg.CONF.system_user.user diff --git a/st2common/tests/unit/test_db_execution.py b/st2common/tests/unit/test_db_execution.py index d78776736f..5b0f73701e 100644 --- a/st2common/tests/unit/test_db_execution.py +++ b/st2common/tests/unit/test_db_execution.py @@ -140,7 +140,7 @@ def setUp(self): created.action = execution["action"] created.status = execution["status"] created.runner = execution["runner"] - created.liveaction = execution["liveaction"] + created.liveaction_id = execution["liveaction_id"] created.result = execution["result"] saved = ActionExecutionModelTest._save_execution(created) diff --git a/st2tests/st2tests/api.py b/st2tests/st2tests/api.py index 1f04d7929b..07e6b60266 100644 --- a/st2tests/st2tests/api.py +++ b/st2tests/st2tests/api.py @@ -87,7 +87,6 @@ def do_request(self, req, **kwargs): if req.environ["REQUEST_METHOD"] != "OPTIONS": # Making sure endpoint handles OPTIONS method properly self.options(req.environ["PATH_INFO"]) - res = super(TestApp, self).do_request(req, **kwargs) if res.headers.get("Warning", None): From 0192c1da272e5b0148810b5a5833576dcdd5c445 Mon Sep 17 00:00:00 2001 From: AJ Date: Fri, 14 Jul 2023 00:26:05 +0000 Subject: [PATCH 36/70] Update st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb Co-authored-by: Jacob Floyd --- .../bin/migrations/v3.9/st2-migrate-liveaction-executiondb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb index 25e1e76b22..91241a9454 100755 --- a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb +++ b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb @@ -14,7 +14,7 @@ # limitations under the License. """ -Migration which which migrates data for existing objects in the database which utilize +Migration which migrates data for existing objects in the database which utilize liveaction to a string Migration step is idempotent and can be retried on failures / partial runs. From d855f39de9e9dc2a44e5fdae59a73f953d93a95d Mon Sep 17 00:00:00 2001 From: AJ Date: Fri, 14 Jul 2023 00:27:12 +0000 Subject: [PATCH 37/70] Update st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb Co-authored-by: Jacob Floyd --- .../bin/migrations/v3.9/st2-migrate-liveaction-executiondb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb index 91241a9454..a6bcb86d11 100755 --- a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb +++ b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb @@ -23,7 +23,7 @@ Right now the script utilizes no concurrency and performs migration one object b for simplicity reasons and also to avoid massive CPU usage spikes when running this script with large concurrency on large objects. -Keep in mind that only "completed" objects are processes - this means Executions in "final" states +Keep in mind that only "completed" objects are processed - this means Executions in "final" states (succeeded, failed, timeout, etc.). We determine if an object should be migrating using mongodb $type query (for execution objects we From f7cd0d2c4256d7a89ac63953cbe70cd0b2be8d15 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Fri, 14 Jul 2023 18:38:05 +0000 Subject: [PATCH 38/70] black fixes --- .../st2api/controllers/v1/actionexecutions.py | 8 +++--- st2common/tests/unit/test_executions.py | 28 ++++++++++++++----- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/st2api/st2api/controllers/v1/actionexecutions.py b/st2api/st2api/controllers/v1/actionexecutions.py index 1f51a75f65..cb22335bca 100644 --- a/st2api/st2api/controllers/v1/actionexecutions.py +++ b/st2api/st2api/controllers/v1/actionexecutions.py @@ -425,9 +425,7 @@ def get( # For backward compatibility we also support old non JSON field storage format if pretty_format: - response_body = orjson.dumps( - result, option=orjson.OPT_INDENT_2 - ) + response_body = orjson.dumps(result, option=orjson.OPT_INDENT_2) else: response_body = orjson.dumps(result) @@ -856,7 +854,9 @@ def update_status(liveaction_api, liveaction_db): liveaction_db = action_service.update_status( liveaction_db, status, result, set_result_size=True ) - actionexecution_db = ActionExecution.get(liveaction_id=str(liveaction_db.id)) + actionexecution_db = ActionExecution.get( + liveaction_id=str(liveaction_db.id) + ) return (liveaction_db, actionexecution_db) try: diff --git a/st2common/tests/unit/test_executions.py b/st2common/tests/unit/test_executions.py index aba9903cac..5b736cbbb0 100644 --- a/st2common/tests/unit/test_executions.py +++ b/st2common/tests/unit/test_executions.py @@ -53,7 +53,9 @@ def setUp(self): "id": str(bson.ObjectId()), "action": copy.deepcopy(fixture.ARTIFACTS["actions"]["local"]), "runner": copy.deepcopy(fixture.ARTIFACTS["runners"]["run-local"]), - "liveaction_id": copy.deepcopy(fixture.ARTIFACTS["liveactions"]["task2"]), + "liveaction_id": copy.deepcopy( + fixture.ARTIFACTS["liveactions"]["task2"] + ), "status": fixture.ARTIFACTS["liveactions"]["task2"]["status"], "start_timestamp": fixture.ARTIFACTS["liveactions"]["task2"][ "start_timestamp" @@ -121,7 +123,9 @@ def test_model_complete(self): self.assertDictEqual(model.rule, self.fake_history_workflow["rule"]) self.assertDictEqual(model.action, self.fake_history_workflow["action"]) self.assertDictEqual(model.runner, self.fake_history_workflow["runner"]) - self.assertEqual(model.liveaction_id, self.fake_history_workflow["liveaction_id"]) + self.assertEqual( + model.liveaction_id, self.fake_history_workflow["liveaction_id"] + ) self.assertIsNone(getattr(model, "parent", None)) self.assertListEqual(model.children, self.fake_history_workflow["children"]) @@ -158,7 +162,9 @@ def test_crud_complete(self): self.assertDictEqual(model.rule, self.fake_history_workflow["rule"]) self.assertDictEqual(model.action, self.fake_history_workflow["action"]) self.assertDictEqual(model.runner, self.fake_history_workflow["runner"]) - self.assertEqual(model.liveaction_id, self.fake_history_workflow["liveaction_id"]) + self.assertEqual( + model.liveaction_id, self.fake_history_workflow["liveaction_id"] + ) self.assertIsNone(getattr(model, "parent", None)) self.assertListEqual(model.children, self.fake_history_workflow["children"]) @@ -184,7 +190,9 @@ def test_model_partial(self): self.assertIsNone(getattr(obj, "rule", None)) self.assertDictEqual(obj.action, self.fake_history_subtasks[0]["action"]) self.assertDictEqual(obj.runner, self.fake_history_subtasks[0]["runner"]) - self.assertEqual(obj.liveaction_id, self.fake_history_subtasks[0]["liveaction_id"]) + self.assertEqual( + obj.liveaction_id, self.fake_history_subtasks[0]["liveaction_id"] + ) self.assertEqual(obj.parent, self.fake_history_subtasks[0]["parent"]) self.assertIsNone(getattr(obj, "children", None)) @@ -197,7 +205,9 @@ def test_model_partial(self): self.assertDictEqual(model.rule, {}) self.assertDictEqual(model.action, self.fake_history_subtasks[0]["action"]) self.assertDictEqual(model.runner, self.fake_history_subtasks[0]["runner"]) - self.assertEqual(model.liveaction_id, self.fake_history_subtasks[0]["liveaction_id"]) + self.assertEqual( + model.liveaction_id, self.fake_history_subtasks[0]["liveaction_id"] + ) self.assertEqual(model.parent, self.fake_history_subtasks[0]["parent"]) self.assertListEqual(model.children, []) @@ -210,7 +220,9 @@ def test_model_partial(self): self.assertIsNone(getattr(obj, "rule", None)) self.assertDictEqual(obj.action, self.fake_history_subtasks[0]["action"]) self.assertDictEqual(obj.runner, self.fake_history_subtasks[0]["runner"]) - self.assertEqual(obj.liveaction_id, self.fake_history_subtasks[0]["liveaction_id"]) + self.assertEqual( + obj.liveaction_id, self.fake_history_subtasks[0]["liveaction_id"] + ) self.assertEqual(obj.parent, self.fake_history_subtasks[0]["parent"]) self.assertIsNone(getattr(obj, "children", None)) @@ -226,7 +238,9 @@ def test_crud_partial(self): self.assertDictEqual(model.rule, {}) self.assertDictEqual(model.action, self.fake_history_subtasks[0]["action"]) self.assertDictEqual(model.runner, self.fake_history_subtasks[0]["runner"]) - self.assertEqual(model.liveaction_id, self.fake_history_subtasks[0]["liveaction_id"]) + self.assertEqual( + model.liveaction_id, self.fake_history_subtasks[0]["liveaction_id"] + ) self.assertEqual(model.parent, self.fake_history_subtasks[0]["parent"]) self.assertListEqual(model.children, []) From 115e83de2c9bb0b3cf668e50a32c5a8b561063b8 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Fri, 14 Jul 2023 18:48:52 +0000 Subject: [PATCH 39/70] untested migration script --- .../v3.9/st2-migrate-liveaction-executiondb | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb index a6bcb86d11..1ab8d6db6d 100755 --- a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb +++ b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb @@ -38,10 +38,14 @@ import traceback from oslo_config import cfg from st2common import config +from st2common import transport +from st2common.models import stormbase +from st2common.models.db import MongoDBAccess from st2common.service_setup import db_setup from st2common.service_setup import db_teardown from st2common.util import isotime from st2common.models.db.execution import ActionExecutionDB +from st2common.persistence.base import Access from st2common.persistence.execution import ActionExecution from st2common.exceptions.db import StackStormDBObjectNotFoundError from st2common.constants.action import LIVEACTION_COMPLETED_STATES @@ -51,6 +55,29 @@ from st2common.constants.action import LIVEACTION_COMPLETED_STATES # single value +class ActionExecutionDBOLD(ActionExecutionDB): + liveaction = stormbase.EscapedDictField(required=True) + + +class ActionExecutionOLD(Access): + impl = MongoDBAccess(ActionExecutionDBOLD) + publisher = None + + @classmethod + def _get_impl(cls): + return cls.impl + + @classmethod + def _get_publisher(cls): + if not cls.publisher: + cls.publisher = transport.execution.ActionExecutionPublisher() + return cls.publisher + + @classmethod + def delete_by_query(cls, *args, **query): + return cls._get_impl().delete_by_query(*args, **query) + + def migrate_executions(start_dt: datetime.datetime, end_dt: datetime.datetime) -> None: """ Perform migrations for execution related objects (ActionExecutionDB, LiveActionDB). @@ -66,7 +93,7 @@ def migrate_executions(start_dt: datetime.datetime, end_dt: datetime.datetime) - # 1. Migrate ActionExecutionDB objects result = ( - ActionExecutionDB.objects( + ActionExecutionDBOLD.objects( __raw__={ "status": { "$in": LIVEACTION_COMPLETED_STATES, @@ -108,7 +135,7 @@ def migrate_executions(start_dt: datetime.datetime, end_dt: datetime.datetime) - # field has been updated and should be saved. If we don't do, nothing will be re-saved on # .save() call due to mongoengine only trying to save what has changed to make it more # efficient instead of always re-saving the whole object. - execution_db._mark_as_changed("liveaction") + execution_db._mark_as_changed("liveaction_id") # NOTE: If you want to view changed fields, you can access execution_db._changed_fields # will throw an exception if already a string execution_db.liveaction_id = execution_db.liveaction.get("id", None) From 48d974922362d42e3d8e1fd25d5890b22ef2a093 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Fri, 14 Jul 2023 18:53:08 +0000 Subject: [PATCH 40/70] flake fixes --- st2api/st2api/controllers/v1/aliasexecution.py | 3 +-- st2common/st2common/fields.py | 1 - st2common/st2common/models/api/execution.py | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/st2api/st2api/controllers/v1/aliasexecution.py b/st2api/st2api/controllers/v1/aliasexecution.py index b46e6fd32f..f25704e67b 100644 --- a/st2api/st2api/controllers/v1/aliasexecution.py +++ b/st2api/st2api/controllers/v1/aliasexecution.py @@ -25,7 +25,7 @@ from st2common.models.api.action import ActionAliasAPI from st2common.models.api.action import AliasMatchAndExecuteInputAPI from st2common.models.api.auth import get_system_username -from st2common.models.api.execution import ActionExecutionAPI, LiveActionAPI +from st2common.models.api.execution import ActionExecutionAPI from st2common.models.db.auth import UserDB from st2common.models.db.liveaction import LiveActionDB from st2common.models.db.notification import NotificationSchema, NotificationSubSchema @@ -35,7 +35,6 @@ ) from st2common.models.utils.action_alias_utils import inject_immutable_parameters from st2common.persistence.actionalias import ActionAlias -from st2common.persistence.liveaction import LiveAction from st2common.services import action as action_service from st2common.util import action_db as action_utils from st2common.util import reference diff --git a/st2common/st2common/fields.py b/st2common/st2common/fields.py index 33144962df..f0f841fb87 100644 --- a/st2common/st2common/fields.py +++ b/st2common/st2common/fields.py @@ -40,7 +40,6 @@ from oslo_config import cfg from st2common.constants.compression import ( - JSONDictFieldCompressionAlgorithmEnum, MAP_COMPRESS, MAP_UNCOMPRESS, ) diff --git a/st2common/st2common/models/api/execution.py b/st2common/st2common/models/api/execution.py index fb1cdac8b3..f5dbf80539 100644 --- a/st2common/st2common/models/api/execution.py +++ b/st2common/st2common/models/api/execution.py @@ -26,7 +26,7 @@ from st2common.models.db.execution import ActionExecutionOutputDB from st2common.models.api.trigger import TriggerTypeAPI, TriggerAPI, TriggerInstanceAPI from st2common.models.api.rule import RuleAPI -from st2common.models.api.action import RunnerTypeAPI, ActionAPI, LiveActionAPI +from st2common.models.api.action import RunnerTypeAPI, ActionAPI from st2common import log as logging from st2common.util.deep_copy import fast_deepcopy_dict from st2common.fields import JSONDictEscapedFieldCompatibilityField From 090200e1216d0440f93b000fcdb8af8c065c6d97 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Fri, 14 Jul 2023 18:55:35 +0000 Subject: [PATCH 41/70] fix test actionchain liveaction --- .../action_chain_runner/tests/unit/test_actionchain_cancel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/runners/action_chain_runner/tests/unit/test_actionchain_cancel.py b/contrib/runners/action_chain_runner/tests/unit/test_actionchain_cancel.py index 635747ad1c..e04d5c01b1 100644 --- a/contrib/runners/action_chain_runner/tests/unit/test_actionchain_cancel.py +++ b/contrib/runners/action_chain_runner/tests/unit/test_actionchain_cancel.py @@ -206,7 +206,7 @@ def test_chain_cancel_cascade_to_subworkflow(self): # Wait until the subworkflow is canceled. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction) + task1_live = LiveAction.get_by_id(task1_exec.liveaction_id) task1_live = self._wait_on_status( task1_live, action_constants.LIVEACTION_STATUS_CANCELED ) From 2c47689d7f5f179181d8e629fb695b82765eac1a Mon Sep 17 00:00:00 2001 From: guzzijones Date: Fri, 14 Jul 2023 19:08:40 +0000 Subject: [PATCH 42/70] import fixes --- .../bin/migrations/v3.9/st2-migrate-liveaction-executiondb | 3 +-- st2stream/tests/unit/controllers/v1/test_stream.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb index 1ab8d6db6d..0abc2b8a8a 100755 --- a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb +++ b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb @@ -39,8 +39,7 @@ from oslo_config import cfg from st2common import config from st2common import transport -from st2common.models import stormbase -from st2common.models.db import MongoDBAccess +from st2common.models.db import MongoDBAccess, stormbase from st2common.service_setup import db_setup from st2common.service_setup import db_teardown from st2common.util import isotime diff --git a/st2stream/tests/unit/controllers/v1/test_stream.py b/st2stream/tests/unit/controllers/v1/test_stream.py index dbfb6277c1..2d661992f9 100644 --- a/st2stream/tests/unit/controllers/v1/test_stream.py +++ b/st2stream/tests/unit/controllers/v1/test_stream.py @@ -19,8 +19,8 @@ from st2common.models.api.action import ActionAPI from st2common.models.api.action import RunnerTypeAPI +from st2common.models.api.action import LiveActionAPI from st2common.models.api.execution import ActionExecutionAPI -from st2common.models.api.execution import LiveActionAPI from st2common.models.api.execution import ActionExecutionOutputAPI from st2common.models.db.liveaction import LiveActionDB from st2common.models.db.execution import ActionExecutionDB From 0a342a1e9a8b5ac7a0030e475059ca926e4a0e19 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Fri, 14 Jul 2023 20:27:58 +0000 Subject: [PATCH 43/70] fix inquiry ttl liveaction_id --- st2common/st2common/garbage_collection/inquiries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2common/st2common/garbage_collection/inquiries.py b/st2common/st2common/garbage_collection/inquiries.py index 6182d2b491..a447a0f5eb 100644 --- a/st2common/st2common/garbage_collection/inquiries.py +++ b/st2common/st2common/garbage_collection/inquiries.py @@ -78,7 +78,7 @@ def purge_inquiries(logger): liveaction_db = action_utils.update_liveaction_status( status=action_constants.LIVEACTION_STATUS_TIMED_OUT, result=inquiry.result, - liveaction_id=inquiry.liveaction, + liveaction_id=inquiry.liveaction_id, ) executions.update_execution(liveaction_db) From 53367df885af6e4dd0c767adc50c081dc9e8635c Mon Sep 17 00:00:00 2001 From: guzzijones Date: Fri, 14 Jul 2023 20:43:25 +0000 Subject: [PATCH 44/70] zipp <=3.16 for python 3.6 compatibility --- st2client/in-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/st2client/in-requirements.txt b/st2client/in-requirements.txt index b0057916f1..3c571a2168 100644 --- a/st2client/in-requirements.txt +++ b/st2client/in-requirements.txt @@ -1,5 +1,6 @@ # Remember to list implicit packages here, otherwise version won't be fixated! importlib-metadata +zipp<=3.16.0 # importlib-metadata requires typing-extensions typing-extensions argcomplete From 14cc5bd4125950dddbd9e8084086f3da66713aa6 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Fri, 14 Jul 2023 20:49:35 +0000 Subject: [PATCH 45/70] zipp requirement for python 3.6 --- requirements.txt | 2 +- st2client/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index c6ddc77374..b9b937c79f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -78,5 +78,5 @@ unittest2 webob==1.8.7 webtest zake==0.2.2 -zipp<3.16.0 +zipp<=3.16.0 zstandard==0.15.2 diff --git a/st2client/requirements.txt b/st2client/requirements.txt index 143e2ba6f5..fba703b95f 100644 --- a/st2client/requirements.txt +++ b/st2client/requirements.txt @@ -25,4 +25,4 @@ requests[security]==2.25.1 six==1.13.0 sseclient-py==1.7 typing-extensions<4.2 -zipp<3.16.0 +zipp<=3.16.0 From 7e2be10589137cbb713527dcb381f4dbef61bea7 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Fri, 14 Jul 2023 21:08:11 +0000 Subject: [PATCH 46/70] pin zipp < 3.16 --- requirements.txt | 2 +- st2client/in-requirements.txt | 2 +- st2client/requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index b9b937c79f..c6ddc77374 100644 --- a/requirements.txt +++ b/requirements.txt @@ -78,5 +78,5 @@ unittest2 webob==1.8.7 webtest zake==0.2.2 -zipp<=3.16.0 +zipp<3.16.0 zstandard==0.15.2 diff --git a/st2client/in-requirements.txt b/st2client/in-requirements.txt index 3c571a2168..581b999f5f 100644 --- a/st2client/in-requirements.txt +++ b/st2client/in-requirements.txt @@ -1,6 +1,6 @@ # Remember to list implicit packages here, otherwise version won't be fixated! importlib-metadata -zipp<=3.16.0 +zipp<3.16.0 # importlib-metadata requires typing-extensions typing-extensions argcomplete diff --git a/st2client/requirements.txt b/st2client/requirements.txt index fba703b95f..143e2ba6f5 100644 --- a/st2client/requirements.txt +++ b/st2client/requirements.txt @@ -25,4 +25,4 @@ requests[security]==2.25.1 six==1.13.0 sseclient-py==1.7 typing-extensions<4.2 -zipp<=3.16.0 +zipp<3.16.0 From af3548feee22e2bc22e45d56f1714b3b419fb006 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Mon, 17 Jul 2023 16:37:48 +0000 Subject: [PATCH 47/70] liveaction return inside actionexecution api --- .../st2api/controllers/v1/execution_views.py | 4 +++- .../controllers/v1/test_executions_filters.py | 2 -- st2common/st2common/models/api/execution.py | 7 +++++- st2common/tests/unit/test_executions.py | 24 +++++++++++++++++-- .../packs/executions/liveactions.yaml | 6 ++--- 5 files changed, 34 insertions(+), 9 deletions(-) diff --git a/st2api/st2api/controllers/v1/execution_views.py b/st2api/st2api/controllers/v1/execution_views.py index 9c44764bd1..a59a1421c3 100644 --- a/st2api/st2api/controllers/v1/execution_views.py +++ b/st2api/st2api/controllers/v1/execution_views.py @@ -32,6 +32,7 @@ "action": "action.ref", "status": "status", "liveaction_id": "liveaction_id", + "liveaction": "liveaction_id", "parent": "parent", "rule": "rule.name", "runner": "runner.name", @@ -54,7 +55,8 @@ # List of filters that are too broad to distinct by them and are very likely to represent 1 to 1 # relation between filter and particular history record. -IGNORE_FILTERS = ["parent", "timestamp", "liveaction_id", "trigger_instance"] +# tldr: these filters represent MANY distinct possibilities +IGNORE_FILTERS = ["parent", "timestamp", "liveaction", "liveaction_id", "trigger_instance"] class FiltersController(object): diff --git a/st2api/tests/unit/controllers/v1/test_executions_filters.py b/st2api/tests/unit/controllers/v1/test_executions_filters.py index 9a1dab25fd..09e5496871 100644 --- a/st2api/tests/unit/controllers/v1/test_executions_filters.py +++ b/st2api/tests/unit/controllers/v1/test_executions_filters.py @@ -95,8 +95,6 @@ def assign_parent(child): data["id"] = obj_id data["start_timestamp"] = isotime.format(timestamp, offset=False) data["end_timestamp"] = isotime.format(timestamp, offset=False) - data["status"] = data["status"] - data["result"] = data["result"] if fake_type["action"]["name"] == "local" and random.choice([True, False]): assign_parent(data) wb_obj = ActionExecutionAPI(**data) diff --git a/st2common/st2common/models/api/execution.py b/st2common/st2common/models/api/execution.py index f5dbf80539..b060636bad 100644 --- a/st2common/st2common/models/api/execution.py +++ b/st2common/st2common/models/api/execution.py @@ -24,9 +24,10 @@ from st2common.models.api.base import BaseAPI from st2common.models.db.execution import ActionExecutionDB from st2common.models.db.execution import ActionExecutionOutputDB +from st2common.persistence.liveaction import LiveAction from st2common.models.api.trigger import TriggerTypeAPI, TriggerAPI, TriggerInstanceAPI from st2common.models.api.rule import RuleAPI -from st2common.models.api.action import RunnerTypeAPI, ActionAPI +from st2common.models.api.action import RunnerTypeAPI, ActionAPI, LiveActionAPI from st2common import log as logging from st2common.util.deep_copy import fast_deepcopy_dict from st2common.fields import JSONDictEscapedFieldCompatibilityField @@ -39,6 +40,7 @@ REQUIRED_ATTR_SCHEMAS = { "action": copy.deepcopy(ActionAPI.schema), "runner": copy.deepcopy(RunnerTypeAPI.schema), + "liveaction": copy.deepcopy(LiveActionAPI.schema) } for k, v in six.iteritems(REQUIRED_ATTR_SCHEMAS): @@ -61,6 +63,7 @@ class ActionExecutionAPI(BaseAPI): "action": REQUIRED_ATTR_SCHEMAS["action"], "runner": REQUIRED_ATTR_SCHEMAS["runner"], "liveaction_id": {"type": "string", "required": True}, + "liveaction": REQUIRED_ATTR_SCHEMAS["liveaction"], "status": { "description": "The current status of the action execution.", "type": "string", @@ -155,6 +158,8 @@ def from_model(cls, model, mask_secrets=False): start_timestamp = model.start_timestamp start_timestamp_iso = isotime.format(start_timestamp, offset=False) doc["start_timestamp"] = start_timestamp_iso + live_action_model = LiveAction.get_by_id(doc["liveaction_id"]) + doc["liveaction"] = LiveActionAPI._from_model(live_action_model, mask_secrets=mask_secrets) end_timestamp = model.end_timestamp if end_timestamp: diff --git a/st2common/tests/unit/test_executions.py b/st2common/tests/unit/test_executions.py index 5b736cbbb0..6f3aae1b82 100644 --- a/st2common/tests/unit/test_executions.py +++ b/st2common/tests/unit/test_executions.py @@ -23,7 +23,9 @@ from st2common.util import isotime from st2common.util import date as date_utils from st2common.persistence.execution import ActionExecution +from st2common.persistence.liveaction import LiveAction from st2common.models.api.execution import ActionExecutionAPI +from st2common.models.api.action import LiveActionAPI from st2common.exceptions.db import StackStormDBObjectNotFoundError from six.moves import range @@ -33,6 +35,10 @@ def setUp(self): super(TestActionExecutionHistoryModel, self).setUp() # Fake execution record for action liveactions triggered by workflow runner. + self.fake_history_liveactions = [ + fixture.ARTIFACTS["liveactions"]["task1"], + fixture.ARTIFACTS["liveactions"]["task2"], + ] self.fake_history_subtasks = [ { "id": str(bson.ObjectId()), @@ -54,7 +60,7 @@ def setUp(self): "action": copy.deepcopy(fixture.ARTIFACTS["actions"]["local"]), "runner": copy.deepcopy(fixture.ARTIFACTS["runners"]["run-local"]), "liveaction_id": copy.deepcopy( - fixture.ARTIFACTS["liveactions"]["task2"] + fixture.ARTIFACTS["liveactions"]["task2"]["id"] ), "status": fixture.ARTIFACTS["liveactions"]["task2"]["status"], "start_timestamp": fixture.ARTIFACTS["liveactions"]["task2"][ @@ -87,12 +93,14 @@ def setUp(self): "end_timestamp" ], } - + self.fake_history_workflow_liveaction = fixture.ARTIFACTS["liveactions"]["workflow"] # Assign parent to the execution records for the subtasks. for task in self.fake_history_subtasks: task["parent"] = self.fake_history_workflow["id"] def test_model_complete(self): + # create LiveactionApiObject + live_action_obj = LiveActionAPI(**copy.deepcopy(self.fake_history_workflow_liveaction)) # Create API object. obj = ActionExecutionAPI(**copy.deepcopy(self.fake_history_workflow)) @@ -110,6 +118,11 @@ def test_model_complete(self): self.assertIsNone(getattr(obj, "parent", None)) self.assertListEqual(obj.children, self.fake_history_workflow["children"]) + # convert liveaction API to model + live_action_model = LiveActionAPI.to_model(live_action_obj) + live_action_model.id = live_action_obj.id + LiveAction.add_or_update(live_action_model) + # Convert API object to DB model. model = ActionExecutionAPI.to_model(obj) self.assertEqual(str(model.id), obj.id) @@ -182,6 +195,8 @@ def test_crud_complete(self): ) def test_model_partial(self): + # create LiveactionApiObject + live_action_obj = LiveActionAPI(**copy.deepcopy(self.fake_history_liveactions[0])) # Create API object. obj = ActionExecutionAPI(**copy.deepcopy(self.fake_history_subtasks[0])) self.assertIsNone(getattr(obj, "trigger", None)) @@ -196,8 +211,13 @@ def test_model_partial(self): self.assertEqual(obj.parent, self.fake_history_subtasks[0]["parent"]) self.assertIsNone(getattr(obj, "children", None)) + # convert liveaction API to model + live_action_model = LiveActionAPI.to_model(live_action_obj) + live_action_model.id = live_action_obj.id # Convert API object to DB model. model = ActionExecutionAPI.to_model(obj) + LiveAction.add_or_update(live_action_model) + self.assertEqual(str(live_action_model.id), str(live_action_model.id)) self.assertEqual(str(model.id), obj.id) self.assertDictEqual(model.trigger, {}) self.assertDictEqual(model.trigger_type, {}) diff --git a/st2tests/st2tests/fixtures/packs/executions/liveactions.yaml b/st2tests/st2tests/fixtures/packs/executions/liveactions.yaml index 2113d0ea99..5e41bc3c4b 100644 --- a/st2tests/st2tests/fixtures/packs/executions/liveactions.yaml +++ b/st2tests/st2tests/fixtures/packs/executions/liveactions.yaml @@ -1,7 +1,7 @@ --- task1: + id: 54c6b6d60640fd4f5354e74a action: executions.local - id: "liveaction1" callback: {} end_timestamp: '2014-09-01T00:00:05.000000Z' parameters: @@ -19,8 +19,8 @@ task1: start_timestamp: '2014-09-01T00:00:02.000000Z' status: succeeded task2: + id: 54c6b6d60640fd4f5354e74a action: executions.local - id: "liveaction2" callback: {} end_timestamp: '2014-09-01T00:00:05.000000Z' parameters: @@ -38,7 +38,7 @@ task2: start_timestamp: '2014-09-01T00:00:03.000000Z' status: succeeded workflow: - id: "workflow1" + id: 54c6b6d60640fd4f5354e74a action: executions.chain callback: {} end_timestamp: '2014-09-01T00:00:05.000000Z' From a98d25b50f574df431c0ac2c3f7021955c818259 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Mon, 17 Jul 2023 19:32:47 +0000 Subject: [PATCH 48/70] fix unit tests to allow embedded liveaction --- .../st2api/controllers/v1/actionexecutions.py | 1 - .../st2api/controllers/v1/execution_views.py | 8 +++++- .../unit/controllers/v1/test_executions.py | 25 +++++++++++++++++-- .../v1/test_executions_descendants.py | 7 ++++-- .../controllers/v1/test_executions_filters.py | 14 +++++++++++ st2common/st2common/models/api/execution.py | 13 +++++++--- st2common/st2common/models/db/execution.py | 2 +- st2common/st2common/openapi.yaml | 2 ++ st2common/st2common/openapi.yaml.j2 | 2 ++ .../test_v35_migrate_db_dict_field_values.py | 4 +-- st2common/tests/unit/test_executions.py | 12 ++++++--- .../descendants/executions/child1_level1.yaml | 2 +- .../descendants/executions/child1_level2.yaml | 2 +- .../descendants/executions/child1_level3.yaml | 2 +- .../descendants/executions/child2_level1.yaml | 2 +- .../descendants/executions/child2_level2.yaml | 2 +- .../descendants/executions/child2_level3.yaml | 2 +- .../descendants/executions/child3_level2.yaml | 2 +- .../descendants/executions/child3_level3.yaml | 2 +- .../executions/root_execution.yaml | 2 +- .../liveactions/liveaction_fake.yaml | 5 ++++ 21 files changed, 89 insertions(+), 24 deletions(-) create mode 100644 st2tests/st2tests/fixtures/descendants/liveactions/liveaction_fake.yaml diff --git a/st2api/st2api/controllers/v1/actionexecutions.py b/st2api/st2api/controllers/v1/actionexecutions.py index cb22335bca..40f52dbcc5 100644 --- a/st2api/st2api/controllers/v1/actionexecutions.py +++ b/st2api/st2api/controllers/v1/actionexecutions.py @@ -136,7 +136,6 @@ def _handle_schedule_execution( rbac_utils.assert_user_is_admin_if_user_query_param_is_provided( user_db=requester_user, user=user ) - try: return self._schedule_execution( liveaction=liveaction_api, diff --git a/st2api/st2api/controllers/v1/execution_views.py b/st2api/st2api/controllers/v1/execution_views.py index a59a1421c3..a051192515 100644 --- a/st2api/st2api/controllers/v1/execution_views.py +++ b/st2api/st2api/controllers/v1/execution_views.py @@ -56,7 +56,13 @@ # List of filters that are too broad to distinct by them and are very likely to represent 1 to 1 # relation between filter and particular history record. # tldr: these filters represent MANY distinct possibilities -IGNORE_FILTERS = ["parent", "timestamp", "liveaction", "liveaction_id", "trigger_instance"] +IGNORE_FILTERS = [ + "parent", + "timestamp", + "liveaction", + "liveaction_id", + "trigger_instance", +] class FiltersController(object): diff --git a/st2api/tests/unit/controllers/v1/test_executions.py b/st2api/tests/unit/controllers/v1/test_executions.py index f34e78fdbf..c62c7e7c1b 100644 --- a/st2api/tests/unit/controllers/v1/test_executions.py +++ b/st2api/tests/unit/controllers/v1/test_executions.py @@ -32,6 +32,7 @@ from st2common.models.db.auth import UserDB from st2common.models.db.execution import ActionExecutionDB from st2common.models.db.execution import ActionExecutionOutputDB +from st2common.models.db.liveaction import LiveActionDB from st2common.models.db.keyvalue import KeyValuePairDB from st2common.persistence.execution import ActionExecution from st2common.persistence.execution import ActionExecutionOutput @@ -2000,9 +2001,19 @@ def test_get_output_running_execution(self): status=status, action={"ref": "core.local"}, runner={"name": "local-shell-cmd"}, - liveaction_id="ref", + liveaction_id="54c6b6d60640fd4f5354e74a", ) action_execution_db = ActionExecution.add_or_update(action_execution_db) + liveaction_db = LiveActionDB( + id="54c6b6d60640fd4f5354e74a", + start_timestamp=timestamp, + end_timestamp=timestamp, + status=status, + action="core.local", + runner_info={"name": "local-shell-cmd"}, + ) + + LiveAction.add_or_update(liveaction_db) output_params = dict( execution_id=str(action_execution_db.id), @@ -2081,9 +2092,19 @@ def test_get_output_finished_execution(self): status=status, action={"ref": "core.local"}, runner={"name": "local-shell-cmd"}, - liveaction_id="ref", + liveaction_id="54c6b6d60640fd4f5354e74a", ) action_execution_db = ActionExecution.add_or_update(action_execution_db) + liveaction_db = LiveActionDB( + id="54c6b6d60640fd4f5354e74a", + start_timestamp=timestamp, + end_timestamp=timestamp, + status=status, + action="core.local", + runner_info={"name": "local-shell-cmd"}, + ) + + LiveAction.add_or_update(liveaction_db) for i in range(1, 6): stdout_db = ActionExecutionOutputDB( diff --git a/st2api/tests/unit/controllers/v1/test_executions_descendants.py b/st2api/tests/unit/controllers/v1/test_executions_descendants.py index 55b1c12f53..88c1574d29 100644 --- a/st2api/tests/unit/controllers/v1/test_executions_descendants.py +++ b/st2api/tests/unit/controllers/v1/test_executions_descendants.py @@ -31,7 +31,8 @@ "child1_level3.yaml", "child2_level3.yaml", "child3_level3.yaml", - ] + ], + "liveactions": ["liveaction_fake.yaml"], } @@ -40,7 +41,9 @@ class ActionExecutionControllerTestCaseDescendantsTest(FunctionalTest): def setUpClass(cls): super(ActionExecutionControllerTestCaseDescendantsTest, cls).setUpClass() cls.MODELS = FixturesLoader().save_fixtures_to_db( - fixtures_pack=DESCENDANTS_PACK, fixtures_dict=DESCENDANTS_FIXTURES + fixtures_pack=DESCENDANTS_PACK, + fixtures_dict=DESCENDANTS_FIXTURES, + use_object_ids=True, ) def test_get_all_descendants(self): diff --git a/st2api/tests/unit/controllers/v1/test_executions_filters.py b/st2api/tests/unit/controllers/v1/test_executions_filters.py index 09e5496871..c916252c9e 100644 --- a/st2api/tests/unit/controllers/v1/test_executions_filters.py +++ b/st2api/tests/unit/controllers/v1/test_executions_filters.py @@ -33,7 +33,9 @@ from st2api.controllers.v1.actionexecutions import ActionExecutionsController from st2api.controllers.v1.execution_views import FILTERS_WITH_VALID_NULL_VALUES from st2common.persistence.execution import ActionExecution +from st2common.persistence.action import LiveAction from st2common.models.api.execution import ActionExecutionAPI +from st2common.models.api.execution import LiveActionAPI class TestActionExecutionFilters(FunctionalTest): @@ -101,6 +103,18 @@ def assign_parent(child): db_obj = ActionExecutionAPI.to_model(wb_obj) cls.refs[obj_id] = ActionExecution.add_or_update(db_obj) cls.start_timestamps.append(timestamp) + # also add the liveaction to the database so it can be retrieved by + # the actionexecution api + liveaction_data = { + "id": data["liveaction_id"], + "action": fake_type["action"]["name"], + "status": data["status"], + } + wb_live_obj = LiveActionAPI(**liveaction_data) + live_db_obj = LiveActionAPI.to_model(wb_live_obj) + # hard code id of liveaction + live_db_obj.id = data["liveaction_id"] + LiveAction.add_or_update(live_db_obj) cls.start_timestamps = sorted(cls.start_timestamps) diff --git a/st2common/st2common/models/api/execution.py b/st2common/st2common/models/api/execution.py index b060636bad..8591c46b16 100644 --- a/st2common/st2common/models/api/execution.py +++ b/st2common/st2common/models/api/execution.py @@ -40,7 +40,7 @@ REQUIRED_ATTR_SCHEMAS = { "action": copy.deepcopy(ActionAPI.schema), "runner": copy.deepcopy(RunnerTypeAPI.schema), - "liveaction": copy.deepcopy(LiveActionAPI.schema) + "liveaction": copy.deepcopy(LiveActionAPI.schema), } for k, v in six.iteritems(REQUIRED_ATTR_SCHEMAS): @@ -158,8 +158,15 @@ def from_model(cls, model, mask_secrets=False): start_timestamp = model.start_timestamp start_timestamp_iso = isotime.format(start_timestamp, offset=False) doc["start_timestamp"] = start_timestamp_iso - live_action_model = LiveAction.get_by_id(doc["liveaction_id"]) - doc["liveaction"] = LiveActionAPI._from_model(live_action_model, mask_secrets=mask_secrets) + # check to see if liveaction_id has been excluded in output filtering + if doc.get("liveaction_id", False): + live_action_model = LiveAction.get_by_id(doc["liveaction_id"]) + if live_action_model is not None: + doc["liveaction"] = LiveActionAPI.from_model( + live_action_model, mask_secrets=mask_secrets + ) + else: + doc["liveaction"] = {} end_timestamp = model.end_timestamp if end_timestamp: diff --git a/st2common/st2common/models/db/execution.py b/st2common/st2common/models/db/execution.py index eba2cb840e..6b591d310e 100644 --- a/st2common/st2common/models/db/execution.py +++ b/st2common/st2common/models/db/execution.py @@ -79,7 +79,7 @@ class ActionExecutionDB(stormbase.StormFoundationDB): web_url = me.StringField(required=False) # liveaction id - liveaction_id = me.StringField() + liveaction_id = me.StringField(required=True) meta = { "indexes": [ diff --git a/st2common/st2common/openapi.yaml b/st2common/st2common/openapi.yaml index c49242c627..7bcda8562b 100644 --- a/st2common/st2common/openapi.yaml +++ b/st2common/st2common/openapi.yaml @@ -4934,6 +4934,8 @@ definitions: $ref: '#/definitions/Action' runner: $ref: '#/definitions/RunnerType' + liveaction: + $ref: '#/definitions/LiveAction' liveaction_id: type: string task_execution: diff --git a/st2common/st2common/openapi.yaml.j2 b/st2common/st2common/openapi.yaml.j2 index f8b08af4c2..c008d5db33 100644 --- a/st2common/st2common/openapi.yaml.j2 +++ b/st2common/st2common/openapi.yaml.j2 @@ -4930,6 +4930,8 @@ definitions: $ref: '#/definitions/Action' runner: $ref: '#/definitions/RunnerType' + liveaction: + $ref: '#/definitions/LiveAction' liveaction_id: type: string task_execution: diff --git a/st2common/tests/unit/migrations/test_v35_migrate_db_dict_field_values.py b/st2common/tests/unit/migrations/test_v35_migrate_db_dict_field_values.py index 3dfe85bc86..82903a4391 100644 --- a/st2common/tests/unit/migrations/test_v35_migrate_db_dict_field_values.py +++ b/st2common/tests/unit/migrations/test_v35_migrate_db_dict_field_values.py @@ -83,7 +83,7 @@ def test_migrate_executions(self): LiveActionDB._meta["allow_inheritance"] = True class ActionExecutionDB_OldFieldType(ActionExecutionDB): - liveaction_id = None + liveaction_id = me.StringField() # not required; didn't exist result = stormbase.EscapedDynamicField(default={}) liveaction = stormbase.EscapedDictField(required=True) parameters = stormbase.EscapedDynamicField(default={}) @@ -168,7 +168,6 @@ class LiveActionDB_OldFieldType(LiveActionDB): ) class LiveActionDB_NewFieldType(LiveActionDB): - liveaction_id = None result = JSONDictEscapedFieldCompatibilityField( default={}, help_text="Action defined result." ) @@ -217,6 +216,7 @@ class LiveActionDB_NewFieldType(LiveActionDB): ) class ActionExecutionDB_NewFieldType(ActionExecutionDB): + liveaction_id = me.StringField() # not required; didn't exist liveaction = stormbase.EscapedDictField(required=True) parameters = stormbase.EscapedDynamicField(default={}) result = JSONDictEscapedFieldCompatibilityField( diff --git a/st2common/tests/unit/test_executions.py b/st2common/tests/unit/test_executions.py index 6f3aae1b82..24dc0be9c0 100644 --- a/st2common/tests/unit/test_executions.py +++ b/st2common/tests/unit/test_executions.py @@ -93,14 +93,18 @@ def setUp(self): "end_timestamp" ], } - self.fake_history_workflow_liveaction = fixture.ARTIFACTS["liveactions"]["workflow"] + self.fake_history_workflow_liveaction = fixture.ARTIFACTS["liveactions"][ + "workflow" + ] # Assign parent to the execution records for the subtasks. for task in self.fake_history_subtasks: task["parent"] = self.fake_history_workflow["id"] def test_model_complete(self): # create LiveactionApiObject - live_action_obj = LiveActionAPI(**copy.deepcopy(self.fake_history_workflow_liveaction)) + live_action_obj = LiveActionAPI( + **copy.deepcopy(self.fake_history_workflow_liveaction) + ) # Create API object. obj = ActionExecutionAPI(**copy.deepcopy(self.fake_history_workflow)) @@ -196,7 +200,9 @@ def test_crud_complete(self): def test_model_partial(self): # create LiveactionApiObject - live_action_obj = LiveActionAPI(**copy.deepcopy(self.fake_history_liveactions[0])) + live_action_obj = LiveActionAPI( + **copy.deepcopy(self.fake_history_liveactions[0]) + ) # Create API object. obj = ActionExecutionAPI(**copy.deepcopy(self.fake_history_subtasks[0])) self.assertIsNone(getattr(obj, "trigger", None)) diff --git a/st2tests/st2tests/fixtures/descendants/executions/child1_level1.yaml b/st2tests/st2tests/fixtures/descendants/executions/child1_level1.yaml index 49e14c3d07..ebd2708f75 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/child1_level1.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/child1_level1.yaml @@ -7,7 +7,7 @@ children: - 54e6583d0640fd16887d685b end_timestamp: '2014-09-01T00:00:57.000001Z' id: 54e657f20640fd16887d6857 -liveaction_id: pointlessaction +liveaction_id: 54c6b6d60640fd4f5354e74a parent: 54e657d60640fd16887d6855 runner: name: pointlessrunner diff --git a/st2tests/st2tests/fixtures/descendants/executions/child1_level2.yaml b/st2tests/st2tests/fixtures/descendants/executions/child1_level2.yaml index 7a7cb27470..6f330f5547 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/child1_level2.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/child1_level2.yaml @@ -5,7 +5,7 @@ action: children: [] end_timestamp: '2014-09-01T00:00:56.000002Z' id: 54e657fa0640fd16887d6858 -liveaction_id: pointlessaction +liveaction_id: 54c6b6d60640fd4f5354e74a parent: 54e657f20640fd16887d6857 runner: name: pointlessrunner diff --git a/st2tests/st2tests/fixtures/descendants/executions/child1_level3.yaml b/st2tests/st2tests/fixtures/descendants/executions/child1_level3.yaml index c5e6ee0dad..59626f9260 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/child1_level3.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/child1_level3.yaml @@ -5,7 +5,7 @@ action: children: [] end_timestamp: '2014-09-01T00:00:55.100000Z' id: 54e6581b0640fd16887d6859 -liveaction_id: pointlessaction +liveaction_id: 54c6b6d60640fd4f5354e74a parent: 54e6583d0640fd16887d685b runner: name: pointlessrunner diff --git a/st2tests/st2tests/fixtures/descendants/executions/child2_level1.yaml b/st2tests/st2tests/fixtures/descendants/executions/child2_level1.yaml index d7479c4b88..d3b9188507 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/child2_level1.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/child2_level1.yaml @@ -6,7 +6,7 @@ children: - 54e658570640fd16887d685d end_timestamp: '2014-09-01T00:00:55.000000Z' id: 54e658290640fd16887d685a -liveaction_id: pointlessaction +liveaction_id: 54c6b6d60640fd4f5354e74a parent: 54e657d60640fd16887d6855 runner: name: pointlessrunner diff --git a/st2tests/st2tests/fixtures/descendants/executions/child2_level2.yaml b/st2tests/st2tests/fixtures/descendants/executions/child2_level2.yaml index ff7fcf3a2e..d5fe4246c7 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/child2_level2.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/child2_level2.yaml @@ -6,7 +6,7 @@ children: - 54e6581b0640fd16887d6859 end_timestamp: '2014-09-01T00:00:55.000000Z' id: 54e6583d0640fd16887d685b -liveaction_id: pointlessaction +liveaction_id: 54c6b6d60640fd4f5354e74a parent: 54e657f20640fd16887d6857 runner: name: pointlessrunner diff --git a/st2tests/st2tests/fixtures/descendants/executions/child2_level3.yaml b/st2tests/st2tests/fixtures/descendants/executions/child2_level3.yaml index 448d4374df..9dd19be9cb 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/child2_level3.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/child2_level3.yaml @@ -5,7 +5,7 @@ action: children: [] end_timestamp: '2014-09-01T00:00:59.000010Z' id: 54e6584a0640fd16887d685c -liveaction_id: pointlessaction +liveaction_id: 54c6b6d60640fd4f5354e74a parent: 54e658570640fd16887d685d runner: name: pointlessrunner diff --git a/st2tests/st2tests/fixtures/descendants/executions/child3_level2.yaml b/st2tests/st2tests/fixtures/descendants/executions/child3_level2.yaml index 9076d6e41b..cc40bcc4ba 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/child3_level2.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/child3_level2.yaml @@ -7,7 +7,7 @@ children: - 54e6585f0640fd16887d685e end_timestamp: '2014-09-01T00:00:55.000000Z' id: 54e658570640fd16887d685d -liveaction_id: pointlessaction +liveaction_id: 54c6b6d60640fd4f5354e74a parent: 54e658290640fd16887d685a runner: name: pointlessrunner diff --git a/st2tests/st2tests/fixtures/descendants/executions/child3_level3.yaml b/st2tests/st2tests/fixtures/descendants/executions/child3_level3.yaml index 6f1bee7c45..4f28157a71 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/child3_level3.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/child3_level3.yaml @@ -5,7 +5,7 @@ action: children: [] end_timestamp: '2014-09-01T00:00:55.000000Z' id: 54e6585f0640fd16887d685e -liveaction_id: pointlessaction +liveaction_id: 54c6b6d60640fd4f5354e74a parent: 54e658570640fd16887d685d runner: name: pointlessrunner diff --git a/st2tests/st2tests/fixtures/descendants/executions/root_execution.yaml b/st2tests/st2tests/fixtures/descendants/executions/root_execution.yaml index 903aa47f6d..1f0a0bfb48 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/root_execution.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/root_execution.yaml @@ -7,7 +7,7 @@ children: - 54e658290640fd16887d685a end_timestamp: '2014-09-01T00:00:59.000000Z' id: 54e657d60640fd16887d6855 -liveaction_id: pointlessaction +liveaction_id: 54c6b6d60640fd4f5354e74a runner: name: pointlessrunner runner_module: no.module diff --git a/st2tests/st2tests/fixtures/descendants/liveactions/liveaction_fake.yaml b/st2tests/st2tests/fixtures/descendants/liveactions/liveaction_fake.yaml new file mode 100644 index 0000000000..b933dedf63 --- /dev/null +++ b/st2tests/st2tests/fixtures/descendants/liveactions/liveaction_fake.yaml @@ -0,0 +1,5 @@ +--- +action: local +name: "fake" +id: 54c6b6d60640fd4f5354e74a +status: succeeded From ff894083e0d578c7360ba48afe814b8dec88bc39 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Mon, 17 Jul 2023 19:51:47 +0000 Subject: [PATCH 49/70] fix stream test --- .../v1/test_stream_execution_output.py | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/st2stream/tests/unit/controllers/v1/test_stream_execution_output.py b/st2stream/tests/unit/controllers/v1/test_stream_execution_output.py index ace3b11685..ab9088056b 100644 --- a/st2stream/tests/unit/controllers/v1/test_stream_execution_output.py +++ b/st2stream/tests/unit/controllers/v1/test_stream_execution_output.py @@ -21,9 +21,11 @@ from st2common.constants import action as action_constants from st2common.models.db.execution import ActionExecutionDB +from st2common.models.db.liveaction import LiveActionDB from st2common.models.db.execution import ActionExecutionOutputDB from st2common.persistence.execution import ActionExecution from st2common.persistence.execution import ActionExecutionOutput +from st2common.persistence.liveaction import LiveAction from st2common.util import date as date_utils from st2common.stream.listener import get_listener @@ -53,15 +55,21 @@ def test_get_output_running_execution(self): # Test the execution output API endpoint for execution which is running (blocking) status = action_constants.LIVEACTION_STATUS_RUNNING timestamp = date_utils.get_datetime_utc_now() + liveaction_id = "54c6b6d60640fd4f5354e74a" action_execution_db = ActionExecutionDB( start_timestamp=timestamp, end_timestamp=timestamp, status=status, action={"ref": "core.local"}, runner={"name": "local-shell-cmd"}, - liveaction_id="ref", + liveaction_id=liveaction_id, ) action_execution_db = ActionExecution.add_or_update(action_execution_db) + liveaction_db = LiveActionDB( + action="core.local", runner_info={"name": "local-shell-cmd"}, status=status + ) + liveaction_db.id = liveaction_id + LiveAction.add_or_update(liveaction_db) output_params = dict( execution_id=str(action_execution_db.id), @@ -135,15 +143,23 @@ def test_get_output_finished_execution(self): # Insert mock execution and output objects status = action_constants.LIVEACTION_STATUS_SUCCEEDED timestamp = date_utils.get_datetime_utc_now() + liveaction_id = "54c6b6d60640fd4f5354e74a" action_execution_db = ActionExecutionDB( start_timestamp=timestamp, end_timestamp=timestamp, status=status, action={"ref": "core.local"}, runner={"name": "local-shell-cmd"}, - liveaction_id="ref", + liveaction_id=liveaction_id, ) action_execution_db = ActionExecution.add_or_update(action_execution_db) + liveaction_db = LiveActionDB( + action="core.local", + runner_info={"name": "local-shell-cmd"}, + status=status, + ) + liveaction_db.id = liveaction_id + LiveAction.add_or_update(liveaction_db) for i in range(1, 6): stdout_db = ActionExecutionOutputDB( From 1aaf11445c890789524eb52c5cb30e543fa8a5ad Mon Sep 17 00:00:00 2001 From: guzzijones Date: Mon, 17 Jul 2023 20:14:35 +0000 Subject: [PATCH 50/70] add back array params test for alias execution --- .../unit/controllers/v1/test_alias_execution.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/st2api/tests/unit/controllers/v1/test_alias_execution.py b/st2api/tests/unit/controllers/v1/test_alias_execution.py index 664d60bb92..8d8ea0559d 100644 --- a/st2api/tests/unit/controllers/v1/test_alias_execution.py +++ b/st2api/tests/unit/controllers/v1/test_alias_execution.py @@ -322,8 +322,14 @@ def test_match_and_execute_list_action_param_str_cast_to_list(self): self.assertEqual(resp.status_int, 201) result = resp.json["results"][0] + live_action = result["execution"]["liveaction"] action_alias = result["actionalias"] self.assertEqual(resp.status_int, 201) + self.assertTrue(isinstance(live_action["parameters"]["array_param"], list)) + self.assertEqual(live_action["parameters"]["array_param"][0], "one") + self.assertEqual(live_action["parameters"]["array_param"][1], "two") + self.assertEqual(live_action["parameters"]["array_param"][2], "three") + self.assertEqual(live_action["parameters"]["array_param"][3], "four") self.assertTrue( isinstance(action_alias["immutable_parameters"]["array_param"], str) ) @@ -342,9 +348,15 @@ def test_match_and_execute_list_action_param_already_a_list(self): self.assertEqual(resp.status_int, 201) result = resp.json["results"][0] + live_action = result["execution"]["liveaction"] action_alias = result["actionalias"] self.assertEqual(resp.status_int, 201) + self.assertTrue(isinstance(live_action["parameters"]["array_param"], list)) + self.assertEqual(live_action["parameters"]["array_param"][0]["key1"], "one") + self.assertEqual(live_action["parameters"]["array_param"][0]["key2"], "two") + self.assertEqual(live_action["parameters"]["array_param"][1]["key3"], "three") + self.assertEqual(live_action["parameters"]["array_param"][1]["key4"], "four") self.assertTrue( isinstance(action_alias["immutable_parameters"]["array_param"], list) ) From 130cc87b5bc6fb387e9c9a29f80bc07a3c8dbfb9 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Mon, 17 Jul 2023 21:08:51 +0000 Subject: [PATCH 51/70] migration script change --- .../bin/migrations/v3.9/st2-migrate-liveaction-executiondb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb index 0abc2b8a8a..7a8d3921d5 100755 --- a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb +++ b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb @@ -56,7 +56,7 @@ from st2common.constants.action import LIVEACTION_COMPLETED_STATES class ActionExecutionDBOLD(ActionExecutionDB): liveaction = stormbase.EscapedDictField(required=True) - + liveaction_id = me.StringField(required=False) # was not required previously class ActionExecutionOLD(Access): impl = MongoDBAccess(ActionExecutionDBOLD) @@ -117,7 +117,7 @@ def migrate_executions(start_dt: datetime.datetime, end_dt: datetime.datetime) - for index, execution_id in enumerate(execution_ids, 1): try: - execution_db = ActionExecution.get_by_id(execution_id) + execution_db = ActionExecutionOLD.get_by_id(execution_id) except StackStormDBObjectNotFoundError: print( "Skipping ActionExecutionDB with id %s which is missing in the database" From a12fcb5d64f07648e038bb19ab44357ddd3489b7 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Wed, 19 Jul 2023 14:19:30 +0000 Subject: [PATCH 52/70] compress and uncompress methods --- st2common/st2common/constants/compression.py | 27 ++++++++++++++++++++ st2common/st2common/fields.py | 23 +++++------------ 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/st2common/st2common/constants/compression.py b/st2common/st2common/constants/compression.py index 91902e0af9..334c737f5b 100644 --- a/st2common/st2common/constants/compression.py +++ b/st2common/st2common/constants/compression.py @@ -19,6 +19,7 @@ import enum +from oslo_config import cfg import zstandard ZSTANDARD_COMPRESS = "zstandard" @@ -61,3 +62,29 @@ def zstandard_uncompress(data): MAP_UNCOMPRESS = { JSONDictFieldCompressionAlgorithmEnum.ZSTANDARD.value: zstandard_uncompress, } + + +def uncompress(value: bytes): + data = value + try: + uncompression_header = value[0:1] + uncompression_method = MAP_UNCOMPRESS.get(uncompression_header, False) + if uncompression_method: # skip if no compress + data = uncompression_method(value[1:]) + # will need to add additional exceptions if additonal compression methods + # are added in the future; please do not catch the general exception here. + except zstandard.ZstdError: + # skip if already a byte string and not zstandard compressed + pass + return data + + +def compress(value: bytes): + data = value + parameter_result_compression = cfg.CONF.database.parameter_result_compression + compression_method = MAP_COMPRESS.get(parameter_result_compression, False) + # none is not mapped at all so has no compression method + if compression_method: + data = compression_method(value) + return data + diff --git a/st2common/st2common/fields.py b/st2common/st2common/fields.py index f0f841fb87..1f100a4656 100644 --- a/st2common/st2common/fields.py +++ b/st2common/st2common/fields.py @@ -40,6 +40,8 @@ from oslo_config import cfg from st2common.constants.compression import ( + compress, + uncompress, MAP_COMPRESS, MAP_UNCOMPRESS, ) @@ -377,17 +379,8 @@ def parse_field_value(self, value: Optional[Union[bytes, dict]]) -> dict: if isinstance(value, dict): # Already deserializaed return value - data = value - try: - uncompression_header = value[0:1] - uncompression_method = MAP_UNCOMPRESS.get(uncompression_header, False) - if uncompression_method: - data = uncompression_method(value[1:]) - # skip if already a byte string and not compressed - except zstandard.ZstdError: - pass - - data = orjson.loads(data) + + data = orjson.loads(uncompress(value)) return data def _serialize_field_value(self, value: dict, compress=True) -> bytes: @@ -411,12 +404,8 @@ def default(obj): raise TypeError data = orjson.dumps(value, default=default) - parameter_result_compression = cfg.CONF.database.parameter_result_compression - compression_method = MAP_COMPRESS.get(parameter_result_compression, False) - # none is not mapped at all so has no compression method - if compress and compression_method: - data = compression_method(data) - + if compress: + data = compress(data) return data def __get__(self, instance, owner): From af0adf25bc97f3b5bd74b50095431da7718a810a Mon Sep 17 00:00:00 2001 From: AJ Date: Tue, 18 Jul 2023 14:37:46 -0400 Subject: [PATCH 53/70] Update st2common/st2common/models/api/execution.py Co-authored-by: Jacob Floyd --- st2common/st2common/models/api/execution.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2common/st2common/models/api/execution.py b/st2common/st2common/models/api/execution.py index 8591c46b16..930f2e8f08 100644 --- a/st2common/st2common/models/api/execution.py +++ b/st2common/st2common/models/api/execution.py @@ -166,7 +166,7 @@ def from_model(cls, model, mask_secrets=False): live_action_model, mask_secrets=mask_secrets ) else: - doc["liveaction"] = {} + doc["liveaction"] = {"id": doc["liveaction_id"]} end_timestamp = model.end_timestamp if end_timestamp: From e28ddc117ad6f71a13341d7f0925ce82265cf130 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Wed, 19 Jul 2023 15:35:55 +0000 Subject: [PATCH 54/70] fix import error compress and uncompress --- st2common/st2common/constants/compression.py | 3 +-- st2common/st2common/fields.py | 9 ++++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/st2common/st2common/constants/compression.py b/st2common/st2common/constants/compression.py index 334c737f5b..edb3581cf9 100644 --- a/st2common/st2common/constants/compression.py +++ b/st2common/st2common/constants/compression.py @@ -69,7 +69,7 @@ def uncompress(value: bytes): try: uncompression_header = value[0:1] uncompression_method = MAP_UNCOMPRESS.get(uncompression_header, False) - if uncompression_method: # skip if no compress + if uncompression_method: # skip if no compress data = uncompression_method(value[1:]) # will need to add additional exceptions if additonal compression methods # are added in the future; please do not catch the general exception here. @@ -87,4 +87,3 @@ def compress(value: bytes): if compression_method: data = compression_method(value) return data - diff --git a/st2common/st2common/fields.py b/st2common/st2common/fields.py index 1f100a4656..603351ea34 100644 --- a/st2common/st2common/fields.py +++ b/st2common/st2common/fields.py @@ -30,7 +30,6 @@ import weakref import orjson -import zstandard from mongoengine import LongField from mongoengine import BinaryField @@ -40,8 +39,8 @@ from oslo_config import cfg from st2common.constants.compression import ( - compress, - uncompress, + compress as compress_function, + uncompress as uncompress_function, MAP_COMPRESS, MAP_UNCOMPRESS, ) @@ -380,7 +379,7 @@ def parse_field_value(self, value: Optional[Union[bytes, dict]]) -> dict: # Already deserializaed return value - data = orjson.loads(uncompress(value)) + data = orjson.loads(uncompress_function(value)) return data def _serialize_field_value(self, value: dict, compress=True) -> bytes: @@ -405,7 +404,7 @@ def default(obj): data = orjson.dumps(value, default=default) if compress: - data = compress(data) + data = compress_function(data) return data def __get__(self, instance, owner): From 1b64e1cff652919efbbe618b714a17e6be9e01e7 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Wed, 19 Jul 2023 15:54:12 +0000 Subject: [PATCH 55/70] add test for st2.inquiry.respond secret masking --- st2common/tests/unit/test_db_liveaction.py | 28 ++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/st2common/tests/unit/test_db_liveaction.py b/st2common/tests/unit/test_db_liveaction.py index 605aa759f6..34ba6fed4d 100644 --- a/st2common/tests/unit/test_db_liveaction.py +++ b/st2common/tests/unit/test_db_liveaction.py @@ -16,6 +16,7 @@ from __future__ import absolute_import import mock +from st2common.constants.secrets import MASKED_ATTRIBUTE_VALUE from st2common.models.db.liveaction import LiveActionDB from st2common.models.db.notification import NotificationSchema, NotificationSubSchema from st2common.persistence.liveaction import LiveAction @@ -132,6 +133,33 @@ def test_liveaction_create_with_notify_both_on_success_and_on_error(self): self.assertEqual(on_failure.message, retrieved.notify.on_failure.message) self.assertEqual(retrieved.notify.on_complete, None) + def test_liveaction_inquiry_response_action(self): + RESPOND_LIVEACTION = { + "parameters": { + "response": { + "secondfactor": "omgsupersecret", + } + }, + "action": "st2.inquiry.respond", + "id": "54c6b6d60640fd4f5354e74c", + } + + created = LiveActionDB() + created.action = RESPOND_LIVEACTION["action"] + created.status = "succeeded" + created.parameters = RESPOND_LIVEACTION["parameters"] + created.id = RESPOND_LIVEACTION["id"] + saved = LiveActionModelTest._save_liveaction(created) + + retrieved = LiveAction.get_by_id(saved.id) + self.assertEqual( + saved.action, retrieved.action, "Same triggertype was not returned." + ) + import pdb; pdb.set_trace() + masked = retrieved.mask_secrets(retrieved.to_serializable_dict()) + for value in masked["parameters"]["response"].values(): + self.assertEqual(value, MASKED_ATTRIBUTE_VALUE) + @staticmethod def _save_liveaction(liveaction): return LiveAction.add_or_update(liveaction) From 8d7491b0b8050688fb2b6c5da6956b2e443efb8b Mon Sep 17 00:00:00 2001 From: guzzijones Date: Wed, 19 Jul 2023 16:53:52 +0000 Subject: [PATCH 56/70] black formatting fix --- st2common/tests/unit/test_db_liveaction.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/st2common/tests/unit/test_db_liveaction.py b/st2common/tests/unit/test_db_liveaction.py index 34ba6fed4d..0a9e1dedea 100644 --- a/st2common/tests/unit/test_db_liveaction.py +++ b/st2common/tests/unit/test_db_liveaction.py @@ -155,7 +155,9 @@ def test_liveaction_inquiry_response_action(self): self.assertEqual( saved.action, retrieved.action, "Same triggertype was not returned." ) - import pdb; pdb.set_trace() + import pdb + + pdb.set_trace() masked = retrieved.mask_secrets(retrieved.to_serializable_dict()) for value in masked["parameters"]["response"].values(): self.assertEqual(value, MASKED_ATTRIBUTE_VALUE) From 2401dfdbe934315d066417057c7662d7a355db2e Mon Sep 17 00:00:00 2001 From: guzzijones Date: Wed, 19 Jul 2023 17:01:12 +0000 Subject: [PATCH 57/70] lint fixes --- st2common/st2common/fields.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/st2common/st2common/fields.py b/st2common/st2common/fields.py index 603351ea34..370a66400f 100644 --- a/st2common/st2common/fields.py +++ b/st2common/st2common/fields.py @@ -36,13 +36,10 @@ from mongoengine.base.datastructures import mark_as_changed_wrapper from mongoengine.base.datastructures import mark_key_as_changed_wrapper from mongoengine.common import _import_class -from oslo_config import cfg from st2common.constants.compression import ( compress as compress_function, - uncompress as uncompress_function, - MAP_COMPRESS, - MAP_UNCOMPRESS, + uncompress as uncompress_function ) from st2common.util import date as date_utils from st2common.util import mongoescape From 4132b30c15e79c1db705c666183edbabf8ff671d Mon Sep 17 00:00:00 2001 From: guzzijones Date: Wed, 19 Jul 2023 17:11:51 +0000 Subject: [PATCH 58/70] fix migration --- .../bin/migrations/v3.9/st2-migrate-liveaction-executiondb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb index 7a8d3921d5..7780d8152b 100755 --- a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb +++ b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb @@ -35,6 +35,7 @@ import datetime import time import traceback +import mongoengine as me from oslo_config import cfg from st2common import config @@ -45,7 +46,6 @@ from st2common.service_setup import db_teardown from st2common.util import isotime from st2common.models.db.execution import ActionExecutionDB from st2common.persistence.base import Access -from st2common.persistence.execution import ActionExecution from st2common.exceptions.db import StackStormDBObjectNotFoundError from st2common.constants.action import LIVEACTION_COMPLETED_STATES @@ -56,7 +56,8 @@ from st2common.constants.action import LIVEACTION_COMPLETED_STATES class ActionExecutionDBOLD(ActionExecutionDB): liveaction = stormbase.EscapedDictField(required=True) - liveaction_id = me.StringField(required=False) # was not required previously + liveaction_id = me.StringField(required=False) # was not required previously + class ActionExecutionOLD(Access): impl = MongoDBAccess(ActionExecutionDBOLD) @@ -137,7 +138,7 @@ def migrate_executions(start_dt: datetime.datetime, end_dt: datetime.datetime) - execution_db._mark_as_changed("liveaction_id") # NOTE: If you want to view changed fields, you can access execution_db._changed_fields # will throw an exception if already a string - execution_db.liveaction_id = execution_db.liveaction.get("id", None) + execution_db.liveaction_id = execution_db.liveaction.get("id") execution_db.save() print("ActionExecutionDB with id %s has been migrated" % (execution_db.id)) From 4df8a9ab8c10a67a8153353e016fe2f50cb31b85 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Wed, 19 Jul 2023 17:23:27 +0000 Subject: [PATCH 59/70] add inheritance --- .../bin/migrations/v3.9/st2-migrate-liveaction-executiondb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb index 7780d8152b..8c747db77d 100755 --- a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb +++ b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb @@ -55,6 +55,8 @@ from st2common.constants.action import LIVEACTION_COMPLETED_STATES class ActionExecutionDBOLD(ActionExecutionDB): + ActionExecutionDB._meta["allow_inheritance"] = True + liveaction = stormbase.EscapedDictField(required=True) liveaction_id = me.StringField(required=False) # was not required previously From 619496c638c95e5281a1de75627e132012990b6e Mon Sep 17 00:00:00 2001 From: guzzijones Date: Wed, 19 Jul 2023 18:17:14 +0000 Subject: [PATCH 60/70] pymongo query for setting liveaction id --- .../v3.9/st2-migrate-liveaction-executiondb | 100 ++++++------------ 1 file changed, 34 insertions(+), 66 deletions(-) diff --git a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb index 8c747db77d..dde6efa567 100755 --- a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb +++ b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb @@ -35,7 +35,6 @@ import datetime import time import traceback -import mongoengine as me from oslo_config import cfg from st2common import config @@ -54,31 +53,6 @@ from st2common.constants.action import LIVEACTION_COMPLETED_STATES # single value -class ActionExecutionDBOLD(ActionExecutionDB): - ActionExecutionDB._meta["allow_inheritance"] = True - - liveaction = stormbase.EscapedDictField(required=True) - liveaction_id = me.StringField(required=False) # was not required previously - - -class ActionExecutionOLD(Access): - impl = MongoDBAccess(ActionExecutionDBOLD) - publisher = None - - @classmethod - def _get_impl(cls): - return cls.impl - - @classmethod - def _get_publisher(cls): - if not cls.publisher: - cls.publisher = transport.execution.ActionExecutionPublisher() - return cls.publisher - - @classmethod - def delete_by_query(cls, *args, **query): - return cls._get_impl().delete_by_query(*args, **query) - def migrate_executions(start_dt: datetime.datetime, end_dt: datetime.datetime) -> None: """ @@ -92,10 +66,9 @@ def migrate_executions(start_dt: datetime.datetime, end_dt: datetime.datetime) - # objects one by one. # Keep in mind we need to use ModelClass.objects and not PersistanceClass.query() so .only() # works correctly - with PersistanceClass.query().only() all the fields will still be retrieved. - # 1. Migrate ActionExecutionDB objects - result = ( - ActionExecutionDBOLD.objects( + result_add_liveaction_id = ( + ActionExecutionDB.objects( __raw__={ "status": { "$in": LIVEACTION_COMPLETED_STATES, @@ -103,47 +76,42 @@ def migrate_executions(start_dt: datetime.datetime, end_dt: datetime.datetime) - }, start_timestamp__gte=start_dt, start_timestamp__lte=end_dt, - ) - .only("id") - .as_pymongo() + ).update( + __raw__={ + "$set": {'liveaction_id': {"$concat": ["$liveaction.id"]}} + } + ) ) - execution_ids = set([str(item["_id"]) for item in result]) - objects_count = result.count() - - if not execution_ids: - print("Found no ActionExecutionDB objects to migrate.") - print("") - return None - - print("Will migrate %s ActionExecutionDB objects" % (objects_count)) - print("") - - for index, execution_id in enumerate(execution_ids, 1): - try: - execution_db = ActionExecutionOLD.get_by_id(execution_id) - except StackStormDBObjectNotFoundError: - print( - "Skipping ActionExecutionDB with id %s which is missing in the database" - % (execution_id) + result_remove_liveaction = ( + ActionExecutionDB.objects( + __raw__={ + "status": { + "$in": LIVEACTION_COMPLETED_STATES, + }, + }, + start_timestamp__gte=start_dt, + start_timestamp__lte=end_dt, + ).update( + __raw__={ + "$unset": {'liveaction': 1 } + } ) - continue - - print( - "[%s/%s] Migrating ActionExecutionDB with id %s" - % (index, objects_count, execution_id) + ) + + res_count = ActionExecutionDB.objects( + __raw__={ + "status": { + "$in": LIVEACTION_COMPLETED_STATES, + }, + }, + start_timestamp__gte=start_dt, + start_timestamp__lte=end_dt, ) + import pdb; pdb.set_trace() + objects_count = res_count.count() - # This is a bit of a "hack", but it's the easiest way to tell mongoengine that a specific - # field has been updated and should be saved. If we don't do, nothing will be re-saved on - # .save() call due to mongoengine only trying to save what has changed to make it more - # efficient instead of always re-saving the whole object. - execution_db._mark_as_changed("liveaction_id") - # NOTE: If you want to view changed fields, you can access execution_db._changed_fields - # will throw an exception if already a string - execution_db.liveaction_id = execution_db.liveaction.get("id") - execution_db.save() - print("ActionExecutionDB with id %s has been migrated" % (execution_db.id)) - + print("migrated %s ActionExecutionDB objects" % (objects_count)) + print("") def _register_cli_opts(): cfg.CONF.register_cli_opt( From 82a1e302e02aabdee229cd84829ae3f1d8513445 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Wed, 19 Jul 2023 19:10:19 +0000 Subject: [PATCH 61/70] working migration script --- .../v3.9/st2-migrate-liveaction-executiondb | 65 +++++++------------ st2common/st2common/fields.py | 2 +- 2 files changed, 23 insertions(+), 44 deletions(-) diff --git a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb index dde6efa567..cc0d4c1fa0 100755 --- a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb +++ b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb @@ -38,14 +38,10 @@ import traceback from oslo_config import cfg from st2common import config -from st2common import transport -from st2common.models.db import MongoDBAccess, stormbase from st2common.service_setup import db_setup from st2common.service_setup import db_teardown from st2common.util import isotime from st2common.models.db.execution import ActionExecutionDB -from st2common.persistence.base import Access -from st2common.exceptions.db import StackStormDBObjectNotFoundError from st2common.constants.action import LIVEACTION_COMPLETED_STATES # NOTE: To avoid unnecessary mongoengine object churn when retrieving only object ids (aka to avoid @@ -53,7 +49,6 @@ from st2common.constants.action import LIVEACTION_COMPLETED_STATES # single value - def migrate_executions(start_dt: datetime.datetime, end_dt: datetime.datetime) -> None: """ Perform migrations for execution related objects (ActionExecutionDB, LiveActionDB). @@ -67,52 +62,36 @@ def migrate_executions(start_dt: datetime.datetime, end_dt: datetime.datetime) - # Keep in mind we need to use ModelClass.objects and not PersistanceClass.query() so .only() # works correctly - with PersistanceClass.query().only() all the fields will still be retrieved. # 1. Migrate ActionExecutionDB objects - result_add_liveaction_id = ( - ActionExecutionDB.objects( - __raw__={ - "status": { - "$in": LIVEACTION_COMPLETED_STATES, - }, + res_count = ActionExecutionDB.objects( + __raw__={ + "status": { + "$in": LIVEACTION_COMPLETED_STATES, }, - start_timestamp__gte=start_dt, - start_timestamp__lte=end_dt, - ).update( - __raw__={ - "$set": {'liveaction_id': {"$concat": ["$liveaction.id"]}} - } - ) - ) - result_remove_liveaction = ( + }, + start_timestamp__gte=start_dt, + start_timestamp__lte=end_dt, + ).as_pymongo() + for item in res_count: ActionExecutionDB.objects( - __raw__={ - "status": { - "$in": LIVEACTION_COMPLETED_STATES, - }, - }, - start_timestamp__gte=start_dt, - start_timestamp__lte=end_dt, - ).update( - __raw__={ - "$unset": {'liveaction': 1 } - } - ) - ) - - res_count = ActionExecutionDB.objects( - __raw__={ - "status": { - "$in": LIVEACTION_COMPLETED_STATES, - }, + __raw__={"_id": item["_id"]} + ).update(__raw__={"$set": {"liveaction_id": item["liveaction"]["id"]}}) + + ActionExecutionDB.objects( + __raw__={ + "status": { + "$in": LIVEACTION_COMPLETED_STATES, }, - start_timestamp__gte=start_dt, - start_timestamp__lte=end_dt, - ) - import pdb; pdb.set_trace() + }, + start_timestamp__gte=start_dt, + start_timestamp__lte=end_dt, + ).update(__raw__={"$unset": {"liveaction": 1}}) + objects_count = res_count.count() print("migrated %s ActionExecutionDB objects" % (objects_count)) print("") + def _register_cli_opts(): cfg.CONF.register_cli_opt( cfg.BoolOpt( diff --git a/st2common/st2common/fields.py b/st2common/st2common/fields.py index 370a66400f..c94151e45e 100644 --- a/st2common/st2common/fields.py +++ b/st2common/st2common/fields.py @@ -39,7 +39,7 @@ from st2common.constants.compression import ( compress as compress_function, - uncompress as uncompress_function + uncompress as uncompress_function, ) from st2common.util import date as date_utils from st2common.util import mongoescape From 766dab1f09afd8a28bacebf576eb4fab1745a333 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Wed, 19 Jul 2023 19:17:50 +0000 Subject: [PATCH 62/70] black fix --- .../bin/migrations/v3.9/st2-migrate-liveaction-executiondb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb index cc0d4c1fa0..ecb002fa63 100755 --- a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb +++ b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb @@ -72,9 +72,9 @@ def migrate_executions(start_dt: datetime.datetime, end_dt: datetime.datetime) - start_timestamp__lte=end_dt, ).as_pymongo() for item in res_count: - ActionExecutionDB.objects( - __raw__={"_id": item["_id"]} - ).update(__raw__={"$set": {"liveaction_id": item["liveaction"]["id"]}}) + ActionExecutionDB.objects(__raw__={"_id": item["_id"]}).update( + __raw__={"$set": {"liveaction_id": item["liveaction"]["id"]}} + ) ActionExecutionDB.objects( __raw__={ From 27435183d9bd43dbc5d99fab346990d71b04c66d Mon Sep 17 00:00:00 2001 From: guzzijones Date: Wed, 19 Jul 2023 19:40:46 +0000 Subject: [PATCH 63/70] add required fields for liveaction in case where liveaction cannot be found in api response --- st2common/st2common/models/api/execution.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/st2common/st2common/models/api/execution.py b/st2common/st2common/models/api/execution.py index 930f2e8f08..2654d5cc52 100644 --- a/st2common/st2common/models/api/execution.py +++ b/st2common/st2common/models/api/execution.py @@ -166,7 +166,11 @@ def from_model(cls, model, mask_secrets=False): live_action_model, mask_secrets=mask_secrets ) else: - doc["liveaction"] = {"id": doc["liveaction_id"]} + doc["liveaction"] = { + "action": doc["action"]["name"], + "id": doc["liveaction_id"], + "status": doc["status"], + } end_timestamp = model.end_timestamp if end_timestamp: From e6cac68c9fd92ae10c5891c789d812b6099659d5 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Wed, 19 Jul 2023 20:09:45 +0000 Subject: [PATCH 64/70] remove breakpoint --- st2common/tests/unit/test_db_liveaction.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/st2common/tests/unit/test_db_liveaction.py b/st2common/tests/unit/test_db_liveaction.py index 0a9e1dedea..9cc308ad38 100644 --- a/st2common/tests/unit/test_db_liveaction.py +++ b/st2common/tests/unit/test_db_liveaction.py @@ -155,9 +155,6 @@ def test_liveaction_inquiry_response_action(self): self.assertEqual( saved.action, retrieved.action, "Same triggertype was not returned." ) - import pdb - - pdb.set_trace() masked = retrieved.mask_secrets(retrieved.to_serializable_dict()) for value in masked["parameters"]["response"].values(): self.assertEqual(value, MASKED_ATTRIBUTE_VALUE) From 1e109090ab56fc6125646d4be99fad2a8ec15cc4 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Mon, 31 Jul 2023 23:00:09 +0000 Subject: [PATCH 65/70] migrate paused inquiries --- .../bin/migrations/v3.9/st2-migrate-liveaction-executiondb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb index ecb002fa63..434ae2cad8 100755 --- a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb +++ b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb @@ -42,7 +42,7 @@ from st2common.service_setup import db_setup from st2common.service_setup import db_teardown from st2common.util import isotime from st2common.models.db.execution import ActionExecutionDB -from st2common.constants.action import LIVEACTION_COMPLETED_STATES +from st2common.constants.action import LIVEACTION_COMPLETED_STATES, LIVEACTION_STATUS_PAUSED # NOTE: To avoid unnecessary mongoengine object churn when retrieving only object ids (aka to avoid # instantiating model class with a single field), we use raw pymongo value which is a dict with a @@ -65,7 +65,7 @@ def migrate_executions(start_dt: datetime.datetime, end_dt: datetime.datetime) - res_count = ActionExecutionDB.objects( __raw__={ "status": { - "$in": LIVEACTION_COMPLETED_STATES, + "$in": LIVEACTION_COMPLETED_STATES + [LIVEACTION_STATUS_PAUSED], }, }, start_timestamp__gte=start_dt, @@ -79,7 +79,7 @@ def migrate_executions(start_dt: datetime.datetime, end_dt: datetime.datetime) - ActionExecutionDB.objects( __raw__={ "status": { - "$in": LIVEACTION_COMPLETED_STATES, + "$in": LIVEACTION_COMPLETED_STATES + [LIVEACTION_STATUS_PAUSED], }, }, start_timestamp__gte=start_dt, From 78050d06929a9b7ad038286296a6f65a357c23c5 Mon Sep 17 00:00:00 2001 From: AJ Date: Sun, 23 Jul 2023 03:01:20 +0000 Subject: [PATCH 66/70] Update st2common/st2common/fields.py remove comment Co-authored-by: Jacob Floyd --- st2common/st2common/fields.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/st2common/st2common/fields.py b/st2common/st2common/fields.py index c94151e45e..f6a821aa73 100644 --- a/st2common/st2common/fields.py +++ b/st2common/st2common/fields.py @@ -365,9 +365,6 @@ def validate(self, value): def parse_field_value(self, value: Optional[Union[bytes, dict]]) -> dict: """ Parse provided binary field value and return parsed value (dictionary). - - For example: - """ if not value: return self.default From 397e7dc465ed0e407b1aebea868d7b8096216093 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Wed, 2 Aug 2023 14:12:25 +0000 Subject: [PATCH 67/70] black fix migration script --- .../bin/migrations/v3.9/st2-migrate-liveaction-executiondb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb index 434ae2cad8..74a9858edc 100755 --- a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb +++ b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb @@ -42,7 +42,10 @@ from st2common.service_setup import db_setup from st2common.service_setup import db_teardown from st2common.util import isotime from st2common.models.db.execution import ActionExecutionDB -from st2common.constants.action import LIVEACTION_COMPLETED_STATES, LIVEACTION_STATUS_PAUSED +from st2common.constants.action import ( + LIVEACTION_COMPLETED_STATES, + LIVEACTION_STATUS_PAUSED, +) # NOTE: To avoid unnecessary mongoengine object churn when retrieving only object ids (aka to avoid # instantiating model class with a single field), we use raw pymongo value which is a dict with a From d4506dde386fb61016f8a55c1f5196553101fe8c Mon Sep 17 00:00:00 2001 From: guzzijones Date: Tue, 10 Oct 2023 17:55:33 +0000 Subject: [PATCH 68/70] add pending actions to cover inquiries to migration --- .../v3.9/st2-migrate-liveaction-executiondb | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb index 74a9858edc..0fea53a29b 100755 --- a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb +++ b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb @@ -45,6 +45,7 @@ from st2common.models.db.execution import ActionExecutionDB from st2common.constants.action import ( LIVEACTION_COMPLETED_STATES, LIVEACTION_STATUS_PAUSED, + LIVEACTION_STATUS_PENDING ) # NOTE: To avoid unnecessary mongoengine object churn when retrieving only object ids (aka to avoid @@ -68,21 +69,24 @@ def migrate_executions(start_dt: datetime.datetime, end_dt: datetime.datetime) - res_count = ActionExecutionDB.objects( __raw__={ "status": { - "$in": LIVEACTION_COMPLETED_STATES + [LIVEACTION_STATUS_PAUSED], + "$in": LIVEACTION_COMPLETED_STATES + [LIVEACTION_STATUS_PAUSED, LIVEACTION_STATUS_PENDING], }, }, start_timestamp__gte=start_dt, start_timestamp__lte=end_dt, ).as_pymongo() for item in res_count: - ActionExecutionDB.objects(__raw__={"_id": item["_id"]}).update( - __raw__={"$set": {"liveaction_id": item["liveaction"]["id"]}} - ) + try: + ActionExecutionDB.objects(__raw__={"_id": item["_id"]}).update( + __raw__={"$set": {"liveaction_id": item["liveaction"]["id"]}} + ) + except KeyError as e: + pass ActionExecutionDB.objects( __raw__={ "status": { - "$in": LIVEACTION_COMPLETED_STATES + [LIVEACTION_STATUS_PAUSED], + "$in": LIVEACTION_COMPLETED_STATES + [LIVEACTION_STATUS_PAUSED, LIVEACTION_STATUS_PENDING], }, }, start_timestamp__gte=start_dt, From 450294a9a26c27dd89c5e838016a096d33159bd6 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Tue, 10 Oct 2023 18:18:29 +0000 Subject: [PATCH 69/70] black and lint fixes --- .../migrations/v3.9/st2-migrate-liveaction-executiondb | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb index 0fea53a29b..76e69bc845 100755 --- a/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb +++ b/st2common/bin/migrations/v3.9/st2-migrate-liveaction-executiondb @@ -45,7 +45,7 @@ from st2common.models.db.execution import ActionExecutionDB from st2common.constants.action import ( LIVEACTION_COMPLETED_STATES, LIVEACTION_STATUS_PAUSED, - LIVEACTION_STATUS_PENDING + LIVEACTION_STATUS_PENDING, ) # NOTE: To avoid unnecessary mongoengine object churn when retrieving only object ids (aka to avoid @@ -69,7 +69,8 @@ def migrate_executions(start_dt: datetime.datetime, end_dt: datetime.datetime) - res_count = ActionExecutionDB.objects( __raw__={ "status": { - "$in": LIVEACTION_COMPLETED_STATES + [LIVEACTION_STATUS_PAUSED, LIVEACTION_STATUS_PENDING], + "$in": LIVEACTION_COMPLETED_STATES + + [LIVEACTION_STATUS_PAUSED, LIVEACTION_STATUS_PENDING], }, }, start_timestamp__gte=start_dt, @@ -80,13 +81,14 @@ def migrate_executions(start_dt: datetime.datetime, end_dt: datetime.datetime) - ActionExecutionDB.objects(__raw__={"_id": item["_id"]}).update( __raw__={"$set": {"liveaction_id": item["liveaction"]["id"]}} ) - except KeyError as e: + except KeyError: pass ActionExecutionDB.objects( __raw__={ "status": { - "$in": LIVEACTION_COMPLETED_STATES + [LIVEACTION_STATUS_PAUSED, LIVEACTION_STATUS_PENDING], + "$in": LIVEACTION_COMPLETED_STATES + + [LIVEACTION_STATUS_PAUSED, LIVEACTION_STATUS_PENDING], }, }, start_timestamp__gte=start_dt, From 6b3bbd577eacd514b733887f99babb4364e67260 Mon Sep 17 00:00:00 2001 From: rebrowning Date: Thu, 14 Mar 2024 17:53:24 -0700 Subject: [PATCH 70/70] remove the liveaction payload -> id change but leave the zstandard compression --- CHANGELOG.rst | 8 +- .../action_chain_runner.py | 6 +- .../tests/unit/test_actionchain_cancel.py | 12 +- .../unit/test_actionchain_notifications.py | 8 +- .../unit/test_actionchain_pause_resume.py | 14 +- .../inquirer_runner/inquirer_runner.py | 2 +- .../orquesta_runner/orquesta_runner.py | 6 +- .../orquesta_runner/tests/unit/test_basic.py | 20 +- .../orquesta_runner/tests/unit/test_cancel.py | 16 +- .../tests/unit/test_data_flow.py | 8 +- .../tests/unit/test_error_handling.py | 29 ++- .../tests/unit/test_functions_common.py | 2 +- .../tests/unit/test_functions_task.py | 4 +- .../tests/unit/test_inquiries.py | 44 ++-- .../orquesta_runner/tests/unit/test_notify.py | 14 +- .../tests/unit/test_pause_and_resume.py | 58 +++--- .../orquesta_runner/tests/unit/test_rerun.py | 24 +-- .../tests/unit/test_with_items.py | 8 +- st2actions/st2actions/container/base.py | 4 +- st2actions/st2actions/notifier/notifier.py | 2 +- st2actions/st2actions/scheduler/entrypoint.py | 2 +- st2actions/st2actions/scheduler/handler.py | 2 +- st2actions/st2actions/worker.py | 6 +- st2actions/st2actions/workflows/workflows.py | 5 +- .../tests/unit/policies/test_concurrency.py | 2 +- .../tests/unit/policies/test_retry_policy.py | 6 +- st2actions/tests/unit/test_executions.py | 15 +- st2actions/tests/unit/test_notifier.py | 15 +- .../st2api/controllers/v1/actionexecutions.py | 10 +- .../st2api/controllers/v1/aliasexecution.py | 1 + .../st2api/controllers/v1/execution_views.py | 12 +- .../controllers/v1/test_alias_execution.py | 1 + .../unit/controllers/v1/test_executions.py | 25 +-- .../v1/test_executions_descendants.py | 7 +- .../controllers/v1/test_executions_filters.py | 30 +-- st2client/in-requirements.txt | 1 - st2common/BUILD | 1 - st2common/bin/st2-track-result | 4 +- st2common/setup.py | 1 - .../garbage_collection/executions.py | 2 +- .../st2common/garbage_collection/inquiries.py | 2 +- st2common/st2common/models/api/base.py | 1 - st2common/st2common/models/api/execution.py | 20 +- st2common/st2common/models/api/inquiry.py | 7 +- st2common/st2common/models/db/execution.py | 44 +++- st2common/st2common/models/db/liveaction.py | 3 +- st2common/st2common/models/db/stormbase.py | 2 - st2common/st2common/openapi.yaml | 2 - st2common/st2common/openapi.yaml.j2 | 2 - st2common/st2common/services/action.py | 16 +- st2common/st2common/services/executions.py | 9 +- st2common/st2common/services/inquiry.py | 6 +- st2common/st2common/services/trace.py | 2 +- st2common/st2common/services/workflows.py | 23 +-- st2common/st2common/util/param.py | 1 - .../test_v35_migrate_db_dict_field_values.py | 188 +----------------- st2common/tests/unit/services/test_trace.py | 4 +- .../test_workflow_identify_orphans.py | 4 +- .../services/test_workflow_service_retries.py | 8 +- st2common/tests/unit/test_db_execution.py | 24 ++- st2common/tests/unit/test_db_liveaction.py | 27 --- st2common/tests/unit/test_executions.py | 81 +++----- st2common/tests/unit/test_executions_util.py | 14 +- st2common/tests/unit/test_purge_executions.py | 2 +- .../integration/test_garbage_collector.py | 6 +- .../tests/unit/controllers/v1/test_stream.py | 2 +- .../v1/test_stream_execution_output.py | 20 +- .../orquesta/test_wiring_error_handling.py | 70 ++++--- st2tests/st2tests/api.py | 3 +- .../descendants/executions/child1_level1.yaml | 3 +- .../descendants/executions/child1_level2.yaml | 3 +- .../descendants/executions/child1_level3.yaml | 3 +- .../descendants/executions/child2_level1.yaml | 3 +- .../descendants/executions/child2_level2.yaml | 3 +- .../descendants/executions/child2_level3.yaml | 3 +- .../descendants/executions/child3_level2.yaml | 3 +- .../descendants/executions/child3_level3.yaml | 3 +- .../executions/root_execution.yaml | 3 +- .../generic/executions/execution1.yaml | 12 +- .../generic/liveactions/parentliveaction.yaml | 2 +- .../actions/workflows/__init__.py | 0 .../packs/executions/liveactions.yaml | 3 - .../executions/execution1.yaml | 12 +- .../executions/execution_with_parent.yaml | 12 +- .../executions/rule_fired_execution.yaml | 12 +- .../executions/traceable_execution.yaml | 12 +- 86 files changed, 464 insertions(+), 658 deletions(-) create mode 100644 st2tests/st2tests/fixtures/packs/dummy_pack_23/actions/workflows/__init__.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 01acc16fe5..ff364d3094 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -3,14 +3,12 @@ Changelog in development -------------- -* implemented zstandard compression for parameters and results. #5995 - contributed by @guzzijones12 - -* removed embedded liveaction in action execution database table #5995 - contributed by @guzzijones12 Python 3.6 is no longer supported; Stackstorm requires at least Python 3.8. +* implemented zstandard compression for parameters and results. #5995 + contributed by @guzzijones12 + Fixed ~~~~~ * Restore Pack integration testing (it was inadvertently skipped) and stop testing against `bionic` and `el7`. #6135 diff --git a/contrib/runners/action_chain_runner/action_chain_runner/action_chain_runner.py b/contrib/runners/action_chain_runner/action_chain_runner/action_chain_runner.py index b923a38c20..57c015adcf 100644 --- a/contrib/runners/action_chain_runner/action_chain_runner/action_chain_runner.py +++ b/contrib/runners/action_chain_runner/action_chain_runner/action_chain_runner.py @@ -333,7 +333,7 @@ def cancel(self): and child_exec.status in action_constants.LIVEACTION_CANCELABLE_STATES ): action_service.request_cancellation( - LiveAction.get(id=child_exec.liveaction_id), + LiveAction.get(id=child_exec.liveaction["id"]), self.context.get("user", None), ) @@ -353,7 +353,7 @@ def pause(self): and child_exec.status == action_constants.LIVEACTION_STATUS_RUNNING ): action_service.request_pause( - LiveAction.get(id=child_exec.liveaction_id), + LiveAction.get(id=child_exec.liveaction["id"]), self.context.get("user", None), ) @@ -966,7 +966,7 @@ def _format_action_exec_result( execution_db = None if liveaction_db: - execution_db = ActionExecution.get(liveaction_id=str(liveaction_db.id)) + execution_db = ActionExecution.get(liveaction__id=str(liveaction_db.id)) result["id"] = action_node.name result["name"] = action_node.name diff --git a/contrib/runners/action_chain_runner/tests/unit/test_actionchain_cancel.py b/contrib/runners/action_chain_runner/tests/unit/test_actionchain_cancel.py index e04d5c01b1..bf6ffdd47a 100644 --- a/contrib/runners/action_chain_runner/tests/unit/test_actionchain_cancel.py +++ b/contrib/runners/action_chain_runner/tests/unit/test_actionchain_cancel.py @@ -171,7 +171,7 @@ def test_chain_cancel_cascade_to_subworkflow(self): # Wait until the subworkflow is running. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction_id) + task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"]) task1_live = self._wait_on_status( task1_live, action_constants.LIVEACTION_STATUS_RUNNING ) @@ -189,7 +189,7 @@ def test_chain_cancel_cascade_to_subworkflow(self): # Wait until the subworkflow is canceling. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction_id) + task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"]) task1_live = self._wait_on_status( task1_live, action_constants.LIVEACTION_STATUS_CANCELING ) @@ -206,7 +206,7 @@ def test_chain_cancel_cascade_to_subworkflow(self): # Wait until the subworkflow is canceled. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction_id) + task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"]) task1_live = self._wait_on_status( task1_live, action_constants.LIVEACTION_STATUS_CANCELED ) @@ -248,7 +248,7 @@ def test_chain_cancel_cascade_to_parent_workflow(self): # Wait until the subworkflow is running. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction_id) + task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"]) task1_live = self._wait_on_status( task1_live, action_constants.LIVEACTION_STATUS_RUNNING ) @@ -260,7 +260,7 @@ def test_chain_cancel_cascade_to_parent_workflow(self): # Wait until the subworkflow is canceling. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction_id) + task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"]) task1_live = self._wait_on_status( task1_live, action_constants.LIVEACTION_STATUS_CANCELING ) @@ -271,7 +271,7 @@ def test_chain_cancel_cascade_to_parent_workflow(self): # Wait until the subworkflow is canceled. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction_id) + task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"]) task1_live = self._wait_on_status( task1_live, action_constants.LIVEACTION_STATUS_CANCELED ) diff --git a/contrib/runners/action_chain_runner/tests/unit/test_actionchain_notifications.py b/contrib/runners/action_chain_runner/tests/unit/test_actionchain_notifications.py index df1c1567d9..f2f2a680c8 100644 --- a/contrib/runners/action_chain_runner/tests/unit/test_actionchain_notifications.py +++ b/contrib/runners/action_chain_runner/tests/unit/test_actionchain_notifications.py @@ -151,7 +151,7 @@ def test_skip_notify_for_task_with_notify(self): # Assert task1 notify is skipped task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction_id) + task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"]) task1_live = self._wait_on_status( task1_live, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -162,7 +162,7 @@ def test_skip_notify_for_task_with_notify(self): # Assert task2 notify is not skipped task2_exec = ActionExecution.get_by_id(execution.children[1]) - task2_live = LiveAction.get_by_id(task2_exec.liveaction_id) + task2_live = LiveAction.get_by_id(task2_exec.liveaction["id"]) notify = notify_api_models.NotificationsHelper.from_model( notify_model=task2_live.notify ) @@ -186,7 +186,7 @@ def test_skip_notify_default_for_task_with_notify(self): # Assert task1 notify is set. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction_id) + task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"]) task1_live = self._wait_on_status( task1_live, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -200,7 +200,7 @@ def test_skip_notify_default_for_task_with_notify(self): # Assert task2 notify is not skipped by default. task2_exec = ActionExecution.get_by_id(execution.children[1]) - task2_live = LiveAction.get_by_id(task2_exec.liveaction_id) + task2_live = LiveAction.get_by_id(task2_exec.liveaction["id"]) self.assertIsNone(task2_live.notify) MockLiveActionPublisherNonBlocking.wait_all() diff --git a/contrib/runners/action_chain_runner/tests/unit/test_actionchain_pause_resume.py b/contrib/runners/action_chain_runner/tests/unit/test_actionchain_pause_resume.py index 6187522d42..e0dfecc4d6 100644 --- a/contrib/runners/action_chain_runner/tests/unit/test_actionchain_pause_resume.py +++ b/contrib/runners/action_chain_runner/tests/unit/test_actionchain_pause_resume.py @@ -431,7 +431,7 @@ def test_chain_pause_resume_cascade_to_subworkflow(self): # Wait until the subworkflow is running. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction_id) + task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"]) task1_live = self._wait_for_status( task1_live, action_constants.LIVEACTION_STATUS_RUNNING ) @@ -452,7 +452,7 @@ def test_chain_pause_resume_cascade_to_subworkflow(self): # Wait until the subworkflow is pausing. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction_id) + task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"]) task1_live = self._wait_for_status( task1_live, action_constants.LIVEACTION_STATUS_PAUSING ) @@ -477,7 +477,7 @@ def test_chain_pause_resume_cascade_to_subworkflow(self): # Wait until the subworkflow is paused. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction_id) + task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"]) task1_live = self._wait_for_status( task1_live, action_constants.LIVEACTION_STATUS_PAUSED ) @@ -548,7 +548,7 @@ def test_chain_pause_resume_cascade_to_parent_workflow(self): # Wait until the subworkflow is running. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction_id) + task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"]) task1_live = self._wait_for_status( task1_live, action_constants.LIVEACTION_STATUS_RUNNING ) @@ -559,7 +559,7 @@ def test_chain_pause_resume_cascade_to_parent_workflow(self): # Wait until the subworkflow is pausing. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction_id) + task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"]) task1_live = self._wait_for_status( task1_live, action_constants.LIVEACTION_STATUS_PAUSING ) @@ -574,7 +574,7 @@ def test_chain_pause_resume_cascade_to_parent_workflow(self): # Wait until the subworkflow is paused. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction_id) + task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"]) task1_live = self._wait_for_status( task1_live, action_constants.LIVEACTION_STATUS_PAUSED ) @@ -611,7 +611,7 @@ def test_chain_pause_resume_cascade_to_parent_workflow(self): # Wait until the subworkflow is paused. task1_exec = ActionExecution.get_by_id(execution.children[0]) - task1_live = LiveAction.get_by_id(task1_exec.liveaction_id) + task1_live = LiveAction.get_by_id(task1_exec.liveaction["id"]) task1_live = self._wait_for_status( task1_live, action_constants.LIVEACTION_STATUS_SUCCEEDED ) diff --git a/contrib/runners/inquirer_runner/inquirer_runner/inquirer_runner.py b/contrib/runners/inquirer_runner/inquirer_runner/inquirer_runner.py index f080c7ab30..af0f0c6f34 100644 --- a/contrib/runners/inquirer_runner/inquirer_runner/inquirer_runner.py +++ b/contrib/runners/inquirer_runner/inquirer_runner/inquirer_runner.py @@ -74,7 +74,7 @@ def pre_run(self): def run(self, action_parameters): liveaction_db = action_utils.get_liveaction_by_id(self.liveaction_id) - exc = ex_db_access.ActionExecution.get(liveaction_id=str(liveaction_db.id)) + exc = ex_db_access.ActionExecution.get(liveaction__id=str(liveaction_db.id)) # Assemble and dispatch trigger trigger_ref = sys_db_models.ResourceReference.to_string_reference( diff --git a/contrib/runners/orquesta_runner/orquesta_runner/orquesta_runner.py b/contrib/runners/orquesta_runner/orquesta_runner/orquesta_runner.py index 717ec979c4..4e5db3dfe8 100644 --- a/contrib/runners/orquesta_runner/orquesta_runner/orquesta_runner.py +++ b/contrib/runners/orquesta_runner/orquesta_runner/orquesta_runner.py @@ -188,7 +188,7 @@ def pause(self): child_ex = ex_db_access.ActionExecution.get(id=child_ex_id) if self.task_pauseable(child_ex): ac_svc.request_pause( - lv_db_access.LiveAction.get(id=child_ex.liveaction_id), + lv_db_access.LiveAction.get(id=child_ex.liveaction["id"]), self.context.get("user", None), ) @@ -219,7 +219,7 @@ def resume(self): child_ex = ex_db_access.ActionExecution.get(id=child_ex_id) if self.task_resumeable(child_ex): ac_svc.request_resume( - lv_db_access.LiveAction.get(id=child_ex.liveaction_id), + lv_db_access.LiveAction.get(id=child_ex.liveaction["id"]), self.context.get("user", None), ) @@ -280,7 +280,7 @@ def cancel(self): child_ex = ex_db_access.ActionExecution.get(id=child_ex_id) if self.task_cancelable(child_ex): ac_svc.request_cancellation( - lv_db_access.LiveAction.get(id=child_ex.liveaction_id), + lv_db_access.LiveAction.get(id=child_ex.liveaction["id"]), self.context.get("user", None), ) diff --git a/contrib/runners/orquesta_runner/tests/unit/test_basic.py b/contrib/runners/orquesta_runner/tests/unit/test_basic.py index dd4fdd49bf..7c9351423a 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_basic.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_basic.py @@ -184,7 +184,7 @@ def test_run_workflow(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) self.assertEqual(tk1_lv_ac_db.context.get("user"), username) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) self.assertTrue(wf_svc.is_action_execution_under_workflow_context(tk1_ac_ex_db)) @@ -204,7 +204,7 @@ def test_run_workflow(self): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction_id) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction["id"]) self.assertEqual(tk2_lv_ac_db.context.get("user"), username) self.assertEqual(tk2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) self.assertTrue(wf_svc.is_action_execution_under_workflow_context(tk2_ac_ex_db)) @@ -224,7 +224,7 @@ def test_run_workflow(self): tk3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk3_ex_db.id) )[0] - tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction_id) + tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction["id"]) self.assertEqual(tk3_lv_ac_db.context.get("user"), username) self.assertEqual(tk3_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) self.assertTrue(wf_svc.is_action_execution_under_workflow_context(tk3_ac_ex_db)) @@ -274,7 +274,7 @@ def test_run_workflow_with_unicode_input(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) wf_svc.handle_action_execution_completion(tk1_ac_ex_db) tk1_ex_db = wf_db_access.TaskExecution.get_by_id(tk1_ex_db.id) @@ -286,7 +286,7 @@ def test_run_workflow_with_unicode_input(self): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction_id) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction["id"]) self.assertEqual(tk2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) wf_svc.handle_action_execution_completion(tk2_ac_ex_db) tk2_ex_db = wf_db_access.TaskExecution.get_by_id(tk2_ex_db.id) @@ -298,7 +298,7 @@ def test_run_workflow_with_unicode_input(self): tk3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk3_ex_db.id) )[0] - tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction_id) + tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction["id"]) self.assertEqual(tk3_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) wf_svc.handle_action_execution_completion(tk3_ac_ex_db) tk3_ex_db = wf_db_access.TaskExecution.get_by_id(tk3_ex_db.id) @@ -347,7 +347,7 @@ def test_run_workflow_action_config_context(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) self.assertTrue(wf_svc.is_action_execution_under_workflow_context(tk1_ac_ex_db)) @@ -400,7 +400,7 @@ def test_run_workflow_with_action_less_tasks(self): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction_id) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction["id"]) self.assertEqual(tk2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion. @@ -412,7 +412,7 @@ def test_run_workflow_with_action_less_tasks(self): tk3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk3_ex_db.id) )[0] - tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction_id) + tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction["id"]) self.assertEqual(tk3_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion. @@ -433,7 +433,7 @@ def test_run_workflow_with_action_less_tasks(self): tk5_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk5_ex_db.id) )[0] - tk5_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk5_ac_ex_db.liveaction_id) + tk5_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk5_ac_ex_db.liveaction["id"]) self.assertEqual(tk5_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion. diff --git a/contrib/runners/orquesta_runner/tests/unit/test_cancel.py b/contrib/runners/orquesta_runner/tests/unit/test_cancel.py index 1c6df3cf11..cdffd6949d 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_cancel.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_cancel.py @@ -139,7 +139,9 @@ def test_cancel_workflow_cascade_down_to_subworkflow(self): ) self.assertEqual(len(tk_ac_ex_dbs), 1) - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_dbs[0].liveaction_id) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id( + tk_ac_ex_dbs[0].liveaction["id"] + ) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Cancel the main workflow. @@ -180,7 +182,9 @@ def test_cancel_subworkflow_cascade_up_to_workflow(self): ) self.assertEqual(len(tk_ac_ex_dbs), 1) - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_dbs[0].liveaction_id) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id( + tk_ac_ex_dbs[0].liveaction["id"] + ) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Cancel the subworkflow. @@ -226,7 +230,9 @@ def test_cancel_subworkflow_cascade_up_to_workflow_with_other_subworkflows(self) ) self.assertEqual(len(tk1_ac_ex_dbs), 1) - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_dbs[0].liveaction_id) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id( + tk1_ac_ex_dbs[0].liveaction["id"] + ) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) tk2_ac_ex_dbs = ex_db_access.ActionExecution.query( @@ -234,7 +240,9 @@ def test_cancel_subworkflow_cascade_up_to_workflow_with_other_subworkflows(self) ) self.assertEqual(len(tk2_ac_ex_dbs), 1) - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_dbs[0].liveaction_id) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id( + tk2_ac_ex_dbs[0].liveaction["id"] + ) self.assertEqual(tk2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Cancel the subworkflow which should cascade up to the root. diff --git a/contrib/runners/orquesta_runner/tests/unit/test_data_flow.py b/contrib/runners/orquesta_runner/tests/unit/test_data_flow.py index 910ff2cf18..eadadbf469 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_data_flow.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_data_flow.py @@ -143,7 +143,7 @@ def assert_data_flow(self, data): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion. @@ -161,7 +161,7 @@ def assert_data_flow(self, data): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction_id) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction["id"]) self.assertEqual(tk2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion. @@ -179,7 +179,7 @@ def assert_data_flow(self, data): tk3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk3_ex_db.id) )[0] - tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction_id) + tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction["id"]) self.assertEqual(tk3_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion. @@ -197,7 +197,7 @@ def assert_data_flow(self, data): tk4_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk4_ex_db.id) )[0] - tk4_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk4_ac_ex_db.liveaction_id) + tk4_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk4_ac_ex_db.liveaction["id"]) self.assertEqual(tk4_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion. diff --git a/contrib/runners/orquesta_runner/tests/unit/test_error_handling.py b/contrib/runners/orquesta_runner/tests/unit/test_error_handling.py index e1482fb3af..a11d2eed11 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_error_handling.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_error_handling.py @@ -378,7 +378,6 @@ def test_fail_start_task_input_value_type(self): lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id)) self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED) - self.assertDictEqual(lv_ac_db.result, expected_result) ac_ex_db = ex_db_access.ActionExecution.get_by_id(str(ac_ex_db.id)) self.assertEqual(ac_ex_db.status, ac_const.LIVEACTION_STATUS_FAILED) @@ -415,7 +414,7 @@ def test_fail_next_task_action(self): tk_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_db.id) )[0] - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction_id) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction["id"]) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion for task1 which has an error in publish. @@ -471,7 +470,7 @@ def test_fail_next_task_input_expr_eval(self): tk_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_db.id) )[0] - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction_id) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction["id"]) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion for task1 which has an error in publish. @@ -523,7 +522,7 @@ def test_fail_next_task_input_value_type(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) self.assertEqual(wf_ex_db.status, wf_statuses.RUNNING) @@ -607,7 +606,7 @@ def test_fail_task_execution(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED) wf_svc.handle_action_execution_completion(tk1_ac_ex_db) @@ -658,7 +657,7 @@ def test_fail_task_transition(self): tk_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_db.id) )[0] - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction_id) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction["id"]) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion for task1 which has an error in publish. @@ -714,7 +713,7 @@ def test_fail_task_publish(self): tk_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_db.id) )[0] - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction_id) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction["id"]) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion for task1 which has an error in publish. @@ -767,7 +766,7 @@ def test_fail_output_rendering(self): tk_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_db.id) )[0] - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction_id) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction["id"]) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually handle action execution completion for task1 which has an error in publish. @@ -823,7 +822,7 @@ def test_output_on_error(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) wf_svc.handle_action_execution_completion(tk1_ac_ex_db) wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id) @@ -835,7 +834,7 @@ def test_output_on_error(self): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction_id) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction["id"]) self.assertEqual(tk2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED) wf_svc.handle_action_execution_completion(tk2_ac_ex_db) @@ -866,7 +865,7 @@ def test_fail_manually(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED) wf_svc.handle_action_execution_completion(tk1_ac_ex_db) wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id) @@ -878,7 +877,7 @@ def test_fail_manually(self): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction_id) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction["id"]) self.assertEqual(tk2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) wf_svc.handle_action_execution_completion(tk2_ac_ex_db) wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id) @@ -924,7 +923,7 @@ def test_fail_manually_with_recovery_failure(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED) wf_svc.handle_action_execution_completion(tk1_ac_ex_db) wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id) @@ -937,7 +936,7 @@ def test_fail_manually_with_recovery_failure(self): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction_id) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction["id"]) self.assertEqual(tk2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED) wf_svc.handle_action_execution_completion(tk2_ac_ex_db) wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id) @@ -1013,7 +1012,7 @@ def test_include_result_to_error_log(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) self.assertEqual(tk1_lv_ac_db.context.get("user"), username) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED) diff --git a/contrib/runners/orquesta_runner/tests/unit/test_functions_common.py b/contrib/runners/orquesta_runner/tests/unit/test_functions_common.py index 6efd2c0f8b..4019f9a890 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_functions_common.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_functions_common.py @@ -115,7 +115,7 @@ def _execute_workflow(self, wf_name, expected_output): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) self.assertTrue(wf_svc.is_action_execution_under_workflow_context(tk1_ac_ex_db)) diff --git a/contrib/runners/orquesta_runner/tests/unit/test_functions_task.py b/contrib/runners/orquesta_runner/tests/unit/test_functions_task.py index 721f5c5de7..b325839c9d 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_functions_task.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_functions_task.py @@ -129,7 +129,9 @@ def _execute_workflow( tk_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_db.id) )[0] - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction_id) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id( + tk_ac_ex_db.liveaction["id"] + ) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) self.assertTrue( diff --git a/contrib/runners/orquesta_runner/tests/unit/test_inquiries.py b/contrib/runners/orquesta_runner/tests/unit/test_inquiries.py index 9e8ad560c3..f60f9415e8 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_inquiries.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_inquiries.py @@ -113,7 +113,7 @@ def test_inquiry(self): t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t1_ex_db.id) )[0] - t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction_id) + t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction["id"]) self.assertEqual( t1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -134,7 +134,7 @@ def test_inquiry(self): t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t2_ex_db.id) )[0] - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) self.assertEqual(t2_lv_ac_db.status, action_constants.LIVEACTION_STATUS_PENDING) workflows.get_engine().process(t2_ac_ex_db) t2_ex_db = wf_db_access.TaskExecution.get_by_id(t2_ex_db.id) @@ -170,7 +170,7 @@ def test_inquiry(self): t3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t3_ex_db.id) )[0] - t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction_id) + t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction["id"]) self.assertEqual( t3_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -203,7 +203,7 @@ def test_consecutive_inquiries(self): t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t1_ex_db.id) )[0] - t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction_id) + t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction["id"]) self.assertEqual( t1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -224,7 +224,7 @@ def test_consecutive_inquiries(self): t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t2_ex_db.id) )[0] - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) self.assertEqual(t2_lv_ac_db.status, action_constants.LIVEACTION_STATUS_PENDING) workflows.get_engine().process(t2_ac_ex_db) t2_ex_db = wf_db_access.TaskExecution.get_by_id(t2_ex_db.id) @@ -263,7 +263,7 @@ def test_consecutive_inquiries(self): t3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t3_ex_db.id) )[0] - t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction_id) + t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction["id"]) self.assertEqual(t3_lv_ac_db.status, action_constants.LIVEACTION_STATUS_PENDING) workflows.get_engine().process(t3_ac_ex_db) t3_ex_db = wf_db_access.TaskExecution.get_by_id(t3_ex_db.id) @@ -299,7 +299,7 @@ def test_consecutive_inquiries(self): t4_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t4_ex_db.id) )[0] - t4_lv_ac_db = lv_db_access.LiveAction.get_by_id(t4_ac_ex_db.liveaction_id) + t4_lv_ac_db = lv_db_access.LiveAction.get_by_id(t4_ac_ex_db.liveaction["id"]) self.assertEqual( t4_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -332,7 +332,7 @@ def test_parallel_inquiries(self): t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t1_ex_db.id) )[0] - t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction_id) + t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction["id"]) self.assertEqual( t1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -350,7 +350,7 @@ def test_parallel_inquiries(self): t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t2_ex_db.id) )[0] - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) self.assertEqual(t2_lv_ac_db.status, action_constants.LIVEACTION_STATUS_PENDING) workflows.get_engine().process(t2_ac_ex_db) t2_ex_db = wf_db_access.TaskExecution.get_by_id(t2_ex_db.id) @@ -366,7 +366,7 @@ def test_parallel_inquiries(self): t3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t3_ex_db.id) )[0] - t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction_id) + t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction["id"]) self.assertEqual(t3_lv_ac_db.status, action_constants.LIVEACTION_STATUS_PENDING) workflows.get_engine().process(t3_ac_ex_db) t3_ex_db = wf_db_access.TaskExecution.get_by_id(t3_ex_db.id) @@ -423,7 +423,7 @@ def test_parallel_inquiries(self): t4_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t4_ex_db.id) )[0] - t4_lv_ac_db = lv_db_access.LiveAction.get_by_id(t4_ac_ex_db.liveaction_id) + t4_lv_ac_db = lv_db_access.LiveAction.get_by_id(t4_ac_ex_db.liveaction["id"]) self.assertEqual( t4_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -456,7 +456,7 @@ def test_nested_inquiry(self): t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t1_ex_db.id) )[0] - t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction_id) + t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction["id"]) self.assertEqual( t1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -477,7 +477,7 @@ def test_nested_inquiry(self): t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t2_ex_db.id) )[0] - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) self.assertEqual(t2_lv_ac_db.status, action_constants.LIVEACTION_STATUS_RUNNING) workflows.get_engine().process(t2_ac_ex_db) t2_ex_db = wf_db_access.TaskExecution.get_by_id(t2_ex_db.id) @@ -493,7 +493,9 @@ def test_nested_inquiry(self): t2_t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t2_t1_ex_db.id) )[0] - t2_t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_t1_ac_ex_db.liveaction_id) + t2_t1_lv_ac_db = lv_db_access.LiveAction.get_by_id( + t2_t1_ac_ex_db.liveaction["id"] + ) self.assertEqual( t2_t1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -512,7 +514,9 @@ def test_nested_inquiry(self): t2_t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t2_t2_ex_db.id) )[0] - t2_t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_t2_ac_ex_db.liveaction_id) + t2_t2_lv_ac_db = lv_db_access.LiveAction.get_by_id( + t2_t2_ac_ex_db.liveaction["id"] + ) self.assertEqual( t2_t2_lv_ac_db.status, action_constants.LIVEACTION_STATUS_PENDING ) @@ -526,7 +530,7 @@ def test_nested_inquiry(self): t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t2_ex_db.id) )[0] - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) self.assertEqual(t2_lv_ac_db.status, action_constants.LIVEACTION_STATUS_PAUSED) workflows.get_engine().process(t2_ac_ex_db) t2_ex_db = wf_db_access.TaskExecution.get_by_id(t2_ex_db.id) @@ -564,7 +568,9 @@ def test_nested_inquiry(self): t2_t3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t2_t3_ex_db.id) )[0] - t2_t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_t3_ac_ex_db.liveaction_id) + t2_t3_lv_ac_db = lv_db_access.LiveAction.get_by_id( + t2_t3_ac_ex_db.liveaction["id"] + ) self.assertEqual( t2_t3_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -576,7 +582,7 @@ def test_nested_inquiry(self): t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t2_ex_db.id) )[0] - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) self.assertEqual( t2_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -592,7 +598,7 @@ def test_nested_inquiry(self): t3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t3_ex_db.id) )[0] - t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction_id) + t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction["id"]) self.assertEqual( t3_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) diff --git a/contrib/runners/orquesta_runner/tests/unit/test_notify.py b/contrib/runners/orquesta_runner/tests/unit/test_notify.py index 04c786fb78..2e7bfcad4f 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_notify.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_notify.py @@ -279,7 +279,7 @@ def test_cascade_notify_to_tasks(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) self.assertIsNone(tk1_lv_ac_db.notify) self.assertEqual( tk1_ac_ex_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED @@ -300,7 +300,7 @@ def test_cascade_notify_to_tasks(self): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction_id) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction["id"]) notify = notify_api_models.NotificationsHelper.from_model( notify_model=tk2_lv_ac_db.notify ) @@ -324,7 +324,7 @@ def test_cascade_notify_to_tasks(self): tk3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk3_ex_db.id) )[0] - tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction_id) + tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction["id"]) self.assertIsNone(tk3_lv_ac_db.notify) self.assertEqual( tk3_ac_ex_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED @@ -371,7 +371,7 @@ def test_notify_task_list_for_task_with_notify(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) self.assertIsNone(tk1_lv_ac_db.notify) # Assert task2 notify is set. query_filters = {"workflow_execution": str(wf_ex_db.id), "task_id": "task2"} @@ -379,7 +379,7 @@ def test_notify_task_list_for_task_with_notify(self): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction_id) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction["id"]) notify = notify_api_models.NotificationsHelper.from_model( notify_model=tk2_lv_ac_db.notify ) @@ -406,7 +406,7 @@ def test_no_notify_for_task_with_notify(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) self.assertIsNone(tk1_lv_ac_db.notify) # Assert task2 notify is not set. @@ -415,5 +415,5 @@ def test_no_notify_for_task_with_notify(self): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction_id) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction["id"]) self.assertIsNone(tk2_lv_ac_db.notify) diff --git a/contrib/runners/orquesta_runner/tests/unit/test_pause_and_resume.py b/contrib/runners/orquesta_runner/tests/unit/test_pause_and_resume.py index bef405cb1c..7473d9db8e 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_pause_and_resume.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_pause_and_resume.py @@ -153,7 +153,9 @@ def test_pause_subworkflow_not_cascade_up_to_workflow(self): ) self.assertEqual(len(tk_ac_ex_dbs), 1) - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_dbs[0].liveaction_id) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id( + tk_ac_ex_dbs[0].liveaction["id"] + ) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Pause the subworkflow. @@ -194,7 +196,7 @@ def test_pause_workflow_cascade_down_to_subworkflow(self): self.assertEqual(len(tk_ac_ex_dbs), 1) tk_ac_ex_db = tk_ac_ex_dbs[0] - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction_id) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction["id"]) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Identify the records for the subworkflow. @@ -261,7 +263,7 @@ def test_pause_subworkflow_while_another_subworkflow_running(self): t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_dbs[0].id) )[0] - t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction_id) + t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction["id"]) t1_wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(t1_ac_ex_db.id) )[0] @@ -271,7 +273,7 @@ def test_pause_subworkflow_while_another_subworkflow_running(self): t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_dbs[1].id) )[0] - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) t2_wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(t2_ac_ex_db.id) )[0] @@ -289,7 +291,7 @@ def test_pause_subworkflow_while_another_subworkflow_running(self): self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Assert the other subworkflow is still running. - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Manually notify action execution completion for the task in the subworkflow. @@ -314,7 +316,7 @@ def test_pause_subworkflow_while_another_subworkflow_running(self): self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Assert the other subworkflow is still running. - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Manually notify action execution completion for the tasks in the other subworkflow. @@ -373,7 +375,7 @@ def test_pause_subworkflow_while_another_subworkflow_completed(self): t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_dbs[0].id) )[0] - t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction_id) + t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction["id"]) t1_wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(t1_ac_ex_db.id) )[0] @@ -383,7 +385,7 @@ def test_pause_subworkflow_while_another_subworkflow_completed(self): t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_dbs[1].id) )[0] - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) t2_wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(t2_ac_ex_db.id) )[0] @@ -401,7 +403,7 @@ def test_pause_subworkflow_while_another_subworkflow_completed(self): self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Assert the other subworkflow is still running. - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Manually notify action execution completion for the tasks in the other subworkflow. @@ -439,7 +441,7 @@ def test_pause_subworkflow_while_another_subworkflow_completed(self): self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Assert the target subworkflow is still pausing. - t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction_id) + t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction["id"]) self.assertEqual(t1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_PAUSING) # Manually notify action execution completion for the task in the subworkflow. @@ -489,7 +491,9 @@ def test_resume(self): tk_ac_ex_dbs = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_dbs[0].id) ) - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_dbs[0].liveaction_id) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id( + tk_ac_ex_dbs[0].liveaction["id"] + ) self.assertEqual(tk_ac_ex_dbs[0].status, ac_const.LIVEACTION_STATUS_SUCCEEDED) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) wf_svc.handle_action_execution_completion(tk_ac_ex_dbs[0]) @@ -546,7 +550,7 @@ def test_resume_cascade_to_subworkflow(self): self.assertEqual(len(tk_ac_ex_dbs), 1) tk_ac_ex_db = tk_ac_ex_dbs[0] - tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction_id) + tk_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk_ac_ex_db.liveaction["id"]) self.assertEqual(tk_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Identify the records for the subworkflow. @@ -622,7 +626,7 @@ def test_resume_from_each_subworkflow_when_parent_is_paused(self): t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_dbs[0].id) )[0] - t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction_id) + t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction["id"]) t1_wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(t1_ac_ex_db.id) )[0] @@ -632,7 +636,7 @@ def test_resume_from_each_subworkflow_when_parent_is_paused(self): t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_dbs[1].id) )[0] - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) t2_wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(t2_ac_ex_db.id) )[0] @@ -650,7 +654,7 @@ def test_resume_from_each_subworkflow_when_parent_is_paused(self): self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Assert the other subworkflow is still running. - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Manually notify action execution completion for the task in the subworkflow. @@ -675,7 +679,7 @@ def test_resume_from_each_subworkflow_when_parent_is_paused(self): self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Assert the other subworkflow is still running. - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Pause the other subworkflow. @@ -769,7 +773,7 @@ def test_resume_from_subworkflow_when_parent_is_paused(self): t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_dbs[0].id) )[0] - t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction_id) + t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction["id"]) t1_wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(t1_ac_ex_db.id) )[0] @@ -779,7 +783,7 @@ def test_resume_from_subworkflow_when_parent_is_paused(self): t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_dbs[1].id) )[0] - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) t2_wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(t2_ac_ex_db.id) )[0] @@ -797,7 +801,7 @@ def test_resume_from_subworkflow_when_parent_is_paused(self): self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Assert the other subworkflow is still running. - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Manually notify action execution completion for the task in the subworkflow. @@ -822,7 +826,7 @@ def test_resume_from_subworkflow_when_parent_is_paused(self): self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Assert the other subworkflow is still running. - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Manually notify action execution completion for the tasks in the other subworkflow. @@ -903,7 +907,7 @@ def test_resume_from_subworkflow_when_parent_is_paused(self): t3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t3_ex_db.id) )[0] - t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction_id) + t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction["id"]) self.assertEqual(t3_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) wf_svc.handle_action_execution_completion(t3_ac_ex_db) @@ -933,7 +937,7 @@ def test_resume_from_subworkflow_when_parent_is_running(self): t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_dbs[0].id) )[0] - t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction_id) + t1_lv_ac_db = lv_db_access.LiveAction.get_by_id(t1_ac_ex_db.liveaction["id"]) t1_wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(t1_ac_ex_db.id) )[0] @@ -943,7 +947,7 @@ def test_resume_from_subworkflow_when_parent_is_running(self): t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk_ex_dbs[1].id) )[0] - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) t2_wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(t2_ac_ex_db.id) )[0] @@ -961,7 +965,7 @@ def test_resume_from_subworkflow_when_parent_is_running(self): self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Assert the other subworkflow is still running. - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Manually notify action execution completion for the task in the subworkflow. @@ -986,7 +990,7 @@ def test_resume_from_subworkflow_when_parent_is_running(self): self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Assert the other subworkflow is still running. - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Resume the subworkflow and assert it is running. @@ -1001,7 +1005,7 @@ def test_resume_from_subworkflow_when_parent_is_running(self): self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Assert the other subworkflow is still running. - t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction_id) + t2_lv_ac_db = lv_db_access.LiveAction.get_by_id(t2_ac_ex_db.liveaction["id"]) self.assertEqual(t2_lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING) # Manually notify action execution completion for the tasks in the subworkflow. @@ -1067,7 +1071,7 @@ def test_resume_from_subworkflow_when_parent_is_running(self): t3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t3_ex_db.id) )[0] - t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction_id) + t3_lv_ac_db = lv_db_access.LiveAction.get_by_id(t3_ac_ex_db.liveaction["id"]) self.assertEqual(t3_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) wf_svc.handle_action_execution_completion(t3_ac_ex_db) diff --git a/contrib/runners/orquesta_runner/tests/unit/test_rerun.py b/contrib/runners/orquesta_runner/tests/unit/test_rerun.py index 22c40aa8c1..420b909e27 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_rerun.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_rerun.py @@ -127,7 +127,7 @@ def test_rerun_workflow(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) self.assertEqual(tk1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_FAILED) workflow_service.handle_action_execution_completion(tk1_ac_ex_db) tk1_ex_db = wf_db_access.TaskExecution.get_by_id(tk1_ex_db.id) @@ -166,7 +166,7 @@ def test_rerun_workflow(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) self.assertEqual( tk1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -196,7 +196,7 @@ def test_rerun_with_missing_workflow_execution_id(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) self.assertEqual(tk1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_FAILED) workflow_service.handle_action_execution_completion(tk1_ac_ex_db) tk1_ex_db = wf_db_access.TaskExecution.get_by_id(tk1_ex_db.id) @@ -264,7 +264,7 @@ def test_rerun_with_invalid_workflow_execution(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) self.assertEqual(tk1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_FAILED) workflow_service.handle_action_execution_completion(tk1_ac_ex_db) tk1_ex_db = wf_db_access.TaskExecution.get_by_id(tk1_ex_db.id) @@ -322,7 +322,7 @@ def test_rerun_workflow_still_running(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) self.assertEqual( tk1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_RUNNING ) @@ -381,7 +381,7 @@ def test_rerun_with_unexpected_error(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) self.assertEqual(tk1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_FAILED) workflow_service.handle_action_execution_completion(tk1_ac_ex_db) tk1_ex_db = wf_db_access.TaskExecution.get_by_id(tk1_ex_db.id) @@ -436,7 +436,7 @@ def test_rerun_workflow_already_succeeded(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) self.assertEqual( tk1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -450,7 +450,7 @@ def test_rerun_workflow_already_succeeded(self): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction_id) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction["id"]) self.assertEqual( tk2_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -464,7 +464,7 @@ def test_rerun_workflow_already_succeeded(self): tk3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk3_ex_db.id) )[0] - tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction_id) + tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction["id"]) self.assertEqual( tk3_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -505,7 +505,7 @@ def test_rerun_workflow_already_succeeded(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) self.assertEqual( tk1_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -522,7 +522,7 @@ def test_rerun_workflow_already_succeeded(self): tk2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk2_ex_db.id) )[0] - tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction_id) + tk2_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk2_ac_ex_db.liveaction["id"]) self.assertEqual( tk2_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) @@ -539,7 +539,7 @@ def test_rerun_workflow_already_succeeded(self): tk3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk3_ex_db.id) )[0] - tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction_id) + tk3_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk3_ac_ex_db.liveaction["id"]) self.assertEqual( tk3_lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED ) diff --git a/contrib/runners/orquesta_runner/tests/unit/test_with_items.py b/contrib/runners/orquesta_runner/tests/unit/test_with_items.py index de9b0bea07..8e8b67bd94 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_with_items.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_with_items.py @@ -368,7 +368,7 @@ def test_with_items_cancellation(self): # Manually succeed the action executions and process completion. for ac_ex in t1_ac_ex_dbs: self.set_execution_status( - ac_ex.liveaction_id, action_constants.LIVEACTION_STATUS_SUCCEEDED + ac_ex.liveaction["id"], action_constants.LIVEACTION_STATUS_SUCCEEDED ) t1_ac_ex_dbs = ex_db_access.ActionExecution.query( @@ -440,7 +440,7 @@ def test_with_items_concurrency_cancellation(self): # Manually succeed the action executions and process completion. for ac_ex in t1_ac_ex_dbs: self.set_execution_status( - ac_ex.liveaction_id, action_constants.LIVEACTION_STATUS_SUCCEEDED + ac_ex.liveaction["id"], action_constants.LIVEACTION_STATUS_SUCCEEDED ) t1_ac_ex_dbs = ex_db_access.ActionExecution.query( @@ -509,7 +509,7 @@ def test_with_items_pause_and_resume(self): # Manually succeed the action executions and process completion. for ac_ex in t1_ac_ex_dbs: self.set_execution_status( - ac_ex.liveaction_id, action_constants.LIVEACTION_STATUS_SUCCEEDED + ac_ex.liveaction["id"], action_constants.LIVEACTION_STATUS_SUCCEEDED ) t1_ac_ex_dbs = ex_db_access.ActionExecution.query( @@ -599,7 +599,7 @@ def test_with_items_concurrency_pause_and_resume(self): # Manually succeed the action executions and process completion. for ac_ex in t1_ac_ex_dbs: self.set_execution_status( - ac_ex.liveaction_id, action_constants.LIVEACTION_STATUS_SUCCEEDED + ac_ex.liveaction["id"], action_constants.LIVEACTION_STATUS_SUCCEEDED ) t1_ac_ex_dbs = ex_db_access.ActionExecution.query( diff --git a/st2actions/st2actions/container/base.py b/st2actions/st2actions/container/base.py index 330fb072c3..6b441c0277 100644 --- a/st2actions/st2actions/container/base.py +++ b/st2actions/st2actions/container/base.py @@ -144,11 +144,11 @@ def _do_run(self, runner): _, ex, tb = sys.exc_info() # mark execution as failed. status = action_constants.LIVEACTION_STATUS_FAILED - # include the error message and traceback to try and provide some hints. LOG.exception( "Failed to run action. traceback: %s" % "".join(traceback.format_tb(tb, 20)) ) + # include the error message and traceback to try and provide some hints. result = { "error": str(ex), "traceback": "".join(traceback.format_tb(tb, 20)), @@ -459,7 +459,7 @@ def _get_runner(self, runner_type_db, action_db, liveaction_db): runner.action_name = action_db.name runner.liveaction = liveaction_db runner.liveaction_id = str(liveaction_db.id) - runner.execution = ActionExecution.get(liveaction_id=str(runner.liveaction_id)) + runner.execution = ActionExecution.get(liveaction__id=runner.liveaction_id) runner.execution_id = str(runner.execution.id) runner.entry_point = resolved_entry_point runner.context = context diff --git a/st2actions/st2actions/notifier/notifier.py b/st2actions/st2actions/notifier/notifier.py index 2b8fe49059..ea1a537733 100644 --- a/st2actions/st2actions/notifier/notifier.py +++ b/st2actions/st2actions/notifier/notifier.py @@ -83,7 +83,7 @@ def process(self, execution_db): LOG.debug('Processing action execution "%s".', execution_id, extra=extra) # Get the corresponding liveaction record. - liveaction_db = LiveAction.get_by_id(execution_db.liveaction_id) + liveaction_db = LiveAction.get_by_id(execution_db.liveaction["id"]) if execution_db.status in LIVEACTION_COMPLETED_STATES: # If the action execution is executed under an orquesta workflow, policies for the diff --git a/st2actions/st2actions/scheduler/entrypoint.py b/st2actions/st2actions/scheduler/entrypoint.py index 47e3295a5d..14d816ded3 100644 --- a/st2actions/st2actions/scheduler/entrypoint.py +++ b/st2actions/st2actions/scheduler/entrypoint.py @@ -97,7 +97,7 @@ def _create_execution_queue_item_db_from_liveaction(self, liveaction, delay=None """ Create ActionExecutionSchedulingQueueItemDB from live action. """ - execution = ActionExecution.get(liveaction_id=str(liveaction.id)) + execution = ActionExecution.get(liveaction__id=str(liveaction.id)) execution_queue_item_db = ActionExecutionSchedulingQueueItemDB() execution_queue_item_db.action_execution_id = str(execution.id) diff --git a/st2actions/st2actions/scheduler/handler.py b/st2actions/st2actions/scheduler/handler.py index 7e3615434b..2e2598f5da 100644 --- a/st2actions/st2actions/scheduler/handler.py +++ b/st2actions/st2actions/scheduler/handler.py @@ -136,7 +136,7 @@ def _fix_missing_action_execution_id(self): for entry in ActionExecutionSchedulingQueue.query( action_execution_id__in=["", None] ): - execution_db = ActionExecution.get(liveaction_id=entry.liveaction_id) + execution_db = ActionExecution.get(liveaction__id=entry.liveaction_id) if not execution_db: continue diff --git a/st2actions/st2actions/worker.py b/st2actions/st2actions/worker.py index b1d3fc790e..30af0d56a7 100644 --- a/st2actions/st2actions/worker.py +++ b/st2actions/st2actions/worker.py @@ -235,7 +235,7 @@ def _run_action(self, liveaction_db): return result def _cancel_action(self, liveaction_db): - action_execution_db = ActionExecution.get(liveaction_id=str(liveaction_db.id)) + action_execution_db = ActionExecution.get(liveaction__id=str(liveaction_db.id)) extra = { "action_execution_db": action_execution_db, "liveaction_db": liveaction_db, @@ -265,7 +265,7 @@ def _cancel_action(self, liveaction_db): return result def _pause_action(self, liveaction_db): - action_execution_db = ActionExecution.get(liveaction_id=str(liveaction_db.id)) + action_execution_db = ActionExecution.get(liveaction__id=str(liveaction_db.id)) extra = { "action_execution_db": action_execution_db, "liveaction_db": liveaction_db, @@ -294,7 +294,7 @@ def _pause_action(self, liveaction_db): return result def _resume_action(self, liveaction_db): - action_execution_db = ActionExecution.get(liveaction_id=str(liveaction_db.id)) + action_execution_db = ActionExecution.get(liveaction__id=str(liveaction_db.id)) extra = { "action_execution_db": action_execution_db, "liveaction_db": liveaction_db, diff --git a/st2actions/st2actions/workflows/workflows.py b/st2actions/st2actions/workflows/workflows.py index 6672069e6f..700244d058 100644 --- a/st2actions/st2actions/workflows/workflows.py +++ b/st2actions/st2actions/workflows/workflows.py @@ -100,7 +100,6 @@ def process(self, message): # error handling routine will fail as well because it will try to update # the database and fail the workflow execution gracefully. In this case, # the garbage collector will find and cancel these workflow executions. - LOG.error(e, exc_info=True) self.fail_workflow_execution(message, e) finally: with self._semaphore: @@ -133,7 +132,7 @@ def shutdown(self): if cfg.CONF.coordination.service_registry and not member_ids: ac_ex_dbs = self._get_running_workflows() for ac_ex_db in ac_ex_dbs: - lv_ac = action_utils.get_liveaction_by_id(ac_ex_db.liveaction_id) + lv_ac = action_utils.get_liveaction_by_id(ac_ex_db.liveaction["id"]) ac_svc.request_pause(lv_ac, WORKFLOW_ENGINE_START_STOP_SEQ) def _get_running_workflows(self): @@ -252,7 +251,7 @@ def handle_action_execution(self, ac_ex_db): return # Apply post run policies. - lv_ac_db = lv_db_access.LiveAction.get_by_id(ac_ex_db.liveaction_id) + lv_ac_db = lv_db_access.LiveAction.get_by_id(ac_ex_db.liveaction["id"]) pc_svc.apply_post_run_policies(lv_ac_db) # Process completion of the action execution. diff --git a/st2actions/tests/unit/policies/test_concurrency.py b/st2actions/tests/unit/policies/test_concurrency.py index a2991ad862..1be4b86da3 100644 --- a/st2actions/tests/unit/policies/test_concurrency.py +++ b/st2actions/tests/unit/policies/test_concurrency.py @@ -218,7 +218,7 @@ def test_over_threshold_delay_executions(self): self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Check the status changes. - execution = ActionExecution.get(liveaction_id=str(liveaction.id)) + execution = ActionExecution.get(liveaction__id=str(liveaction.id)) expected_status_changes = [ "requested", "delayed", diff --git a/st2actions/tests/unit/policies/test_retry_policy.py b/st2actions/tests/unit/policies/test_retry_policy.py index 609a607b95..8e16a7029c 100644 --- a/st2actions/tests/unit/policies/test_retry_policy.py +++ b/st2actions/tests/unit/policies/test_retry_policy.py @@ -128,7 +128,7 @@ def test_retry_on_timeout_first_retry_is_successful(self): self.assertEqual(action_execution_dbs[1].status, LIVEACTION_STATUS_REQUESTED) # Verify retried execution contains policy related context - original_liveaction_id = action_execution_dbs[0].liveaction_id + original_liveaction_id = action_execution_dbs[0].liveaction["id"] context = action_execution_dbs[1].context self.assertIn("policies", context) @@ -183,7 +183,7 @@ def test_retry_on_timeout_policy_is_retried_twice(self): self.assertEqual(action_execution_dbs[1].status, LIVEACTION_STATUS_REQUESTED) # Verify retried execution contains policy related context - original_liveaction_id = action_execution_dbs[0].liveaction_id + original_liveaction_id = action_execution_dbs[0].liveaction["id"] context = action_execution_dbs[1].context self.assertIn("policies", context) @@ -216,7 +216,7 @@ def test_retry_on_timeout_policy_is_retried_twice(self): self.assertEqual(action_execution_dbs[2].status, LIVEACTION_STATUS_REQUESTED) # Verify retried execution contains policy related context - original_liveaction_id = action_execution_dbs[1].liveaction_id + original_liveaction_id = action_execution_dbs[1].liveaction["id"] context = action_execution_dbs[2].context self.assertIn("policies", context) diff --git a/st2actions/tests/unit/test_executions.py b/st2actions/tests/unit/test_executions.py index 8814b78616..1c95a51061 100644 --- a/st2actions/tests/unit/test_executions.py +++ b/st2actions/tests/unit/test_executions.py @@ -99,7 +99,7 @@ def test_basic_execution(self): ) execution = self._get_action_execution( - liveaction_id=str(liveaction.id), raise_exception=True + liveaction__id=str(liveaction.id), raise_exception=True ) self.assertDictEqual(execution.trigger, {}) @@ -121,7 +121,8 @@ def test_basic_execution(self): self.assertEqual(execution.result, liveaction.result) self.assertEqual(execution.status, liveaction.status) self.assertEqual(execution.context, liveaction.context) - self.assertEqual(execution.liveaction_id, str(liveaction.id)) + self.assertEqual(execution.liveaction["callback"], liveaction.callback) + self.assertEqual(execution.liveaction["action"], liveaction.action) def test_basic_execution_history_create_failed(self): MOCK_FAIL_EXECUTION_CREATE = True # noqa @@ -135,7 +136,7 @@ def test_chained_executions(self): ) execution = self._get_action_execution( - liveaction_id=str(liveaction.id), raise_exception=True + liveaction__id=str(liveaction.id), raise_exception=True ) action = action_utils.get_action_by_ref("executions.chain") @@ -153,7 +154,8 @@ def test_chained_executions(self): self.assertEqual(execution.result, liveaction.result) self.assertEqual(execution.status, liveaction.status) self.assertEqual(execution.context, liveaction.context) - self.assertEqual(execution.liveaction_id, str(liveaction.id)) + self.assertEqual(execution.liveaction["callback"], liveaction.callback) + self.assertEqual(execution.liveaction["action"], liveaction.action) self.assertGreater(len(execution.children), 0) for child in execution.children: @@ -200,7 +202,7 @@ def test_triggered_execution(self): ) execution = self._get_action_execution( - liveaction_id=str(liveaction.id), raise_exception=True + liveaction__id=str(liveaction.id), raise_exception=True ) self.assertDictEqual(execution.trigger, vars(TriggerAPI.from_model(trigger))) @@ -227,7 +229,8 @@ def test_triggered_execution(self): self.assertEqual(execution.result, liveaction.result) self.assertEqual(execution.status, liveaction.status) self.assertEqual(execution.context, liveaction.context) - self.assertEqual(execution.liveaction_id, str(liveaction.id)) + self.assertEqual(execution.liveaction["callback"], liveaction.callback) + self.assertEqual(execution.liveaction["action"], liveaction.action) def _get_action_execution(self, **kwargs): return ActionExecution.get(**kwargs) diff --git a/st2actions/tests/unit/test_notifier.py b/st2actions/tests/unit/test_notifier.py index 4bce9e294d..b648d7fad3 100644 --- a/st2actions/tests/unit/test_notifier.py +++ b/st2actions/tests/unit/test_notifier.py @@ -27,6 +27,7 @@ from st2common.constants.action import LIVEACTION_COMPLETED_STATES from st2common.constants.action import LIVEACTION_STATUSES from st2common.constants.triggers import INTERNAL_TRIGGER_TYPES +from st2common.models.api.action import LiveActionAPI from st2common.models.db.action import ActionDB from st2common.models.db.execution import ActionExecutionDB from st2common.models.db.liveaction import LiveActionDB @@ -134,7 +135,7 @@ def test_notify_triggers(self): LiveAction.add_or_update(liveaction_db) execution = MOCK_EXECUTION - execution.liveaction_id = str(liveaction_db.id) + execution.liveaction = vars(LiveActionAPI.from_model(liveaction_db)) execution.status = liveaction_db.status dispatcher = NotifierTestCase.MockDispatcher(self) @@ -184,7 +185,7 @@ def test_notify_triggers_end_timestamp_none(self): LiveAction.add_or_update(liveaction_db) execution = MOCK_EXECUTION - execution.liveaction_id = str(liveaction_db.id) + execution.liveaction = vars(LiveActionAPI.from_model(liveaction_db)) execution.status = liveaction_db.status dispatcher = NotifierTestCase.MockDispatcher(self) @@ -237,7 +238,7 @@ def test_notify_triggers_jinja_patterns(self, dispatch): LiveAction.add_or_update(liveaction_db) execution = MOCK_EXECUTION - execution.liveaction_id = str(liveaction_db.id) + execution.liveaction = vars(LiveActionAPI.from_model(liveaction_db)) execution.status = liveaction_db.status notifier = Notifier(connection=None, queues=[]) @@ -269,7 +270,7 @@ def test_post_generic_trigger_emit_when_default_value_is_used(self, dispatch): liveaction_db = LiveActionDB(action="core.local") liveaction_db.status = status execution = MOCK_EXECUTION - execution.liveaction_id = str(liveaction_db.id) + execution.liveaction = vars(LiveActionAPI.from_model(liveaction_db)) execution.status = liveaction_db.status notifier = Notifier(connection=None, queues=[]) @@ -306,7 +307,7 @@ def test_post_generic_trigger_with_emit_condition(self, dispatch): liveaction_db = LiveActionDB(action="core.local") liveaction_db.status = status execution = MOCK_EXECUTION - execution.liveaction_id = str(liveaction_db.id) + execution.liveaction = vars(LiveActionAPI.from_model(liveaction_db)) execution.status = liveaction_db.status notifier = Notifier(connection=None, queues=[]) @@ -353,7 +354,7 @@ def test_process_post_generic_notify_trigger_on_completed_state_default( liveaction_db = LiveActionDB(id=bson.ObjectId(), action="core.local") liveaction_db.status = status execution = MOCK_EXECUTION - execution.liveaction_id = str(liveaction_db.id) + execution.liveaction = vars(LiveActionAPI.from_model(liveaction_db)) execution.status = liveaction_db.status mock_LiveAction.get_by_id.return_value = liveaction_db @@ -403,7 +404,7 @@ def test_process_post_generic_notify_trigger_on_custom_emit_when_states( liveaction_db = LiveActionDB(id=bson.ObjectId(), action="core.local") liveaction_db.status = status execution = MOCK_EXECUTION - execution.liveaction_id = str(liveaction_db.id) + execution.liveaction = vars(LiveActionAPI.from_model(liveaction_db)) execution.status = liveaction_db.status mock_LiveAction.get_by_id.return_value = liveaction_db diff --git a/st2api/st2api/controllers/v1/actionexecutions.py b/st2api/st2api/controllers/v1/actionexecutions.py index 40f52dbcc5..32533f2430 100644 --- a/st2api/st2api/controllers/v1/actionexecutions.py +++ b/st2api/st2api/controllers/v1/actionexecutions.py @@ -136,6 +136,7 @@ def _handle_schedule_execution( rbac_utils.assert_user_is_admin_if_user_query_param_is_provided( user_db=requester_user, user=user ) + try: return self._schedule_execution( liveaction=liveaction_api, @@ -205,6 +206,7 @@ def _schedule_execution( runnertype_db = action_utils.get_runnertype_by_name( action_db.runner_type["name"] ) + try: liveaction_db.parameters = param_utils.render_live_params( runnertype_db.runner_parameters, @@ -240,6 +242,7 @@ def _schedule_execution( liveaction_db, actionexecution_db = action_service.create_request( liveaction=liveaction_db, action_db=action_db, runnertype_db=runnertype_db ) + _, actionexecution_db = action_service.publish_request( liveaction_db, actionexecution_db ) @@ -422,7 +425,6 @@ def get( except IndexError: raise NotFoundException("Execution with id %s not found" % (id)) - # For backward compatibility we also support old non JSON field storage format if pretty_format: response_body = orjson.dumps(result, option=orjson.OPT_INDENT_2) else: @@ -829,7 +831,7 @@ def put(self, id, liveaction_api, requester_user, show_secrets=False): if not execution_api: abort(http_client.NOT_FOUND, "Execution with id %s not found." % id) - liveaction_id = execution_api.liveaction_id + liveaction_id = execution_api.liveaction["id"] if not liveaction_id: abort( http_client.INTERNAL_SERVER_ERROR, @@ -854,7 +856,7 @@ def update_status(liveaction_api, liveaction_db): liveaction_db, status, result, set_result_size=True ) actionexecution_db = ActionExecution.get( - liveaction_id=str(liveaction_db.id) + liveaction__id=str(liveaction_db.id) ) return (liveaction_db, actionexecution_db) @@ -958,7 +960,7 @@ def delete(self, id, requester_user, show_secrets=False): if not execution_api: abort(http_client.NOT_FOUND, "Execution with id %s not found." % id) - liveaction_id = execution_api.liveaction_id + liveaction_id = execution_api.liveaction["id"] if not liveaction_id: abort( http_client.INTERNAL_SERVER_ERROR, diff --git a/st2api/st2api/controllers/v1/aliasexecution.py b/st2api/st2api/controllers/v1/aliasexecution.py index f25704e67b..4e0f780896 100644 --- a/st2api/st2api/controllers/v1/aliasexecution.py +++ b/st2api/st2api/controllers/v1/aliasexecution.py @@ -182,6 +182,7 @@ def _post(self, payload, requester_user, show_secrets=False, match_multiple=Fals show_secrets=show_secrets, requester_user=requester_user, ) + result = { "execution": execution, "actionalias": ActionAliasAPI.from_model(action_alias_db), diff --git a/st2api/st2api/controllers/v1/execution_views.py b/st2api/st2api/controllers/v1/execution_views.py index a051192515..f4240b94ab 100644 --- a/st2api/st2api/controllers/v1/execution_views.py +++ b/st2api/st2api/controllers/v1/execution_views.py @@ -31,8 +31,7 @@ SUPPORTED_FILTERS = { "action": "action.ref", "status": "status", - "liveaction_id": "liveaction_id", - "liveaction": "liveaction_id", + "liveaction": "liveaction.id", "parent": "parent", "rule": "rule.name", "runner": "runner.name", @@ -55,14 +54,7 @@ # List of filters that are too broad to distinct by them and are very likely to represent 1 to 1 # relation between filter and particular history record. -# tldr: these filters represent MANY distinct possibilities -IGNORE_FILTERS = [ - "parent", - "timestamp", - "liveaction", - "liveaction_id", - "trigger_instance", -] +IGNORE_FILTERS = ["parent", "timestamp", "liveaction", "trigger_instance"] class FiltersController(object): diff --git a/st2api/tests/unit/controllers/v1/test_alias_execution.py b/st2api/tests/unit/controllers/v1/test_alias_execution.py index 8d8ea0559d..44261fde3f 100644 --- a/st2api/tests/unit/controllers/v1/test_alias_execution.py +++ b/st2api/tests/unit/controllers/v1/test_alias_execution.py @@ -324,6 +324,7 @@ def test_match_and_execute_list_action_param_str_cast_to_list(self): result = resp.json["results"][0] live_action = result["execution"]["liveaction"] action_alias = result["actionalias"] + self.assertEqual(resp.status_int, 201) self.assertTrue(isinstance(live_action["parameters"]["array_param"], list)) self.assertEqual(live_action["parameters"]["array_param"][0], "one") diff --git a/st2api/tests/unit/controllers/v1/test_executions.py b/st2api/tests/unit/controllers/v1/test_executions.py index c62c7e7c1b..eb3face2ba 100644 --- a/st2api/tests/unit/controllers/v1/test_executions.py +++ b/st2api/tests/unit/controllers/v1/test_executions.py @@ -32,7 +32,6 @@ from st2common.models.db.auth import UserDB from st2common.models.db.execution import ActionExecutionDB from st2common.models.db.execution import ActionExecutionOutputDB -from st2common.models.db.liveaction import LiveActionDB from st2common.models.db.keyvalue import KeyValuePairDB from st2common.persistence.execution import ActionExecution from st2common.persistence.execution import ActionExecutionOutput @@ -2001,19 +2000,9 @@ def test_get_output_running_execution(self): status=status, action={"ref": "core.local"}, runner={"name": "local-shell-cmd"}, - liveaction_id="54c6b6d60640fd4f5354e74a", + liveaction={"ref": "foo"}, ) action_execution_db = ActionExecution.add_or_update(action_execution_db) - liveaction_db = LiveActionDB( - id="54c6b6d60640fd4f5354e74a", - start_timestamp=timestamp, - end_timestamp=timestamp, - status=status, - action="core.local", - runner_info={"name": "local-shell-cmd"}, - ) - - LiveAction.add_or_update(liveaction_db) output_params = dict( execution_id=str(action_execution_db.id), @@ -2092,19 +2081,9 @@ def test_get_output_finished_execution(self): status=status, action={"ref": "core.local"}, runner={"name": "local-shell-cmd"}, - liveaction_id="54c6b6d60640fd4f5354e74a", + liveaction={"ref": "foo"}, ) action_execution_db = ActionExecution.add_or_update(action_execution_db) - liveaction_db = LiveActionDB( - id="54c6b6d60640fd4f5354e74a", - start_timestamp=timestamp, - end_timestamp=timestamp, - status=status, - action="core.local", - runner_info={"name": "local-shell-cmd"}, - ) - - LiveAction.add_or_update(liveaction_db) for i in range(1, 6): stdout_db = ActionExecutionOutputDB( diff --git a/st2api/tests/unit/controllers/v1/test_executions_descendants.py b/st2api/tests/unit/controllers/v1/test_executions_descendants.py index 88c1574d29..55b1c12f53 100644 --- a/st2api/tests/unit/controllers/v1/test_executions_descendants.py +++ b/st2api/tests/unit/controllers/v1/test_executions_descendants.py @@ -31,8 +31,7 @@ "child1_level3.yaml", "child2_level3.yaml", "child3_level3.yaml", - ], - "liveactions": ["liveaction_fake.yaml"], + ] } @@ -41,9 +40,7 @@ class ActionExecutionControllerTestCaseDescendantsTest(FunctionalTest): def setUpClass(cls): super(ActionExecutionControllerTestCaseDescendantsTest, cls).setUpClass() cls.MODELS = FixturesLoader().save_fixtures_to_db( - fixtures_pack=DESCENDANTS_PACK, - fixtures_dict=DESCENDANTS_FIXTURES, - use_object_ids=True, + fixtures_pack=DESCENDANTS_PACK, fixtures_dict=DESCENDANTS_FIXTURES ) def test_get_all_descendants(self): diff --git a/st2api/tests/unit/controllers/v1/test_executions_filters.py b/st2api/tests/unit/controllers/v1/test_executions_filters.py index c916252c9e..af451ca519 100644 --- a/st2api/tests/unit/controllers/v1/test_executions_filters.py +++ b/st2api/tests/unit/controllers/v1/test_executions_filters.py @@ -33,9 +33,7 @@ from st2api.controllers.v1.actionexecutions import ActionExecutionsController from st2api.controllers.v1.execution_views import FILTERS_WITH_VALID_NULL_VALUES from st2common.persistence.execution import ActionExecution -from st2common.persistence.action import LiveAction from st2common.models.api.execution import ActionExecutionAPI -from st2common.models.api.execution import LiveActionAPI class TestActionExecutionFilters(FunctionalTest): @@ -62,10 +60,8 @@ def setUpClass(cls): "rule": copy.deepcopy(fixture.ARTIFACTS["rule"]), "action": copy.deepcopy(fixture.ARTIFACTS["actions"]["chain"]), "runner": copy.deepcopy(fixture.ARTIFACTS["runners"]["action-chain"]), - "liveaction_id": fixture.ARTIFACTS["liveactions"]["workflow"]["id"], - "status": fixture.ARTIFACTS["liveactions"]["workflow"]["status"], - "result": copy.deepcopy( - fixture.ARTIFACTS["liveactions"]["workflow"]["result"] + "liveaction": copy.deepcopy( + fixture.ARTIFACTS["liveactions"]["workflow"] ), "context": copy.deepcopy(fixture.ARTIFACTS["context"]), "children": [], @@ -73,11 +69,7 @@ def setUpClass(cls): { "action": copy.deepcopy(fixture.ARTIFACTS["actions"]["local"]), "runner": copy.deepcopy(fixture.ARTIFACTS["runners"]["run-local"]), - "liveaction_id": fixture.ARTIFACTS["liveactions"]["task1"]["id"], - "status": fixture.ARTIFACTS["liveactions"]["task1"]["status"], - "result": copy.deepcopy( - fixture.ARTIFACTS["liveactions"]["task1"]["result"] - ), + "liveaction": copy.deepcopy(fixture.ARTIFACTS["liveactions"]["task1"]), }, ] @@ -97,24 +89,14 @@ def assign_parent(child): data["id"] = obj_id data["start_timestamp"] = isotime.format(timestamp, offset=False) data["end_timestamp"] = isotime.format(timestamp, offset=False) + data["status"] = data["liveaction"]["status"] + data["result"] = data["liveaction"]["result"] if fake_type["action"]["name"] == "local" and random.choice([True, False]): assign_parent(data) wb_obj = ActionExecutionAPI(**data) db_obj = ActionExecutionAPI.to_model(wb_obj) cls.refs[obj_id] = ActionExecution.add_or_update(db_obj) cls.start_timestamps.append(timestamp) - # also add the liveaction to the database so it can be retrieved by - # the actionexecution api - liveaction_data = { - "id": data["liveaction_id"], - "action": fake_type["action"]["name"], - "status": data["status"], - } - wb_live_obj = LiveActionAPI(**liveaction_data) - live_db_obj = LiveActionAPI.to_model(wb_live_obj) - # hard code id of liveaction - live_db_obj.id = data["liveaction_id"] - LiveAction.add_or_update(live_db_obj) cls.start_timestamps = sorted(cls.start_timestamps) @@ -153,7 +135,7 @@ def test_get_one(self): self.assertEqual(record["id"], obj_id) self.assertDictEqual(record["action"], fake_record.action) self.assertDictEqual(record["runner"], fake_record.runner) - self.assertEqual(record["liveaction_id"], fake_record.liveaction_id) + self.assertDictEqual(record["liveaction"], fake_record.liveaction) def test_get_one_failed(self): response = self.app.get( diff --git a/st2client/in-requirements.txt b/st2client/in-requirements.txt index 581b999f5f..b0057916f1 100644 --- a/st2client/in-requirements.txt +++ b/st2client/in-requirements.txt @@ -1,6 +1,5 @@ # Remember to list implicit packages here, otherwise version won't be fixated! importlib-metadata -zipp<3.16.0 # importlib-metadata requires typing-extensions typing-extensions argcomplete diff --git a/st2common/BUILD b/st2common/BUILD index 48a0726dfd..832a17e483 100644 --- a/st2common/BUILD +++ b/st2common/BUILD @@ -21,7 +21,6 @@ st2_component_python_distribution( "bin/st2-pack-setup-virtualenv", "bin/migrations/v3.5/st2-migrate-db-dict-field-values", "bin/migrations/v3.8/st2-drop-st2exporter-marker-collections", - "bin/migrations/v3.9/st2-migrate-liveaction-executiondb", "bin/st2-run-pack-tests:shell", "bin/st2ctl:shell", "bin/st2-self-check:shell", diff --git a/st2common/bin/st2-track-result b/st2common/bin/st2-track-result index 7f158a2453..773421bf82 100755 --- a/st2common/bin/st2-track-result +++ b/st2common/bin/st2-track-result @@ -68,7 +68,7 @@ def add_result_tracker(exec_id): LOG.info("Retrieving runner type and liveaction records...") runnertype_db = action_db.get_runnertype_by_name(exec_db.action.get("runner_type")) - liveaction_db = action_db.get_liveaction_by_id(exec_db.liveaction_id) + liveaction_db = action_db.get_liveaction_by_id(exec_db.liveaction["id"]) # Skip if liveaction is completed. if liveaction_db.status in action_constants.LIVEACTION_COMPLETED_STATES: @@ -100,7 +100,7 @@ def del_result_tracker(exec_id): LOG.info('Found action execution record for "%s".', exec_id) LOG.info("Retrieving runner type and liveaction records...") - liveaction_db = action_db.get_liveaction_by_id(exec_db.liveaction_id) + liveaction_db = action_db.get_liveaction_by_id(exec_db.liveaction["id"]) LOG.info("Removing result tracker entry...") removed = queries.remove_query(liveaction_db.id) diff --git a/st2common/setup.py b/st2common/setup.py index 5e2764286d..e67c846b90 100644 --- a/st2common/setup.py +++ b/st2common/setup.py @@ -69,7 +69,6 @@ "bin/st2-pack-download", "bin/st2-pack-setup-virtualenv", "bin/migrations/v3.5/st2-migrate-db-dict-field-values", - "bin/migrations/v3.9/st2-migrate-liveaction-executiondb", ], entry_points={ "st2common.metrics.driver": [ diff --git a/st2common/st2common/garbage_collection/executions.py b/st2common/st2common/garbage_collection/executions.py index bc0e85f1f0..ae0f3296f4 100644 --- a/st2common/st2common/garbage_collection/executions.py +++ b/st2common/st2common/garbage_collection/executions.py @@ -223,5 +223,5 @@ def purge_orphaned_workflow_executions(logger): # as a result of the original failure, the garbage collection routine here cancels # the workflow execution so it cannot be rerun from failed task(s). for ac_ex_db in workflow_service.identify_orphaned_workflows(): - lv_ac_db = LiveAction.get(id=ac_ex_db.liveaction_id) + lv_ac_db = LiveAction.get(id=ac_ex_db.liveaction["id"]) action_service.request_cancellation(lv_ac_db, None) diff --git a/st2common/st2common/garbage_collection/inquiries.py b/st2common/st2common/garbage_collection/inquiries.py index a447a0f5eb..2381472a73 100644 --- a/st2common/st2common/garbage_collection/inquiries.py +++ b/st2common/st2common/garbage_collection/inquiries.py @@ -78,7 +78,7 @@ def purge_inquiries(logger): liveaction_db = action_utils.update_liveaction_status( status=action_constants.LIVEACTION_STATUS_TIMED_OUT, result=inquiry.result, - liveaction_id=inquiry.liveaction_id, + liveaction_id=inquiry.liveaction.get("id"), ) executions.update_execution(liveaction_db) diff --git a/st2common/st2common/models/api/base.py b/st2common/st2common/models/api/base.py index 3be5b2d69a..445fe39023 100644 --- a/st2common/st2common/models/api/base.py +++ b/st2common/st2common/models/api/base.py @@ -126,7 +126,6 @@ def convert_raw(cls, doc, raw_values): :param doc: dict :param raw_values: dict[field]:bytestring """ - for field_name, field_value in raw_values.items(): doc[field_name] = field_value return doc diff --git a/st2common/st2common/models/api/execution.py b/st2common/st2common/models/api/execution.py index 2654d5cc52..019b367eed 100644 --- a/st2common/st2common/models/api/execution.py +++ b/st2common/st2common/models/api/execution.py @@ -24,7 +24,6 @@ from st2common.models.api.base import BaseAPI from st2common.models.db.execution import ActionExecutionDB from st2common.models.db.execution import ActionExecutionOutputDB -from st2common.persistence.liveaction import LiveAction from st2common.models.api.trigger import TriggerTypeAPI, TriggerAPI, TriggerInstanceAPI from st2common.models.api.rule import RuleAPI from st2common.models.api.action import RunnerTypeAPI, ActionAPI, LiveActionAPI @@ -62,7 +61,6 @@ class ActionExecutionAPI(BaseAPI): "rule": RuleAPI.schema, "action": REQUIRED_ATTR_SCHEMAS["action"], "runner": REQUIRED_ATTR_SCHEMAS["runner"], - "liveaction_id": {"type": "string", "required": True}, "liveaction": REQUIRED_ATTR_SCHEMAS["liveaction"], "status": { "description": "The current status of the action execution.", @@ -152,25 +150,13 @@ class ActionExecutionAPI(BaseAPI): @classmethod def from_model(cls, model, mask_secrets=False): - doc = cls._from_model(model, mask_secrets=mask_secrets) + doc["result"] = ActionExecutionDB.result.parse_field_value(doc["result"]) + start_timestamp = model.start_timestamp start_timestamp_iso = isotime.format(start_timestamp, offset=False) doc["start_timestamp"] = start_timestamp_iso - # check to see if liveaction_id has been excluded in output filtering - if doc.get("liveaction_id", False): - live_action_model = LiveAction.get_by_id(doc["liveaction_id"]) - if live_action_model is not None: - doc["liveaction"] = LiveActionAPI.from_model( - live_action_model, mask_secrets=mask_secrets - ) - else: - doc["liveaction"] = { - "action": doc["action"]["name"], - "id": doc["liveaction_id"], - "status": doc["status"], - } end_timestamp = model.end_timestamp if end_timestamp: @@ -189,11 +175,9 @@ def convert_raw(cls, doc, raw_values): """ override this class to convert any raw byte values into dict - Now add the JSON string field value which shouldn't be escaped back. We don't JSON parse the field value here because that happens inside the model specific "from_model()" method where we also parse and convert all the other field values. - :param doc: dict :param raw_values: dict[field]:bytestring """ diff --git a/st2common/st2common/models/api/inquiry.py b/st2common/st2common/models/api/inquiry.py index f5a3be92b8..d04c8efbf1 100644 --- a/st2common/st2common/models/api/inquiry.py +++ b/st2common/st2common/models/api/inquiry.py @@ -21,7 +21,7 @@ from st2common.constants.action import LIVEACTION_STATUSES from st2common.models.api.base import BaseAPI -from st2common.models.api.action import RunnerTypeAPI, ActionAPI +from st2common.models.api.action import RunnerTypeAPI, ActionAPI, LiveActionAPI from st2common.models.db.execution import ActionExecutionDB from st2common import log as logging @@ -31,6 +31,7 @@ REQUIRED_ATTR_SCHEMAS = { "action": copy.deepcopy(ActionAPI.schema), "runner": copy.deepcopy(RunnerTypeAPI.schema), + "liveaction": copy.deepcopy(LiveActionAPI.schema), } for k, v in six.iteritems(REQUIRED_ATTR_SCHEMAS): @@ -75,7 +76,7 @@ class InquiryAPI(BaseAPI): }, "required": True, }, - "liveaction_id": {"type": "string", "required": True}, + "liveaction": REQUIRED_ATTR_SCHEMAS["liveaction"], "runner": REQUIRED_ATTR_SCHEMAS["runner"], "status": { "description": "The current status of the action execution.", @@ -111,7 +112,7 @@ def from_model(cls, model, mask_secrets=False): "id": doc["id"], "runner": doc.get("runner", None), "status": doc.get("status", None), - "liveaction_id": doc.get("liveaction_id", None), + "liveaction": doc.get("liveaction", None), "parent": doc.get("parent", None), "result": doc.get("result", None), } diff --git a/st2common/st2common/models/db/execution.py b/st2common/st2common/models/db/execution.py index 6b591d310e..c0ffb4ab8f 100644 --- a/st2common/st2common/models/db/execution.py +++ b/st2common/st2common/models/db/execution.py @@ -39,7 +39,16 @@ class ActionExecutionDB(stormbase.StormFoundationDB): RESOURCE_TYPE = ResourceType.EXECUTION UID_FIELDS = ["id"] - # SAME as liveaction + + trigger = stormbase.EscapedDictField() + trigger_type = stormbase.EscapedDictField() + trigger_instance = stormbase.EscapedDictField() + rule = stormbase.EscapedDictField() + action = stormbase.EscapedDictField(required=True) + runner = stormbase.EscapedDictField(required=True) + # Only the diff between the liveaction type and what is replicated + # in the ActionExecutionDB object. + liveaction = stormbase.EscapedDictField(required=True) workflow_execution = me.StringField() task_execution = me.StringField() status = me.StringField( @@ -60,6 +69,7 @@ class ActionExecutionDB(stormbase.StormFoundationDB): result = JSONDictEscapedFieldCompatibilityField( default={}, help_text="Action defined result." ) + result_size = me.IntField(default=0, help_text="Serialized result size in bytes") context = me.DictField( default={}, help_text="Contextual information on the action execution." ) @@ -75,17 +85,15 @@ class ActionExecutionDB(stormbase.StormFoundationDB): parent = me.StringField() children = me.ListField(field=me.StringField()) log = me.ListField(field=me.DictField()) + delay = me.IntField(min_value=0) # Do not use URLField for web_url. If host doesn't have FQDN set, URLField validation blows. web_url = me.StringField(required=False) - # liveaction id - liveaction_id = me.StringField(required=True) - meta = { "indexes": [ {"fields": ["rule.ref"]}, {"fields": ["action.ref"]}, - {"fields": ["liveaction_id"]}, + {"fields": ["liveaction.id"]}, {"fields": ["start_timestamp"]}, {"fields": ["end_timestamp"]}, {"fields": ["status"]}, @@ -119,6 +127,7 @@ def mask_secrets(self, value): """ result = copy.deepcopy(value) + liveaction = result["liveaction"] parameters = {} # pylint: disable=no-member parameters.update(value.get("action", {}).get("parameters", {})) @@ -128,6 +137,31 @@ def mask_secrets(self, value): result["parameters"] = mask_secret_parameters( parameters=result.get("parameters", {}), secret_parameters=secret_parameters ) + + if "parameters" in liveaction: + liveaction["parameters"] = mask_secret_parameters( + parameters=liveaction["parameters"], secret_parameters=secret_parameters + ) + + if liveaction.get("action", "") == "st2.inquiry.respond": + # Special case to mask parameters for `st2.inquiry.respond` action + # In this case, this execution is just a plain python action, not + # an inquiry, so we don't natively have a handle on the response + # schema. + # + # To prevent leakage, we can just mask all response fields. + # + # Note: The 'string' type in secret_parameters doesn't matter, + # it's just a placeholder to tell mask_secret_parameters() + # that this parameter is indeed a secret parameter and to + # mask it. + result["parameters"]["response"] = mask_secret_parameters( + parameters=liveaction["parameters"]["response"], + secret_parameters={ + p: "string" for p in liveaction["parameters"]["response"] + }, + ) + output_value = ActionExecutionDB.result.parse_field_value(result["result"]) masked_output_value = output_schema.mask_secret_output(result, output_value) result["result"] = masked_output_value diff --git a/st2common/st2common/models/db/liveaction.py b/st2common/st2common/models/db/liveaction.py index 4f4ebcaa37..99cad1982d 100644 --- a/st2common/st2common/models/db/liveaction.py +++ b/st2common/st2common/models/db/liveaction.py @@ -38,7 +38,6 @@ class LiveActionDB(stormbase.StormFoundationDB): - # same as action execution workflow_execution = me.StringField() task_execution = me.StringField() # TODO: Can status be an enum at the Mongo layer? @@ -69,7 +68,6 @@ class LiveActionDB(stormbase.StormFoundationDB): min_value=0, help_text="How long (in milliseconds) to delay the execution before scheduling.", ) - # diff from action execution action_is_workflow = me.BooleanField( default=False, @@ -84,6 +82,7 @@ class LiveActionDB(stormbase.StormFoundationDB): default={}, help_text="Information about the runner which executed this live action (hostname, pid).", ) + meta = { "indexes": [ {"fields": ["-start_timestamp", "action"]}, diff --git a/st2common/st2common/models/db/stormbase.py b/st2common/st2common/models/db/stormbase.py index e75e381941..67c93f42a1 100644 --- a/st2common/st2common/models/db/stormbase.py +++ b/st2common/st2common/models/db/stormbase.py @@ -108,10 +108,8 @@ def to_serializable_dict(self, mask_secrets=False): v = json_decode(v.to_json()) serializable_dict[k] = v - if mask_secrets and cfg.CONF.log.mask_secrets: serializable_dict = self.mask_secrets(value=serializable_dict) - return serializable_dict diff --git a/st2common/st2common/openapi.yaml b/st2common/st2common/openapi.yaml index 7bcda8562b..93d4bcdec6 100644 --- a/st2common/st2common/openapi.yaml +++ b/st2common/st2common/openapi.yaml @@ -4936,8 +4936,6 @@ definitions: $ref: '#/definitions/RunnerType' liveaction: $ref: '#/definitions/LiveAction' - liveaction_id: - type: string task_execution: type: string workflow_execution: diff --git a/st2common/st2common/openapi.yaml.j2 b/st2common/st2common/openapi.yaml.j2 index c008d5db33..6b10362640 100644 --- a/st2common/st2common/openapi.yaml.j2 +++ b/st2common/st2common/openapi.yaml.j2 @@ -4932,8 +4932,6 @@ definitions: $ref: '#/definitions/RunnerType' liveaction: $ref: '#/definitions/LiveAction' - liveaction_id: - type: string task_execution: type: string workflow_execution: diff --git a/st2common/st2common/services/action.py b/st2common/st2common/services/action.py index 9aa33ee542..9c026f5507 100644 --- a/st2common/st2common/services/action.py +++ b/st2common/st2common/services/action.py @@ -61,7 +61,6 @@ def create_request( ): """ Create an action execution. - :param liveaction: LiveActionDB :param action_db: Action model to operate one. If not provided, one is retrieved from the database using values from "liveaction". @@ -168,6 +167,7 @@ def create_request( runnertype_db=runnertype_db, publish=False, ) + if trace_db: trace_service.add_or_update_given_trace_db( trace_db=trace_db, @@ -316,7 +316,7 @@ def request_cancellation(liveaction, requester): liveaction, status, result=result, context=liveaction.context ) - execution = ActionExecution.get(liveaction_id=str(liveaction.id)) + execution = ActionExecution.get(liveaction__id=str(liveaction.id)) return (liveaction, execution) @@ -347,7 +347,7 @@ def request_pause(liveaction, requester): liveaction.status == action_constants.LIVEACTION_STATUS_PAUSING or liveaction.status == action_constants.LIVEACTION_STATUS_PAUSED ): - execution = ActionExecution.get(liveaction_id=str(liveaction.id)) + execution = ActionExecution.get(liveaction__id=str(liveaction.id)) return (liveaction, execution) if liveaction.status != action_constants.LIVEACTION_STATUS_RUNNING: @@ -363,7 +363,7 @@ def request_pause(liveaction, requester): context=liveaction.context, ) - execution = ActionExecution.get(liveaction_id=str(liveaction.id)) + execution = ActionExecution.get(liveaction__id=str(liveaction.id)) return (liveaction, execution) @@ -396,7 +396,7 @@ def request_resume(liveaction, requester): ] if liveaction.status in running_states: - execution = ActionExecution.get(liveaction_id=str(liveaction.id)) + execution = ActionExecution.get(liveaction__id=str(liveaction.id)) return (liveaction, execution) if liveaction.status != action_constants.LIVEACTION_STATUS_PAUSED: @@ -412,7 +412,7 @@ def request_resume(liveaction, requester): context=liveaction.context, ) - execution = ActionExecution.get(liveaction_id=str(liveaction.id)) + execution = ActionExecution.get(liveaction__id=str(liveaction.id)) return (liveaction, execution) @@ -433,7 +433,7 @@ def get_parent_liveaction(liveaction_db): return None parent_execution_db = ActionExecution.get(id=parent["execution_id"]) - parent_liveaction_db = LiveAction.get(id=parent_execution_db.liveaction_id) + parent_liveaction_db = LiveAction.get(id=parent_execution_db.liveaction["id"]) return parent_liveaction_db @@ -541,7 +541,7 @@ def store_execution_output_data_ex( def is_children_active(liveaction_id): - execution_db = ActionExecution.get(liveaction_id=str(liveaction_id)) + execution_db = ActionExecution.get(liveaction__id=str(liveaction_id)) if execution_db.runner["name"] not in action_constants.WORKFLOW_RUNNER_TYPES: return False diff --git a/st2common/st2common/services/executions.py b/st2common/st2common/services/executions.py index 926c8ac808..4b984cd587 100644 --- a/st2common/st2common/services/executions.py +++ b/st2common/st2common/services/executions.py @@ -82,10 +82,12 @@ def _decompose_liveaction(liveaction_db): """ Splits the liveaction into an ActionExecution compatible dict. """ - decomposed = {"liveaction_id": str(liveaction_db.id)} + decomposed = {"liveaction": {}} liveaction_api = vars(LiveActionAPI.from_model(liveaction_db)) for k in liveaction_api.keys(): - if k not in LIVEACTION_ATTRIBUTES: + if k in LIVEACTION_ATTRIBUTES: + decomposed["liveaction"][k] = liveaction_api[k] + else: decomposed[k] = getattr(liveaction_db, k) return decomposed @@ -153,7 +155,6 @@ def create_execution_object( # NOTE: User input data is already validate as part of the API request, # other data is set by us. Skipping validation here makes operation 10%-30% faster - execution.liveaction_id = str(liveaction.id) execution = ActionExecution.add_or_update( execution, publish=publish, validate=False ) @@ -193,7 +194,7 @@ def update_execution(liveaction_db, publish=True, set_result_size=False): :param set_result_size: True to calculate size of the serialized result field value and set it on the "result_size" database field. """ - execution = ActionExecution.get(liveaction_id=str(liveaction_db.id)) + execution = ActionExecution.get(liveaction__id=str(liveaction_db.id)) with coordination.get_coordinator().get_lock(str(liveaction_db.id).encode()): # Skip execution object update when action is already in completed state. diff --git a/st2common/st2common/services/inquiry.py b/st2common/st2common/services/inquiry.py index e181faa5a0..1301a62c4b 100644 --- a/st2common/st2common/services/inquiry.py +++ b/st2common/st2common/services/inquiry.py @@ -121,16 +121,12 @@ def validate_response(inquiry, response): def respond(inquiry, response, requester=None): - """ - :param inquiry: InquiryAPI - :param response: dict - """ # Set requester to system user is not provided. if not requester: requester = cfg.CONF.system_user.user # Retrieve the liveaction from the database. - liveaction_db = lv_db_access.LiveAction.get_by_id(inquiry.liveaction_id) + liveaction_db = lv_db_access.LiveAction.get_by_id(inquiry.liveaction.get("id")) # Resume the parent workflow first. If the action execution for the inquiry is updated first, # it triggers handling of the action execution completion which will interact with the paused diff --git a/st2common/st2common/services/trace.py b/st2common/st2common/services/trace.py index 37d435f36f..2d51161838 100644 --- a/st2common/st2common/services/trace.py +++ b/st2common/st2common/services/trace.py @@ -197,7 +197,7 @@ def get_trace_db_by_live_action(liveaction): ) return (created, trace_db) # 3. Check if the action_execution associated with liveaction leads to a trace_db - execution = ActionExecution.get(liveaction_id=str(liveaction.id)) + execution = ActionExecution.get(liveaction__id=str(liveaction.id)) if execution: trace_db = get_trace_db_by_action_execution(action_execution=execution) # 4. No trace_db found, therefore create one. This typically happens diff --git a/st2common/st2common/services/workflows.py b/st2common/st2common/services/workflows.py index c99fb896b5..2aa50cee5f 100644 --- a/st2common/st2common/services/workflows.py +++ b/st2common/st2common/services/workflows.py @@ -19,8 +19,6 @@ import datetime import retrying import six -import sys -import traceback from orquesta import conducting from orquesta import events @@ -472,7 +470,7 @@ def request_cancellation(ac_ex_db): and root_ac_ex_db.status not in ac_const.LIVEACTION_CANCEL_STATES ): LOG.info("[%s] Cascading cancelation request to parent workflow.", wf_ac_ex_id) - root_lv_ac_db = lv_db_access.LiveAction.get(id=root_ac_ex_db.liveaction_id) + root_lv_ac_db = lv_db_access.LiveAction.get(id=root_ac_ex_db.liveaction["id"]) ac_svc.request_cancellation(root_lv_ac_db, None) LOG.debug("[%s] %s", wf_ac_ex_id, conductor.serialize()) @@ -679,13 +677,7 @@ def request_task_execution(wf_ex_db, st2_ctx, task_ex_req): "task_id": task_id, "route": task_route, } - exc_type, exc_value, exc_traceback = sys.exc_info() - traceback_in_var = traceback.format_tb(exc_traceback) - update_task_execution( - str(task_ex_db.id), - statuses.FAILED, - {"errors": [error], "traceback": traceback_in_var}, - ) + update_task_execution(str(task_ex_db.id), statuses.FAILED, {"errors": [error]}) raise e return task_ex_db @@ -915,7 +907,7 @@ def handle_action_execution_resume(ac_ex_db): if parent_ac_ex_db.status == ac_const.LIVEACTION_STATUS_PAUSED: action_utils.update_liveaction_status( - liveaction_id=parent_ac_ex_db.liveaction_id, + liveaction_id=parent_ac_ex_db.liveaction["id"], status=ac_const.LIVEACTION_STATUS_RUNNING, publish=False, ) @@ -1193,15 +1185,12 @@ def request_next_tasks(wf_ex_db, task_ex_id=None): # Request the task execution. request_task_execution(wf_ex_db, st2_ctx, task) except Exception as e: - - exc_type, exc_value, exc_traceback = sys.exc_info() msg = 'Failed task execution for task "%s", route "%s".' msg = msg % (task["id"], str(task["route"])) update_progress( wf_ex_db, "%s %s" % (msg, str(e)), severity="error", log=False ) LOG.exception(msg, exc_info=True) - fail_workflow_execution(str(wf_ex_db.id), e, task=task) return @@ -1215,9 +1204,7 @@ def request_next_tasks(wf_ex_db, task_ex_id=None): next_tasks = conductor.get_next_tasks() if not next_tasks: - update_progress( - wf_ex_db, "end of while No tasks identified to execute next." - ) + update_progress(wf_ex_db, "No tasks identified to execute next.") update_progress(wf_ex_db, "\n", log=False) @@ -1449,7 +1436,7 @@ def update_execution_records( # Update the corresponding liveaction and action execution for the workflow. wf_ac_ex_db = ex_db_access.ActionExecution.get_by_id(wf_ex_db.action_execution) - wf_lv_ac_db = action_utils.get_liveaction_by_id(wf_ac_ex_db.liveaction_id) + wf_lv_ac_db = action_utils.get_liveaction_by_id(wf_ac_ex_db.liveaction["id"]) # Gather result for liveaction and action execution. result = {"output": wf_ex_db.output or None} diff --git a/st2common/st2common/util/param.py b/st2common/st2common/util/param.py index 104a3c5479..67fb83e9ac 100644 --- a/st2common/st2common/util/param.py +++ b/st2common/st2common/util/param.py @@ -310,7 +310,6 @@ def render_live_params( additional_contexts=None, ): """ - :param params: BaseDict Renders list of parameters. Ensures that there's no cyclic or missing dependencies. Returns a dict of plain rendered parameters. """ diff --git a/st2common/tests/unit/migrations/test_v35_migrate_db_dict_field_values.py b/st2common/tests/unit/migrations/test_v35_migrate_db_dict_field_values.py index 82903a4391..84347fcbab 100644 --- a/st2common/tests/unit/migrations/test_v35_migrate_db_dict_field_values.py +++ b/st2common/tests/unit/migrations/test_v35_migrate_db_dict_field_values.py @@ -16,15 +16,11 @@ import sys import datetime -import mongoengine as me from st2common.constants import action as action_constants -from st2common.fields import ComplexDateTimeField -from st2common.fields import JSONDictEscapedFieldCompatibilityField from st2common.models.db import stormbase from st2common.models.db.execution import ActionExecutionDB from st2common.models.db.liveaction import LiveActionDB -from st2common.models.db.notification import NotificationSchema from st2common.models.db.workflow import WorkflowExecutionDB from st2common.models.db.workflow import TaskExecutionDB from st2common.models.db.trigger import TriggerInstanceDB @@ -35,7 +31,6 @@ from st2common.persistence.trigger import TriggerInstance from st2common.constants.triggers import TRIGGER_INSTANCE_PROCESSED from st2common.constants.triggers import TRIGGER_INSTANCE_PENDING -from st2common.util import date as date_utils from st2tests import DbTestCase @@ -46,6 +41,7 @@ import st2_migrate_db_dict_field_values as migration_module + MOCK_RESULT_1 = { "foo": "bar1", "bar": 1, @@ -83,178 +79,10 @@ def test_migrate_executions(self): LiveActionDB._meta["allow_inheritance"] = True class ActionExecutionDB_OldFieldType(ActionExecutionDB): - liveaction_id = me.StringField() # not required; didn't exist result = stormbase.EscapedDynamicField(default={}) - liveaction = stormbase.EscapedDictField(required=True) - parameters = stormbase.EscapedDynamicField(default={}) - - workflow_execution = me.StringField() - task_execution = me.StringField() - status = me.StringField( - required=True, help_text="The current status of the liveaction." - ) - start_timestamp = ComplexDateTimeField( - default=date_utils.get_datetime_utc_now, - help_text="The timestamp when the liveaction was created.", - ) - end_timestamp = ComplexDateTimeField( - help_text="The timestamp when the liveaction has finished." - ) - action = stormbase.EscapedDictField(required=True) - context = me.DictField( - default={}, help_text="Contextual information on the action execution." - ) - delay = me.IntField(min_value=0) - - # diff from liveaction - runner = stormbase.EscapedDictField(required=True) - trigger = stormbase.EscapedDictField() - trigger_type = stormbase.EscapedDictField() - trigger_instance = stormbase.EscapedDictField() - rule = stormbase.EscapedDictField() - result_size = me.IntField( - default=0, help_text="Serialized result size in bytes" - ) - parent = me.StringField() - children = me.ListField(field=me.StringField()) - log = me.ListField(field=me.DictField()) - # Do not use URLField for web_url. If host doesn't have FQDN set, URLField validation blows. - web_url = me.StringField(required=False) class LiveActionDB_OldFieldType(LiveActionDB): result = stormbase.EscapedDynamicField(default={}) - workflow_execution = me.StringField() - task_execution = me.StringField() - # TODO: Can status be an enum at the Mongo layer? - status = me.StringField( - required=True, help_text="The current status of the liveaction." - ) - start_timestamp = ComplexDateTimeField( - default=date_utils.get_datetime_utc_now, - help_text="The timestamp when the liveaction was created.", - ) - end_timestamp = ComplexDateTimeField( - help_text="The timestamp when the liveaction has finished." - ) - action = me.StringField( - required=True, - help_text="Reference to the action that has to be executed.", - ) - parameters = JSONDictEscapedFieldCompatibilityField( - default={}, - help_text="The key-value pairs passed as to the action runner & execution.", - ) - context = me.DictField( - default={}, help_text="Contextual information on the action execution." - ) - delay = me.IntField( - min_value=0, - help_text="How long (in milliseconds) to delay the execution before scheduling.", - ) - - # diff from action execution - action_is_workflow = me.BooleanField( - default=False, - help_text="A flag indicating whether the referenced action is a workflow.", - ) - callback = me.DictField( - default={}, - help_text="Callback information for the on completion of action execution.", - ) - notify = me.EmbeddedDocumentField(NotificationSchema) - runner_info = me.DictField( - default={}, - help_text="Information about the runner which executed this live action (hostname, pid).", - ) - - class LiveActionDB_NewFieldType(LiveActionDB): - result = JSONDictEscapedFieldCompatibilityField( - default={}, help_text="Action defined result." - ) - workflow_execution = me.StringField() - task_execution = me.StringField() - # TODO: Can status be an enum at the Mongo layer? - status = me.StringField( - required=True, help_text="The current status of the liveaction." - ) - start_timestamp = ComplexDateTimeField( - default=date_utils.get_datetime_utc_now, - help_text="The timestamp when the liveaction was created.", - ) - end_timestamp = ComplexDateTimeField( - help_text="The timestamp when the liveaction has finished." - ) - action = me.StringField( - required=True, - help_text="Reference to the action that has to be executed.", - ) - parameters = JSONDictEscapedFieldCompatibilityField( - default={}, - help_text="The key-value pairs passed as to the action runner & execution.", - ) - context = me.DictField( - default={}, help_text="Contextual information on the action execution." - ) - delay = me.IntField( - min_value=0, - help_text="How long (in milliseconds) to delay the execution before scheduling.", - ) - - # diff from action execution - action_is_workflow = me.BooleanField( - default=False, - help_text="A flag indicating whether the referenced action is a workflow.", - ) - callback = me.DictField( - default={}, - help_text="Callback information for the on completion of action execution.", - ) - notify = me.EmbeddedDocumentField(NotificationSchema) - runner_info = me.DictField( - default={}, - help_text="Information about the runner which executed this live action (hostname, pid).", - ) - - class ActionExecutionDB_NewFieldType(ActionExecutionDB): - liveaction_id = me.StringField() # not required; didn't exist - liveaction = stormbase.EscapedDictField(required=True) - parameters = stormbase.EscapedDynamicField(default={}) - result = JSONDictEscapedFieldCompatibilityField( - default={}, help_text="Action defined result." - ) - - workflow_execution = me.StringField() - task_execution = me.StringField() - status = me.StringField( - required=True, help_text="The current status of the liveaction." - ) - start_timestamp = ComplexDateTimeField( - default=date_utils.get_datetime_utc_now, - help_text="The timestamp when the liveaction was created.", - ) - end_timestamp = ComplexDateTimeField( - help_text="The timestamp when the liveaction has finished." - ) - action = stormbase.EscapedDictField(required=True) - context = me.DictField( - default={}, help_text="Contextual information on the action execution." - ) - delay = me.IntField(min_value=0) - - # diff from liveaction - runner = stormbase.EscapedDictField(required=True) - trigger = stormbase.EscapedDictField() - trigger_type = stormbase.EscapedDictField() - trigger_instance = stormbase.EscapedDictField() - rule = stormbase.EscapedDictField() - result_size = me.IntField( - default=0, help_text="Serialized result size in bytes" - ) - parent = me.StringField() - children = me.ListField(field=me.StringField()) - log = me.ListField(field=me.DictField()) - # Do not use URLField for web_url. If host doesn't have FQDN set, URLField validation blows. - web_url = me.StringField(required=False) execution_dbs = ActionExecution.query( __raw__={ @@ -396,23 +224,15 @@ class ActionExecutionDB_NewFieldType(ActionExecutionDB): "$type": "object", }, } - ).update(set___cls="ActionExecutionDB.ActionExecutionDB_NewFieldType") - execution_dbs = ActionExecution.query( - __raw__={ - "result": { - "$not": { - "$type": "binData", - }, - } - } - ) + ).update(set___cls="ActionExecutionDB") + LiveAction.query( __raw__={ "result": { "$type": "object", }, } - ).update(set___cls="LiveActionDB.LiveActionDB_NewFieldType") + ).update(set___cls="LiveActionDB") # 2. Run migration start_dt = datetime.datetime.utcnow().replace( diff --git a/st2common/tests/unit/services/test_trace.py b/st2common/tests/unit/services/test_trace.py index a19f8c3552..39db4c7ade 100644 --- a/st2common/tests/unit/services/test_trace.py +++ b/st2common/tests/unit/services/test_trace.py @@ -253,7 +253,9 @@ def test_get_trace_db_by_live_action_parent_fail(self): def test_get_trace_db_by_live_action_from_execution(self): traceable_liveaction = copy.copy(self.traceable_liveaction) # fixtures id value in liveaction is not persisted in DB. - traceable_liveaction.id = bson.ObjectId(self.traceable_execution.liveaction_id) + traceable_liveaction.id = bson.ObjectId( + self.traceable_execution.liveaction["id"] + ) created, trace_db = trace_service.get_trace_db_by_live_action( traceable_liveaction ) diff --git a/st2common/tests/unit/services/test_workflow_identify_orphans.py b/st2common/tests/unit/services/test_workflow_identify_orphans.py index 8c3de819d3..7110b509c9 100644 --- a/st2common/tests/unit/services/test_workflow_identify_orphans.py +++ b/st2common/tests/unit/services/test_workflow_identify_orphans.py @@ -175,7 +175,7 @@ def mock_workflow_records(self, completed=False, expired=True, log=True): workflow_execution=str(wf_ex_db.id), action={"runner_type": runner, "ref": action_ref}, runner={"name": runner}, - liveaction_id=str(lv_ac_db.id), + liveaction={"id": str(lv_ac_db.id)}, context={"user": user, "workflow_execution": str(wf_ex_db.id)}, status=status, start_timestamp=start_timestamp, @@ -269,7 +269,7 @@ def mock_task_records( task_execution=str(tk_ex_db.id), action={"runner_type": runner, "ref": action_ref}, runner={"name": runner}, - liveaction_id=str(lv_ac_db.id), + liveaction={"id": str(lv_ac_db.id)}, context=context, status=status, start_timestamp=tk_ex_db.start_timestamp, diff --git a/st2common/tests/unit/services/test_workflow_service_retries.py b/st2common/tests/unit/services/test_workflow_service_retries.py index bfc6250e28..0e322fe573 100644 --- a/st2common/tests/unit/services/test_workflow_service_retries.py +++ b/st2common/tests/unit/services/test_workflow_service_retries.py @@ -144,7 +144,7 @@ def test_recover_from_coordinator_connection_error(self, mock_get_lock): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) mock_get_lock.side_effect = [ coordination.ToozConnectionError("foobar"), @@ -178,7 +178,7 @@ def test_retries_exhausted_from_coordinator_connection_error(self, mock_get_lock tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) mock_get_lock.side_effect = [ @@ -220,7 +220,7 @@ def test_recover_from_database_connection_error(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) wf_svc.handle_action_execution_completion(tk1_ac_ex_db) @@ -247,7 +247,7 @@ def test_retries_exhausted_from_database_connection_error(self): tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id) )[0] - tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction_id) + tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(tk1_ac_ex_db.liveaction["id"]) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # The connection error should raise if retries are exhaused. diff --git a/st2common/tests/unit/test_db_execution.py b/st2common/tests/unit/test_db_execution.py index 5b0f73701e..ca322c0f1b 100644 --- a/st2common/tests/unit/test_db_execution.py +++ b/st2common/tests/unit/test_db_execution.py @@ -60,7 +60,6 @@ }, }, }, - "id": "liveaction_inquiry", "action": "core.ask", } @@ -71,7 +70,6 @@ } }, "action": "st2.inquiry.respond", - "id": "liveaction_respond", } OUTPUT_SCHEMA_RESULT = { @@ -84,7 +82,6 @@ } OUTPUT_SCHEMA_LIVEACTION = { - "id": "output_schema", "action": "core.ask", "parameters": {}, } @@ -94,14 +91,14 @@ "action": {"uid": "action:core:ask", "output_schema": {}}, "status": "succeeded", "runner": {"name": "inquirer"}, - "liveaction_id": INQUIRY_LIVEACTION["id"], + "liveaction": INQUIRY_LIVEACTION, "result": INQUIRY_RESULT, }, "execution_2": { "action": {"uid": "action:st2:inquiry.respond", "output_schema": {}}, "status": "succeeded", "runner": {"name": "python-script"}, - "liveaction_id": RESPOND_LIVEACTION["id"], + "liveaction": RESPOND_LIVEACTION, "result": {"exit_code": 0, "result": None, "stderr": "", "stdout": ""}, }, "execution_3": { @@ -122,7 +119,7 @@ }, "status": "succeeded", "runner": {"name": "inquirer", "output_key": "result"}, - "liveaction_id": OUTPUT_SCHEMA_LIVEACTION["id"], + "liveaction": OUTPUT_SCHEMA_LIVEACTION, "result": OUTPUT_SCHEMA_RESULT, }, } @@ -140,7 +137,7 @@ def setUp(self): created.action = execution["action"] created.status = execution["status"] created.runner = execution["runner"] - created.liveaction_id = execution["liveaction_id"] + created.liveaction = execution["liveaction"] created.result = execution["result"] saved = ActionExecutionModelTest._save_execution(created) @@ -191,6 +188,19 @@ def test_execution_inquiry_secrets(self): "supersecretvalue", ) + def test_execution_inquiry_response_action(self): + """Test that the response parameters for any `st2.inquiry.respond` executions are masked + + We aren't bothering to get the inquiry schema in the `st2.inquiry.respond` action, + so we mask all response values. This test ensures this happens. + """ + + masked = self.executions["execution_2"].mask_secrets( + self.executions["execution_2"].to_serializable_dict() + ) + for value in masked["parameters"]["response"].values(): + self.assertEqual(value, MASKED_ATTRIBUTE_VALUE) + def test_output_schema_secret_param_masking(self): """Test that the output marked as secret in the output schema is masked in the output result diff --git a/st2common/tests/unit/test_db_liveaction.py b/st2common/tests/unit/test_db_liveaction.py index 9cc308ad38..605aa759f6 100644 --- a/st2common/tests/unit/test_db_liveaction.py +++ b/st2common/tests/unit/test_db_liveaction.py @@ -16,7 +16,6 @@ from __future__ import absolute_import import mock -from st2common.constants.secrets import MASKED_ATTRIBUTE_VALUE from st2common.models.db.liveaction import LiveActionDB from st2common.models.db.notification import NotificationSchema, NotificationSubSchema from st2common.persistence.liveaction import LiveAction @@ -133,32 +132,6 @@ def test_liveaction_create_with_notify_both_on_success_and_on_error(self): self.assertEqual(on_failure.message, retrieved.notify.on_failure.message) self.assertEqual(retrieved.notify.on_complete, None) - def test_liveaction_inquiry_response_action(self): - RESPOND_LIVEACTION = { - "parameters": { - "response": { - "secondfactor": "omgsupersecret", - } - }, - "action": "st2.inquiry.respond", - "id": "54c6b6d60640fd4f5354e74c", - } - - created = LiveActionDB() - created.action = RESPOND_LIVEACTION["action"] - created.status = "succeeded" - created.parameters = RESPOND_LIVEACTION["parameters"] - created.id = RESPOND_LIVEACTION["id"] - saved = LiveActionModelTest._save_liveaction(created) - - retrieved = LiveAction.get_by_id(saved.id) - self.assertEqual( - saved.action, retrieved.action, "Same triggertype was not returned." - ) - masked = retrieved.mask_secrets(retrieved.to_serializable_dict()) - for value in masked["parameters"]["response"].values(): - self.assertEqual(value, MASKED_ATTRIBUTE_VALUE) - @staticmethod def _save_liveaction(liveaction): return LiveAction.add_or_update(liveaction) diff --git a/st2common/tests/unit/test_executions.py b/st2common/tests/unit/test_executions.py index 24dc0be9c0..0be1ca7c9d 100644 --- a/st2common/tests/unit/test_executions.py +++ b/st2common/tests/unit/test_executions.py @@ -23,9 +23,7 @@ from st2common.util import isotime from st2common.util import date as date_utils from st2common.persistence.execution import ActionExecution -from st2common.persistence.liveaction import LiveAction from st2common.models.api.execution import ActionExecutionAPI -from st2common.models.api.action import LiveActionAPI from st2common.exceptions.db import StackStormDBObjectNotFoundError from six.moves import range @@ -35,18 +33,12 @@ def setUp(self): super(TestActionExecutionHistoryModel, self).setUp() # Fake execution record for action liveactions triggered by workflow runner. - self.fake_history_liveactions = [ - fixture.ARTIFACTS["liveactions"]["task1"], - fixture.ARTIFACTS["liveactions"]["task2"], - ] self.fake_history_subtasks = [ { "id": str(bson.ObjectId()), "action": copy.deepcopy(fixture.ARTIFACTS["actions"]["local"]), "runner": copy.deepcopy(fixture.ARTIFACTS["runners"]["run-local"]), - "liveaction_id": copy.deepcopy( - fixture.ARTIFACTS["liveactions"]["task1"]["id"] - ), + "liveaction": copy.deepcopy(fixture.ARTIFACTS["liveactions"]["task1"]), "status": fixture.ARTIFACTS["liveactions"]["task1"]["status"], "start_timestamp": fixture.ARTIFACTS["liveactions"]["task1"][ "start_timestamp" @@ -59,9 +51,7 @@ def setUp(self): "id": str(bson.ObjectId()), "action": copy.deepcopy(fixture.ARTIFACTS["actions"]["local"]), "runner": copy.deepcopy(fixture.ARTIFACTS["runners"]["run-local"]), - "liveaction_id": copy.deepcopy( - fixture.ARTIFACTS["liveactions"]["task2"]["id"] - ), + "liveaction": copy.deepcopy(fixture.ARTIFACTS["liveactions"]["task2"]), "status": fixture.ARTIFACTS["liveactions"]["task2"]["status"], "start_timestamp": fixture.ARTIFACTS["liveactions"]["task2"][ "start_timestamp" @@ -81,9 +71,7 @@ def setUp(self): "rule": copy.deepcopy(fixture.ARTIFACTS["rule"]), "action": copy.deepcopy(fixture.ARTIFACTS["actions"]["chain"]), "runner": copy.deepcopy(fixture.ARTIFACTS["runners"]["action-chain"]), - "liveaction_id": copy.deepcopy( - fixture.ARTIFACTS["liveactions"]["workflow"]["id"] - ), + "liveaction": copy.deepcopy(fixture.ARTIFACTS["liveactions"]["workflow"]), "children": [task["id"] for task in self.fake_history_subtasks], "status": fixture.ARTIFACTS["liveactions"]["workflow"]["status"], "start_timestamp": fixture.ARTIFACTS["liveactions"]["workflow"][ @@ -93,18 +81,12 @@ def setUp(self): "end_timestamp" ], } - self.fake_history_workflow_liveaction = fixture.ARTIFACTS["liveactions"][ - "workflow" - ] + # Assign parent to the execution records for the subtasks. for task in self.fake_history_subtasks: task["parent"] = self.fake_history_workflow["id"] def test_model_complete(self): - # create LiveactionApiObject - live_action_obj = LiveActionAPI( - **copy.deepcopy(self.fake_history_workflow_liveaction) - ) # Create API object. obj = ActionExecutionAPI(**copy.deepcopy(self.fake_history_workflow)) @@ -118,15 +100,10 @@ def test_model_complete(self): self.assertDictEqual(obj.rule, self.fake_history_workflow["rule"]) self.assertDictEqual(obj.action, self.fake_history_workflow["action"]) self.assertDictEqual(obj.runner, self.fake_history_workflow["runner"]) - self.assertEqual(obj.liveaction_id, self.fake_history_workflow["liveaction_id"]) + self.assertEqual(obj.liveaction, self.fake_history_workflow["liveaction"]) self.assertIsNone(getattr(obj, "parent", None)) self.assertListEqual(obj.children, self.fake_history_workflow["children"]) - # convert liveaction API to model - live_action_model = LiveActionAPI.to_model(live_action_obj) - live_action_model.id = live_action_obj.id - LiveAction.add_or_update(live_action_model) - # Convert API object to DB model. model = ActionExecutionAPI.to_model(obj) self.assertEqual(str(model.id), obj.id) @@ -140,9 +117,10 @@ def test_model_complete(self): self.assertDictEqual(model.rule, self.fake_history_workflow["rule"]) self.assertDictEqual(model.action, self.fake_history_workflow["action"]) self.assertDictEqual(model.runner, self.fake_history_workflow["runner"]) - self.assertEqual( - model.liveaction_id, self.fake_history_workflow["liveaction_id"] - ) + doc = copy.deepcopy(self.fake_history_workflow["liveaction"]) + doc["start_timestamp"] = doc["start_timestamp"] + doc["end_timestamp"] = doc["end_timestamp"] + self.assertDictEqual(model.liveaction, doc) self.assertIsNone(getattr(model, "parent", None)) self.assertListEqual(model.children, self.fake_history_workflow["children"]) @@ -159,7 +137,7 @@ def test_model_complete(self): self.assertDictEqual(obj.rule, self.fake_history_workflow["rule"]) self.assertDictEqual(obj.action, self.fake_history_workflow["action"]) self.assertDictEqual(obj.runner, self.fake_history_workflow["runner"]) - self.assertEqual(obj.liveaction_id, self.fake_history_workflow["liveaction_id"]) + self.assertDictEqual(obj.liveaction, self.fake_history_workflow["liveaction"]) self.assertIsNone(getattr(obj, "parent", None)) self.assertListEqual(obj.children, self.fake_history_workflow["children"]) @@ -179,9 +157,10 @@ def test_crud_complete(self): self.assertDictEqual(model.rule, self.fake_history_workflow["rule"]) self.assertDictEqual(model.action, self.fake_history_workflow["action"]) self.assertDictEqual(model.runner, self.fake_history_workflow["runner"]) - self.assertEqual( - model.liveaction_id, self.fake_history_workflow["liveaction_id"] - ) + doc = copy.deepcopy(self.fake_history_workflow["liveaction"]) + doc["start_timestamp"] = doc["start_timestamp"] + doc["end_timestamp"] = doc["end_timestamp"] + self.assertDictEqual(model.liveaction, doc) self.assertIsNone(getattr(model, "parent", None)) self.assertListEqual(model.children, self.fake_history_workflow["children"]) @@ -199,10 +178,6 @@ def test_crud_complete(self): ) def test_model_partial(self): - # create LiveactionApiObject - live_action_obj = LiveActionAPI( - **copy.deepcopy(self.fake_history_liveactions[0]) - ) # Create API object. obj = ActionExecutionAPI(**copy.deepcopy(self.fake_history_subtasks[0])) self.assertIsNone(getattr(obj, "trigger", None)) @@ -211,19 +186,14 @@ def test_model_partial(self): self.assertIsNone(getattr(obj, "rule", None)) self.assertDictEqual(obj.action, self.fake_history_subtasks[0]["action"]) self.assertDictEqual(obj.runner, self.fake_history_subtasks[0]["runner"]) - self.assertEqual( - obj.liveaction_id, self.fake_history_subtasks[0]["liveaction_id"] + self.assertDictEqual( + obj.liveaction, self.fake_history_subtasks[0]["liveaction"] ) self.assertEqual(obj.parent, self.fake_history_subtasks[0]["parent"]) self.assertIsNone(getattr(obj, "children", None)) - # convert liveaction API to model - live_action_model = LiveActionAPI.to_model(live_action_obj) - live_action_model.id = live_action_obj.id # Convert API object to DB model. model = ActionExecutionAPI.to_model(obj) - LiveAction.add_or_update(live_action_model) - self.assertEqual(str(live_action_model.id), str(live_action_model.id)) self.assertEqual(str(model.id), obj.id) self.assertDictEqual(model.trigger, {}) self.assertDictEqual(model.trigger_type, {}) @@ -231,9 +201,11 @@ def test_model_partial(self): self.assertDictEqual(model.rule, {}) self.assertDictEqual(model.action, self.fake_history_subtasks[0]["action"]) self.assertDictEqual(model.runner, self.fake_history_subtasks[0]["runner"]) - self.assertEqual( - model.liveaction_id, self.fake_history_subtasks[0]["liveaction_id"] - ) + doc = copy.deepcopy(self.fake_history_subtasks[0]["liveaction"]) + doc["start_timestamp"] = doc["start_timestamp"] + doc["end_timestamp"] = doc["end_timestamp"] + + self.assertDictEqual(model.liveaction, doc) self.assertEqual(model.parent, self.fake_history_subtasks[0]["parent"]) self.assertListEqual(model.children, []) @@ -246,8 +218,8 @@ def test_model_partial(self): self.assertIsNone(getattr(obj, "rule", None)) self.assertDictEqual(obj.action, self.fake_history_subtasks[0]["action"]) self.assertDictEqual(obj.runner, self.fake_history_subtasks[0]["runner"]) - self.assertEqual( - obj.liveaction_id, self.fake_history_subtasks[0]["liveaction_id"] + self.assertDictEqual( + obj.liveaction, self.fake_history_subtasks[0]["liveaction"] ) self.assertEqual(obj.parent, self.fake_history_subtasks[0]["parent"]) self.assertIsNone(getattr(obj, "children", None)) @@ -264,9 +236,10 @@ def test_crud_partial(self): self.assertDictEqual(model.rule, {}) self.assertDictEqual(model.action, self.fake_history_subtasks[0]["action"]) self.assertDictEqual(model.runner, self.fake_history_subtasks[0]["runner"]) - self.assertEqual( - model.liveaction_id, self.fake_history_subtasks[0]["liveaction_id"] - ) + doc = copy.deepcopy(self.fake_history_subtasks[0]["liveaction"]) + doc["start_timestamp"] = doc["start_timestamp"] + doc["end_timestamp"] = doc["end_timestamp"] + self.assertDictEqual(model.liveaction, doc) self.assertEqual(model.parent, self.fake_history_subtasks[0]["parent"]) self.assertListEqual(model.children, []) diff --git a/st2common/tests/unit/test_executions_util.py b/st2common/tests/unit/test_executions_util.py index 6e180db1e0..4c2530155a 100644 --- a/st2common/tests/unit/test_executions_util.py +++ b/st2common/tests/unit/test_executions_util.py @@ -79,7 +79,7 @@ def test_execution_creation_manual_action_run(self): executions_util.create_execution_object(liveaction) post_creation_timestamp = date_utils.get_datetime_utc_now() execution = self._get_action_execution( - liveaction_id=str(liveaction.id), raise_exception=True + liveaction__id=str(liveaction.id), raise_exception=True ) self.assertDictEqual(execution.trigger, {}) self.assertDictEqual(execution.trigger_type, {}) @@ -90,7 +90,7 @@ def test_execution_creation_manual_action_run(self): runner = RunnerType.get_by_name(action.runner_type["name"]) self.assertDictEqual(execution.runner, vars(RunnerTypeAPI.from_model(runner))) liveaction = LiveAction.get_by_id(str(liveaction.id)) - self.assertEqual(execution.liveaction_id, str(liveaction.id)) + self.assertEqual(execution.liveaction["id"], str(liveaction.id)) self.assertEqual(len(execution.log), 1) self.assertEqual(execution.log[0]["status"], liveaction.status) self.assertGreater(execution.log[0]["timestamp"], pre_creation_timestamp) @@ -120,7 +120,7 @@ def test_execution_creation_action_triggered_by_rule(self): ) executions_util.create_execution_object(liveaction) execution = self._get_action_execution( - liveaction_id=str(liveaction.id), raise_exception=True + liveaction__id=str(liveaction.id), raise_exception=True ) self.assertDictEqual(execution.trigger, vars(TriggerAPI.from_model(trigger))) self.assertDictEqual( @@ -136,13 +136,13 @@ def test_execution_creation_action_triggered_by_rule(self): runner = RunnerType.get_by_name(action.runner_type["name"]) self.assertDictEqual(execution.runner, vars(RunnerTypeAPI.from_model(runner))) liveaction = LiveAction.get_by_id(str(liveaction.id)) - self.assertEqual(execution.liveaction_id, str(liveaction.id)) + self.assertEqual(execution.liveaction["id"], str(liveaction.id)) def test_execution_creation_with_web_url(self): liveaction = self.MODELS["liveactions"]["liveaction1.yaml"] executions_util.create_execution_object(liveaction) execution = self._get_action_execution( - liveaction_id=str(liveaction.id), raise_exception=True + liveaction__id=str(liveaction.id), raise_exception=True ) self.assertIsNotNone(execution.web_url) execution_id = str(execution.id) @@ -164,7 +164,7 @@ def test_execution_update(self): executions_util.update_execution(liveaction) post_update_timestamp = date_utils.get_datetime_utc_now() execution = self._get_action_execution( - liveaction_id=str(liveaction.id), raise_exception=True + liveaction__id=str(liveaction.id), raise_exception=True ) self.assertEqual(len(execution.log), 2) self.assertEqual(execution.log[1]["status"], liveaction.status) @@ -178,7 +178,7 @@ def test_skip_execution_update(self): liveaction.status = "running" executions_util.update_execution(liveaction) execution = self._get_action_execution( - liveaction_id=str(liveaction.id), raise_exception=True + liveaction__id=str(liveaction.id), raise_exception=True ) self.assertEqual(len(execution.log), 1) # Check status is not updated if it's already in completed state. diff --git a/st2common/tests/unit/test_purge_executions.py b/st2common/tests/unit/test_purge_executions.py index 559494c705..f43266a121 100644 --- a/st2common/tests/unit/test_purge_executions.py +++ b/st2common/tests/unit/test_purge_executions.py @@ -194,7 +194,7 @@ def test_liveaction_gets_deleted(self): exec_model["end_timestamp"] = end_ts exec_model["status"] = action_constants.LIVEACTION_STATUS_SUCCEEDED exec_model["id"] = bson.ObjectId() - exec_model["liveaction_id"] = str(liveaction.id) + exec_model["liveaction"]["id"] = str(liveaction.id) ActionExecution.add_or_update(exec_model) liveactions = LiveAction.get_all() diff --git a/st2reactor/tests/integration/test_garbage_collector.py b/st2reactor/tests/integration/test_garbage_collector.py index 1435125e1a..649d09e534 100644 --- a/st2reactor/tests/integration/test_garbage_collector.py +++ b/st2reactor/tests/integration/test_garbage_collector.py @@ -88,7 +88,7 @@ def test_garbage_collection(self): status=status, action={"ref": "core.local"}, runner={"name": "local-shell-cmd"}, - liveaction_id="ref", + liveaction={"ref": "foo"}, ) ActionExecution.add_or_update(action_execution_db) @@ -124,7 +124,7 @@ def test_garbage_collection(self): status=status, action={"ref": "core.local"}, runner={"name": "local-shell-cmd"}, - liveaction_id="ref", + liveaction={"ref": "foo"}, ) ActionExecution.add_or_update(action_execution_db) @@ -159,7 +159,7 @@ def test_garbage_collection(self): status=status, action={"ref": "core.local"}, runner={"name": "local-shell-cmd"}, - liveaction_id="ref", + liveaction={"ref": "foo"}, ) ActionExecution.add_or_update(action_execution_db) diff --git a/st2stream/tests/unit/controllers/v1/test_stream.py b/st2stream/tests/unit/controllers/v1/test_stream.py index 2d661992f9..dbfb6277c1 100644 --- a/st2stream/tests/unit/controllers/v1/test_stream.py +++ b/st2stream/tests/unit/controllers/v1/test_stream.py @@ -19,8 +19,8 @@ from st2common.models.api.action import ActionAPI from st2common.models.api.action import RunnerTypeAPI -from st2common.models.api.action import LiveActionAPI from st2common.models.api.execution import ActionExecutionAPI +from st2common.models.api.execution import LiveActionAPI from st2common.models.api.execution import ActionExecutionOutputAPI from st2common.models.db.liveaction import LiveActionDB from st2common.models.db.execution import ActionExecutionDB diff --git a/st2stream/tests/unit/controllers/v1/test_stream_execution_output.py b/st2stream/tests/unit/controllers/v1/test_stream_execution_output.py index ab9088056b..9a135d1789 100644 --- a/st2stream/tests/unit/controllers/v1/test_stream_execution_output.py +++ b/st2stream/tests/unit/controllers/v1/test_stream_execution_output.py @@ -21,11 +21,9 @@ from st2common.constants import action as action_constants from st2common.models.db.execution import ActionExecutionDB -from st2common.models.db.liveaction import LiveActionDB from st2common.models.db.execution import ActionExecutionOutputDB from st2common.persistence.execution import ActionExecution from st2common.persistence.execution import ActionExecutionOutput -from st2common.persistence.liveaction import LiveAction from st2common.util import date as date_utils from st2common.stream.listener import get_listener @@ -55,21 +53,15 @@ def test_get_output_running_execution(self): # Test the execution output API endpoint for execution which is running (blocking) status = action_constants.LIVEACTION_STATUS_RUNNING timestamp = date_utils.get_datetime_utc_now() - liveaction_id = "54c6b6d60640fd4f5354e74a" action_execution_db = ActionExecutionDB( start_timestamp=timestamp, end_timestamp=timestamp, status=status, action={"ref": "core.local"}, runner={"name": "local-shell-cmd"}, - liveaction_id=liveaction_id, + liveaction={"ref": "foo"}, ) action_execution_db = ActionExecution.add_or_update(action_execution_db) - liveaction_db = LiveActionDB( - action="core.local", runner_info={"name": "local-shell-cmd"}, status=status - ) - liveaction_db.id = liveaction_id - LiveAction.add_or_update(liveaction_db) output_params = dict( execution_id=str(action_execution_db.id), @@ -143,23 +135,15 @@ def test_get_output_finished_execution(self): # Insert mock execution and output objects status = action_constants.LIVEACTION_STATUS_SUCCEEDED timestamp = date_utils.get_datetime_utc_now() - liveaction_id = "54c6b6d60640fd4f5354e74a" action_execution_db = ActionExecutionDB( start_timestamp=timestamp, end_timestamp=timestamp, status=status, action={"ref": "core.local"}, runner={"name": "local-shell-cmd"}, - liveaction_id=liveaction_id, + liveaction={"ref": "foo"}, ) action_execution_db = ActionExecution.add_or_update(action_execution_db) - liveaction_db = LiveActionDB( - action="core.local", - runner_info={"name": "local-shell-cmd"}, - status=status, - ) - liveaction_db.id = liveaction_id - LiveAction.add_or_update(liveaction_db) for i in range(1, 6): stdout_db = ActionExecutionOutputDB( diff --git a/st2tests/integration/orquesta/test_wiring_error_handling.py b/st2tests/integration/orquesta/test_wiring_error_handling.py index edaba668a1..1c906ce20d 100644 --- a/st2tests/integration/orquesta/test_wiring_error_handling.py +++ b/st2tests/integration/orquesta/test_wiring_error_handling.py @@ -22,16 +22,7 @@ class ErrorHandlingTest(base.TestWorkflowExecution): - def error_inspect(self, ex, expected_errors): - errors = [] - for i in ex.result.get("errors"): - i.pop("traceback", None) - errors.append(i) - for index, i in enumerate(errors): - self.assertDictEqual(i, expected_errors[index]) - def test_inspection_error(self): - self.maxDiff = None expected_errors = [ { "type": "content", @@ -75,8 +66,27 @@ def test_inspection_error(self): ex = self._execute_workflow("examples.orquesta-fail-inspection") ex = self._wait_for_completion(ex) self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) - self.error_inspect(ex, expected_errors) - self.assertIsNone(ex.result["output"]) + self.assertEqual(len(ex.result["errors"]), len(expected_errors)) + for i in range(len(ex.result["errors"])): + self.assertEqual(ex.result["errors"][i]["type"], expected_errors[i]["type"]) + self.assertEqual( + ex.result["errors"][i]["message"], expected_errors[i]["message"] + ) + self.assertEqual( + ex.result["errors"][i]["schema_path"], expected_errors[i]["schema_path"] + ) + self.assertEqual( + ex.result["errors"][i]["spec_path"], expected_errors[i]["spec_path"] + ) + if "language" in expected_errors[i]: + self.assertEqual( + ex.result["errors"][i]["language"], expected_errors[i]["language"] + ) + if "expression" in expected_errors[i]: + self.assertEqual( + ex.result["errors"][i]["expression"], + expected_errors[i]["expression"], + ) def test_input_error(self): expected_errors = [ @@ -93,7 +103,7 @@ def test_input_error(self): ex = self._execute_workflow("examples.orquesta-fail-input-rendering") ex = self._wait_for_completion(ex) self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) - self.error_inspect(ex, expected_errors) + self.assertDictEqual(ex.result, {"errors": expected_errors, "output": None}) def test_vars_error(self): expected_errors = [ @@ -109,8 +119,8 @@ def test_vars_error(self): ex = self._execute_workflow("examples.orquesta-fail-vars-rendering") ex = self._wait_for_completion(ex) - self.error_inspect(ex, expected_errors) self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) + self.assertDictEqual(ex.result, {"errors": expected_errors, "output": None}) def test_start_task_error(self): self.maxDiff = None @@ -137,8 +147,8 @@ def test_start_task_error(self): ex = self._execute_workflow("examples.orquesta-fail-start-task") ex = self._wait_for_completion(ex) - self.error_inspect(ex, expected_errors) self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) + self.assertDictEqual(ex.result, {"errors": expected_errors, "output": None}) def test_task_transition_error(self): expected_errors = [ @@ -159,8 +169,9 @@ def test_task_transition_error(self): ex = self._execute_workflow("examples.orquesta-fail-task-transition") ex = self._wait_for_completion(ex) self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) - self.assertDictEqual(ex.result["output"], expected_output) - self.error_inspect(ex, expected_errors) + self.assertDictEqual( + ex.result, {"errors": expected_errors, "output": expected_output} + ) def test_task_publish_error(self): expected_errors = [ @@ -180,10 +191,10 @@ def test_task_publish_error(self): ex = self._execute_workflow("examples.orquesta-fail-task-publish") ex = self._wait_for_completion(ex) - self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) - self.assertDictEqual(ex.result["output"], expected_output) - self.error_inspect(ex, expected_errors) + self.assertDictEqual( + ex.result, {"errors": expected_errors, "output": expected_output} + ) def test_output_error(self): expected_errors = [ @@ -200,10 +211,9 @@ def test_output_error(self): ex = self._execute_workflow("examples.orquesta-fail-output-rendering") ex = self._wait_for_completion(ex) self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) - self.error_inspect(ex, expected_errors) + self.assertDictEqual(ex.result, {"errors": expected_errors, "output": None}) def test_task_content_errors(self): - self.maxDiff = None expected_errors = [ { "type": "content", @@ -236,9 +246,19 @@ def test_task_content_errors(self): ex = self._execute_workflow("examples.orquesta-fail-inspection-task-contents") ex = self._wait_for_completion(ex) - self.error_inspect(ex, expected_errors) - self.assertIsNone(ex.result["output"]) self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) + self.assertEqual(len(ex.result["errors"]), len(expected_errors)) + for i in range(len(ex.result["errors"])): + self.assertEqual(ex.result["errors"][i]["type"], expected_errors[i]["type"]) + self.assertEqual( + ex.result["errors"][i]["message"], expected_errors[i]["message"] + ) + self.assertEqual( + ex.result["errors"][i]["schema_path"], expected_errors[i]["schema_path"] + ) + self.assertEqual( + ex.result["errors"][i]["spec_path"], expected_errors[i]["spec_path"] + ) def test_remediate_then_fail(self): expected_errors = [ @@ -273,9 +293,10 @@ def test_remediate_then_fail(self): self._wait_for_task(ex, "task1", ac_const.LIVEACTION_STATUS_FAILED) self._wait_for_task(ex, "log", ac_const.LIVEACTION_STATUS_SUCCEEDED) + # Assert workflow status and result. self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) - self.error_inspect(ex, expected_errors) + self.assertDictEqual(ex.result, {"errors": expected_errors, "output": None}) def test_fail_manually(self): expected_errors = [ @@ -342,6 +363,7 @@ def test_fail_continue(self): # Assert task status. self._wait_for_task(ex, "task1", ac_const.LIVEACTION_STATUS_FAILED) + # Assert workflow status and result. self.assertEqual(ex.status, ac_const.LIVEACTION_STATUS_FAILED) self.assertDictEqual( diff --git a/st2tests/st2tests/api.py b/st2tests/st2tests/api.py index 07e6b60266..dae8eb7831 100644 --- a/st2tests/st2tests/api.py +++ b/st2tests/st2tests/api.py @@ -87,6 +87,7 @@ def do_request(self, req, **kwargs): if req.environ["REQUEST_METHOD"] != "OPTIONS": # Making sure endpoint handles OPTIONS method properly self.options(req.environ["PATH_INFO"]) + res = super(TestApp, self).do_request(req, **kwargs) if res.headers.get("Warning", None): @@ -394,7 +395,7 @@ def _get_actionexecution_id(resp): @staticmethod def _get_liveaction_id(resp): - return resp.json["liveaction_id"] + return resp.json["liveaction"]["id"] def _do_get_one(self, actionexecution_id, *args, **kwargs): return self.app.get("/v1/executions/%s" % actionexecution_id, *args, **kwargs) diff --git a/st2tests/st2tests/fixtures/descendants/executions/child1_level1.yaml b/st2tests/st2tests/fixtures/descendants/executions/child1_level1.yaml index ebd2708f75..30467cfb18 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/child1_level1.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/child1_level1.yaml @@ -7,7 +7,8 @@ children: - 54e6583d0640fd16887d685b end_timestamp: '2014-09-01T00:00:57.000001Z' id: 54e657f20640fd16887d6857 -liveaction_id: 54c6b6d60640fd4f5354e74a +liveaction: + action: pointlessaction parent: 54e657d60640fd16887d6855 runner: name: pointlessrunner diff --git a/st2tests/st2tests/fixtures/descendants/executions/child1_level2.yaml b/st2tests/st2tests/fixtures/descendants/executions/child1_level2.yaml index 6f330f5547..2ee548ccb2 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/child1_level2.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/child1_level2.yaml @@ -5,7 +5,8 @@ action: children: [] end_timestamp: '2014-09-01T00:00:56.000002Z' id: 54e657fa0640fd16887d6858 -liveaction_id: 54c6b6d60640fd4f5354e74a +liveaction: + action: pointlessaction parent: 54e657f20640fd16887d6857 runner: name: pointlessrunner diff --git a/st2tests/st2tests/fixtures/descendants/executions/child1_level3.yaml b/st2tests/st2tests/fixtures/descendants/executions/child1_level3.yaml index 59626f9260..276c5aa0b7 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/child1_level3.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/child1_level3.yaml @@ -5,7 +5,8 @@ action: children: [] end_timestamp: '2014-09-01T00:00:55.100000Z' id: 54e6581b0640fd16887d6859 -liveaction_id: 54c6b6d60640fd4f5354e74a +liveaction: + action: pointlessaction parent: 54e6583d0640fd16887d685b runner: name: pointlessrunner diff --git a/st2tests/st2tests/fixtures/descendants/executions/child2_level1.yaml b/st2tests/st2tests/fixtures/descendants/executions/child2_level1.yaml index d3b9188507..35050c15cd 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/child2_level1.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/child2_level1.yaml @@ -6,7 +6,8 @@ children: - 54e658570640fd16887d685d end_timestamp: '2014-09-01T00:00:55.000000Z' id: 54e658290640fd16887d685a -liveaction_id: 54c6b6d60640fd4f5354e74a +liveaction: + action: pointlessaction parent: 54e657d60640fd16887d6855 runner: name: pointlessrunner diff --git a/st2tests/st2tests/fixtures/descendants/executions/child2_level2.yaml b/st2tests/st2tests/fixtures/descendants/executions/child2_level2.yaml index d5fe4246c7..7d57ceb171 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/child2_level2.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/child2_level2.yaml @@ -6,7 +6,8 @@ children: - 54e6581b0640fd16887d6859 end_timestamp: '2014-09-01T00:00:55.000000Z' id: 54e6583d0640fd16887d685b -liveaction_id: 54c6b6d60640fd4f5354e74a +liveaction: + action: pointlessaction parent: 54e657f20640fd16887d6857 runner: name: pointlessrunner diff --git a/st2tests/st2tests/fixtures/descendants/executions/child2_level3.yaml b/st2tests/st2tests/fixtures/descendants/executions/child2_level3.yaml index 9dd19be9cb..a10bcd016b 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/child2_level3.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/child2_level3.yaml @@ -5,7 +5,8 @@ action: children: [] end_timestamp: '2014-09-01T00:00:59.000010Z' id: 54e6584a0640fd16887d685c -liveaction_id: 54c6b6d60640fd4f5354e74a +liveaction: + action: pointlessaction parent: 54e658570640fd16887d685d runner: name: pointlessrunner diff --git a/st2tests/st2tests/fixtures/descendants/executions/child3_level2.yaml b/st2tests/st2tests/fixtures/descendants/executions/child3_level2.yaml index cc40bcc4ba..d803654d6b 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/child3_level2.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/child3_level2.yaml @@ -7,7 +7,8 @@ children: - 54e6585f0640fd16887d685e end_timestamp: '2014-09-01T00:00:55.000000Z' id: 54e658570640fd16887d685d -liveaction_id: 54c6b6d60640fd4f5354e74a +liveaction: + action: pointlessaction parent: 54e658290640fd16887d685a runner: name: pointlessrunner diff --git a/st2tests/st2tests/fixtures/descendants/executions/child3_level3.yaml b/st2tests/st2tests/fixtures/descendants/executions/child3_level3.yaml index 4f28157a71..ad1aae1bad 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/child3_level3.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/child3_level3.yaml @@ -5,7 +5,8 @@ action: children: [] end_timestamp: '2014-09-01T00:00:55.000000Z' id: 54e6585f0640fd16887d685e -liveaction_id: 54c6b6d60640fd4f5354e74a +liveaction: + action: pointlessaction parent: 54e658570640fd16887d685d runner: name: pointlessrunner diff --git a/st2tests/st2tests/fixtures/descendants/executions/root_execution.yaml b/st2tests/st2tests/fixtures/descendants/executions/root_execution.yaml index 1f0a0bfb48..d993f9c140 100644 --- a/st2tests/st2tests/fixtures/descendants/executions/root_execution.yaml +++ b/st2tests/st2tests/fixtures/descendants/executions/root_execution.yaml @@ -7,7 +7,8 @@ children: - 54e658290640fd16887d685a end_timestamp: '2014-09-01T00:00:59.000000Z' id: 54e657d60640fd16887d6855 -liveaction_id: 54c6b6d60640fd4f5354e74a +liveaction: + action: pointlessaction runner: name: pointlessrunner runner_module: no.module diff --git a/st2tests/st2tests/fixtures/generic/executions/execution1.yaml b/st2tests/st2tests/fixtures/generic/executions/execution1.yaml index 4aef0aac05..8d519fad7a 100644 --- a/st2tests/st2tests/fixtures/generic/executions/execution1.yaml +++ b/st2tests/st2tests/fixtures/generic/executions/execution1.yaml @@ -13,7 +13,17 @@ action: runner_type: run-local end_timestamp: '2014-09-01T00:00:05.000000Z' id: 54c6bb640640fd5211edef0d -liveaction_id: 54c6b6d60640fd4f5354e74a +liveaction: + action: core.someworkflow + callback: {} + context: + user: system + end_timestamp: '2014-09-01T00:00:05.000000Z' + id: 54c6b6d60640fd4f5354e74a + parameters: {} + result: {} + start_timestamp: '2014-09-01T00:00:01.000000Z' + status: scheduled parameters: {} result: {} runner: diff --git a/st2tests/st2tests/fixtures/generic/liveactions/parentliveaction.yaml b/st2tests/st2tests/fixtures/generic/liveactions/parentliveaction.yaml index 087be792b1..ed56d4c449 100644 --- a/st2tests/st2tests/fixtures/generic/liveactions/parentliveaction.yaml +++ b/st2tests/st2tests/fixtures/generic/liveactions/parentliveaction.yaml @@ -1,10 +1,10 @@ --- action: core.someworkflow -id: 54c6b6d60640fd4f5354e74a callback: {} context: user: system end_timestamp: '2014-09-01T00:00:05.000000Z' +id: 54c6b6d60640fd4f5354e74a parameters: {} result: {} start_timestamp: '2014-09-01T00:00:01.000000Z' diff --git a/st2tests/st2tests/fixtures/packs/dummy_pack_23/actions/workflows/__init__.py b/st2tests/st2tests/fixtures/packs/dummy_pack_23/actions/workflows/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/st2tests/st2tests/fixtures/packs/executions/liveactions.yaml b/st2tests/st2tests/fixtures/packs/executions/liveactions.yaml index 5e41bc3c4b..6f87578650 100644 --- a/st2tests/st2tests/fixtures/packs/executions/liveactions.yaml +++ b/st2tests/st2tests/fixtures/packs/executions/liveactions.yaml @@ -1,6 +1,5 @@ --- task1: - id: 54c6b6d60640fd4f5354e74a action: executions.local callback: {} end_timestamp: '2014-09-01T00:00:05.000000Z' @@ -19,7 +18,6 @@ task1: start_timestamp: '2014-09-01T00:00:02.000000Z' status: succeeded task2: - id: 54c6b6d60640fd4f5354e74a action: executions.local callback: {} end_timestamp: '2014-09-01T00:00:05.000000Z' @@ -38,7 +36,6 @@ task2: start_timestamp: '2014-09-01T00:00:03.000000Z' status: succeeded workflow: - id: 54c6b6d60640fd4f5354e74a action: executions.chain callback: {} end_timestamp: '2014-09-01T00:00:05.000000Z' diff --git a/st2tests/st2tests/fixtures/rule_enforcements/executions/execution1.yaml b/st2tests/st2tests/fixtures/rule_enforcements/executions/execution1.yaml index 291eab7bcb..b76bd99d57 100644 --- a/st2tests/st2tests/fixtures/rule_enforcements/executions/execution1.yaml +++ b/st2tests/st2tests/fixtures/rule_enforcements/executions/execution1.yaml @@ -13,7 +13,17 @@ action: runner_type: run-local end_timestamp: '2014-09-01T00:00:05.000000Z' id: 565e15ce32ed350857dfa626 -liveaction_id: 54c6b6d60640fd4f5354e74a +liveaction: + action: core.someworkflow + callback: {} + context: + user: system + end_timestamp: '2014-09-01T00:00:05.000000Z' + id: 54c6b6d60640fd4f5354e74a + parameters: {} + result: {} + start_timestamp: '2014-09-01T00:00:01.000000Z' + status: scheduled parameters: cmd: echo bar result: {} diff --git a/st2tests/st2tests/fixtures/traces/executions/execution_with_parent.yaml b/st2tests/st2tests/fixtures/traces/executions/execution_with_parent.yaml index 279e799b8b..d627ef3777 100644 --- a/st2tests/st2tests/fixtures/traces/executions/execution_with_parent.yaml +++ b/st2tests/st2tests/fixtures/traces/executions/execution_with_parent.yaml @@ -17,7 +17,17 @@ action: runner_type: action-chain end_timestamp: '2014-09-01T00:00:05.000000Z' id: 54c6bb640640fd5211edef3d -liveaction_id: 54c6b6d60640fd4f5354e75a +liveaction: + action: traces.someworkflow + callback: {} + context: + user: system + end_timestamp: '2014-09-01T00:00:05.000000Z' + id: 54c6b6d60640fd4f5354e75a + parameters: {} + result: {} + start_timestamp: '2014-09-01T00:00:01.000000Z' + status: scheduled parameters: {} result: {} runner: diff --git a/st2tests/st2tests/fixtures/traces/executions/rule_fired_execution.yaml b/st2tests/st2tests/fixtures/traces/executions/rule_fired_execution.yaml index d09ee9faaa..9e5f3af967 100644 --- a/st2tests/st2tests/fixtures/traces/executions/rule_fired_execution.yaml +++ b/st2tests/st2tests/fixtures/traces/executions/rule_fired_execution.yaml @@ -17,7 +17,17 @@ action: runner_type: action-chain end_timestamp: '2014-09-01T00:00:05.000000Z' id: 54c6bb640640fd5211edef0d -liveaction_id: 54c6b6d60640fd4f5354e74a +liveaction: + action: traces.someworkflow + callback: {} + context: + user: system + end_timestamp: '2014-09-01T00:00:05.000000Z' + id: 54c6b6d60640fd4f5354e74a + parameters: {} + result: {} + start_timestamp: '2014-09-01T00:00:01.000000Z' + status: scheduled parameters: {} result: {} runner: diff --git a/st2tests/st2tests/fixtures/traces/executions/traceable_execution.yaml b/st2tests/st2tests/fixtures/traces/executions/traceable_execution.yaml index 70a0667e7b..55d7d404fa 100644 --- a/st2tests/st2tests/fixtures/traces/executions/traceable_execution.yaml +++ b/st2tests/st2tests/fixtures/traces/executions/traceable_execution.yaml @@ -17,7 +17,17 @@ action: runner_type: action-chain end_timestamp: '2014-09-01T00:00:05.000000Z' id: 54c6bb640640fd5211edef0d -liveaction_id: 54c6b6d60640fd4f5354e74a +liveaction: + action: traces.someworkflow + callback: {} + context: + user: system + end_timestamp: '2014-09-01T00:00:05.000000Z' + id: 54c6b6d60640fd4f5354e74a + parameters: {} + result: {} + start_timestamp: '2014-09-01T00:00:01.000000Z' + status: scheduled parameters: {} result: {} runner: