From 7aed0eabe9f16278ff562242988f4665c7f49375 Mon Sep 17 00:00:00 2001 From: Liz Gehret Date: Thu, 13 Jul 2023 16:47:21 -0700 Subject: [PATCH 01/22] NEW: adds support for ResultCollection --- q2_mystery_stew/generators/__init__.py | 17 ++++++------- q2_mystery_stew/generators/collections.py | 18 +++++++------- q2_mystery_stew/generators/typemaps.py | 7 ++++-- q2_mystery_stew/usage.py | 29 +++++++++++++++-------- 4 files changed, 42 insertions(+), 29 deletions(-) diff --git a/q2_mystery_stew/generators/__init__.py b/q2_mystery_stew/generators/__init__.py index 15d8938..56793bb 100644 --- a/q2_mystery_stew/generators/__init__.py +++ b/q2_mystery_stew/generators/__init__.py @@ -10,7 +10,7 @@ primitive_union_params) from .metadata import metadata_params from .artifacts import artifact_params -from .collections import list_paramgen, set_paramgen +from .collections import list_paramgen, collection_paramgen from .actions import (generate_single_type_methods, generate_multiple_output_methods) from .base import ParamTemplate, ActionTemplate, ParamSpec, Invocation @@ -38,23 +38,24 @@ def should_add(filter_): add_collections = should_add('collections') lists = [] - sets = [] + collections = [] for key, generator in BASIC_GENERATORS.items(): if should_add(key): selected_generators.append(generator()) if add_collections and key != 'metadata': lists.append(list_paramgen(generator())) - sets.append(set_paramgen(generator())) + collections.append(collection_paramgen(generator())) selected_generators.extend(lists) - selected_generators.extend(sets) + selected_generators.extend(collections) return selected_generators __all__ = ['int_params', 'float_params', 'string_params', 'bool_params', 'primitive_union_params', 'metadata_params', 'artifact_params', - 'list_paramgen', 'set_paramgen', 'generate_single_type_methods', - 'generate_multiple_output_methods', 'generate_typemap_methods', - 'BASIC_GENERATORS', 'FILTERS', 'get_param_generators', - 'ParamTemplate', 'ParamSpec', 'ActionTemplate', 'Invocation'] + 'list_paramgen', 'collection_paramgen', + 'generate_single_type_methods', 'generate_multiple_output_methods', + 'generate_typemap_methods', 'BASIC_GENERATORS', 'FILTERS', + 'get_param_generators', 'ParamTemplate', 'ParamSpec', + 'ActionTemplate', 'Invocation'] diff --git a/q2_mystery_stew/generators/collections.py b/q2_mystery_stew/generators/collections.py index d05dd11..028d961 100644 --- a/q2_mystery_stew/generators/collections.py +++ b/q2_mystery_stew/generators/collections.py @@ -8,8 +8,7 @@ import itertools - -from qiime2.plugin import List, Set +from qiime2.plugin import List, Collection from q2_mystery_stew.generators.base import ParamTemplate @@ -39,13 +38,14 @@ def make_list(): return make_list() -def set_paramgen(generator): - def make_set(): +def collection_paramgen(generator): + def make_collection(): for param in generator: yield ParamTemplate( - param.base_name + "_set", - Set[param.qiime_type], + param.base_name + "_collection", + Collection[param.qiime_type], param.view_type, - tuple(set(x) for x in underpowered_set(param.domain))) - make_set.__name__ = 'set_' + generator.__name__ - return make_set() + tuple({str(k): v for k, v in enumerate(x)} + for x in underpowered_set(param.domain))) + make_collection.__name__ = 'collection_' + generator.__name__ + return make_collection() diff --git a/q2_mystery_stew/generators/typemaps.py b/q2_mystery_stew/generators/typemaps.py index 45d3401..36803d5 100644 --- a/q2_mystery_stew/generators/typemaps.py +++ b/q2_mystery_stew/generators/typemaps.py @@ -26,7 +26,8 @@ from q2_mystery_stew.generators.artifacts import ( single_int1_1, single_int1_2, single_int2_1, single_int2_2, wrapped_int1_1, wrapped_int1_2, wrapped_int2_1, wrapped_int2_2) -from q2_mystery_stew.generators.collections import list_paramgen, set_paramgen +from q2_mystery_stew.generators.collections import (list_paramgen, + collection_paramgen) OUTPUT_STATES = [EchoOutputBranch1, EchoOutputBranch2, EchoOutputBranch3] @@ -64,8 +65,10 @@ def should_add(filter_): if should_add('collections'): yield from generate_the_matrix( 'typemap_lists', [list_paramgen(x()) for x in selected_types]) + # TODO: this probably needs to change for collections yield from generate_the_matrix( - 'typemap_sets', [set_paramgen(x()) for x in selected_types]) + 'typemap_collections', [collection_paramgen(x()) + for x in selected_types]) def _to_action(factory): diff --git a/q2_mystery_stew/usage.py b/q2_mystery_stew/usage.py index e671eee..2a6b930 100644 --- a/q2_mystery_stew/usage.py +++ b/q2_mystery_stew/usage.py @@ -46,23 +46,32 @@ def do(use_method, *args): inputs[name] = realized_arguments[name] = None elif is_semantic_type(spec.qiime_type): - if type(argument) == list or type(argument) == set: + if type(argument) == list or type(argument) == dict: collection_type = type(argument) realized_arguments[name] = collection_type() inputs[name] = collection_type() - for arg in argument: - artifact = arg() - view = artifact.view(spec.view_type) - view.__hide_from_garbage_collector = artifact - var = do(use.init_artifact, arg.__name__, arg) + if collection_type == list: + for arg in argument: + artifact = arg() + view = artifact.view(spec.view_type) + view.__hide_from_garbage_collector = artifact + var = do(use.init_artifact, arg.__name__, arg) - if collection_type == list: realized_arguments[name].append(view) inputs[name].append(var) - elif collection_type == set: - realized_arguments[name].add(view) - inputs[name].add(var) + + # we know that if we're not a list, we'll be a dict + else: + for key, arg in argument.items(): + artifact = arg() + view = artifact.view(spec.view_type) + view.__hide_from_garbage_collector = artifact + var = do(use.init_artifact, arg.__name__, arg) + + realized_arguments[name][key] = view + inputs[name][key] = var + else: artifact = argument() view = artifact.view(spec.view_type) From a13ebcdbc2f93c16e87805bbba35f10ea5667146 Mon Sep 17 00:00:00 2001 From: Anthony Date: Mon, 17 Jul 2023 16:34:37 -0700 Subject: [PATCH 02/22] Preliminary implementation that kinda works --- q2_mystery_stew/template.py | 12 +++++++++++- q2_mystery_stew/usage.py | 13 +++++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/q2_mystery_stew/template.py b/q2_mystery_stew/template.py index bc25227..9ea1629 100644 --- a/q2_mystery_stew/template.py +++ b/q2_mystery_stew/template.py @@ -57,7 +57,7 @@ def argument_to_line(name, arg): # We need a list so we can jsonize it (cannot jsonize sets) sort = False - if type(arg) is list or type(arg) is set: + if type(arg) is list: temp = [] for i in value: # If we are given a set of artifacts it will be turned into a list @@ -74,6 +74,16 @@ def argument_to_line(name, arg): value = sorted(temp, key=repr) else: value = temp + elif type(arg) is qiime2.ResultCollection or type(arg) is dict: + temp = {} + for k, v in value.items(): + if isinstance(v, SingleIntFormat): + temp[k] = v.get_int() + expected_type = 'dict' + else: + temp[k] = v + + value = temp return json.dumps([name, value, expected_type]) + '\n' diff --git a/q2_mystery_stew/usage.py b/q2_mystery_stew/usage.py index 2a6b930..0618c02 100644 --- a/q2_mystery_stew/usage.py +++ b/q2_mystery_stew/usage.py @@ -67,10 +67,19 @@ def do(use_method, *args): artifact = arg() view = artifact.view(spec.view_type) view.__hide_from_garbage_collector = artifact - var = do(use.init_artifact, arg.__name__, arg) realized_arguments[name][key] = view - inputs[name][key] = var + + def factory(): + _input = {} + for k, v in argument.items(): + if callable(v): + v = v() + _input[k] = v + return _input + + var = do(use.init_result_collection, name, factory) + inputs[name] = var else: artifact = argument() From a8531629cb1bbd9aa7349509762740b3f08c94da Mon Sep 17 00:00:00 2001 From: Anthony Date: Tue, 18 Jul 2023 16:01:45 -0700 Subject: [PATCH 03/22] Start generating things that return collections --- q2_mystery_stew/generators/__init__.py | 9 +++++---- q2_mystery_stew/generators/actions.py | 27 ++++++++++++++++++++++++++ q2_mystery_stew/plugin_setup.py | 7 ++++++- 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/q2_mystery_stew/generators/__init__.py b/q2_mystery_stew/generators/__init__.py index 56793bb..e21f418 100644 --- a/q2_mystery_stew/generators/__init__.py +++ b/q2_mystery_stew/generators/__init__.py @@ -12,7 +12,8 @@ from .artifacts import artifact_params from .collections import list_paramgen, collection_paramgen from .actions import (generate_single_type_methods, - generate_multiple_output_methods) + generate_multiple_output_methods, + generate_output_collection_methods) from .base import ParamTemplate, ActionTemplate, ParamSpec, Invocation BASIC_GENERATORS = { @@ -56,6 +57,6 @@ def should_add(filter_): 'primitive_union_params', 'metadata_params', 'artifact_params', 'list_paramgen', 'collection_paramgen', 'generate_single_type_methods', 'generate_multiple_output_methods', - 'generate_typemap_methods', 'BASIC_GENERATORS', 'FILTERS', - 'get_param_generators', 'ParamTemplate', 'ParamSpec', - 'ActionTemplate', 'Invocation'] + 'generate_output_collection_methods', 'generate_typemap_methods', + 'BASIC_GENERATORS', 'FILTERS', 'get_param_generators', + 'ParamTemplate', 'ParamSpec', 'ActionTemplate', 'Invocation'] diff --git a/q2_mystery_stew/generators/actions.py b/q2_mystery_stew/generators/actions.py index c9fc252..fbef647 100644 --- a/q2_mystery_stew/generators/actions.py +++ b/q2_mystery_stew/generators/actions.py @@ -8,6 +8,7 @@ from collections import deque +from qiime2.core.type import Collection from qiime2.sdk.util import is_metadata_type, is_semantic_type from q2_mystery_stew.type import EchoOutput @@ -59,6 +60,7 @@ def generate_single_type_methods(generator): invocation_domain=domain) +# TODO: Add functions to do with generating output collection methods here def generate_multiple_output_methods(): for num_outputs in range(1, 5+1): action_id = f'multiple_outputs_{num_outputs}' @@ -71,3 +73,28 @@ def generate_multiple_output_methods(): parameter_specs={}, registered_outputs=qiime_outputs, invocation_domain=[Invocation({}, qiime_outputs)]) + + +def generate_output_collection_methods(): + action_id = 'collection_only' + qiime_outputs = [('output', Collection[EchoOutput])] + yield ActionTemplate(action_id=action_id, + parameter_specs={}, + registered_outputs=qiime_outputs, + invocation_domain=[Invocation({}, qiime_outputs)]) + + action_id = 'collection_first' + qiime_outputs = [('output_collection', Collection[EchoOutput]), + ('output', EchoOutput)] + yield ActionTemplate(action_id=action_id, + parameter_specs={}, + registered_outputs=qiime_outputs, + invocation_domain=[Invocation({}, qiime_outputs)]) + + action_id = 'collection_second' + qiime_outputs = [('output', EchoOutput), + ('output_collection', Collection[EchoOutput])] + yield ActionTemplate(action_id=action_id, + parameter_specs={}, + registered_outputs=qiime_outputs, + invocation_domain=[Invocation({}, qiime_outputs)]) diff --git a/q2_mystery_stew/plugin_setup.py b/q2_mystery_stew/plugin_setup.py index 95439e0..81487ba 100644 --- a/q2_mystery_stew/plugin_setup.py +++ b/q2_mystery_stew/plugin_setup.py @@ -23,7 +23,8 @@ from q2_mystery_stew.template import get_disguised_echo_function from q2_mystery_stew.generators import ( get_param_generators, generate_single_type_methods, - generate_multiple_output_methods, generate_typemap_methods, FILTERS) + generate_multiple_output_methods, generate_typemap_methods, + generate_output_collection_methods, FILTERS) from q2_mystery_stew.transformers import ( to_single_int_format, transform_from_metatadata, transform_to_metadata) @@ -61,6 +62,10 @@ def create_plugin(**filters): for action_template in generate_typemap_methods(filters): register_test_method(plugin, action_template) + if not filters or filters.get('output_collections', False): + for action_template in generate_output_collection_methods(): + register_test_method(plugin, action_template) + return plugin From 3bfa424353f0c85d52c36e053a9aef8e85d6ce38 Mon Sep 17 00:00:00 2001 From: Anthony Date: Tue, 18 Jul 2023 16:02:43 -0700 Subject: [PATCH 04/22] Start templating output collcetions --- q2_mystery_stew/template.py | 97 ++++++++++++++++++++++++++++++++----- 1 file changed, 84 insertions(+), 13 deletions(-) diff --git a/q2_mystery_stew/template.py b/q2_mystery_stew/template.py index 9ea1629..ffd06f8 100644 --- a/q2_mystery_stew/template.py +++ b/q2_mystery_stew/template.py @@ -14,7 +14,7 @@ from q2_mystery_stew.format import SingleIntFormat, EchoOutputFmt -def get_disguised_echo_function(id, python_parameters, num_outputs): +def get_disguised_echo_function(id, python_parameters, qiime_outputs): TEMPLATES = [ _function_template_1output, _function_template_2output, @@ -23,8 +23,32 @@ def get_disguised_echo_function(id, python_parameters, num_outputs): _function_template_5output, ] - function = TEMPLATES[num_outputs - 1] - disguise_echo_function(function, id, python_parameters, num_outputs) + # raise ValueError(qiime_outputs) + # raise ValueError(str(qiime_outputs[0][1])) + # qiime_outputs is a list of (name, type) tuples. If all output types are + # EchoFormat, get template based on number of outputs + types = [] + for output in qiime_outputs: + _, _type = output + types.append(str(_type)) + + if all(_type == 'EchoOutput' for _type in types): + function = TEMPLATES[len(qiime_outputs) - 1] + # Otherwise if we only have have one output it must be a + # Collection[EchoOutput] + elif len(qiime_outputs) == 1: + function = _function_template_collection_only + # Otherwise, we need to figure out if the Collection is first or second in + # the outputs + else: + # Check if type of first output is collection + if str(qiime_outputs[0][1]) == 'Collection[EchoOutput]': + function = _function_template_collection_first + # Otherwise, it must be the second + else: + function = _function_template_collection_second + + disguise_echo_function(function, id, python_parameters, len(qiime_outputs)) return function @@ -88,18 +112,53 @@ def argument_to_line(name, arg): return json.dumps([name, value, expected_type]) + '\n' -def _echo_outputs(kwargs, num_outputs): - output = EchoOutputFmt() - with output.open() as fh: +def _echo_outputs(kwargs, num_outputs, collection_idx=None): + """ NOTE: We used 1 based indexing here for reasons I do not remember, so + collection_idx uses 1 based indexing. + """ + outputs = [] + + if collection_idx == 1: + output = _echo_collection(kwargs=kwargs) + else: + output = _echo_single(kwargs=kwargs) + + outputs.append(output) + + # We already handled the 1st output above, and we do 1 based indexing + for idx in range(2, num_outputs + 1): + if idx == collection_idx: + output = _echo_collection(idx=idx) + else: + output = _echo_single(idx=idx) + + outputs.append(output) + + return tuple(outputs) + + +def _echo_collection(kwargs=None, idx=None): + outputs = {} + + if kwargs: for name, arg in kwargs.items(): - fh.write(argument_to_line(name, arg)) + outputs[name] = _echo_single(kwargs={name: arg}) + else: + for i in range(2): + outputs[i] = _echo_single(kwargs={idx: i}) + + return outputs + - if num_outputs > 1: - output = (output, *map(lambda x: EchoOutputFmt(), - range(1, num_outputs))) - for idx, fmt in enumerate(output[1:], 2): - with fmt.open() as fh: - fh.write(str(idx)) +def _echo_single(kwargs=None, idx=None): + output = EchoOutputFmt() + + with output.open() as fh: + if kwargs: + for name, arg in kwargs.items(): + fh.write(argument_to_line(name, arg)) + else: + fh.write(str(idx)) return output @@ -122,3 +181,15 @@ def _function_template_4output(**kwargs): def _function_template_5output(**kwargs): return _echo_outputs(kwargs, 5) + + +def _function_template_collection_only(**kwargs): + return _echo_outputs(kwargs, 1, 1) + + +def _function_template_collection_first(**kwargs): + return _echo_outputs(kwargs, 2, 2) + + +def _function_template_collection_second(**kwargs): + return _echo_outputs(kwargs, 2, 2) From 1b92e6e0e00f3e9ae970f1388339531c4b6e442e Mon Sep 17 00:00:00 2001 From: Anthony Date: Tue, 18 Jul 2023 16:04:43 -0700 Subject: [PATCH 05/22] qiime_outputs not num_outputs --- q2_mystery_stew/plugin_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/q2_mystery_stew/plugin_setup.py b/q2_mystery_stew/plugin_setup.py index 81487ba..39d87d3 100644 --- a/q2_mystery_stew/plugin_setup.py +++ b/q2_mystery_stew/plugin_setup.py @@ -115,7 +115,7 @@ def register_test_method(plugin, action_template): function = get_disguised_echo_function(id=action_template.action_id, python_parameters=python_parameters, - num_outputs=len(qiime_outputs)) + qiime_outputs=qiime_outputs) usage_examples = {} for idx, invocation in enumerate(action_template.invocation_domain): usage_examples[f'example_{idx}'] = UsageInstantiator( From 57d9f34502ade680f921d39c3d6cfa9f9ed5fb98 Mon Sep 17 00:00:00 2001 From: Anthony Date: Tue, 18 Jul 2023 16:20:59 -0700 Subject: [PATCH 06/22] Refactor logic to acknowledge that TypeMatch uses multiple varients of EchoOutputFmt --- q2_mystery_stew/template.py | 53 +++++++++++++++---------------------- q2_mystery_stew/usage.py | 17 ++++++++++-- 2 files changed, 36 insertions(+), 34 deletions(-) diff --git a/q2_mystery_stew/template.py b/q2_mystery_stew/template.py index ffd06f8..05ab71f 100644 --- a/q2_mystery_stew/template.py +++ b/q2_mystery_stew/template.py @@ -23,30 +23,22 @@ def get_disguised_echo_function(id, python_parameters, qiime_outputs): _function_template_5output, ] - # raise ValueError(qiime_outputs) - # raise ValueError(str(qiime_outputs[0][1])) - # qiime_outputs is a list of (name, type) tuples. If all output types are - # EchoFormat, get template based on number of outputs - types = [] - for output in qiime_outputs: - _, _type = output - types.append(str(_type)) - - if all(_type == 'EchoOutput' for _type in types): - function = TEMPLATES[len(qiime_outputs) - 1] - # Otherwise if we only have have one output it must be a - # Collection[EchoOutput] - elif len(qiime_outputs) == 1: - function = _function_template_collection_only - # Otherwise, we need to figure out if the Collection is first or second in - # the outputs - else: - # Check if type of first output is collection - if str(qiime_outputs[0][1]) == 'Collection[EchoOutput]': - function = _function_template_collection_first - # Otherwise, it must be the second + # If the first output is a Collection we check how many outputs we have + if str(qiime_outputs[0][1]) == 'Collection[EchoOutput]': + # If we only have the collection we use this template + if len(qiime_outputs) == 1: + function = _function_template_collection_only + # Otherwise, the collection is first of several, so we use this one else: - function = _function_template_collection_second + function = _function_template_collection_first + # Now we need to check if the second argument is a collection, only the + # first or second ever will be + elif len(qiime_outputs) > 1 \ + and str(qiime_outputs[1][1]) == 'Collection[EchoOutput]': + function = _function_template_collection_second + # In all other cases, we do not need a collection output template + else: + function = TEMPLATES[len(qiime_outputs) - 1] disguise_echo_function(function, id, python_parameters, len(qiime_outputs)) @@ -113,20 +105,17 @@ def argument_to_line(name, arg): def _echo_outputs(kwargs, num_outputs, collection_idx=None): - """ NOTE: We used 1 based indexing here for reasons I do not remember, so - collection_idx uses 1 based indexing. - """ outputs = [] - if collection_idx == 1: + if collection_idx == 0: output = _echo_collection(kwargs=kwargs) else: output = _echo_single(kwargs=kwargs) outputs.append(output) - # We already handled the 1st output above, and we do 1 based indexing - for idx in range(2, num_outputs + 1): + # We already handled the 1st output above + for idx in range(1, num_outputs): if idx == collection_idx: output = _echo_collection(idx=idx) else: @@ -184,12 +173,12 @@ def _function_template_5output(**kwargs): def _function_template_collection_only(**kwargs): - return _echo_outputs(kwargs, 1, 1) + return _echo_outputs(kwargs, 1, 0) def _function_template_collection_first(**kwargs): - return _echo_outputs(kwargs, 2, 2) + return _echo_outputs(kwargs, 2, 0) def _function_template_collection_second(**kwargs): - return _echo_outputs(kwargs, 2, 2) + return _echo_outputs(kwargs, 2, 1) diff --git a/q2_mystery_stew/usage.py b/q2_mystery_stew/usage.py index 0618c02..35a103b 100644 --- a/q2_mystery_stew/usage.py +++ b/q2_mystery_stew/usage.py @@ -166,11 +166,24 @@ def _assert_output(self, computed_results, output_name, expected_type, idx, output = computed_results[idx] output.assert_output_type(semantic_type=expected_type) + if output.var_type == 'result_collection': + self._assert_output_collection(output, idx, realized_arguments) + else: + self._assert_output_single(output, idx, realized_arguments) + + def _assert_output_collection(self, output, idx, realized_arguments): + for i in range(2): + self._assert_output_single( + output, idx, realized_arguments, key=str(i)) + + def _assert_output_single(self, output, idx, realized_arguments, key=None): if idx == 0: for name, arg in realized_arguments.items(): regex = self._fmt_regex(name, arg) output.assert_has_line_matching(path='echo.txt', - expression=regex) + expression=regex, + key=key) else: output.assert_has_line_matching(path='echo.txt', - expression=str(idx + 1)) + expression=str(idx), + key=key) From f43934fce164c6ea4bb1990023978b6108906254 Mon Sep 17 00:00:00 2001 From: Anthony Date: Tue, 18 Jul 2023 16:23:17 -0700 Subject: [PATCH 07/22] no magic numbers --- q2_mystery_stew/template.py | 4 +++- q2_mystery_stew/usage.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/q2_mystery_stew/template.py b/q2_mystery_stew/template.py index 05ab71f..9209786 100644 --- a/q2_mystery_stew/template.py +++ b/q2_mystery_stew/template.py @@ -13,6 +13,8 @@ from q2_mystery_stew.format import SingleIntFormat, EchoOutputFmt +OUTPUT_COLLECTION_SIZE = 2 + def get_disguised_echo_function(id, python_parameters, qiime_outputs): TEMPLATES = [ @@ -133,7 +135,7 @@ def _echo_collection(kwargs=None, idx=None): for name, arg in kwargs.items(): outputs[name] = _echo_single(kwargs={name: arg}) else: - for i in range(2): + for i in range(OUTPUT_COLLECTION_SIZE): outputs[i] = _echo_single(kwargs={idx: i}) return outputs diff --git a/q2_mystery_stew/usage.py b/q2_mystery_stew/usage.py index 35a103b..058118b 100644 --- a/q2_mystery_stew/usage.py +++ b/q2_mystery_stew/usage.py @@ -12,7 +12,7 @@ from qiime2.sdk.util import (is_semantic_type, is_metadata_type, is_metadata_column_type) -from q2_mystery_stew.template import argument_to_line +from q2_mystery_stew.template import argument_to_line, OUTPUT_COLLECTION_SIZE class UsageInstantiator: @@ -172,7 +172,7 @@ def _assert_output(self, computed_results, output_name, expected_type, idx, self._assert_output_single(output, idx, realized_arguments) def _assert_output_collection(self, output, idx, realized_arguments): - for i in range(2): + for i in range(OUTPUT_COLLECTION_SIZE): self._assert_output_single( output, idx, realized_arguments, key=str(i)) From 31a34bf3021e8e909e8063491b45de9de4bdf82e Mon Sep 17 00:00:00 2001 From: Liz Gehret Date: Wed, 19 Jul 2023 16:48:59 -0700 Subject: [PATCH 08/22] some debugging for galaxy errors --- q2_mystery_stew/plugin_setup.py | 12 ++++++------ q2_mystery_stew/usage.py | 5 +++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/q2_mystery_stew/plugin_setup.py b/q2_mystery_stew/plugin_setup.py index 39d87d3..b6eacaf 100644 --- a/q2_mystery_stew/plugin_setup.py +++ b/q2_mystery_stew/plugin_setup.py @@ -49,18 +49,18 @@ def create_plugin(**filters): ) register_base_implementation(plugin) - if not filters or filters.get('outputs', False): - for action_template in generate_multiple_output_methods(): - register_test_method(plugin, action_template) + # if not filters or filters.get('outputs', False): + # for action_template in generate_multiple_output_methods(): + # register_test_method(plugin, action_template) selected_types = get_param_generators(**filters) for generator in selected_types: for action_template in generate_single_type_methods(generator): register_test_method(plugin, action_template) - if not filters or filters.get('typemaps', False): - for action_template in generate_typemap_methods(filters): - register_test_method(plugin, action_template) + # if not filters or filters.get('typemaps', False): + # for action_template in generate_typemap_methods(filters): + # register_test_method(plugin, action_template) if not filters or filters.get('output_collections', False): for action_template in generate_output_collection_methods(): diff --git a/q2_mystery_stew/usage.py b/q2_mystery_stew/usage.py index 058118b..a0a6f9f 100644 --- a/q2_mystery_stew/usage.py +++ b/q2_mystery_stew/usage.py @@ -9,6 +9,7 @@ import re import qiime2 +from qiime2.sdk import ResultCollection, Result from qiime2.sdk.util import (is_semantic_type, is_metadata_type, is_metadata_column_type) @@ -76,6 +77,10 @@ def factory(): if callable(v): v = v() _input[k] = v + if all(isinstance(v, Result) + for v in _input.values()): + _input = ResultCollection(_input) + return _input var = do(use.init_result_collection, name, factory) From 5d49a9986940dc589a00c407c93604aa0b6d2236 Mon Sep 17 00:00:00 2001 From: Anthony Date: Mon, 24 Jul 2023 11:26:39 -0700 Subject: [PATCH 09/22] Uncomment --- q2_mystery_stew/plugin_setup.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/q2_mystery_stew/plugin_setup.py b/q2_mystery_stew/plugin_setup.py index b6eacaf..39d87d3 100644 --- a/q2_mystery_stew/plugin_setup.py +++ b/q2_mystery_stew/plugin_setup.py @@ -49,18 +49,18 @@ def create_plugin(**filters): ) register_base_implementation(plugin) - # if not filters or filters.get('outputs', False): - # for action_template in generate_multiple_output_methods(): - # register_test_method(plugin, action_template) + if not filters or filters.get('outputs', False): + for action_template in generate_multiple_output_methods(): + register_test_method(plugin, action_template) selected_types = get_param_generators(**filters) for generator in selected_types: for action_template in generate_single_type_methods(generator): register_test_method(plugin, action_template) - # if not filters or filters.get('typemaps', False): - # for action_template in generate_typemap_methods(filters): - # register_test_method(plugin, action_template) + if not filters or filters.get('typemaps', False): + for action_template in generate_typemap_methods(filters): + register_test_method(plugin, action_template) if not filters or filters.get('output_collections', False): for action_template in generate_output_collection_methods(): From 2ca7a2bbaa58519bd303477f5336629f96d49fd1 Mon Sep 17 00:00:00 2001 From: Anthony Date: Mon, 24 Jul 2023 11:27:00 -0700 Subject: [PATCH 10/22] Add output_collections to filters --- q2_mystery_stew/generators/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/q2_mystery_stew/generators/__init__.py b/q2_mystery_stew/generators/__init__.py index e21f418..7526806 100644 --- a/q2_mystery_stew/generators/__init__.py +++ b/q2_mystery_stew/generators/__init__.py @@ -25,7 +25,8 @@ 'strings': string_params, 'primitive_unions': primitive_union_params, } -FILTERS = {*BASIC_GENERATORS.keys(), 'collections', 'typemaps', 'outputs'} +FILTERS = {*BASIC_GENERATORS.keys(), 'collections', 'typemaps', 'outputs', + 'output_collections'} from .typemaps import generate_typemap_methods # noqa: E402 From 59df0bc8c62928f00ddc1a0b5c0cfb3d1ef2675b Mon Sep 17 00:00:00 2001 From: Anthony Date: Tue, 25 Jul 2023 11:29:24 -0700 Subject: [PATCH 11/22] Clean up some of the usage assertions --- q2_mystery_stew/template.py | 4 ++-- q2_mystery_stew/usage.py | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/q2_mystery_stew/template.py b/q2_mystery_stew/template.py index 9209786..e6413b7 100644 --- a/q2_mystery_stew/template.py +++ b/q2_mystery_stew/template.py @@ -110,9 +110,9 @@ def _echo_outputs(kwargs, num_outputs, collection_idx=None): outputs = [] if collection_idx == 0: - output = _echo_collection(kwargs=kwargs) + output = _echo_collection(kwargs=kwargs, idx=0) else: - output = _echo_single(kwargs=kwargs) + output = _echo_single(kwargs=kwargs, idx=0) outputs.append(output) diff --git a/q2_mystery_stew/usage.py b/q2_mystery_stew/usage.py index a0a6f9f..f822a3b 100644 --- a/q2_mystery_stew/usage.py +++ b/q2_mystery_stew/usage.py @@ -172,17 +172,19 @@ def _assert_output(self, computed_results, output_name, expected_type, idx, output.assert_output_type(semantic_type=expected_type) if output.var_type == 'result_collection': - self._assert_output_collection(output, idx, realized_arguments) + self._assert_output_collection(output, idx, realized_arguments, expected_type) else: self._assert_output_single(output, idx, realized_arguments) - def _assert_output_collection(self, output, idx, realized_arguments): + def _assert_output_collection(self, output, idx, realized_arguments, expected_type): + inner_type = expected_type.fields[0] for i in range(OUTPUT_COLLECTION_SIZE): + output.assert_output_type(semantic_type=inner_type, key=idx) self._assert_output_single( - output, idx, realized_arguments, key=str(i)) + output, idx, realized_arguments, key=i) def _assert_output_single(self, output, idx, realized_arguments, key=None): - if idx == 0: + if idx == 0 and realized_arguments: for name, arg in realized_arguments.items(): regex = self._fmt_regex(name, arg) output.assert_has_line_matching(path='echo.txt', From 81721ca81cd22f23be0250d70f4b0154ace28467 Mon Sep 17 00:00:00 2001 From: Anthony Date: Tue, 25 Jul 2023 14:14:22 -0700 Subject: [PATCH 12/22] Ensure we are echoing both indices to collection elements --- q2_mystery_stew/template.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/q2_mystery_stew/template.py b/q2_mystery_stew/template.py index e6413b7..2899896 100644 --- a/q2_mystery_stew/template.py +++ b/q2_mystery_stew/template.py @@ -136,7 +136,10 @@ def _echo_collection(kwargs=None, idx=None): outputs[name] = _echo_single(kwargs={name: arg}) else: for i in range(OUTPUT_COLLECTION_SIZE): - outputs[i] = _echo_single(kwargs={idx: i}) + # Elements within a collection have a dual index, the index of the + # entire output followed by the index of the element within the + # collection + outputs[i] = _echo_single(idx=f'{idx}: {i}') return outputs From 9aeeca02f38c06c85693a973d4676d9a6fe90752 Mon Sep 17 00:00:00 2001 From: Anthony Date: Tue, 25 Jul 2023 14:14:38 -0700 Subject: [PATCH 13/22] Ensure that we are writing usage assertions correctly --- q2_mystery_stew/usage.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/q2_mystery_stew/usage.py b/q2_mystery_stew/usage.py index f822a3b..3d04c7d 100644 --- a/q2_mystery_stew/usage.py +++ b/q2_mystery_stew/usage.py @@ -179,11 +179,11 @@ def _assert_output(self, computed_results, output_name, expected_type, idx, def _assert_output_collection(self, output, idx, realized_arguments, expected_type): inner_type = expected_type.fields[0] for i in range(OUTPUT_COLLECTION_SIZE): - output.assert_output_type(semantic_type=inner_type, key=idx) + output.assert_output_type(semantic_type=inner_type, key=i) self._assert_output_single( - output, idx, realized_arguments, key=i) + output, idx, realized_arguments, key=i, expression=f"{idx}: {i}") - def _assert_output_single(self, output, idx, realized_arguments, key=None): + def _assert_output_single(self, output, idx, realized_arguments, key=None, expression=None): if idx == 0 and realized_arguments: for name, arg in realized_arguments.items(): regex = self._fmt_regex(name, arg) @@ -191,6 +191,9 @@ def _assert_output_single(self, output, idx, realized_arguments, key=None): expression=regex, key=key) else: + if expression is None: + expression = str(idx) + output.assert_has_line_matching(path='echo.txt', - expression=str(idx), + expression=expression, key=key) From 3d0233cc3db7bbf4a229a6a30da9d9ed742c69fe Mon Sep 17 00:00:00 2001 From: Anthony Date: Tue, 25 Jul 2023 14:15:25 -0700 Subject: [PATCH 14/22] Sets are dead --- q2_mystery_stew/template.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/q2_mystery_stew/template.py b/q2_mystery_stew/template.py index 2899896..e75ef2d 100644 --- a/q2_mystery_stew/template.py +++ b/q2_mystery_stew/template.py @@ -78,18 +78,12 @@ def argument_to_line(name, arg): if type(arg) is list: temp = [] for i in value: - # If we are given a set of artifacts it will be turned into a list - # by the framework, so we need to be ready to accept a list if isinstance(i, SingleIntFormat): temp.append(i.get_int()) expected_type = 'list' sort = True else: temp.append(i) - # If we turned a set into a list for json purposes, we need to sort it - # to ensure it is always in the same order - if type(arg) is set or sort: - value = sorted(temp, key=repr) else: value = temp elif type(arg) is qiime2.ResultCollection or type(arg) is dict: From 99368b5a56c847a34a4e532c4f26b07e79c457e5 Mon Sep 17 00:00:00 2001 From: Anthony Date: Wed, 26 Jul 2023 10:50:38 -0700 Subject: [PATCH 15/22] some set leftovers --- q2_mystery_stew/template.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/q2_mystery_stew/template.py b/q2_mystery_stew/template.py index e75ef2d..c11fb43 100644 --- a/q2_mystery_stew/template.py +++ b/q2_mystery_stew/template.py @@ -73,15 +73,12 @@ def argument_to_line(name, arg): qiime2.NumericMetadataColumn)): value = arg.to_series().to_json() - # We need a list so we can jsonize it (cannot jsonize sets) - sort = False if type(arg) is list: temp = [] for i in value: if isinstance(i, SingleIntFormat): temp.append(i.get_int()) expected_type = 'list' - sort = True else: temp.append(i) else: From b26a673dd550c2c219c292acb0ecc106da91febd Mon Sep 17 00:00:00 2001 From: Anthony Date: Wed, 26 Jul 2023 10:55:03 -0700 Subject: [PATCH 16/22] Set view_type of collection params to the collection type and of input params to the artifact view --- q2_mystery_stew/generators/collections.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/q2_mystery_stew/generators/collections.py b/q2_mystery_stew/generators/collections.py index 028d961..d8dd3a7 100644 --- a/q2_mystery_stew/generators/collections.py +++ b/q2_mystery_stew/generators/collections.py @@ -9,6 +9,7 @@ import itertools from qiime2.plugin import List, Collection +from qiime2.core.type.util import is_semantic_type from q2_mystery_stew.generators.base import ParamTemplate @@ -29,10 +30,15 @@ def underpowered_set(iterable): def list_paramgen(generator): def make_list(): for param in generator: + if is_semantic_type(param.qiime_type): + view_type = param.view_type + else: + view_type = list + yield ParamTemplate( param.base_name + "_list", List[param.qiime_type], - param.view_type, + view_type, tuple(list(x) for x in underpowered_set(param.domain))) make_list.__name__ = 'list_' + generator.__name__ return make_list() @@ -41,10 +47,15 @@ def make_list(): def collection_paramgen(generator): def make_collection(): for param in generator: + if is_semantic_type(param.qiime_type): + view_type = param.view_type + else: + view_type = dict + yield ParamTemplate( param.base_name + "_collection", Collection[param.qiime_type], - param.view_type, + view_type, tuple({str(k): v for k, v in enumerate(x)} for x in underpowered_set(param.domain))) make_collection.__name__ = 'collection_' + generator.__name__ From 20d11d53b59067732388f9f5f70d09014da4a09b Mon Sep 17 00:00:00 2001 From: Anthony Date: Wed, 26 Jul 2023 10:58:09 -0700 Subject: [PATCH 17/22] Lint --- q2_mystery_stew/usage.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/q2_mystery_stew/usage.py b/q2_mystery_stew/usage.py index 3d04c7d..3b9a118 100644 --- a/q2_mystery_stew/usage.py +++ b/q2_mystery_stew/usage.py @@ -172,18 +172,22 @@ def _assert_output(self, computed_results, output_name, expected_type, idx, output.assert_output_type(semantic_type=expected_type) if output.var_type == 'result_collection': - self._assert_output_collection(output, idx, realized_arguments, expected_type) + self._assert_output_collection(output, idx, realized_arguments, + expected_type) else: self._assert_output_single(output, idx, realized_arguments) - def _assert_output_collection(self, output, idx, realized_arguments, expected_type): + def _assert_output_collection(self, output, idx, realized_arguments, + expected_type): inner_type = expected_type.fields[0] for i in range(OUTPUT_COLLECTION_SIZE): output.assert_output_type(semantic_type=inner_type, key=i) self._assert_output_single( - output, idx, realized_arguments, key=i, expression=f"{idx}: {i}") + output, idx, realized_arguments, key=i, + expression=f"{idx}: {i}") - def _assert_output_single(self, output, idx, realized_arguments, key=None, expression=None): + def _assert_output_single(self, output, idx, realized_arguments, key=None, + expression=None): if idx == 0 and realized_arguments: for name, arg in realized_arguments.items(): regex = self._fmt_regex(name, arg) From b44fce9992a92ea03c24e5b83653ca7ba8224d3a Mon Sep 17 00:00:00 2001 From: Anthony Date: Thu, 10 Aug 2023 11:07:17 -0700 Subject: [PATCH 18/22] Make collection keys less obvious numbers --- q2_mystery_stew/template.py | 4 +++- q2_mystery_stew/usage.py | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/q2_mystery_stew/template.py b/q2_mystery_stew/template.py index c11fb43..e288155 100644 --- a/q2_mystery_stew/template.py +++ b/q2_mystery_stew/template.py @@ -14,6 +14,8 @@ from q2_mystery_stew.format import SingleIntFormat, EchoOutputFmt OUTPUT_COLLECTION_SIZE = 2 +OUTPUT_COLLECTION_START = 42 +OUTPUT_COLLECTION_END = OUTPUT_COLLECTION_START + OUTPUT_COLLECTION_SIZE def get_disguised_echo_function(id, python_parameters, qiime_outputs): @@ -126,7 +128,7 @@ def _echo_collection(kwargs=None, idx=None): for name, arg in kwargs.items(): outputs[name] = _echo_single(kwargs={name: arg}) else: - for i in range(OUTPUT_COLLECTION_SIZE): + for i in range(OUTPUT_COLLECTION_START, OUTPUT_COLLECTION_END): # Elements within a collection have a dual index, the index of the # entire output followed by the index of the element within the # collection diff --git a/q2_mystery_stew/usage.py b/q2_mystery_stew/usage.py index 3b9a118..859fc37 100644 --- a/q2_mystery_stew/usage.py +++ b/q2_mystery_stew/usage.py @@ -13,7 +13,8 @@ from qiime2.sdk.util import (is_semantic_type, is_metadata_type, is_metadata_column_type) -from q2_mystery_stew.template import argument_to_line, OUTPUT_COLLECTION_SIZE +from q2_mystery_stew.template import ( + argument_to_line, OUTPUT_COLLECTION_START, OUTPUT_COLLECTION_END) class UsageInstantiator: @@ -171,7 +172,8 @@ def _assert_output(self, computed_results, output_name, expected_type, idx, output = computed_results[idx] output.assert_output_type(semantic_type=expected_type) - if output.var_type == 'result_collection': + if output.var_type in ('artifact_collection', + 'visualization_collection'): self._assert_output_collection(output, idx, realized_arguments, expected_type) else: @@ -180,7 +182,7 @@ def _assert_output(self, computed_results, output_name, expected_type, idx, def _assert_output_collection(self, output, idx, realized_arguments, expected_type): inner_type = expected_type.fields[0] - for i in range(OUTPUT_COLLECTION_SIZE): + for i in range(OUTPUT_COLLECTION_START, OUTPUT_COLLECTION_END): output.assert_output_type(semantic_type=inner_type, key=i) self._assert_output_single( output, idx, realized_arguments, key=i, From 44efa59b2f239e7fa1eacac3c59293fae232a980 Mon Sep 17 00:00:00 2001 From: Anthony Date: Mon, 14 Aug 2023 13:52:45 -0700 Subject: [PATCH 19/22] COLLECTION_VAR_TYPES --- q2_mystery_stew/usage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/q2_mystery_stew/usage.py b/q2_mystery_stew/usage.py index 859fc37..cbb7a59 100644 --- a/q2_mystery_stew/usage.py +++ b/q2_mystery_stew/usage.py @@ -12,6 +12,7 @@ from qiime2.sdk import ResultCollection, Result from qiime2.sdk.util import (is_semantic_type, is_metadata_type, is_metadata_column_type) +from qiime2.sdk.usage import COLLECTION_VAR_TYPES from q2_mystery_stew.template import ( argument_to_line, OUTPUT_COLLECTION_START, OUTPUT_COLLECTION_END) @@ -172,8 +173,7 @@ def _assert_output(self, computed_results, output_name, expected_type, idx, output = computed_results[idx] output.assert_output_type(semantic_type=expected_type) - if output.var_type in ('artifact_collection', - 'visualization_collection'): + if output.var_type in COLLECTION_VAR_TYPES: self._assert_output_collection(output, idx, realized_arguments, expected_type) else: From 1a1150cf62a9f442298d8531ad41bef7cdae90f4 Mon Sep 17 00:00:00 2001 From: Anthony Date: Mon, 14 Aug 2023 15:14:43 -0700 Subject: [PATCH 20/22] Remove dead TODOs --- q2_mystery_stew/generators/actions.py | 1 - q2_mystery_stew/generators/typemaps.py | 1 - 2 files changed, 2 deletions(-) diff --git a/q2_mystery_stew/generators/actions.py b/q2_mystery_stew/generators/actions.py index fbef647..486e437 100644 --- a/q2_mystery_stew/generators/actions.py +++ b/q2_mystery_stew/generators/actions.py @@ -60,7 +60,6 @@ def generate_single_type_methods(generator): invocation_domain=domain) -# TODO: Add functions to do with generating output collection methods here def generate_multiple_output_methods(): for num_outputs in range(1, 5+1): action_id = f'multiple_outputs_{num_outputs}' diff --git a/q2_mystery_stew/generators/typemaps.py b/q2_mystery_stew/generators/typemaps.py index 36803d5..71b84fe 100644 --- a/q2_mystery_stew/generators/typemaps.py +++ b/q2_mystery_stew/generators/typemaps.py @@ -65,7 +65,6 @@ def should_add(filter_): if should_add('collections'): yield from generate_the_matrix( 'typemap_lists', [list_paramgen(x()) for x in selected_types]) - # TODO: this probably needs to change for collections yield from generate_the_matrix( 'typemap_collections', [collection_paramgen(x()) for x in selected_types]) From 2604baa18b179af4b8712f736513252e7ac96a99 Mon Sep 17 00:00:00 2001 From: Liz Gehret Date: Tue, 15 Aug 2023 15:58:09 -0700 Subject: [PATCH 21/22] lint --- q2_mystery_stew/usage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/q2_mystery_stew/usage.py b/q2_mystery_stew/usage.py index cbb7a59..2d881fe 100644 --- a/q2_mystery_stew/usage.py +++ b/q2_mystery_stew/usage.py @@ -49,7 +49,7 @@ def do(use_method, *args): inputs[name] = realized_arguments[name] = None elif is_semantic_type(spec.qiime_type): - if type(argument) == list or type(argument) == dict: + if type(argument) is list or type(argument) is dict: collection_type = type(argument) realized_arguments[name] = collection_type() inputs[name] = collection_type() From 56b4558c6ec7e83109f20272f37d1101ba9656b1 Mon Sep 17 00:00:00 2001 From: Evan Bolyen Date: Wed, 16 Aug 2023 15:32:47 -0700 Subject: [PATCH 22/22] bug: need a closure --- q2_mystery_stew/usage.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/q2_mystery_stew/usage.py b/q2_mystery_stew/usage.py index 2d881fe..dbcd51b 100644 --- a/q2_mystery_stew/usage.py +++ b/q2_mystery_stew/usage.py @@ -73,17 +73,25 @@ def do(use_method, *args): realized_arguments[name][key] = view - def factory(): - _input = {} - for k, v in argument.items(): - if callable(v): - v = v() - _input[k] = v - if all(isinstance(v, Result) - for v in _input.values()): - _input = ResultCollection(_input) - - return _input + def _closure(argument): + # We need to bind the argument from the loop above + # for the factory to use the correct one. + # Otherwise the argument will always be the last + # element. + def factory(): + _input = {} + for k, v in argument.items(): + if callable(v): + v = v() + _input[k] = v + if all(isinstance(v, Result) + for v in _input.values()): + _input = ResultCollection(_input) + + return _input + return factory + # neato! + factory = _closure(argument) var = do(use.init_result_collection, name, factory) inputs[name] = var