From ecc9c89b3af9fc7bb20277f027aecdc3e31d4513 Mon Sep 17 00:00:00 2001 From: Gerald Walter Irsiegler Date: Tue, 16 Apr 2024 15:39:36 +0200 Subject: [PATCH] Fix: fixed parsing of non-dict arguments for resolution --- openeo_pg_parser_networkx/resolving_utils.py | 6 +- resolved_gfm_graph.json | 1 - .../res_tests/resolved/resolved_sen2like.json | 32 +++ .../udps/sen2like_original_outputs.json | 205 ++++++++++++++++++ .../unresolved/unresolved_sen2like.json | 23 ++ tests/test_pg_resolving.py | 29 ++- 6 files changed, 292 insertions(+), 4 deletions(-) delete mode 100644 resolved_gfm_graph.json create mode 100644 tests/data/res_tests/resolved/resolved_sen2like.json create mode 100644 tests/data/res_tests/udps/sen2like_original_outputs.json create mode 100644 tests/data/res_tests/unresolved/unresolved_sen2like.json diff --git a/openeo_pg_parser_networkx/resolving_utils.py b/openeo_pg_parser_networkx/resolving_utils.py index 78d8ff8..1cc0023 100644 --- a/openeo_pg_parser_networkx/resolving_utils.py +++ b/openeo_pg_parser_networkx/resolving_utils.py @@ -231,10 +231,14 @@ def _adjust_parameters(process_graph, process_replacement_id, arguments): for node_key, node in process_graph[process_replacement_id].items(): for arg_key, from_param in node['arguments'].items(): # Find from_parameter value in arguments list and replace with arguments[from_parameter value] value - if "from_parameter" in from_param: + if isinstance(from_param, dict) and "from_parameter" in from_param: process_graph[process_replacement_id][node_key]['arguments'][ arg_key ] = arguments[from_param['from_parameter']] + else: + process_graph[process_replacement_id][node_key]['arguments'][ + arg_key + ] = from_param def _adjust_references(input_graph): diff --git a/resolved_gfm_graph.json b/resolved_gfm_graph.json deleted file mode 100644 index c3eebc4..0000000 --- a/resolved_gfm_graph.json +++ /dev/null @@ -1 +0,0 @@ -{"GFM_load1": {"process_id": "load_collection", "arguments": {"id": "GFM", "spatial_extent": {"west": 65.27044369351682, "east": 69.21281566288451, "south": 28.076233929760804, "north": 29.369117066086332}, "temporal_extent": ["2022-08-01T00:00:00Z", "2022-10-01T00:00:00Z"], "properties": {}}}, "GFM_reduce1": {"process_id": "reduce_dimension", "arguments": {"data": {"from_node": "GFM_load1"}, "reducer": {"process_graph": {"sum1": {"process_id": "sum", "arguments": {"data": {"from_parameter": "data"}}, "result": true}}}, "dimension": "time"}}, "GFM_save2": {"process_id": "save_result", "arguments": {"format": "GTIFF", "data": {"from_node": "GFM_reduce1"}}, "result": true}} diff --git a/tests/data/res_tests/resolved/resolved_sen2like.json b/tests/data/res_tests/resolved/resolved_sen2like.json new file mode 100644 index 0000000..218467a --- /dev/null +++ b/tests/data/res_tests/resolved/resolved_sen2like.json @@ -0,0 +1,32 @@ +{ + "sen2like_original_outputs_load1": { + "process_id": "load_collection", + "arguments": { + "id": "SENTINEL2_L1C", + "spatial_extent": { + "west": -4.919340483677538, + "east": 36.248628266322456, + "south": 41.43373541041478, + "north": 53.27118132212786 + }, + "temporal_extent": [ + "2024-04-01T00:00:00Z", + "2024-04-09T00:00:00Z" + ], + "bands": [ + "bo1" + ], + "properties": {} + } + }, + "sen2like_original_outputs_sen2": { + "process_id": "sen2like", + "arguments": { + "data": { + "from_node": "sen2like_original_outputs_load1" + }, + "export_original_files": true + }, + "result": true + } +} diff --git a/tests/data/res_tests/udps/sen2like_original_outputs.json b/tests/data/res_tests/udps/sen2like_original_outputs.json new file mode 100644 index 0000000..ec91fef --- /dev/null +++ b/tests/data/res_tests/udps/sen2like_original_outputs.json @@ -0,0 +1,205 @@ +{ + "id": "sen2like_original_outputs", + "summary": "Create Sentinel 2 - like .SAFE outputs from Sentinel 2 L1C and Landsat 8 and 9 datasets. ", + "description": "Process sen2like and create Sentinel 2 - like .SAFE outputs from Sentinel 2 L1C and Landsat 8 and 9 datasets for all bands. Area of interest and time need to be specified. ", + "parameters": [ + { + "schema": { + "type": "object", + "subtype": "bounding-box", + "title": "Bounding Box", + "description": "A bounding box with the required fields `west`, `south`, `east`, `north` and optionally `base`, `height`, `crs`. The `crs` is a EPSG code, a WKT2:2018 string or a PROJ definition (deprecated).", + "required": [ + "west", + "south", + "east", + "north" + ], + "properties": { + "west": { + "description": "West (lower left corner, coordinate axis 1).", + "type": "number" + }, + "south": { + "description": "South (lower left corner, coordinate axis 2).", + "type": "number" + }, + "east": { + "description": "East (upper right corner, coordinate axis 1).", + "type": "number" + }, + "north": { + "description": "North (upper right corner, coordinate axis 2).", + "type": "number" + }, + "base": { + "description": "Base (optional, lower left corner, coordinate axis 3).", + "type": [ + "number", + "null" + ] + }, + "height": { + "description": "Height (optional, upper right corner, coordinate axis 3).", + "type": [ + "number", + "null" + ] + }, + "crs": { + "description": "Coordinate reference system of the extent, specified as as [EPSG code](http://www.epsg-registry.org/), [WKT2 (ISO 19162) string](http://docs.opengeospatial.org/is/18-010r7/18-010r7.html) or [PROJ definition (deprecated)](https://proj.org/usage/quickstart.html). Defaults to `4326` (EPSG code 4326) unless the client explicitly requests a different coordinate reference system.", + "anyOf": [ + { + "type": "integer", + "subtype": "epsg-code", + "title": "EPSG Code", + "description": "Specifies details about cartographic projections as [EPSG](http://www.epsg.org) code.", + "minimum": 1000, + "examples": [ + 3857 + ] + }, + { + "type": "string", + "subtype": "wkt2-definition", + "title": "WKT2 definition", + "description": "Specifies details about cartographic projections as WKT2 string. Refers to the latest WKT2 version (currently [WKT2:2018](http://docs.opengeospatial.org/is/18-010r7/18-010r7.html) / ISO 19162:2018) unless otherwise stated by the process." + }, + { + "type": "string", + "subtype": "proj-definition", + "title": "PROJ definition", + "description": "**DEPRECATED.** Specifies details about cartographic projections as [PROJ](https://proj.org/usage/quickstart.html) definition." + } + ], + "default": 4326 + } + } + }, + "name": "spatial_extent", + "description": "Bounding box for the area of interest" + }, + { + "schema": { + "type": "array", + "subtype": "temporal-interval", + "title": "Single temporal interval", + "description": "Left-closed temporal interval, represented as two-element array with the following elements:\n\n1. The first element is the start of the temporal interval. The specified instance in time is **included** in the interval.\n2. The second element is the end of the temporal interval. The specified instance in time is **excluded** from the interval.\n\nThe specified temporal strings follow [RFC 3339](https://www.rfc-editor.org/rfc/rfc3339.html). Although [RFC 3339 prohibits the hour to be '24'](https://www.rfc-editor.org/rfc/rfc3339.html#section-5.7), **this process allows the value '24' for the hour** of an end time in order to make it possible that left-closed time intervals can fully cover the day. `null` can be used to specify open intervals.", + "minItems": 2, + "maxItems": 2, + "items": { + "description": "Processes and implementations may choose to only implement a subset of the subtypes specified here. Clients must check what back-ends / processes actually support.", + "anyOf": [ + { + "type": "string", + "subtype": "date-time", + "format": "date-time", + "title": "Date with Time", + "description": "Date and time representation, as defined for `date-time` by [RFC 3339 in section 5.6](https://www.rfc-editor.org/rfc/rfc3339.html#section-5.6)." + }, + { + "type": "string", + "subtype": "date", + "format": "date", + "title": "Date only", + "description": "Date only representation, as defined for `full-date` by [RFC 3339 in section 5.6](https://www.rfc-editor.org/rfc/rfc3339.html#section-5.6). The time zone is UTC." + }, + { + "type": "string", + "subtype": "time", + "format": "time", + "title": "Time only", + "description": "Time only representation, as defined for `full-time` by [RFC 3339 in section 5.6](https://www.rfc-editor.org/rfc/rfc3339.html#section-5.6). Although [RFC 3339 prohibits the hour to be '24'](https://www.rfc-editor.org/rfc/rfc3339.html#section-5.7), this definition allows the value '24' for the hour as end time in an interval in order to make it possible that left-closed time intervals can fully cover the day." + }, + { + "type": "string", + "subtype": "year", + "minLength": 4, + "maxLength": 4, + "pattern": "^\\d{4}$", + "title": "Year only", + "description": "Year representation, as defined for `date-fullyear` by [RFC 3339 in section 5.6](https://www.rfc-editor.org/rfc/rfc3339.html#section-5.6)." + }, + { + "type": "null" + } + ] + }, + "examples": [ + [ + "2015-01-01T00:00:00Z", + "2016-01-01T00:00:00Z" + ], + [ + "2015-01-01", + "2016-01-01" + ], + [ + "00:00:00Z", + "12:00:00Z" + ], + [ + "2015-01-01", + null + ] + ] + }, + "name": "temporal_extent", + "description": "Time span of interest" + }, + { + "schema": [ + { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "subtype": "band-name" + } + }, + { + "title": "No filter", + "description": "Don't filter bands. All bands are included in the data cube.", + "type": "null" + } + ], + "name": "bands", + "description": "Bands to load and process. Note: sen2like does not process `B09`, `B10`. ", + "optional": true + } + ], + "returns": { + "description": "A list of .zip files containing the Sentinel 2 - like .SAFE folders. ", + "schema": { + "type": "list" + } + }, + "process_graph": { + "load1": { + "process_id": "load_collection", + "arguments": { + "id": "SENTINEL2_L1C", + "spatial_extent": { + "from_parameter": "spatial_extent" + }, + "temporal_extent": { + "from_parameter": "temporal_extent" + }, + "bands": { + "from_parameter": "bands" + }, + "properties": {} + } + }, + "sen2": { + "process_id": "sen2like", + "arguments": { + "data": { + "from_node": "load1" + }, + "export_original_files": true + }, + "result": true + } + } + } diff --git a/tests/data/res_tests/unresolved/unresolved_sen2like.json b/tests/data/res_tests/unresolved/unresolved_sen2like.json new file mode 100644 index 0000000..f4fd4ef --- /dev/null +++ b/tests/data/res_tests/unresolved/unresolved_sen2like.json @@ -0,0 +1,23 @@ +{ + "sen2like_original_outputs": { + "process_id": "sen2like_original_outputs", + "arguments": { + "bands": [ + "bo1" + ], + "spatial_extent": { + "west": -4.919340483677538, + "east": 36.248628266322456, + "south": 41.43373541041478, + "north": 53.27118132212786 + }, + "temporal_extent": [ + "2024-04-01T00:00:00Z", + "2024-04-09T00:00:00Z" + ] + }, + "result": true, + "namespace": "user", + "description": "Create Sentinel 2 - like .SAFE outputs from Sentinel 2 L1C and Landsat 8 and 9 datasets. " + } +} diff --git a/tests/test_pg_resolving.py b/tests/test_pg_resolving.py index 42cea47..c17fc24 100644 --- a/tests/test_pg_resolving.py +++ b/tests/test_pg_resolving.py @@ -20,6 +20,7 @@ def get_predefined_process_registry(): predefined_process_registry = ProcessRegistry() predefined_processes_specs = [ + ('sen2like', {}), ('add', {}), ('apply', {}), ('load_collection', {}), @@ -67,6 +68,12 @@ def unresolved_gfm_pg() -> dict: return dict(json.loads(f.read())) +@pytest.fixture +def unresolved_sen2like_pg() -> dict: + with open('tests/data/res_tests/unresolved/unresolved_sen2like.json') as f: + return dict(json.loads(f.read())) + + @pytest.fixture def correctly_resolved_pg() -> dict: with open('tests/data/res_tests/resolved/resolved_complex.json') as f: @@ -79,6 +86,12 @@ def correctly_resolved_gfm_pg() -> dict: return dict(json.loads(f.read())) +@pytest.fixture +def correctly_resolved_sen2like_pg() -> dict: + with open('tests/data/res_tests/resolved/resolved_sen2like.json') as f: + return dict(json.loads(f.read())) + + def test_resolve_graph_with_predefined_process_registry( predefined_process_registry: ProcessRegistry, unresolved_pg: dict, @@ -160,6 +173,18 @@ def test_resolve_gfm_graph_with_predefined_process_registry( get_udp_spec=get_udp, ) - with open('resolved_gfm_graph.json', 'w') as f: - json.dump(resolved_pg, f) assert correctly_resolved_gfm_pg == resolved_pg + + +def test_resolve_sen2like_graph_with_predefined_process_registry( + predefined_process_registry: ProcessRegistry, + unresolved_sen2like_pg: dict, + correctly_resolved_sen2like_pg: dict, +): + resolved_pg = resolving_utils.resolve_process_graph( + process_graph=unresolved_sen2like_pg, + process_registry=predefined_process_registry, + get_udp_spec=get_udp, + ) + + assert correctly_resolved_sen2like_pg == resolved_pg