Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add a default suffix to ELP #1378

Merged
merged 11 commits into from
Aug 23, 2024
7 changes: 6 additions & 1 deletion CHANGES.rst
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
0.16.2 (unreleased)
===================

-
pipeline
--------

- Added ``suffix`` to the spec of ExposurePipeline with a
default value of ``cal``. Removed explicit setting of ``suffix``
so that it can be passed as an argument to ``strun``. [#1378]

0.16.1 (2024-08-13)
===================
Expand Down
13 changes: 1 addition & 12 deletions romancal/pipeline/exposure_pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ class ExposurePipeline(RomanPipeline):
spec = """
save_calibrated_ramp = boolean(default=False)
save_results = boolean(default=False)
suffix = string(default="cal")
"""

# Define aliases to steps
Expand Down Expand Up @@ -149,8 +150,6 @@ def process(self, input):
]:
result.meta.cal_step[step_str] = "SKIPPED"

# Set suffix for proper output naming
self.suffix = "cal"
results.append(result)
return results

Expand All @@ -177,9 +176,6 @@ def process(self, input):
result.meta.cal_step.photom = "SKIPPED"
result.meta.cal_step.source_detection = "SKIPPED"
result.meta.cal_step.tweakreg = "SKIPPED"
self.suffix = "cal"

self.setup_output(result)

self.output_use_model = True
results.append(result)
Expand All @@ -193,10 +189,6 @@ def process(self, input):

return results

def setup_output(self, input):
"""Determine the proper file name suffix to use later"""
self.suffix = "cal"

def create_fully_saturated_zeroed_image(self, input_model):
"""
Create zeroed-out image file
Expand Down Expand Up @@ -226,8 +218,5 @@ def create_fully_saturated_zeroed_image(self, input_model):
]:
fully_saturated_model.meta.cal_step[step_str] = "SKIPPED"

# Set suffix for proper output naming
self.suffix = "cal"

# Return zeroed-out image file
return fully_saturated_model
50 changes: 49 additions & 1 deletion romancal/regtest/test_wfi_pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -517,7 +517,11 @@ def test_elp_input_dm(rtdata, ignore_asdf_paths):

@pytest.mark.bigdata
def test_processing_pipeline_all_saturated(rtdata, ignore_asdf_paths):
"""Tests for fully saturated data skipping steps in the pipeline"""
"""Tests for fully saturated data skipping steps in the pipeline

Note that this test mimics how the pipeline is run in OPS.
Any changes to this test should be coordinated with OPS.
"""
input_data = "r0000101001001001001_01101_0001_WFI01_ALL_SATURATED_uncal.asdf"
rtdata.get_data(f"WFI/image/{input_data}")
rtdata.input = input_data
Expand Down Expand Up @@ -547,3 +551,47 @@ def test_processing_pipeline_all_saturated(rtdata, ignore_asdf_paths):
assert model.meta.cal_step.assign_wcs == "SKIPPED"
assert model.meta.cal_step.flat_field == "SKIPPED"
assert model.meta.cal_step.photom == "SKIPPED"


@pytest.mark.bigdata
def test_pipeline_suffix(rtdata, ignore_asdf_paths):
"""
Tests passing suffix to the pipeline

Note that this test mimics how the pipeline is run in OPS.

Any changes to this test should be coordinated with OPS.
"""
input_data = "r0000101001001001001_01101_0001_WFI01_uncal.asdf"
rtdata.get_data(f"WFI/image/{input_data}")

output = "r0000101001001001001_01101_0001_WFI01_star.asdf"
rtdata.output = output

args = [
"roman_elp",
rtdata.input,
"--steps.tweakreg.skip=True",
"--suffix=star",
]
ExposurePipeline.from_cmdline(args)
rtdata.get_truth(f"truth/WFI/image/{output}")

diff = compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths)
assert diff.identical, diff.report()

# Ensure step completion is as expected
model = rdm.open(rtdata.output)

assert model.meta.cal_step.dq_init == "COMPLETE"
assert model.meta.cal_step.saturation == "COMPLETE"
assert model.meta.cal_step.linearity == "COMPLETE"
assert model.meta.cal_step.dark == "COMPLETE"
assert model.meta.cal_step.jump == "COMPLETE"
assert model.meta.cal_step.ramp_fit == "COMPLETE"
assert model.meta.cal_step.assign_wcs == "COMPLETE"
assert model.meta.cal_step.flat_field == "COMPLETE"
assert model.meta.cal_step.photom == "COMPLETE"
assert model.meta.cal_step.source_detection == "COMPLETE"
assert model.meta.cal_step.tweakreg == "INCOMPLETE"
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch.

There's an open issue already for why this isn't SKIPPED: spacetelescope/roman_datamodels#343

assert model.meta.filename == output