diff --git a/README.md b/README.md index b0b0884d..9cc26c77 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ usage: WfExS-backend.py [-h] [--log-file LOGFILENAME] [-q] [-v] [-d] [-L LOCALCO {init,cache,staged-workdir,export,list-fetchers,list-exporters,config-validate,stage,mount-workdir,export-stage,offline-execute,execute,export-results,export-crate} ... -WfExS (workflow execution service) backend 0.9.3-117-g69322b8 (69322b80ab3d794003c60b4ca951a84c4ead8844) +WfExS (workflow execution service) backend 0.9.3-131-gcb4803d (cb4803d833ccaf7370a96e559bb997a8426936f0) optional arguments: -h, --help show this help message and exit @@ -141,7 +141,7 @@ optional arguments: Subparser 'staged-workdir' usage: WfExS-backend.py staged-workdir [-h] [--private-key-file PRIVATE_KEY_FILE] [--private-key-passphrase-envvar PRIVATE_KEY_PASSPHRASE_ENVVAR] [--inputs] [--outputs] - [--workflow] [--containers] [--prospective] [--full] [-g] + [--workflow] [--containers] [--prospective] [--full] [--licences LICENCES] [-g] {offline-exec,ls,mount,rm,shell,status,create-staged-crate,create-prov-crate} [staged_workdir_command_args [staged_workdir_command_args ...]] @@ -189,11 +189,13 @@ ro-crate-payload: --containers Should the RO-Crate contain a containers copy (of everything)? (default: []) --prospective Should the RO-Crate contain a prospective copy (of everything)? (default: []) --full Should the RO-Crate contain a full copy (of everything)? (default: []) + --licences LICENCES Licence(s) to attach to the generated RO-Crate (default: []) Subparser 'export' usage: WfExS-backend.py export [-h] [-Z SECURITYCONTEXTSCONFIGFILENAME] [-E EXPORTSCONFIGFILENAME] [--public-key-file PUBLIC_KEY_FILES] [--private-key-file PRIVATE_KEY_FILE] [--private-key-passphrase-envvar PRIVATE_KEY_PASSPHRASE_ENVVAR] -J WORKFLOWWORKINGDIRECTORY + [--licences LICENCES] {ls,run} [export_contents_command_args [export_contents_command_args ...]] positional arguments: @@ -215,6 +217,7 @@ optional arguments: working directory (default: []) -J WORKFLOWWORKINGDIRECTORY, --staged-job-dir WORKFLOWWORKINGDIRECTORY Already staged job directory (default: None) + --licences LICENCES Licence(s) to attach to the exported contents (default: []) secure workdir arguments: Private key and passphrase to access secured working directories @@ -301,6 +304,7 @@ Subparser 'export-stage' usage: WfExS-backend.py export-stage [-h] [--private-key-file PRIVATE_KEY_FILE] [--private-key-passphrase-envvar PRIVATE_KEY_PASSPHRASE_ENVVAR] -J WORKFLOWWORKINGDIRECTORY [--inputs] [--outputs] [--workflow] [--containers] [--prospective] [--full] + [--licences LICENCES] optional arguments: -h, --help show this help message and exit @@ -326,6 +330,7 @@ ro-crate-payload: --containers Should the RO-Crate contain a containers copy (of everything)? (default: []) --prospective Should the RO-Crate contain a prospective copy (of everything)? (default: []) --full Should the RO-Crate contain a full copy (of everything)? (default: []) + --licences LICENCES Licence(s) to attach to the generated RO-Crate (default: []) Subparser 'offline-execute' usage: WfExS-backend.py offline-execute [-h] [--private-key-file PRIVATE_KEY_FILE] @@ -351,7 +356,7 @@ Subparser 'execute' usage: WfExS-backend.py execute [-h] -W WORKFLOWCONFIGFILENAME [-Z SECURITYCONTEXTSCONFIGFILENAME] [-E EXPORTSCONFIGFILENAME] [-n NICKNAME_PREFIX] [--public-key-file PUBLIC_KEY_FILES] [--private-key-file PRIVATE_KEY_FILE] [--private-key-passphrase-envvar PRIVATE_KEY_PASSPHRASE_ENVVAR] [--inputs] [--outputs] - [--workflow] [--containers] [--prospective] [--full] + [--workflow] [--containers] [--prospective] [--full] [--licences LICENCES] optional arguments: -h, --help show this help message and exit @@ -386,16 +391,18 @@ ro-crate-payload: --containers Should the RO-Crate contain a containers copy (of everything)? (default: []) --prospective Should the RO-Crate contain a prospective copy (of everything)? (default: []) --full Should the RO-Crate contain a full copy (of everything)? (default: []) + --licences LICENCES Licence(s) to attach to the generated RO-Crate (default: []) Subparser 'export-results' usage: WfExS-backend.py export-results [-h] [--private-key-file PRIVATE_KEY_FILE] [--private-key-passphrase-envvar PRIVATE_KEY_PASSPHRASE_ENVVAR] -J - WORKFLOWWORKINGDIRECTORY + WORKFLOWWORKINGDIRECTORY [--licences LICENCES] optional arguments: -h, --help show this help message and exit -J WORKFLOWWORKINGDIRECTORY, --staged-job-dir WORKFLOWWORKINGDIRECTORY Already staged job directory (default: None) + --licences LICENCES Licence(s) to attach to the exported contents (default: []) secure workdir arguments: Private key and passphrase to access secured working directories @@ -411,6 +418,7 @@ Subparser 'export-crate' usage: WfExS-backend.py export-crate [-h] [--private-key-file PRIVATE_KEY_FILE] [--private-key-passphrase-envvar PRIVATE_KEY_PASSPHRASE_ENVVAR] -J WORKFLOWWORKINGDIRECTORY [--inputs] [--outputs] [--workflow] [--containers] [--prospective] [--full] + [--licences LICENCES] optional arguments: -h, --help show this help message and exit @@ -436,6 +444,7 @@ ro-crate-payload: --containers Should the RO-Crate contain a containers copy (of everything)? (default: []) --prospective Should the RO-Crate contain a prospective copy (of everything)? (default: []) --full Should the RO-Crate contain a full copy (of everything)? (default: []) + --licences LICENCES Licence(s) to attach to the generated RO-Crate (default: []) ``` WfExS commands are: diff --git a/wfexs_backend/__main__.py b/wfexs_backend/__main__.py index 2e7ccda9..5ec4a6f8 100644 --- a/wfexs_backend/__main__.py +++ b/wfexs_backend/__main__.py @@ -305,6 +305,22 @@ def genParserSub( const=val_mat, help=f"Should the RO-Crate contain a {key_mat} copy (of everything)?", ) + mat_opts.add_argument( + "--licences", + dest="licences", + action="append", + default=[], + help="Licence(s) to attach to the generated RO-Crate", + ) + + if (exportParams or command == WfExS_Commands.ExportResults) and not crateParams: + ap_.add_argument( + "--licences", + dest="licences", + action="append", + default=[], + help="Licence(s) to attach to the exported contents", + ) return ap_ @@ -525,6 +541,12 @@ def processStagedWorkdirCommand( else: private_key_passphrase = "" + # Getting the list of licences (in case they are needed) + if hasattr(args, "licences") and args.licences is not None: + op_licences = args.licences + else: + op_licences = [] + if args.staged_workdir_command == WfExS_Staged_WorkDir_Commands.Mount: if len(args.staged_workdir_command_args) > 0: for ( @@ -739,6 +761,7 @@ def processStagedWorkdirCommand( wfInstance.createStageResearchObject( filename=args.staged_workdir_command_args[1], payloads=doMaterializedROCrate, + licences=op_licences, ) else: mStatus = wfInstance.getMarshallingStatus(reread_stats=True) @@ -752,6 +775,7 @@ def processStagedWorkdirCommand( wfInstance.createResultsResearchObject( filename=args.staged_workdir_command_args[1], payloads=doMaterializedROCrate, + licences=op_licences, ) else: print( @@ -1131,7 +1155,7 @@ def main() -> None: if command == WfExS_Commands.StagedWorkDir: sys.exit(processStagedWorkdirCommand(wfBackend, args, logLevel)) - # This can be needed in more than one place + # These can be needed in more than one place if ( hasattr(args, "private_key_passphrase_envvar") and args.private_key_passphrase_envvar is not None @@ -1140,6 +1164,11 @@ def main() -> None: else: private_key_passphrase = "" + if hasattr(args, "licences") and args.licences is not None: + op_licences = args.licences + else: + op_licences = [] + wfInstance = None if command in ( WfExS_Commands.MountWorkDir, @@ -1239,7 +1268,10 @@ def main() -> None: doMaterializedROCrate = WF.ExportROCrate2Payloads[""] if command in (WfExS_Commands.ExportStage, WfExS_Commands.Execute): - wfInstance.createStageResearchObject(payloads=doMaterializedROCrate) + wfInstance.createStageResearchObject( + payloads=doMaterializedROCrate, + licences=op_licences, + ) if command in (WfExS_Commands.OfflineExecute, WfExS_Commands.Execute): print( @@ -1261,10 +1293,13 @@ def main() -> None: ) if command in (WfExS_Commands.ExportResults, WfExS_Commands.Execute): - wfInstance.exportResults() + wfInstance.exportResults(op_licences=op_licences) if command in (WfExS_Commands.ExportCrate, WfExS_Commands.Execute): - wfInstance.createResultsResearchObject(payloads=doMaterializedROCrate) + wfInstance.createResultsResearchObject( + payloads=doMaterializedROCrate, + licences=op_licences, + ) if __name__ == "__main__": diff --git a/wfexs_backend/wfexs_backend.py b/wfexs_backend/wfexs_backend.py index 895bfc32..adca630c 100644 --- a/wfexs_backend/wfexs_backend.py +++ b/wfexs_backend/wfexs_backend.py @@ -662,6 +662,7 @@ def instantiateExportPlugin( wfInstance: "WF", plugin_id: "SymbolicName", sec_context: "Optional[SecurityContextConfig]", + licences: "Sequence[str]", ) -> "AbstractExportPlugin": """ This method instantiates an stateful export plugin @@ -680,7 +681,9 @@ def instantiateExportPlugin( if stagedSetup.is_damaged: raise ValueError(f"Staged setup from {stagedSetup.instance_id} is damaged") - return self._export_plugins[plugin_id](wfInstance, setup_block=sec_context) + return self._export_plugins[plugin_id]( + wfInstance, setup_block=sec_context, licences=licences + ) def addStatefulSchemeHandlers( self, diff --git a/wfexs_backend/workflow.py b/wfexs_backend/workflow.py index dac7bb34..69c119a7 100644 --- a/wfexs_backend/workflow.py +++ b/wfexs_backend/workflow.py @@ -2637,6 +2637,7 @@ def parseExportActions( setup=actionDesc.get("setup"), preferred_scheme=actionDesc.get("preferred-scheme"), preferred_id=actionDesc.get("preferred-pid"), + licences=actionDesc.get("licences", []), ) actions.append(action) @@ -2719,6 +2720,7 @@ def exportResults( creds_config: "Optional[SecurityContextConfigBlock]" = None, action_ids: "Sequence[SymbolicName]" = [], fail_ok: "bool" = False, + op_licences: "Sequence[str]" = [], ) -> "Tuple[Sequence[MaterializedExportAction], Sequence[Tuple[ExportAction, Exception]]]": # The precondition if self.unmarshallExport(offline=True, fail_ok=True) is None: @@ -2750,7 +2752,10 @@ def exportResults( for action in filtered_actions: try: # check the export items are available - elems = self.locateExportItems(action.what) + the_licences = ( + action.licences if len(action.licences) > 0 else op_licences + ) + elems = self.locateExportItems(action.what, licences=the_licences) # check the security context is available a_setup_block: "Optional[WritableSecurityContextConfig]" @@ -2784,7 +2789,10 @@ def exportResults( # check whether plugin is available export_p = self.wfexs.instantiateExportPlugin( - self, action.plugin_id, a_setup_block + self, + action.plugin_id, + sec_context=a_setup_block, + licences=the_licences, ) # Export the contents and obtain a PID @@ -3426,7 +3434,9 @@ def unmarshallExport( } def locateExportItems( - self, items: "Sequence[ExportItem]" + self, + items: "Sequence[ExportItem]", + licences: "Sequence[str]" = [], ) -> "Sequence[AnyContent]": """ The located paths in the contents should be relative to the working directory @@ -3651,6 +3661,7 @@ def locateExportItems( create_rocrate( filename=cast("AbsPath", temp_rocrate_file), payloads=self.ExportROCrate2Payloads[item.block], + licences=licences, ) retval.append( MaterializedContent( @@ -3679,6 +3690,7 @@ def createStageResearchObject( self, filename: "Optional[AnyPath]" = None, payloads: "CratableItem" = NoCratableItem, + licences: "Sequence[str]" = [], ) -> "AnyPath": """ Create RO-crate from stage provenance. @@ -3707,6 +3719,7 @@ def createStageResearchObject( self.arch, staged_setup=self.stagedSetup, payloads=payloads, + licences=licences, ) wrroc.addWorkflowInputs( @@ -3736,6 +3749,7 @@ def createResultsResearchObject( self, filename: "Optional[AnyPath]" = None, payloads: "CratableItem" = NoCratableItem, + licences: "Sequence[str]" = [], ) -> "AnyPath": """ Create RO-crate from stage provenance. @@ -3763,6 +3777,7 @@ def createResultsResearchObject( self.arch, staged_setup=self.stagedSetup, payloads=payloads, + licences=licences, ) for stagedExec in self.stagedExecutions: