Added --licences parameter.

Allow propagating licences from command line to both RO-Crate generation and export plugins being instantiated.
inab · Sep 8, 2023 · 82ed931 · 82ed931
1 parent cb4803d
commit 82ed931
Show file tree

Hide file tree

Showing 4 changed files with 74 additions and 12 deletions.
diff --git a/README.md b/README.md
@@ -67,7 +67,7 @@ usage: WfExS-backend.py [-h] [--log-file LOGFILENAME] [-q] [-v] [-d] [-L LOCALCO
                         {init,cache,staged-workdir,export,list-fetchers,list-exporters,config-validate,stage,mount-workdir,export-stage,offline-execute,execute,export-results,export-crate}
                         ...
 
-WfExS (workflow execution service) backend 0.9.3-117-g69322b8 (69322b80ab3d794003c60b4ca951a84c4ead8844)
+WfExS (workflow execution service) backend 0.9.3-131-gcb4803d (cb4803d833ccaf7370a96e559bb997a8426936f0)
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -141,7 +141,7 @@ optional arguments:
 Subparser 'staged-workdir'
 usage: WfExS-backend.py staged-workdir [-h] [--private-key-file PRIVATE_KEY_FILE]
                                        [--private-key-passphrase-envvar PRIVATE_KEY_PASSPHRASE_ENVVAR] [--inputs] [--outputs]
-                                       [--workflow] [--containers] [--prospective] [--full] [-g]
+                                       [--workflow] [--containers] [--prospective] [--full] [--licences LICENCES] [-g]
                                        {offline-exec,ls,mount,rm,shell,status,create-staged-crate,create-prov-crate}
                                        [staged_workdir_command_args [staged_workdir_command_args ...]]
 
@@ -189,11 +189,13 @@ ro-crate-payload:
   --containers          Should the RO-Crate contain a containers copy (of everything)? (default: [])
   --prospective         Should the RO-Crate contain a prospective copy (of everything)? (default: [])
   --full                Should the RO-Crate contain a full copy (of everything)? (default: [])
+  --licences LICENCES   Licence(s) to attach to the generated RO-Crate (default: [])
 
 Subparser 'export'
 usage: WfExS-backend.py export [-h] [-Z SECURITYCONTEXTSCONFIGFILENAME] [-E EXPORTSCONFIGFILENAME]
                                [--public-key-file PUBLIC_KEY_FILES] [--private-key-file PRIVATE_KEY_FILE]
                                [--private-key-passphrase-envvar PRIVATE_KEY_PASSPHRASE_ENVVAR] -J WORKFLOWWORKINGDIRECTORY
+                               [--licences LICENCES]
                                {ls,run} [export_contents_command_args [export_contents_command_args ...]]
 
 positional arguments:
@@ -215,6 +217,7 @@ optional arguments:
                         working directory (default: [])
   -J WORKFLOWWORKINGDIRECTORY, --staged-job-dir WORKFLOWWORKINGDIRECTORY
                         Already staged job directory (default: None)
+  --licences LICENCES   Licence(s) to attach to the exported contents (default: [])
 
 secure workdir arguments:
   Private key and passphrase to access secured working directories
@@ -301,6 +304,7 @@ Subparser 'export-stage'
 usage: WfExS-backend.py export-stage [-h] [--private-key-file PRIVATE_KEY_FILE]
                                      [--private-key-passphrase-envvar PRIVATE_KEY_PASSPHRASE_ENVVAR] -J WORKFLOWWORKINGDIRECTORY
                                      [--inputs] [--outputs] [--workflow] [--containers] [--prospective] [--full]
+                                     [--licences LICENCES]
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -326,6 +330,7 @@ ro-crate-payload:
   --containers          Should the RO-Crate contain a containers copy (of everything)? (default: [])
   --prospective         Should the RO-Crate contain a prospective copy (of everything)? (default: [])
   --full                Should the RO-Crate contain a full copy (of everything)? (default: [])
+  --licences LICENCES   Licence(s) to attach to the generated RO-Crate (default: [])
 
 Subparser 'offline-execute'
 usage: WfExS-backend.py offline-execute [-h] [--private-key-file PRIVATE_KEY_FILE]
@@ -351,7 +356,7 @@ Subparser 'execute'
 usage: WfExS-backend.py execute [-h] -W WORKFLOWCONFIGFILENAME [-Z SECURITYCONTEXTSCONFIGFILENAME] [-E EXPORTSCONFIGFILENAME]
                                 [-n NICKNAME_PREFIX] [--public-key-file PUBLIC_KEY_FILES] [--private-key-file PRIVATE_KEY_FILE]
                                 [--private-key-passphrase-envvar PRIVATE_KEY_PASSPHRASE_ENVVAR] [--inputs] [--outputs]
-                                [--workflow] [--containers] [--prospective] [--full]
+                                [--workflow] [--containers] [--prospective] [--full] [--licences LICENCES]
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -386,16 +391,18 @@ ro-crate-payload:
   --containers          Should the RO-Crate contain a containers copy (of everything)? (default: [])
   --prospective         Should the RO-Crate contain a prospective copy (of everything)? (default: [])
   --full                Should the RO-Crate contain a full copy (of everything)? (default: [])
+  --licences LICENCES   Licence(s) to attach to the generated RO-Crate (default: [])
 
 Subparser 'export-results'
 usage: WfExS-backend.py export-results [-h] [--private-key-file PRIVATE_KEY_FILE]
                                        [--private-key-passphrase-envvar PRIVATE_KEY_PASSPHRASE_ENVVAR] -J
-                                       WORKFLOWWORKINGDIRECTORY
+                                       WORKFLOWWORKINGDIRECTORY [--licences LICENCES]
 
 optional arguments:
   -h, --help            show this help message and exit
   -J WORKFLOWWORKINGDIRECTORY, --staged-job-dir WORKFLOWWORKINGDIRECTORY
                         Already staged job directory (default: None)
+  --licences LICENCES   Licence(s) to attach to the exported contents (default: [])
 
 secure workdir arguments:
   Private key and passphrase to access secured working directories
@@ -411,6 +418,7 @@ Subparser 'export-crate'
 usage: WfExS-backend.py export-crate [-h] [--private-key-file PRIVATE_KEY_FILE]
                                      [--private-key-passphrase-envvar PRIVATE_KEY_PASSPHRASE_ENVVAR] -J WORKFLOWWORKINGDIRECTORY
                                      [--inputs] [--outputs] [--workflow] [--containers] [--prospective] [--full]
+                                     [--licences LICENCES]
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -436,6 +444,7 @@ ro-crate-payload:
   --containers          Should the RO-Crate contain a containers copy (of everything)? (default: [])
   --prospective         Should the RO-Crate contain a prospective copy (of everything)? (default: [])
   --full                Should the RO-Crate contain a full copy (of everything)? (default: [])
+  --licences LICENCES   Licence(s) to attach to the generated RO-Crate (default: [])
 ```
 
 WfExS commands are:

diff --git a/wfexs_backend/__main__.py b/wfexs_backend/__main__.py
@@ -305,6 +305,22 @@ def genParserSub(
                     const=val_mat,
                     help=f"Should the RO-Crate contain a {key_mat} copy (of everything)?",
                 )
+        mat_opts.add_argument(
+            "--licences",
+            dest="licences",
+            action="append",
+            default=[],
+            help="Licence(s) to attach to the generated RO-Crate",
+        )
+
+    if (exportParams or command == WfExS_Commands.ExportResults) and not crateParams:
+        ap_.add_argument(
+            "--licences",
+            dest="licences",
+            action="append",
+            default=[],
+            help="Licence(s) to attach to the exported contents",
+        )
 
     return ap_
 
@@ -525,6 +541,12 @@ def processStagedWorkdirCommand(
     else:
         private_key_passphrase = ""
 
+    # Getting the list of licences (in case they are needed)
+    if hasattr(args, "licences") and args.licences is not None:
+        op_licences = args.licences
+    else:
+        op_licences = []
+
     if args.staged_workdir_command == WfExS_Staged_WorkDir_Commands.Mount:
         if len(args.staged_workdir_command_args) > 0:
             for (
@@ -739,6 +761,7 @@ def processStagedWorkdirCommand(
                             wfInstance.createStageResearchObject(
                                 filename=args.staged_workdir_command_args[1],
                                 payloads=doMaterializedROCrate,
+                                licences=op_licences,
                             )
                         else:
                             mStatus = wfInstance.getMarshallingStatus(reread_stats=True)
@@ -752,6 +775,7 @@ def processStagedWorkdirCommand(
                                 wfInstance.createResultsResearchObject(
                                     filename=args.staged_workdir_command_args[1],
                                     payloads=doMaterializedROCrate,
+                                    licences=op_licences,
                                 )
                             else:
                                 print(
@@ -1131,7 +1155,7 @@ def main() -> None:
     if command == WfExS_Commands.StagedWorkDir:
         sys.exit(processStagedWorkdirCommand(wfBackend, args, logLevel))
 
-    # This can be needed in more than one place
+    # These can be needed in more than one place
     if (
         hasattr(args, "private_key_passphrase_envvar")
         and args.private_key_passphrase_envvar is not None
@@ -1140,6 +1164,11 @@ def main() -> None:
     else:
         private_key_passphrase = ""
 
+    if hasattr(args, "licences") and args.licences is not None:
+        op_licences = args.licences
+    else:
+        op_licences = []
+
     wfInstance = None
     if command in (
         WfExS_Commands.MountWorkDir,
@@ -1239,7 +1268,10 @@ def main() -> None:
         doMaterializedROCrate = WF.ExportROCrate2Payloads[""]
 
     if command in (WfExS_Commands.ExportStage, WfExS_Commands.Execute):
-        wfInstance.createStageResearchObject(payloads=doMaterializedROCrate)
+        wfInstance.createStageResearchObject(
+            payloads=doMaterializedROCrate,
+            licences=op_licences,
+        )
 
     if command in (WfExS_Commands.OfflineExecute, WfExS_Commands.Execute):
         print(
@@ -1261,10 +1293,13 @@ def main() -> None:
         )
 
     if command in (WfExS_Commands.ExportResults, WfExS_Commands.Execute):
-        wfInstance.exportResults()
+        wfInstance.exportResults(op_licences=op_licences)
 
     if command in (WfExS_Commands.ExportCrate, WfExS_Commands.Execute):
-        wfInstance.createResultsResearchObject(payloads=doMaterializedROCrate)
+        wfInstance.createResultsResearchObject(
+            payloads=doMaterializedROCrate,
+            licences=op_licences,
+        )
 
 
 if __name__ == "__main__":

diff --git a/wfexs_backend/wfexs_backend.py b/wfexs_backend/wfexs_backend.py
@@ -662,6 +662,7 @@ def instantiateExportPlugin(
         wfInstance: "WF",
         plugin_id: "SymbolicName",
         sec_context: "Optional[SecurityContextConfig]",
+        licences: "Sequence[str]",
     ) -> "AbstractExportPlugin":
         """
         This method instantiates an stateful export plugin
@@ -680,7 +681,9 @@ def instantiateExportPlugin(
         if stagedSetup.is_damaged:
             raise ValueError(f"Staged setup from {stagedSetup.instance_id} is damaged")
 
-        return self._export_plugins[plugin_id](wfInstance, setup_block=sec_context)
+        return self._export_plugins[plugin_id](
+            wfInstance, setup_block=sec_context, licences=licences
+        )
 
     def addStatefulSchemeHandlers(
         self,

diff --git a/wfexs_backend/workflow.py b/wfexs_backend/workflow.py
@@ -2637,6 +2637,7 @@ def parseExportActions(
                 setup=actionDesc.get("setup"),
                 preferred_scheme=actionDesc.get("preferred-scheme"),
                 preferred_id=actionDesc.get("preferred-pid"),
+                licences=actionDesc.get("licences", []),
             )
             actions.append(action)
 
@@ -2719,6 +2720,7 @@ def exportResults(
         creds_config: "Optional[SecurityContextConfigBlock]" = None,
         action_ids: "Sequence[SymbolicName]" = [],
         fail_ok: "bool" = False,
+        op_licences: "Sequence[str]" = [],
     ) -> "Tuple[Sequence[MaterializedExportAction], Sequence[Tuple[ExportAction, Exception]]]":
         # The precondition
         if self.unmarshallExport(offline=True, fail_ok=True) is None:
@@ -2750,7 +2752,10 @@ def exportResults(
         for action in filtered_actions:
             try:
                 # check the export items are available
-                elems = self.locateExportItems(action.what)
+                the_licences = (
+                    action.licences if len(action.licences) > 0 else op_licences
+                )
+                elems = self.locateExportItems(action.what, licences=the_licences)
 
                 # check the security context is available
                 a_setup_block: "Optional[WritableSecurityContextConfig]"
@@ -2784,7 +2789,10 @@ def exportResults(
 
                 # check whether plugin is available
                 export_p = self.wfexs.instantiateExportPlugin(
-                    self, action.plugin_id, a_setup_block
+                    self,
+                    action.plugin_id,
+                    sec_context=a_setup_block,
+                    licences=the_licences,
                 )
 
                 # Export the contents and obtain a PID
@@ -3426,7 +3434,9 @@ def unmarshallExport(
     }
 
     def locateExportItems(
-        self, items: "Sequence[ExportItem]"
+        self,
+        items: "Sequence[ExportItem]",
+        licences: "Sequence[str]" = [],
     ) -> "Sequence[AnyContent]":
         """
         The located paths in the contents should be relative to the working directory
@@ -3651,6 +3661,7 @@ def locateExportItems(
                 create_rocrate(
                     filename=cast("AbsPath", temp_rocrate_file),
                     payloads=self.ExportROCrate2Payloads[item.block],
+                    licences=licences,
                 )
                 retval.append(
                     MaterializedContent(
@@ -3679,6 +3690,7 @@ def createStageResearchObject(
         self,
         filename: "Optional[AnyPath]" = None,
         payloads: "CratableItem" = NoCratableItem,
+        licences: "Sequence[str]" = [],
     ) -> "AnyPath":
         """
         Create RO-crate from stage provenance.
@@ -3707,6 +3719,7 @@ def createStageResearchObject(
             self.arch,
             staged_setup=self.stagedSetup,
             payloads=payloads,
+            licences=licences,
         )
 
         wrroc.addWorkflowInputs(
@@ -3736,6 +3749,7 @@ def createResultsResearchObject(
         self,
         filename: "Optional[AnyPath]" = None,
         payloads: "CratableItem" = NoCratableItem,
+        licences: "Sequence[str]" = [],
     ) -> "AnyPath":
         """
         Create RO-crate from stage provenance.
@@ -3763,6 +3777,7 @@ def createResultsResearchObject(
             self.arch,
             staged_setup=self.stagedSetup,
             payloads=payloads,
+            licences=licences,
         )
 
         for stagedExec in self.stagedExecutions: