Merge pull request #79 from rabix/feature/sample_sheet_support

Feature/sample sheet support

.gitignore

@@ -2,3 +2,4 @@
 /dist/
 /sbpack.egg-info/
 /.idea/
+/venv/

MANIFEST.in

@@ -0,0 +1 @@
+include sbpack/noncwl/js_templates/*.js

requirements.txt

@@ -2,4 +2,4 @@ ruamel.yaml >= 0.16
 sevenbridges-python >= 2.0
 nf-core==2.1
 cwlformat
-packaging
+packaging

sbpack/lib.py

@@ -1,7 +1,8 @@
-from typing import Union
+from typing import Union, Optional
 from copy import deepcopy
-import urllib.parse
-import urllib.request
+from urllib.parse import ParseResult, urlparse, urljoin
+from urllib.request import urlopen
+from urllib.error import HTTPError
 import pathlib
 import sys
 
@@ -71,7 +72,8 @@ def normalize_to_map(obj: Union[list, dict], key_field: str):
  raise RuntimeError("Expecting a dictionary or a list here")
 
 
-def normalize_to_list(obj: Union[list, dict], key_field: str, value_field: str):
+def normalize_to_list(
+ obj: Union[list, dict], key_field: str, value_field: Optional[str]):
  if isinstance(obj, list):
  return deepcopy(obj)
  elif isinstance(obj, dict):
@@ -89,8 +91,8 @@ def normalize_to_list(obj: Union[list, dict], key_field: str, value_field: str):
 
 
 # To deprecate
-def normalized_path(link: str, base_url: urllib.parse.ParseResult):
- link_url = urllib.parse.urlparse(link)
+def normalized_path(link: str, base_url: ParseResult):
+ link_url = urlparse(link)
  if link_url.scheme in ["file://", ""]:
  new_url = base_url._replace(
  path=str((pathlib.Path(base_url.path) / pathlib.Path(link)).resolve())
@@ -101,7 +103,7 @@ def normalized_path(link: str, base_url: urllib.parse.ParseResult):
  return new_url
 
 
-def resolved_path(base_url: urllib.parse.ParseResult, link: str):
+def resolved_path(base_url: ParseResult, link: str):
  """
  Given a base_url ("this document") and a link ("string in this document")
  return a new url (urllib.parse.ParseResult) that allows us to retrieve the
@@ -110,12 +112,12 @@ def resolved_path(base_url: urllib.parse.ParseResult, link: str):
  2. Use the OS appropriate path resolution for local paths, and network
  apropriate resolution for network paths
  """
- link_url = urllib.parse.urlparse(link)
+ link_url = urlparse(link)
  # The link will always Posix
 
  if link_url.scheme == "file://":
  # Absolute local path
- new_url = urllib.parse.ParseResult(link_url)
+ new_url = ParseResult(str(link_url))
 
  elif link_url.scheme == "":
  # Relative path, can be local or remote
@@ -130,7 +132,7 @@ def resolved_path(base_url: urllib.parse.ParseResult, link: str):
 
  else:
  # Remote relative path
- new_url = urllib.parse.urlparse(urllib.parse.urljoin(base_url.geturl(), link_url.path))
+ new_url = urlparse(urljoin(base_url.geturl(), link_url.path))
  # We need urljoin because we need to resolve relative links in a
  # platform independent manner
 
@@ -141,16 +143,16 @@ def resolved_path(base_url: urllib.parse.ParseResult, link: str):
  return new_url
 
 
-def load_linked_file(base_url: urllib.parse.ParseResult, link: str, is_import=False):
+def load_linked_file(base_url: ParseResult, link: str, is_import=False):
 
  new_url = resolved_path(base_url, link)
 
  if new_url.scheme in ["file://", ""]:
  contents = pathlib.Path(new_url.path).open().read()
  else:
  try:
- contents = urllib.request.urlopen(new_url.geturl()).read().decode("utf-8")
- except urllib.error.HTTPError as e:
+ contents = urlopen(new_url.geturl()).read().decode("utf-8")
+ except HTTPError as e:
  e.msg += f"\n===\nCould not find linked file: {new_url.geturl()}\n===\n"
  raise SystemExit(e)
 
@@ -177,7 +179,7 @@ def load_linked_file(base_url: urllib.parse.ParseResult, link: str, is_import=Fa
  return _node, new_url
 
 
-def _is_github_symbolic_link(base_url: urllib.parse.ParseResult, contents: str):
+def _is_github_symbolic_link(base_url: ParseResult, contents: str):
  """Look for remote path with contents that is a single line with no new
  line with an extension."""
  if base_url.scheme in ["file://", ""]:

sbpack/noncwl/Readme.md

@@ -7,25 +7,21 @@ to the Platform.
 ```
 $ sbpack_wdl -h
 
-usage: sbpack_wdl [-h] [--profile PROFILE] --appid APPID --entrypoint ENTRYPOINT --workflow-path WORKFLOW_PATH [--sb-package-id SB_PACKAGE_ID] [--womtool-input WOMTOOL_INPUT]
- [--sb-inputs SB_INPUTS] [--sb-outputs SB_OUTPUTS] [--sb-doc SB_DOC] [--dump-sb-app] [--no-package] [--womtool-path WOMTOOL_PATH] [--json] [--sb-schema SB_SCHEMA]
+usage: sbpack_wdl [-h] [--profile PROFILE] --appid APPID --entrypoint ENTRYPOINT --workflow-path WORKFLOW_PATH [--sb-package-id SB_PACKAGE_ID] [--womtool-input WOMTOOL_INPUT] [--sb-doc SB_DOC] [--dump-sb-app] [--no-package]
+ [--womtool-path WOMTOOL_PATH] [--json] [--sb-schema SB_SCHEMA]
 
 optional arguments:
  -h, --help show this help message and exit
  --profile PROFILE SB platform profile as set in the SB API credentials file.
  --appid APPID Takes the form {user or division}/{project}/{app_id}.
  --entrypoint ENTRYPOINT
- Relative path to the workflow from the main workflow directory. If not provided, 'main.nf' will be used if available. If not available, but a single '*.nf' is located in the workflow-path will be used. If more than one '*.nf' script is detected, an error is raised.
+ Relative path to the workflow from the main workflow directory
  --workflow-path WORKFLOW_PATH
  Path to the main workflow directory
  --sb-package-id SB_PACKAGE_ID
  Id of an already uploaded package
  --womtool-input WOMTOOL_INPUT
  Path to inputs .JSON generated by womtool
- --sb-inputs SB_INPUTS
- Path to pre built sb app inputs schema
- --sb-outputs SB_OUTPUTS
- Path to pre build sb app outputs schema
  --sb-doc SB_DOC Path to a doc file for sb app. If not provided, README.md will be used if available
  --dump-sb-app Dump created sb app to file if true and exit
  --no-package Only provide a sb app schema and a git URL for entrypoint
@@ -53,14 +49,20 @@ to the Platform.
 ```
 $ sbpack_nf -h
 
-usage: nextflow.py [-h] [--profile PROFILE] --appid APPID --workflow-path
- WORKFLOW_PATH [--entrypoint ENTRYPOINT]
- [--sb-package-id SB_PACKAGE_ID] [--sb-doc SB_DOC]
- [--dump-sb-app] [--no-package]
- [--executor-version EXECUTOR_VERSION]
- [--execution-mode {single-instance,multi-instance}]
- [--json] [--sb-schema SB_SCHEMA]
- [--revision-note REVISION_NOTE]
+usage: sbpack_nf [-h]
+ [--profile PROFILE]
+ --appid APPID 
+ --workflow-path WORKFLOW_PATH
+ [--entrypoint ENTRYPOINT]
+ [--sb-package-id SB_PACKAGE_ID]
+ [--sb-doc SB_DOC]
+ [--dump-sb-app]
+ [--no-package]
+ [--executor-version EXECUTOR_VERSION]
+ [--execution-mode {single-instance,multi-instance}]
+ [--json]
+ [--sb-schema SB_SCHEMA]
+ [--revision-note REVISION_NOTE]
 
 optional arguments:
  -h, --help show this help message and exit
@@ -113,8 +115,10 @@ Note: The link between the original, and the new app will not be available.
 
 ```
 sbcopy -h
-usage: copy_app.py [-h] [--profile PROFILE [PROFILE ...]] --appid APPID
- --projectid PROJECTID
+usage: sbcopy [-h]
+ [--profile PROFILE [PROFILE ...]]
+ --appid APPID 
+ --projectid PROJECTID
 
 optional arguments:
  -h, --help show this help message and exit

sbpack/noncwl/__init__.py

@@ -0,0 +1,9 @@
+import os
+
+PACKAGE_PATH = os.path.abspath(os.path.dirname(__file__))
+JS_PATH = os.path.join(PACKAGE_PATH, 'js_templates')
+
+
+def read_js_template(file_name):
+ with open(os.path.join(JS_PATH, file_name), 'r') as f:
+ return f.read()

sbpack/noncwl/constants.py

@@ -1,7 +1,10 @@
 from enum import Enum
+from sbpack.noncwl import read_js_template
+
 
 # ############################## Generic Bits ############################### #
 PACKAGE_SIZE_LIMIT = 100 * 1024 * 1024 # 100 MB
+REMOVE_INPUT_KEY = "REMOVE_THIS_KEY"
 
 
 # keep track of what extensions are applicable for processing
@@ -19,6 +22,9 @@ class EXTENSIONS:
 # ############################ CWL Standard Bits ############################ #
 # A generic SB input array of files that should be available on the
 # instance but are not explicitly provided to the execution as wdl params.
+SAMPLE_SHEET_FUNCTION = read_js_template("sample_sheet_generator.js")
+SAMPLE_SHEET_SWITCH = read_js_template("sample_sheet_switch.js")
+
 GENERIC_FILE_ARRAY_INPUT = {
  "id": "auxiliary_files",
  "type": "File[]?",
@@ -27,6 +33,14 @@ class EXTENSIONS:
  "required for workflow execution."
 }
 
+SAMPLE_SHEET_FILE_ARRAY_INPUT = {
+ "id": "file_input",
+ "type": "File[]?",
+ "label": "Input files",
+ "doc": "List of files that will be used to autogenerate the sample sheet "
+ "that is required for workflow execution."
+}
+
 GENERIC_NF_OUTPUT_DIRECTORY = {
  "id": "nf_workdir",
  "type": "Directory?",
@@ -50,22 +64,52 @@ class EXTENSIONS:
  }
 }
 
-# Requirements to be added to sb wrapper
+# Requirements for sb wrapper
+INLINE_JS_REQUIREMENT = {
+ 'class': "InlineJavascriptRequirement"
+}
+LOAD_LISTING_REQUIREMENT = {
+ 'class': "LoadListingRequirement"
+}
+AUX_FILES_REQUIREMENT = {
+ "class": "InitialWorkDirRequirement",
+ "listing": [
+ "$(inputs.auxiliary_files)"
+ ]
+}
+
+# Legacy - Delete after updating wdl.py
 WRAPPER_REQUIREMENTS = [
- {
- "class": "InlineJavascriptRequirement"
- },
- {
+ INLINE_JS_REQUIREMENT,
+ AUX_FILES_REQUIREMENT
+]
+
+
+def sample_sheet(
+ file_name, sample_sheet_input, format_, input_source, header, rows,
+ defaults, group_by):
+ basename = ".".join(file_name.split(".")[:-1])
+ ext = file_name.split(".")[-1]
+ new_name = f"{basename}.new.{ext}"
+
+ return {
  "class": "InitialWorkDirRequirement",
  "listing": [
- "$(inputs.auxiliary_files)"
+ {
+ "entryname": f"${{ return {sample_sheet_input} ? {sample_sheet_input}.nameroot + '.new' + {sample_sheet_input}.nameext : '{file_name}' }}",
+ "entry": SAMPLE_SHEET_FUNCTION.format_map(locals()),
+ "writable": False
+ }
  ]
  }
-]
+
 
 # ############################## Nextflow Bits ############################## #
 # Keys that should be skipped when parsing nextflow tower yaml file
 
+NF_SCHEMA_DEFAULT_NAME = 'nextflow_schema.json'
+SB_SCHEMA_DEFAULT_NAME = 'sb_nextflow_schema'
+
 # Mappings of nextflow input fields to SB input fields
 # nextflow_key: cwl_key mapping
 NF_TO_CWL_PORT_MAP = {
@@ -96,3 +140,6 @@ class EXTENSIONS:
 class ExecMode(Enum):
  single = 'single-instance'
  multi = 'multi-instance'
+
+ def __str__(self):
+ return self.value