Add js templates

Add method to get templates
Add Wrapper class to store app wrapper (wrapper.py)
Support enums
Add --auto mode
Cleanup
First try to make app wrapper, then try to upload, instead of the other way around
Give warning if detected version is not matching provided
Profile is not required
pep8 pack
appid not required if dump-sb-app is used

.gitignore

@@ -2,3 +2,4 @@
 /dist/
 /sbpack.egg-info/
 /.idea/
+/venv/

requirements.txt

@@ -2,4 +2,4 @@ ruamel.yaml >= 0.16
 sevenbridges-python >= 2.0
 nf-core==2.1
 cwlformat
-packaging
+packaging

sbpack/lib.py

@@ -1,7 +1,8 @@
 from typing import Union
 from copy import deepcopy
-import urllib.parse
-import urllib.request
+from urllib.parse import ParseResult, urlparse, urljoin
+from urllib.request import urlopen
+from urllib.error import HTTPError
 import pathlib
 import sys
 
@@ -71,7 +72,8 @@ def normalize_to_map(obj: Union[list, dict], key_field: str):
  raise RuntimeError("Expecting a dictionary or a list here")
 
 
-def normalize_to_list(obj: Union[list, dict], key_field: str, value_field: str):
+def normalize_to_list(
+ obj: Union[list, dict], key_field: str, value_field: Union[str, None]):
  if isinstance(obj, list):
  return deepcopy(obj)
  elif isinstance(obj, dict):
@@ -89,8 +91,8 @@ def normalize_to_list(obj: Union[list, dict], key_field: str, value_field: str):
 
 
 # To deprecate
-def normalized_path(link: str, base_url: urllib.parse.ParseResult):
- link_url = urllib.parse.urlparse(link)
+def normalized_path(link: str, base_url: ParseResult):
+ link_url = urlparse(link)
  if link_url.scheme in ["file://", ""]:
  new_url = base_url._replace(
  path=str((pathlib.Path(base_url.path) / pathlib.Path(link)).resolve())
@@ -101,7 +103,7 @@ def normalized_path(link: str, base_url: urllib.parse.ParseResult):
  return new_url
 
 
-def resolved_path(base_url: urllib.parse.ParseResult, link: str):
+def resolved_path(base_url: ParseResult, link: str):
  """
  Given a base_url ("this document") and a link ("string in this document")
  return a new url (urllib.parse.ParseResult) that allows us to retrieve the
@@ -110,12 +112,12 @@ def resolved_path(base_url: urllib.parse.ParseResult, link: str):
  2. Use the OS appropriate path resolution for local paths, and network
  apropriate resolution for network paths
  """
- link_url = urllib.parse.urlparse(link)
+ link_url = urlparse(link)
  # The link will always Posix
 
  if link_url.scheme == "file://":
  # Absolute local path
- new_url = urllib.parse.ParseResult(link_url)
+ new_url = ParseResult(str(link_url))
 
  elif link_url.scheme == "":
  # Relative path, can be local or remote
@@ -130,7 +132,7 @@ def resolved_path(base_url: urllib.parse.ParseResult, link: str):
 
  else:
  # Remote relative path
- new_url = urllib.parse.urlparse(urllib.parse.urljoin(base_url.geturl(), link_url.path))
+ new_url = urlparse(urljoin(base_url.geturl(), link_url.path))
  # We need urljoin because we need to resolve relative links in a
  # platform independent manner
 
@@ -141,16 +143,16 @@ def resolved_path(base_url: urllib.parse.ParseResult, link: str):
  return new_url
 
 
-def load_linked_file(base_url: urllib.parse.ParseResult, link: str, is_import=False):
+def load_linked_file(base_url: ParseResult, link: str, is_import=False):
 
  new_url = resolved_path(base_url, link)
 
  if new_url.scheme in ["file://", ""]:
  contents = pathlib.Path(new_url.path).open().read()
  else:
  try:
- contents = urllib.request.urlopen(new_url.geturl()).read().decode("utf-8")
- except urllib.error.HTTPError as e:
+ contents = urlopen(new_url.geturl()).read().decode("utf-8")
+ except HTTPError as e:
  e.msg += f"\n===\nCould not find linked file: {new_url.geturl()}\n===\n"
  raise SystemExit(e)
 
@@ -177,7 +179,7 @@ def load_linked_file(base_url: urllib.parse.ParseResult, link: str, is_import=Fa
  return _node, new_url
 
 
-def _is_github_symbolic_link(base_url: urllib.parse.ParseResult, contents: str):
+def _is_github_symbolic_link(base_url: ParseResult, contents: str):
  """Look for remote path with contents that is a single line with no new
  line with an extension."""
  if base_url.scheme in ["file://", ""]:

sbpack/noncwl/nextflow.py

@@ -492,7 +492,7 @@ def main():
  help="SB platform profile as set in the SB API credentials file.",
  )
  parser.add_argument(
- "--appid", required=True,
+ "--appid", required=False,
  help="Takes the form {user or division}/{project}/{app_id}.",
  )
  parser.add_argument(
@@ -574,6 +574,7 @@ def main():
  revision_note = args.revision_note or \
  f"Uploaded using sbpack v{__version__}"
  sample_sheet_schema = args.sample_sheet_schema or None
+ dump_sb_app = args.dump_sb_app or False
 
  sb_doc = None
  if args.sb_doc:
@@ -631,6 +632,19 @@ def main():
  if not sample_sheet_schema:
  sample_sheet_schema = get_sample_sheet_schema(args.workflow_path)
 
+ # if appid is not provided, dump the app
+ if not args.appid:
+ dump_sb_app = True
+
+ # Input validation
+ if not dump_sb_app:
+ # appid is required
+ if not args.appid:
+ raise Exception(
+ "The --appid argument is required if "
+ "--dump-sb-app is not used"
+ )
+
  nf_wrapper = SBNextflowWrapper(
  workflow_path=args.workflow_path,
  sb_doc=sb_doc
@@ -641,7 +655,6 @@ def main():
  nf_wrapper.generate_sb_app(
  sb_schema=sb_schema
  )
-
  else:
  # build schema
  nf_wrapper.nf_schema_build()
@@ -655,7 +668,11 @@ def main():
  )
 
  # Install app
- if not args.dump_sb_app:
+ if dump_sb_app:
+ # Dump app to local file
+ out_format = EXTENSIONS.json if args.json else EXTENSIONS.yaml
+ nf_wrapper.dump_sb_wrapper(out_format=out_format)
+ else:
  api = lib.get_profile(args.profile)
 
  sb_package_id = None
@@ -678,7 +695,8 @@ def main():
 
  # Dump app to local file
  out_format = EXTENSIONS.json if args.json else EXTENSIONS.yaml
- nf_wrapper.dump_sb_wrapper(out_format=out_format)
+ if not sb_schema:
+ nf_wrapper.dump_sb_wrapper(out_format=out_format)
  install_or_upgrade_app(api, args.appid, nf_wrapper.sb_wrapper.dump())
 
 

sbpack/noncwl/utils.py

@@ -35,9 +35,9 @@ def nf_to_sb_input_mapper(port_id, port_data, category=None, required=False):
  """
  sb_input = dict()
  sb_input['id'] = port_id
- # Do not convert outdir
- if port_id == 'outdir':
- port_data['format'] = ''
+ # # Do not convert outdir
+ # if port_id == 'outdir':
+ #  port_data['format'] = ''
 
  enum_symbols = port_data.get('enum', [])
 

sbpack/pack.py

@@ -17,7 +17,6 @@
 import urllib.parse
 import urllib.request
 from typing import Union
-from copy import deepcopy
 import json
 import enum
 
@@ -201,7 +200,7 @@ def resolve_steps(
 ):
 
  if isinstance(cwl, str):
- raise RuntimeError(f"{base_url.getulr()}: Expecting a process, found a string")
+ raise RuntimeError(f"{base_url.geturl()}: Expecting a process, found a string")
 
  if not isinstance(cwl, dict):
  return cwl
@@ -226,9 +225,8 @@ def resolve_steps(
  add_ids=add_ids,
  )
  if "id" not in v["run"] and add_ids:
- v["run"][
- "id"
- ] = f"{workflow_id}@step_{v['id']}@{os.path.basename(_run)}"
+ v["run"]["id"] = f"{workflow_id}@step_{v['id']}" \
+ f"@{os.path.basename(_run)}"
  else:
  v["run"] = pack_process(
  v["run"],
@@ -297,7 +295,8 @@ def filter_out_non_sbg_tags(cwl: Union[list, dict]):
 
 
 def get_git_info(cwl_path: str) -> str:
- import subprocess, os
+ import subprocess
+ import os
 
  source_str = cwl_path
 
@@ -307,7 +306,9 @@ def get_git_info(cwl_path: str) -> str:
  os.chdir(source_path.parent)
  try:
  origin = (
- subprocess.check_output(["git", "config", "--get", "remote.origin.url"])
+ subprocess.check_output(
+ ["git", "config", "--get", "remote.origin.url"]
+ )
  .strip()
  .decode()
  )
@@ -319,13 +320,17 @@ def get_git_info(cwl_path: str) -> str:
  .decode()
  )
  changed = (
- subprocess.check_output(["git", "status", source_path.name, "-s"])
+ subprocess.check_output(
+ ["git", "status", source_path.name, "-s"]
+ )
  .strip()
  .decode()
  )
  if changed == "":
  tag = (
- subprocess.check_output(["git", "describe", "--always"])
+ subprocess.check_output(
+ ["git", "describe", "--always"]
+ )
  .strip()
  .decode()
  )
@@ -376,7 +381,6 @@ def pack(cwl_path: str, filter_non_sbg_tags=False, add_ids=False):
 
 
 def main():
-
  logging.basicConfig()
  logger.setLevel(logging.INFO)
  print(
@@ -386,12 +390,23 @@ def main():
  )
 
  parser = argparse.ArgumentParser()
- parser.add_argument("profile", help="SB platform profile as set in the SB API credentials file.")
- parser.add_argument("appid", help="Takes the form {user}/{project}/{app_id}.")
- parser.add_argument("cwl_path", help="Path or URL to the main CWL file to be uploaded.")
- parser.add_argument("--filter-non-sbg-tags",
- action="store_true",
- help="Filter out custom tags that are not 'sbg:'")
+ parser.add_argument(
+ "profile", "--profile",
+ help="SB platform profile as set in the SB API credentials file."
+ )
+ parser.add_argument(
+ "appid", "--appid",
+ help="Takes the form {user}/{project}/{app_id}."
+ )
+ parser.add_argument(
+ "cwl_path", "--cwl-path",
+ help="Path or URL to the main CWL file to be uploaded."
+ )
+ parser.add_argument(
+ "--filter-non-sbg-tags",
+ action="store_true",
+ help="Filter out custom tags that are not 'sbg:'"
+ )
 
  args = parser.parse_args()
 
@@ -410,9 +425,8 @@ def main():
 
  api = lib.get_profile(profile)
 
- cwl[
- "sbg:revisionNotes"
- ] = f"Uploaded using sbpack v{__version__}. \nSource: {get_git_info(cwl_path)}"
+ cwl["sbg:revisionNotes"] = f"Uploaded using sbpack v{__version__}.\n" \
+ f"Source: {get_git_info(cwl_path)}"
  try:
  app = api.apps.get(appid)
  logger.debug(f"Creating revised app: {appid}")
@@ -425,6 +439,7 @@ def main():
 def localpack():
  _localpack(sys.argv[1:])
 
+
 def _localpack(args):
  logging.basicConfig()
  logger.setLevel(logging.INFO)
@@ -452,7 +467,9 @@ def _localpack(args):
  cwl_path = args.cwl_path
 
  cwl = pack(
- cwl_path, filter_non_sbg_tags=args.filter_non_sbg_tags, add_ids=args.add_ids
+ cwl_path,
+ filter_non_sbg_tags=args.filter_non_sbg_tags,
+ add_ids=args.add_ids
  )
  if args.json:
  json.dump(cwl, sys.stdout, indent=4)