From 9d620e566a63f938acace668d603a1fdcf889351 Mon Sep 17 00:00:00 2001 From: pavlemarinkovic Date: Thu, 18 Jan 2024 16:44:47 +0100 Subject: [PATCH] Add js templates Add method to get templates Add Wrapper class to store app wrapper (wrapper.py) Support enums Add --auto mode Cleanup First try to make app wrapper, then try to upload, instead of the other way around Give warning if detected version is not matching provided Profile is not required pep8 pack appid not required if dump-sb-app is used --- .gitignore | 1 + requirements.txt | 2 +- sbpack/lib.py | 28 ++++++++++--------- sbpack/noncwl/nextflow.py | 26 +++++++++++++++--- sbpack/noncwl/utils.py | 6 ++--- sbpack/pack.py | 57 +++++++++++++++++++++++++-------------- 6 files changed, 79 insertions(+), 41 deletions(-) diff --git a/.gitignore b/.gitignore index ecf3118..220202f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /dist/ /sbpack.egg-info/ /.idea/ +/venv/ diff --git a/requirements.txt b/requirements.txt index 8050873..f48d423 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,4 @@ ruamel.yaml >= 0.16 sevenbridges-python >= 2.0 nf-core==2.1 cwlformat -packaging +packaging \ No newline at end of file diff --git a/sbpack/lib.py b/sbpack/lib.py index 021b0e2..009b25b 100644 --- a/sbpack/lib.py +++ b/sbpack/lib.py @@ -1,7 +1,8 @@ from typing import Union from copy import deepcopy -import urllib.parse -import urllib.request +from urllib.parse import ParseResult, urlparse, urljoin +from urllib.request import urlopen +from urllib.error import HTTPError import pathlib import sys @@ -71,7 +72,8 @@ def normalize_to_map(obj: Union[list, dict], key_field: str): raise RuntimeError("Expecting a dictionary or a list here") -def normalize_to_list(obj: Union[list, dict], key_field: str, value_field: str): +def normalize_to_list( + obj: Union[list, dict], key_field: str, value_field: Union[str, None]): if isinstance(obj, list): return deepcopy(obj) elif isinstance(obj, dict): @@ -89,8 +91,8 @@ def normalize_to_list(obj: Union[list, dict], key_field: str, value_field: str): # To deprecate -def normalized_path(link: str, base_url: urllib.parse.ParseResult): - link_url = urllib.parse.urlparse(link) +def normalized_path(link: str, base_url: ParseResult): + link_url = urlparse(link) if link_url.scheme in ["file://", ""]: new_url = base_url._replace( path=str((pathlib.Path(base_url.path) / pathlib.Path(link)).resolve()) @@ -101,7 +103,7 @@ def normalized_path(link: str, base_url: urllib.parse.ParseResult): return new_url -def resolved_path(base_url: urllib.parse.ParseResult, link: str): +def resolved_path(base_url: ParseResult, link: str): """ Given a base_url ("this document") and a link ("string in this document") return a new url (urllib.parse.ParseResult) that allows us to retrieve the @@ -110,12 +112,12 @@ def resolved_path(base_url: urllib.parse.ParseResult, link: str): 2. Use the OS appropriate path resolution for local paths, and network apropriate resolution for network paths """ - link_url = urllib.parse.urlparse(link) + link_url = urlparse(link) # The link will always Posix if link_url.scheme == "file://": # Absolute local path - new_url = urllib.parse.ParseResult(link_url) + new_url = ParseResult(str(link_url)) elif link_url.scheme == "": # Relative path, can be local or remote @@ -130,7 +132,7 @@ def resolved_path(base_url: urllib.parse.ParseResult, link: str): else: # Remote relative path - new_url = urllib.parse.urlparse(urllib.parse.urljoin(base_url.geturl(), link_url.path)) + new_url = urlparse(urljoin(base_url.geturl(), link_url.path)) # We need urljoin because we need to resolve relative links in a # platform independent manner @@ -141,7 +143,7 @@ def resolved_path(base_url: urllib.parse.ParseResult, link: str): return new_url -def load_linked_file(base_url: urllib.parse.ParseResult, link: str, is_import=False): +def load_linked_file(base_url: ParseResult, link: str, is_import=False): new_url = resolved_path(base_url, link) @@ -149,8 +151,8 @@ def load_linked_file(base_url: urllib.parse.ParseResult, link: str, is_import=Fa contents = pathlib.Path(new_url.path).open().read() else: try: - contents = urllib.request.urlopen(new_url.geturl()).read().decode("utf-8") - except urllib.error.HTTPError as e: + contents = urlopen(new_url.geturl()).read().decode("utf-8") + except HTTPError as e: e.msg += f"\n===\nCould not find linked file: {new_url.geturl()}\n===\n" raise SystemExit(e) @@ -177,7 +179,7 @@ def load_linked_file(base_url: urllib.parse.ParseResult, link: str, is_import=Fa return _node, new_url -def _is_github_symbolic_link(base_url: urllib.parse.ParseResult, contents: str): +def _is_github_symbolic_link(base_url: ParseResult, contents: str): """Look for remote path with contents that is a single line with no new line with an extension.""" if base_url.scheme in ["file://", ""]: diff --git a/sbpack/noncwl/nextflow.py b/sbpack/noncwl/nextflow.py index 3f2ea47..c4a8c65 100644 --- a/sbpack/noncwl/nextflow.py +++ b/sbpack/noncwl/nextflow.py @@ -492,7 +492,7 @@ def main(): help="SB platform profile as set in the SB API credentials file.", ) parser.add_argument( - "--appid", required=True, + "--appid", required=False, help="Takes the form {user or division}/{project}/{app_id}.", ) parser.add_argument( @@ -574,6 +574,7 @@ def main(): revision_note = args.revision_note or \ f"Uploaded using sbpack v{__version__}" sample_sheet_schema = args.sample_sheet_schema or None + dump_sb_app = args.dump_sb_app or False sb_doc = None if args.sb_doc: @@ -631,6 +632,19 @@ def main(): if not sample_sheet_schema: sample_sheet_schema = get_sample_sheet_schema(args.workflow_path) + # if appid is not provided, dump the app + if not args.appid: + dump_sb_app = True + + # Input validation + if not dump_sb_app: + # appid is required + if not args.appid: + raise Exception( + "The --appid argument is required if " + "--dump-sb-app is not used" + ) + nf_wrapper = SBNextflowWrapper( workflow_path=args.workflow_path, sb_doc=sb_doc @@ -641,7 +655,6 @@ def main(): nf_wrapper.generate_sb_app( sb_schema=sb_schema ) - else: # build schema nf_wrapper.nf_schema_build() @@ -655,7 +668,11 @@ def main(): ) # Install app - if not args.dump_sb_app: + if dump_sb_app: + # Dump app to local file + out_format = EXTENSIONS.json if args.json else EXTENSIONS.yaml + nf_wrapper.dump_sb_wrapper(out_format=out_format) + else: api = lib.get_profile(args.profile) sb_package_id = None @@ -678,7 +695,8 @@ def main(): # Dump app to local file out_format = EXTENSIONS.json if args.json else EXTENSIONS.yaml - nf_wrapper.dump_sb_wrapper(out_format=out_format) + if not sb_schema: + nf_wrapper.dump_sb_wrapper(out_format=out_format) install_or_upgrade_app(api, args.appid, nf_wrapper.sb_wrapper.dump()) diff --git a/sbpack/noncwl/utils.py b/sbpack/noncwl/utils.py index 1ffc155..29d6bbf 100644 --- a/sbpack/noncwl/utils.py +++ b/sbpack/noncwl/utils.py @@ -35,9 +35,9 @@ def nf_to_sb_input_mapper(port_id, port_data, category=None, required=False): """ sb_input = dict() sb_input['id'] = port_id - # Do not convert outdir - if port_id == 'outdir': - port_data['format'] = '' + # # Do not convert outdir + # if port_id == 'outdir': + # port_data['format'] = '' enum_symbols = port_data.get('enum', []) diff --git a/sbpack/pack.py b/sbpack/pack.py index 90b6937..49c8709 100644 --- a/sbpack/pack.py +++ b/sbpack/pack.py @@ -17,7 +17,6 @@ import urllib.parse import urllib.request from typing import Union -from copy import deepcopy import json import enum @@ -201,7 +200,7 @@ def resolve_steps( ): if isinstance(cwl, str): - raise RuntimeError(f"{base_url.getulr()}: Expecting a process, found a string") + raise RuntimeError(f"{base_url.geturl()}: Expecting a process, found a string") if not isinstance(cwl, dict): return cwl @@ -226,9 +225,8 @@ def resolve_steps( add_ids=add_ids, ) if "id" not in v["run"] and add_ids: - v["run"][ - "id" - ] = f"{workflow_id}@step_{v['id']}@{os.path.basename(_run)}" + v["run"]["id"] = f"{workflow_id}@step_{v['id']}" \ + f"@{os.path.basename(_run)}" else: v["run"] = pack_process( v["run"], @@ -297,7 +295,8 @@ def filter_out_non_sbg_tags(cwl: Union[list, dict]): def get_git_info(cwl_path: str) -> str: - import subprocess, os + import subprocess + import os source_str = cwl_path @@ -307,7 +306,9 @@ def get_git_info(cwl_path: str) -> str: os.chdir(source_path.parent) try: origin = ( - subprocess.check_output(["git", "config", "--get", "remote.origin.url"]) + subprocess.check_output( + ["git", "config", "--get", "remote.origin.url"] + ) .strip() .decode() ) @@ -319,13 +320,17 @@ def get_git_info(cwl_path: str) -> str: .decode() ) changed = ( - subprocess.check_output(["git", "status", source_path.name, "-s"]) + subprocess.check_output( + ["git", "status", source_path.name, "-s"] + ) .strip() .decode() ) if changed == "": tag = ( - subprocess.check_output(["git", "describe", "--always"]) + subprocess.check_output( + ["git", "describe", "--always"] + ) .strip() .decode() ) @@ -376,7 +381,6 @@ def pack(cwl_path: str, filter_non_sbg_tags=False, add_ids=False): def main(): - logging.basicConfig() logger.setLevel(logging.INFO) print( @@ -386,12 +390,23 @@ def main(): ) parser = argparse.ArgumentParser() - parser.add_argument("profile", help="SB platform profile as set in the SB API credentials file.") - parser.add_argument("appid", help="Takes the form {user}/{project}/{app_id}.") - parser.add_argument("cwl_path", help="Path or URL to the main CWL file to be uploaded.") - parser.add_argument("--filter-non-sbg-tags", - action="store_true", - help="Filter out custom tags that are not 'sbg:'") + parser.add_argument( + "profile", "--profile", + help="SB platform profile as set in the SB API credentials file." + ) + parser.add_argument( + "appid", "--appid", + help="Takes the form {user}/{project}/{app_id}." + ) + parser.add_argument( + "cwl_path", "--cwl-path", + help="Path or URL to the main CWL file to be uploaded." + ) + parser.add_argument( + "--filter-non-sbg-tags", + action="store_true", + help="Filter out custom tags that are not 'sbg:'" + ) args = parser.parse_args() @@ -410,9 +425,8 @@ def main(): api = lib.get_profile(profile) - cwl[ - "sbg:revisionNotes" - ] = f"Uploaded using sbpack v{__version__}. \nSource: {get_git_info(cwl_path)}" + cwl["sbg:revisionNotes"] = f"Uploaded using sbpack v{__version__}.\n" \ + f"Source: {get_git_info(cwl_path)}" try: app = api.apps.get(appid) logger.debug(f"Creating revised app: {appid}") @@ -425,6 +439,7 @@ def main(): def localpack(): _localpack(sys.argv[1:]) + def _localpack(args): logging.basicConfig() logger.setLevel(logging.INFO) @@ -452,7 +467,9 @@ def _localpack(args): cwl_path = args.cwl_path cwl = pack( - cwl_path, filter_non_sbg_tags=args.filter_non_sbg_tags, add_ids=args.add_ids + cwl_path, + filter_non_sbg_tags=args.filter_non_sbg_tags, + add_ids=args.add_ids ) if args.json: json.dump(cwl, sys.stdout, indent=4)