From 6af5d395253da76981f63e4ef05eead616eabf06 Mon Sep 17 00:00:00 2001 From: Bogdan Gavrilovic Date: Fri, 21 May 2021 10:52:33 +0200 Subject: [PATCH 1/8] Fix referencing git URL in local wf --- sbpack/lib.py | 2 +- sbpack/pack.py | 4 +--- sbpack/version.py | 2 +- tests/test_packing.py | 7 +++++++ tests/workflows/wf-with-git.cwl | 13 +++++++++++++ 5 files changed, 23 insertions(+), 5 deletions(-) create mode 100644 tests/workflows/wf-with-git.cwl diff --git a/sbpack/lib.py b/sbpack/lib.py index 1833023..09b59c2 100644 --- a/sbpack/lib.py +++ b/sbpack/lib.py @@ -122,7 +122,7 @@ def resolved_path(base_url: urllib.parse.ParseResult, link: str): else: # Absolute remote path - new_url = urllib.parse.ParseResult(link_url) + new_url = link_url return new_url diff --git a/sbpack/pack.py b/sbpack/pack.py index 0892169..9105386 100644 --- a/sbpack/pack.py +++ b/sbpack/pack.py @@ -20,14 +20,12 @@ import enum from ruamel.yaml import YAML - -import sevenbridges as sbg import sevenbridges.errors as sbgerr import sbpack.schemadef as schemadef import sbpack.lib as lib -from .version import __version__ +from sbpack.version import __version__ import logging diff --git a/sbpack/version.py b/sbpack/version.py index 7c86064..da87d83 100644 --- a/sbpack/version.py +++ b/sbpack/version.py @@ -1 +1 @@ -__version__ = "2020.10.05" +__version__ = "2021.05.22" diff --git a/tests/test_packing.py b/tests/test_packing.py index 8ea3ad3..a550c64 100644 --- a/tests/test_packing.py +++ b/tests/test_packing.py @@ -72,3 +72,10 @@ def test_remote_packing_github_soft_links(): s1 = _find(cwl.get("steps"), "id", "s1") tool1 = s1.get("run") assert tool1.get("class") == "CommandLineTool" + + +def test_git_ref_in_local_workflow(): + cwl = pack("workflows/wf-with-git.cwl") + git_step = _find(cwl.get("steps"), "id", "git") + tool = git_step.get("run") + assert tool.get("class") == "CommandLineTool" \ No newline at end of file diff --git a/tests/workflows/wf-with-git.cwl b/tests/workflows/wf-with-git.cwl new file mode 100644 index 0000000..70c8d0e --- /dev/null +++ b/tests/workflows/wf-with-git.cwl @@ -0,0 +1,13 @@ +class: Workflow +cwlVersion: v1.2 +inputs: + in: File +steps: + git: + in: + - id: file1 + source: in + out: + - id: output_file + run: https://raw.githubusercontent.com/common-workflow-language/cwl-v1.2/main/tests/cat3-tool-docker.cwl + From 28429711b2f8fa29003281ca93ddd7b49b1d8b23 Mon Sep 17 00:00:00 2001 From: Bogdan Gavrilovic Date: Fri, 21 May 2021 11:16:02 +0200 Subject: [PATCH 2/8] Refactor setup.py --- requirements.txt | 4 ++++ setup.py | 8 +++----- tests/test_packing.py | 7 ------- tests/test_validation_battery.py | 1 + 4 files changed, 8 insertions(+), 12 deletions(-) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..eef97b1 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +ruamel.yaml >= 0.16 +sevenbridges-python >= 0.20 +cwltool >= 3.0 +cwlformat \ No newline at end of file diff --git a/setup.py b/setup.py index fc4d1e4..c645c4e 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ # Copyright (c) 2020 Seven Bridges. See LICENSE import pathlib +import os from datetime import datetime from setuptools import setup, find_packages @@ -11,6 +12,7 @@ now = datetime.utcnow() desc_path = pathlib.Path(current_path, "Readme.md") long_description = desc_path.open("r").read() +requirements = os.path.join(current_path, 'requirements.txt') setup( name=name, @@ -18,11 +20,7 @@ packages=find_packages(), platforms=['POSIX', 'MacOS', 'Windows'], python_requires='>=3.6', - install_requires=[ - "ruamel.yaml >= 0.16.12", - "sevenbridges-python >= 0.20.2", - "cwlformat" - ], + install_requires=open(requirements).read().splitlines(), entry_points={ 'console_scripts': [ 'sbpack = sbpack.pack:main', diff --git a/tests/test_packing.py b/tests/test_packing.py index a550c64..8ea3ad3 100644 --- a/tests/test_packing.py +++ b/tests/test_packing.py @@ -72,10 +72,3 @@ def test_remote_packing_github_soft_links(): s1 = _find(cwl.get("steps"), "id", "s1") tool1 = s1.get("run") assert tool1.get("class") == "CommandLineTool" - - -def test_git_ref_in_local_workflow(): - cwl = pack("workflows/wf-with-git.cwl") - git_step = _find(cwl.get("steps"), "id", "git") - tool = git_step.get("run") - assert tool.get("class") == "CommandLineTool" \ No newline at end of file diff --git a/tests/test_validation_battery.py b/tests/test_validation_battery.py index 35e6d2a..ac27bcb 100644 --- a/tests/test_validation_battery.py +++ b/tests/test_validation_battery.py @@ -25,6 +25,7 @@ def cwl_is_valid(fname): ('f',), [("tools/clt1.cwl",), ("tools/clt2.cwl",), ("tools/clt3.cwl",), ("workflows/wf1.cwl",), ("workflows/wf2.cwl",), ("workflows/wf4.cwl",), + ("workflows/wf-with-git.cwl",), ("https://raw.githubusercontent.com/rabix/sbpack/master/tests/workflows/wf1.cwl",), ("https://raw.githubusercontent.com/rabix/sbpack/master/tests/workflows/wf2.cwl",), ("https://raw.githubusercontent.com/rabix/sbpack/master/tests/workflows/wf4.cwl",), From 1f6c3138d4bebdbe9a473ef17d4c35537f721717 Mon Sep 17 00:00:00 2001 From: Bogdan Gavrilovic Date: Fri, 21 May 2021 11:22:20 +0200 Subject: [PATCH 3/8] Remove cwltool from requirements --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index eef97b1..4a4d9e5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ ruamel.yaml >= 0.16 sevenbridges-python >= 0.20 -cwltool >= 3.0 cwlformat \ No newline at end of file From 0822b069d213737909360cd104aed19ccf0dc3b6 Mon Sep 17 00:00:00 2001 From: Bogdan Gavrilovic Date: Fri, 21 May 2021 11:27:33 +0200 Subject: [PATCH 4/8] Lower cwltool version in test deps --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a3c0dc3..e5a4b08 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,7 +34,7 @@ jobs: run: | pip install pytest pip install pipx - pipx install cwltool + pipx install cwltool >= 1.0 pipx ensurepath # Need a new shell so that ensurepath takes - name: Test with pytest From ecb05a4c82e2a3ca1323a441bd6980bab179e85b Mon Sep 17 00:00:00 2001 From: Bogdan Gavrilovic Date: Fri, 21 May 2021 11:29:17 +0200 Subject: [PATCH 5/8] Lower cwltool version in test deps --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e5a4b08..2e40b82 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,7 +34,7 @@ jobs: run: | pip install pytest pip install pipx - pipx install cwltool >= 1.0 + pipx install cwltool>=1.0 pipx ensurepath # Need a new shell so that ensurepath takes - name: Test with pytest From 6fdf80fee32be49bbe48830ccc5c63ead02ef851 Mon Sep 17 00:00:00 2001 From: Bogdan Gavrilovic Date: Fri, 21 May 2021 12:36:40 +0200 Subject: [PATCH 6/8] Skip validation tests on windows --- tests/test_validation_battery.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_validation_battery.py b/tests/test_validation_battery.py index ac27bcb..93a061d 100644 --- a/tests/test_validation_battery.py +++ b/tests/test_validation_battery.py @@ -32,6 +32,8 @@ def cwl_is_valid(fname): ("remote-cwl/tool1.cwl",), ("remote-cwl/tool2.cwl",), ("remote-cwl/wf1.cwl",) ] ) +@pytest.mark.skipif(sys.platform == 'win32', + reason='Skip on windows due to errors in cwltool with import pwd') def test_local_packing_with_validation(f): url = urllib.parse.urlparse(f) packed_name = pathlib.Path(url.path).stem + "-packed.cwl" From bb2774aa15c1a1cb8756f262f05cf77a7d43e380 Mon Sep 17 00:00:00 2001 From: Bogdan Gavrilovic Date: Thu, 7 Oct 2021 10:26:23 +0200 Subject: [PATCH 7/8] Bump package and dep versions --- .github/workflows/tests.yml | 2 +- requirements.txt | 2 +- sbpack/version.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2e40b82..e3a056a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,7 +34,7 @@ jobs: run: | pip install pytest pip install pipx - pipx install cwltool>=1.0 + pipx install cwltool>=3.0 pipx ensurepath # Need a new shell so that ensurepath takes - name: Test with pytest diff --git a/requirements.txt b/requirements.txt index 4a4d9e5..85d1737 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ ruamel.yaml >= 0.16 -sevenbridges-python >= 0.20 +sevenbridges-python >= 2.0 cwlformat \ No newline at end of file diff --git a/sbpack/version.py b/sbpack/version.py index da87d83..f1e8ea6 100644 --- a/sbpack/version.py +++ b/sbpack/version.py @@ -1 +1 @@ -__version__ = "2021.05.22" +__version__ = "2021.10.07" From 80da42630166f2fed145838546af7077a7a819ef Mon Sep 17 00:00:00 2001 From: Bogdan Gavrilovic Date: Thu, 7 Oct 2021 10:39:58 +0200 Subject: [PATCH 8/8] Add docs on reading credentials from env --- Readme.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Readme.md b/Readme.md index 0cc0893..e15df19 100644 --- a/Readme.md +++ b/Readme.md @@ -175,6 +175,19 @@ You can have several profiles on the same platform if, for example, you are an enterprise user and you belong to several divisions. Please refer to the API documentation for more detail. +### Reading credentials from env variables + +Instead of using the credentials file, you can specify environment variables +`SB_API_ENDPOINT` and `SB_AUTH_TOKEN`. To use the env variables in `sbpack` simply +specify profile `.` in the command, e.g. + +```bash +sbpack . kghosesbg/sbpla-31744/ATAC-seq-pipeline-se https://raw.githubusercontent.com/Duke-GCB/GGR-cwl/master/v1.0/ATAC-seq_pipeline/pipeline-se.cwl +``` + +By specifying `.` profile, `sbpack` will use env variables. If these are not found, the default profile +from the credentials file is used. + ### Running the test suite The pulling test requires two environment variables to be set