Merge pull request #103 from i-VRESSE/quick-close

Dont close a connection that is not there

.github/workflows/dirac.yml

@@ -5,6 +5,7 @@ on:
  # Run tests only if dirac code is changed or workflow is triggerd manually
  paths:
  - '**dirac**'
+ - .github/workflows/dirac.yml
  workflow_dispatch:
 
 jobs:

.pre-commit-config.yaml

@@ -15,7 +15,7 @@ repos:
  - id: add-trailing-comma
 
 - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
- rev: v2.1.0
+ rev: v2.14.0
  hooks:
  - id: pretty-format-yaml
  args:

README.md

@@ -35,16 +35,18 @@ at <https://i-vresse-bartender.readthedocs.io> .
 
 ## Quickstart
 
-1. Install bartender via github:
+1. Install bartender via GitHub:
 
  ```bash
- pip install git+https://github.com/i-VRESSE/bartender.git
+ # Inside an empty directory
+ git clone https://github.com/i-VRESSE/bartender.git .
+ pip install .
  ```
 
 1. Obtain a copy of the example configuration file
 
  ```bash
- curl -o config.yaml https://raw.githubusercontent.com/i-VRESSE/bartender/main/config-example.yaml
+ cp config-example.yaml config.yaml
  ```
 
 1. In another terminal, start up a database for storing jobs.
@@ -109,8 +111,8 @@ Bartender is by default configured with a word count applicaton. Use the
 following steps to run a job:
 
 1. Create an archive to submit. The zip file should contain a file called
- `README.md`. A zip file could be created in a clone of this repo with `zip
- README.zip README.md`.
+ `README.md`. A zip file could be created in a clone of this repo with
+ `zip README.zip README.md`.
 2. Start [bartender web service and postgresql
  server](https://i-vresse-bartender.readthedocs.io/en/latest/index.html#quickstart)
 3. Generate token & authorize
@@ -123,7 +125,7 @@ following steps to run a job:
  4. Try out the `GET /api/whoami` route.
  It should return JSON document with `someone` as username.
 4. Submit archive.
- 1. Try out the `PUT /api/application/wc/job` route.
+ 1. Try out the `PUT /api/application/wc` route.
  2. Upload the `README.zip` as request body.
  3. Press execute button
  4. The response contains a job identifier (`id` property) that can be used

deploy/Dockerfile

@@ -3,7 +3,7 @@ FROM python:3.10-slim-buster
 # Install healthcheck dependency
 RUN apt update && apt install -y curl unzip
 
-RUN pip install poetry==1.7.0
+RUN pip install poetry==1.8.3
 
 # Configuring poetry
 RUN poetry config virtualenvs.create false

deploy/Dockerfile.dirac

@@ -8,7 +8,7 @@ RUN apt install -y unzip
 
 USER diracuser
 
-RUN pip install poetry==1.4.2
+RUN pip install poetry==1.8.3
 
 # Configuring poetry
 RUN poetry config virtualenvs.create false

docs/configuration.md

@@ -340,7 +340,7 @@ If you need [DIRAC](http://diracgrid.org/) support create a conda environment
 instead of creating a virtual environment.
 
 ```bash
-mamba create --name bartender dirac-grid python=3.10 poetry=1.5.1
+mamba create --name bartender dirac-grid python=3.10 poetry=1.8.3
 conda activate bartender
 poetry install
 ```

docs/develop.md

@@ -203,7 +203,7 @@ the `WorkloadManagement_SiteDirector` service starting a pilot.
 To look around inside the DIRAC server use
 
 ```shell
-docker-compose -f tests_dirac/docker-compose.yml exec dirac-tuto bash
+docker compose -f tests_dirac/docker-compose.yml exec dirac-tuto bash
 ```
 
 Sometimes the DIRAC server needs clearing of its state,

pyproject.toml

@@ -21,7 +21,7 @@ yarl = "^1.7.2"
 ujson = "^5.1.0"
 sqlalchemy = {extras = ["asyncio"], version = "^2.0.7"}
 alembic = "^1.10.2"
-asyncpg = "^0.27.0"
+asyncpg = "^0.29.0"
 python-multipart = "^0.0.6"
 aiofiles = "^23.1.0"
 asyncssh = "^2.13.1"
@@ -38,12 +38,13 @@ redis = {extras = ["hiredis"], version ="5.0.8"}
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.0"
-flake8 = "^4.0.1"
+flake8 = "^7.1.1"
 mypy = "^1.1.1"
 isort = "^5.10.1"
 yesqa = "^1.3.0"
 pre-commit = "^2.19.0"
-wemake-python-styleguide = "^0.16.1"
+# released version not python 3.12 compatible due to flake8-commas, use git version
+wemake-python-styleguide = {git = "https://github.com/wemake-services/wemake-python-styleguide.git", rev = "56121b39a190c0a0fb575f4066cd8bafddfe2211"}
 black = "^22.3.0"
 autoflake = "^1.4"
 pytest-cov = "^3.0.0"
@@ -99,5 +100,5 @@ env = [
 ]
 
 [build-system]
-requires = ["poetry-core>=1.5.2"]
+requires = ["poetry-core>=1.8.3"]
 build-backend = "poetry.core.masonry.api"

src/bartender/schedulers/arq.py

@@ -42,10 +42,11 @@ class ArqSchedulerConfig(BaseModel):
 
  Default is one hour.
 
- In seconds or string in `ISO 8601 duration format <https://en.wikipedia.org/wiki/ISO_8601#Durations>`_.
+ In seconds or string in
+ `ISO 8601 duration format <https://en.wikipedia.org/wiki/ISO_8601#Durations>`_.
 
  For example, "PT12H" represents a max runtime of "twelve hours".
- """ # noqa: E501, WPS428
+ """ # noqa: WPS428
 
  @property
  def redis_settings(self) -> RedisSettings:
@@ -73,8 +74,8 @@ def __init__(self, config: ArqSchedulerConfig) -> None:
  self.connection: Optional[ArqRedis] = None
 
  async def close(self) -> None: # noqa: D102
- pool = await self._pool()
- await pool.close()
+ if self.connection is not None:
+  await self.connection.close()
 
  async def submit(self, description: JobDescription) -> str: # noqa: D102
  pool = await self._pool()

src/bartender/schedulers/dirac.py

@@ -298,7 +298,7 @@ async def _jdl_script(self, description: JobDescription, scriptdir: Path) -> str
  StdOutput = "jobstdout.txt";
  StdError = "jobstderr.txt";
  OutputSandbox = {{ "jobstdout.txt", "jobstderr.txt" }};
- """, # noqa: E501, WPS237
+ """, # noqa: E501, WPS237, WPS318
  )
 
 

tests_dirac/Dockerfile

@@ -1,7 +1,7 @@
-FROM ghcr.io/xenon-middleware/diracclient:8.0.39
+FROM ghcr.io/xenon-middleware/diracclient:8.0.49
 
 RUN pip install --upgrade pip wheel setuptools && \
- pip install poetry==1.7.0
+ pip install poetry==1.8.3
 
 # Configuring poetry
 RUN poetry config virtualenvs.create false
@@ -12,7 +12,9 @@ WORKDIR /workspace
 COPY pyproject.toml poetry.lock ./
 
 # Installing requirements
-RUN poetry install --no-root --no-interaction --no-ansi --with dev
+RUN poetry install --no-root --no-interaction --no-ansi --without dev
+
+RUN pip install pytest anyio pytest-env
 
 # Workaround `ImportWarning: _SixMetaPathImporter.find_spec() not found; falling back to find_module()` error
 RUN /home/diracuser/diracos/bin/micromamba install m2crypto==0.041.0

tests_dirac/docker-compose.yml

@@ -1,8 +1,6 @@
-version: '3.9'
-
 services:
  dirac-tuto:
- image: ghcr.io/xenon-middleware/dirac:8.0.39
+ image: ghcr.io/xenon-middleware/dirac:8.0.49
  privileged: true
  hostname: dirac-tuto
  test:

tests_dirac/test_it.py

@@ -127,7 +127,20 @@ async def apptainer_image() -> Path:
  return image
 
 
+# Skip test
+# su - diracpilot
+# . /opt/dirac/bashrc
+# apptainer run docker://alpine cat /etc/os-release
+# Gives
+# ERROR [U=65534,P=5871] setup_userns_mappings()
+# Could not write info to setgroups: Permission denied
+# ERROR [U=65534,P=5892] init()
+# Error while waiting event for user namespace mappings: no event received
+# image does not run bashrc or has apptainer suid installed
 @pytest.mark.anyio
+@pytest.mark.skip(
+ reason="diracpilot user in DIRAC server container does not have permission to run apptainer", # noqa: E501
+)
 async def test_it_with_apptainer( # noqa: WPS217 single piece of code for readablilty
  apptainer_image: Path,
  tmp_path: Path,

tests_dirac/test_renewer.py

@@ -66,7 +66,6 @@ async def test_make_valid_dirac_proxy_given_bad_cert(tmp_path: Path) -> None:
 @pytest.mark.anyio
 async def test_renewer_setup_teardown() -> None:
  setup_proxy_renewer(ProxyConfig(valid="00:03", min_life=30))
-
  # Give renewer task time to run
  await asyncio.sleep(0.1)