continue...

integration_tests.py

@@ -325,9 +325,10 @@ def install_server():
         check=True,
     )
 
-    typer.secho("Exporting configuration for diracx", fg=c.GREEN)
+    typer.secho("Starting configuration export loop for diracx", fg=c.GREEN)
+    base_cmd = _build_docker_cmd("server", tty=False, daemon=True)
     subprocess.run(
-        base_cmd + ["bash", "/home/dirac/LocalRepo/ALTERNATIVE_MODULES/DIRAC/tests/CI/exportCS.sh"],
+        base_cmd + ["bash", "/home/dirac/LocalRepo/ALTERNATIVE_MODULES/DIRAC/tests/CI/exportCSLoop.sh"],
         check=True,
     )
 
@@ -989,6 +990,8 @@ def _make_config(modules, flags, release_var, editable):
         "CLIENT_HOST": "client",
         # Test specific variables
         "WORKSPACE": "/home/dirac",
+        # DiracX variable
+        "DIRACX_URL": "http://diracx:8000",
     }
 
     if editable:
@@ -1035,7 +1038,7 @@ def _load_module_configs(modules):
     return module_ci_configs
 
 
-def _build_docker_cmd(container_name, *, use_root=False, cwd="/home/dirac", tty=True):
+def _build_docker_cmd(container_name, *, use_root=False, cwd="/home/dirac", tty=True, daemon=False):
     if use_root or os.getuid() == 0:
         user = "root"
     else:
@@ -1050,6 +1053,8 @@ def _build_docker_cmd(container_name, *, use_root=False, cwd="/home/dirac", tty=
                 err=True,
                 fg=c.YELLOW,
             )
+    if daemon:
+        cmd += ["-d"]
     cmd += [
         "-e=TERM=xterm-color",
         "-e=INSTALLROOT=/home/dirac",

src/DIRAC/FrameworkSystem/Service/ProxyManagerHandler.py

@@ -423,7 +423,7 @@ def export_exchangeProxyForToken(self):
             "vo": vo,
             "aud": authSettings.token_audience,
             "iss": authSettings.token_issuer,
-            "dirac_properties": list(set(credDict.get("groupProperties", [])) + set(credDict.get("properties", []))),
+            "dirac_properties": list(set(credDict.get("groupProperties", [])) | set(credDict.get("properties", []))),
             "jti": str(uuid4()),
             "preferred_username": credDict["username"],
             "dirac_group": credDict["group"],

src/DIRAC/WorkloadManagementSystem/Client/JobMonitoringClient.py

@@ -4,9 +4,9 @@
 from DIRAC.Core.Utilities.DEncode import ignoreEncodeWarning
 from DIRAC.Core.Utilities.JEncode import strToIntDict
 
-# from DIRAC.WorkloadManagementSystem.FutureClient.JobMonitoringClient import (
-#     JobMonitoringClient as futureJobMonitoringClient,
-# )
+from DIRAC.WorkloadManagementSystem.FutureClient.JobMonitoringClient import (
+    JobMonitoringClient as futureJobMonitoringClient,
+)
 
 
 @createClient("WorkloadManagement/JobMonitoring")
@@ -15,7 +15,7 @@ def __init__(self, **kwargs):
         super().__init__(**kwargs)
         self.setServer("WorkloadManagement/JobMonitoring")
 
-    # httpsClient = futureJobMonitoringClient
+    httpsClient = futureJobMonitoringClient
 
     @ignoreEncodeWarning
     def getJobsStatus(self, jobIDs):

src/DIRAC/WorkloadManagementSystem/FutureClient/JobMonitoringClient.py

@@ -1,34 +1,33 @@
-# from diracx.client import Dirac
-# from diracx.client.models import JobSearchParams
+from diracx.client import Dirac
+from diracx.client.models import JobSearchParams
 
-# from diracx.cli.utils import get_auth_headers
+from diracx.cli.utils import get_auth_headers
+from diracx.core.preferences import DiracxPreferences
 
-# from DIRAC.Core.Utilities.ReturnValues import convertToReturnValue
+from DIRAC.Core.Utilities.ReturnValues import convertToReturnValue
 
 
-# def fetch(parameters, jobIDs):
-#     # breakpoint()
-#     with Dirac(endpoint="http://localhost:8000") as api:
-#         jobs = api.jobs.search(
-#             parameters=["JobID"] + parameters,
-#             search=[{"parameter": "JobID", "operator": "in", "values": jobIDs}],
-#             headers=get_auth_headers(),
-#         )
-#         return {j["JobID"]: {param: j[param] for param in parameters} for j in jobs}
+class JobMonitoringClient:
+    def __init__(self, *args, **kwargs):
+        self.endpoint = DiracxPreferences().url
 
+    def fetch(self, parameters, jobIDs):
+        with Dirac(endpoint=self.endpoint) as api:
+            jobs = api.jobs.search(
+                parameters=["JobID"] + parameters,
+                search=[{"parameter": "JobID", "operator": "in", "values": jobIDs}],
+                headers=get_auth_headers(),
+            )
+            return {j["JobID"]: {param: j[param] for param in parameters} for j in jobs}
 
-# class JobMonitoringClient:
-#     def __init__(self, *args, **kwargs):
-#         """TODO"""
+    @convertToReturnValue
+    def getJobsMinorStatus(self, jobIDs):
+        return self.fetch(["MinorStatus"], jobIDs)
 
-#     @convertToReturnValue
-#     def getJobsMinorStatus(self, jobIDs):
-#         return fetch(["MinorStatus"], jobIDs)
+    @convertToReturnValue
+    def getJobsStates(self, jobIDs):
+        return self.fetch(["Status", "MinorStatus", "ApplicationStatus"], jobIDs)
 
-#     @convertToReturnValue
-#     def getJobsStates(self, jobIDs):
-#         return fetch(["Status", "MinorStatus", "ApplicationStatus"], jobIDs)
-
-#     @convertToReturnValue
-#     def getJobsSites(self, jobIDs):
-#         return fetch(["Site"], jobIDs)
+    @convertToReturnValue
+    def getJobsSites(self, jobIDs):
+        return self.fetch(["Site"], jobIDs)

tests/CI/docker-compose.yml

@@ -73,6 +73,9 @@ services:
     volumes:
       - cs-store-diracx:/cs_store
       - key-store:/signing-key
+    environment:
+      - DIRACX_CONFIG_BACKEND_URL=git+file:///cs_store/initialRepo
+      - DIRACX_SERVICE_AUTH_TOKEN_KEY=file:///signing-key/rs256.key
 
   dirac-client:
     image: ${CI_REGISTRY_IMAGE}/${HOST_OS}-dirac
@@ -85,6 +88,7 @@ services:
       nofile: 8192
 
 
+
   diracx-init-key:
     image: gitlab-registry.cern.ch/chaen/chrissquare-hack-a-ton/diracx
     container_name: diracx-init-key
@@ -111,8 +115,8 @@ services:
     image: gitlab-registry.cern.ch/chaen/chrissquare-hack-a-ton/diracx
     container_name: diracx-init-cs
     environment:
-      - DIRACX_CONFIG_BACKEND_URL="git+file:///cs_store/initialRepo"
-      - DIRACX_SERVICE_AUTH_TOKEN_KEY="file:///signing-key/rs256.key"
+      - DIRACX_CONFIG_BACKEND_URL=git+file:///cs_store/initialRepo
+      - DIRACX_SERVICE_AUTH_TOKEN_KEY=file:///signing-key/rs256.key
     volumes:
       - cs-store-diracx:/cs_store/
       - key-store:/signing-key/
@@ -147,6 +151,7 @@ services:
     environment:
       - DIRACX_CONFIG_BACKEND_URL=git+file:///cs_store/initialRepo
       - "DIRACX_DB_URL_AUTHDB=sqlite+aiosqlite:///:memory:"
+      # - DIRACX_DB_URL_JOBDB=mysql+aiomysql://Dirac:Dirac@mysql:3306/JobBD
       - "DIRACX_DB_URL_JOBDB=sqlite+aiosqlite:///:memory:"
       - DIRACX_SERVICE_AUTH_TOKEN_KEY=file:///signing-key/rs256.key
       - DIRACX_SERVICE_AUTH_ALLOWED_REDIRECTS=["http://pclhcb211:8000/docs/oauth2-redirect"]

tests/CI/exportCSLoop.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# This script will export to the `Production.cfg` file to the
+# yaml format for diracx every 5 seconds
+
+source /home/dirac/ServerInstallDIR/bashrc
+git config --global user.name "DIRAC Server CI"
+git config --global user.email "dirac-server-ci@invalid"
+
+while true;
+do
+    curl -L https://gitlab.cern.ch/chaen/chris-hackaton-cs/-/raw/master/convert-from-legacy.py |DIRAC_COMPAT_ENABLE_CS_CONVERSION=True  ~/ServerInstallDIR/diracos/bin/python - ~/ServerInstallDIR/etc/Production.cfg /cs_store/initialRepo/
+    git -C /cs_store/initialRepo/ commit -am "export $(date)"
+    sleep 5;
+done