From f967379195ee97bf1ebf911745f8b5442f0fcfb2 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Wed, 2 Aug 2023 15:15:42 +0200 Subject: [PATCH] continue... --- integration_tests.py | 11 ++-- .../Service/ProxyManagerHandler.py | 2 +- .../Client/JobMonitoringClient.py | 8 +-- .../FutureClient/JobMonitoringClient.py | 51 +++++++++---------- tests/CI/docker-compose.yml | 9 +++- tests/CI/exportCS.sh | 6 --- tests/CI/exportCSLoop.sh | 14 +++++ 7 files changed, 59 insertions(+), 42 deletions(-) delete mode 100755 tests/CI/exportCS.sh create mode 100755 tests/CI/exportCSLoop.sh diff --git a/integration_tests.py b/integration_tests.py index d474d537448..ee49a19fc39 100755 --- a/integration_tests.py +++ b/integration_tests.py @@ -325,9 +325,10 @@ def install_server(): check=True, ) - typer.secho("Exporting configuration for diracx", fg=c.GREEN) + typer.secho("Starting configuration export loop for diracx", fg=c.GREEN) + base_cmd = _build_docker_cmd("server", tty=False, daemon=True) subprocess.run( - base_cmd + ["bash", "/home/dirac/LocalRepo/ALTERNATIVE_MODULES/DIRAC/tests/CI/exportCS.sh"], + base_cmd + ["bash", "/home/dirac/LocalRepo/ALTERNATIVE_MODULES/DIRAC/tests/CI/exportCSLoop.sh"], check=True, ) @@ -989,6 +990,8 @@ def _make_config(modules, flags, release_var, editable): "CLIENT_HOST": "client", # Test specific variables "WORKSPACE": "/home/dirac", + # DiracX variable + "DIRACX_URL": "http://diracx:8000", } if editable: @@ -1035,7 +1038,7 @@ def _load_module_configs(modules): return module_ci_configs -def _build_docker_cmd(container_name, *, use_root=False, cwd="/home/dirac", tty=True): +def _build_docker_cmd(container_name, *, use_root=False, cwd="/home/dirac", tty=True, daemon=False): if use_root or os.getuid() == 0: user = "root" else: @@ -1050,6 +1053,8 @@ def _build_docker_cmd(container_name, *, use_root=False, cwd="/home/dirac", tty= err=True, fg=c.YELLOW, ) + if daemon: + cmd += ["-d"] cmd += [ "-e=TERM=xterm-color", "-e=INSTALLROOT=/home/dirac", diff --git a/src/DIRAC/FrameworkSystem/Service/ProxyManagerHandler.py b/src/DIRAC/FrameworkSystem/Service/ProxyManagerHandler.py index 354a570d13c..fd6e19952c3 100644 --- a/src/DIRAC/FrameworkSystem/Service/ProxyManagerHandler.py +++ b/src/DIRAC/FrameworkSystem/Service/ProxyManagerHandler.py @@ -423,7 +423,7 @@ def export_exchangeProxyForToken(self): "vo": vo, "aud": authSettings.token_audience, "iss": authSettings.token_issuer, - "dirac_properties": list(set(credDict.get("groupProperties", [])) + set(credDict.get("properties", []))), + "dirac_properties": list(set(credDict.get("groupProperties", [])) | set(credDict.get("properties", []))), "jti": str(uuid4()), "preferred_username": credDict["username"], "dirac_group": credDict["group"], diff --git a/src/DIRAC/WorkloadManagementSystem/Client/JobMonitoringClient.py b/src/DIRAC/WorkloadManagementSystem/Client/JobMonitoringClient.py index e90cb2bf443..2a2f1505071 100755 --- a/src/DIRAC/WorkloadManagementSystem/Client/JobMonitoringClient.py +++ b/src/DIRAC/WorkloadManagementSystem/Client/JobMonitoringClient.py @@ -4,9 +4,9 @@ from DIRAC.Core.Utilities.DEncode import ignoreEncodeWarning from DIRAC.Core.Utilities.JEncode import strToIntDict -# from DIRAC.WorkloadManagementSystem.FutureClient.JobMonitoringClient import ( -# JobMonitoringClient as futureJobMonitoringClient, -# ) +from DIRAC.WorkloadManagementSystem.FutureClient.JobMonitoringClient import ( + JobMonitoringClient as futureJobMonitoringClient, +) @createClient("WorkloadManagement/JobMonitoring") @@ -15,7 +15,7 @@ def __init__(self, **kwargs): super().__init__(**kwargs) self.setServer("WorkloadManagement/JobMonitoring") - # httpsClient = futureJobMonitoringClient + httpsClient = futureJobMonitoringClient @ignoreEncodeWarning def getJobsStatus(self, jobIDs): diff --git a/src/DIRAC/WorkloadManagementSystem/FutureClient/JobMonitoringClient.py b/src/DIRAC/WorkloadManagementSystem/FutureClient/JobMonitoringClient.py index 79302b21cc1..ad980666c6e 100644 --- a/src/DIRAC/WorkloadManagementSystem/FutureClient/JobMonitoringClient.py +++ b/src/DIRAC/WorkloadManagementSystem/FutureClient/JobMonitoringClient.py @@ -1,34 +1,33 @@ -# from diracx.client import Dirac -# from diracx.client.models import JobSearchParams +from diracx.client import Dirac +from diracx.client.models import JobSearchParams -# from diracx.cli.utils import get_auth_headers +from diracx.cli.utils import get_auth_headers +from diracx.core.preferences import DiracxPreferences -# from DIRAC.Core.Utilities.ReturnValues import convertToReturnValue +from DIRAC.Core.Utilities.ReturnValues import convertToReturnValue -# def fetch(parameters, jobIDs): -# # breakpoint() -# with Dirac(endpoint="http://localhost:8000") as api: -# jobs = api.jobs.search( -# parameters=["JobID"] + parameters, -# search=[{"parameter": "JobID", "operator": "in", "values": jobIDs}], -# headers=get_auth_headers(), -# ) -# return {j["JobID"]: {param: j[param] for param in parameters} for j in jobs} +class JobMonitoringClient: + def __init__(self, *args, **kwargs): + self.endpoint = DiracxPreferences().url + def fetch(self, parameters, jobIDs): + with Dirac(endpoint=self.endpoint) as api: + jobs = api.jobs.search( + parameters=["JobID"] + parameters, + search=[{"parameter": "JobID", "operator": "in", "values": jobIDs}], + headers=get_auth_headers(), + ) + return {j["JobID"]: {param: j[param] for param in parameters} for j in jobs} -# class JobMonitoringClient: -# def __init__(self, *args, **kwargs): -# """TODO""" + @convertToReturnValue + def getJobsMinorStatus(self, jobIDs): + return self.fetch(["MinorStatus"], jobIDs) -# @convertToReturnValue -# def getJobsMinorStatus(self, jobIDs): -# return fetch(["MinorStatus"], jobIDs) + @convertToReturnValue + def getJobsStates(self, jobIDs): + return self.fetch(["Status", "MinorStatus", "ApplicationStatus"], jobIDs) -# @convertToReturnValue -# def getJobsStates(self, jobIDs): -# return fetch(["Status", "MinorStatus", "ApplicationStatus"], jobIDs) - -# @convertToReturnValue -# def getJobsSites(self, jobIDs): -# return fetch(["Site"], jobIDs) + @convertToReturnValue + def getJobsSites(self, jobIDs): + return self.fetch(["Site"], jobIDs) diff --git a/tests/CI/docker-compose.yml b/tests/CI/docker-compose.yml index 3be8cd70f04..0b220609334 100644 --- a/tests/CI/docker-compose.yml +++ b/tests/CI/docker-compose.yml @@ -73,6 +73,9 @@ services: volumes: - cs-store-diracx:/cs_store - key-store:/signing-key + environment: + - DIRACX_CONFIG_BACKEND_URL=git+file:///cs_store/initialRepo + - DIRACX_SERVICE_AUTH_TOKEN_KEY=file:///signing-key/rs256.key dirac-client: image: ${CI_REGISTRY_IMAGE}/${HOST_OS}-dirac @@ -85,6 +88,7 @@ services: nofile: 8192 + diracx-init-key: image: gitlab-registry.cern.ch/chaen/chrissquare-hack-a-ton/diracx container_name: diracx-init-key @@ -111,8 +115,8 @@ services: image: gitlab-registry.cern.ch/chaen/chrissquare-hack-a-ton/diracx container_name: diracx-init-cs environment: - - DIRACX_CONFIG_BACKEND_URL="git+file:///cs_store/initialRepo" - - DIRACX_SERVICE_AUTH_TOKEN_KEY="file:///signing-key/rs256.key" + - DIRACX_CONFIG_BACKEND_URL=git+file:///cs_store/initialRepo + - DIRACX_SERVICE_AUTH_TOKEN_KEY=file:///signing-key/rs256.key volumes: - cs-store-diracx:/cs_store/ - key-store:/signing-key/ @@ -147,6 +151,7 @@ services: environment: - DIRACX_CONFIG_BACKEND_URL=git+file:///cs_store/initialRepo - "DIRACX_DB_URL_AUTHDB=sqlite+aiosqlite:///:memory:" + # - DIRACX_DB_URL_JOBDB=mysql+aiomysql://Dirac:Dirac@mysql:3306/JobBD - "DIRACX_DB_URL_JOBDB=sqlite+aiosqlite:///:memory:" - DIRACX_SERVICE_AUTH_TOKEN_KEY=file:///signing-key/rs256.key - DIRACX_SERVICE_AUTH_ALLOWED_REDIRECTS=["http://pclhcb211:8000/docs/oauth2-redirect"] diff --git a/tests/CI/exportCS.sh b/tests/CI/exportCS.sh deleted file mode 100755 index b32e847ec28..00000000000 --- a/tests/CI/exportCS.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -source /home/dirac/ServerInstallDIR/bashrc -git config --global user.name "DIRAC Server CI" -git config --global user.email "dirac-server-ci@invalid" -curl -L https://gitlab.cern.ch/chaen/chris-hackaton-cs/-/raw/master/convert-from-legacy.py |DIRAC_COMPAT_ENABLE_CS_CONVERSION=True ~/ServerInstallDIR/diracos/bin/python - ~/ServerInstallDIR/etc/Production.cfg /cs_store/initialRepo/ -git -C /cs_store/initialRepo/ commit -am "export $(date)" diff --git a/tests/CI/exportCSLoop.sh b/tests/CI/exportCSLoop.sh new file mode 100755 index 00000000000..c5eb5132ad8 --- /dev/null +++ b/tests/CI/exportCSLoop.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# This script will export to the `Production.cfg` file to the +# yaml format for diracx every 5 seconds + +source /home/dirac/ServerInstallDIR/bashrc +git config --global user.name "DIRAC Server CI" +git config --global user.email "dirac-server-ci@invalid" + +while true; +do + curl -L https://gitlab.cern.ch/chaen/chris-hackaton-cs/-/raw/master/convert-from-legacy.py |DIRAC_COMPAT_ENABLE_CS_CONVERSION=True ~/ServerInstallDIR/diracos/bin/python - ~/ServerInstallDIR/etc/Production.cfg /cs_store/initialRepo/ + git -C /cs_store/initialRepo/ commit -am "export $(date)" + sleep 5; +done