From 3481554db77ce7f27b9bdc42991c0be6ed5efd8a Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Tue, 1 Aug 2023 14:10:46 +0200 Subject: [PATCH] Getting there to adda diracx instance... --- integration_tests.py | 20 +++- src/DIRAC/ConfigurationSystem/Client/CSAPI.py | 15 +-- .../Client/JobMonitoringClient.py | 8 +- .../FutureClient/JobMonitoringClient.py | 50 ++++----- tests/CI/docker-compose.yml | 104 ++++++++++++++++-- tests/CI/exportCS.sh | 6 + 6 files changed, 150 insertions(+), 53 deletions(-) create mode 100755 tests/CI/exportCS.sh diff --git a/integration_tests.py b/integration_tests.py index 14290d81197..d474d537448 100755 --- a/integration_tests.py +++ b/integration_tests.py @@ -33,9 +33,7 @@ "DIRAC_USE_JSON_ENCODE": None, "INSTALLATION_BRANCH": "", } -DEFAULT_MODULES = { - "DIRAC": Path(__file__).parent.absolute(), -} +DEFAULT_MODULES = {"DIRAC": Path(__file__).parent.absolute(), "diracx": Path("/home/chaen/dirac/diracx-project/diracx")} # Static configuration DB_USER = "Dirac" @@ -180,7 +178,7 @@ def destroy(): with _gen_docker_compose(DEFAULT_MODULES) as docker_compose_fn: os.execvpe( "docker-compose", - ["docker-compose", "-f", docker_compose_fn, "down", "--remove-orphans", "-t", "0"], + ["docker-compose", "-f", docker_compose_fn, "down", "--remove-orphans", "-t", "0", "--volumes"], _make_env({}), ) @@ -226,6 +224,7 @@ def prepare_environment( subprocess.run( ["docker-compose", "-f", docker_compose_fn, "up", "-d"], check=True, + capture_output=True, env=docker_compose_env, ) @@ -326,6 +325,12 @@ def install_server(): check=True, ) + typer.secho("Exporting configuration for diracx", fg=c.GREEN) + subprocess.run( + base_cmd + ["bash", "/home/dirac/LocalRepo/ALTERNATIVE_MODULES/DIRAC/tests/CI/exportCS.sh"], + check=True, + ) + typer.secho("Copying credentials and certificates", fg=c.GREEN) base_cmd = _build_docker_cmd("client", tty=False) subprocess.run( @@ -508,10 +513,13 @@ def _gen_docker_compose(modules): # Load the docker-compose configuration and mount the necessary volumes input_fn = Path(__file__).parent / "tests/CI/docker-compose.yml" docker_compose = yaml.safe_load(input_fn.read_text()) + for ctn in ("dirac-server", "dirac-client"): + if "volumes" not in docker_compose["services"][ctn]: + docker_compose["services"][ctn]["volumes"] = [] volumes = [f"{path}:/home/dirac/LocalRepo/ALTERNATIVE_MODULES/{name}" for name, path in modules.items()] volumes += [f"{path}:/home/dirac/LocalRepo/TestCode/{name}" for name, path in modules.items()] - docker_compose["services"]["dirac-server"]["volumes"] = volumes[:] - docker_compose["services"]["dirac-client"]["volumes"] = volumes[:] + docker_compose["services"]["dirac-server"]["volumes"].extend(volumes[:]) + docker_compose["services"]["dirac-client"]["volumes"].extend(volumes[:]) # Add any extension services for module_name, module_configs in _load_module_configs(modules).items(): diff --git a/src/DIRAC/ConfigurationSystem/Client/CSAPI.py b/src/DIRAC/ConfigurationSystem/Client/CSAPI.py index 79ad7515c34..310086b80fe 100644 --- a/src/DIRAC/ConfigurationSystem/Client/CSAPI.py +++ b/src/DIRAC/ConfigurationSystem/Client/CSAPI.py @@ -612,22 +612,17 @@ def getOpsSection(): Where is the shifters section? """ vo = CSGlobals.getVO() - setup = CSGlobals.getSetup() if vo: - res = gConfig.getSections(f"/Operations/{vo}/{setup}/Shifter") + res = gConfig.getSections(f"/Operations/{vo}/Shifter") if res["OK"]: - return S_OK(f"/Operations/{vo}/{setup}/Shifter") + return S_OK(f"/Operations/{vo}/Shifter") - res = gConfig.getSections(f"/Operations/{vo}/Defaults/Shifter") + res = gConfig.getSections(f"/Operations/{vo}/Shifter") if res["OK"]: - return S_OK(f"/Operations/{vo}/Defaults/Shifter") + return S_OK(f"/Operations/{vo}/Shifter") else: - res = gConfig.getSections(f"/Operations/{setup}/Shifter") - if res["OK"]: - return S_OK(f"/Operations/{setup}/Shifter") - res = gConfig.getSections("/Operations/Defaults/Shifter") if res["OK"]: return S_OK("/Operations/Defaults/Shifter") @@ -671,7 +666,7 @@ def getOpsSection(): gLogger.info("Adding shifter section") vo = CSGlobals.getVO() if vo: - section = f"/Operations/{vo}/Defaults/Shifter" + section = f"/Operations/{vo}/Shifter" else: section = "/Operations/Defaults/Shifter" res = self.__csMod.createSection(section) diff --git a/src/DIRAC/WorkloadManagementSystem/Client/JobMonitoringClient.py b/src/DIRAC/WorkloadManagementSystem/Client/JobMonitoringClient.py index 2a2f1505071..e90cb2bf443 100755 --- a/src/DIRAC/WorkloadManagementSystem/Client/JobMonitoringClient.py +++ b/src/DIRAC/WorkloadManagementSystem/Client/JobMonitoringClient.py @@ -4,9 +4,9 @@ from DIRAC.Core.Utilities.DEncode import ignoreEncodeWarning from DIRAC.Core.Utilities.JEncode import strToIntDict -from DIRAC.WorkloadManagementSystem.FutureClient.JobMonitoringClient import ( - JobMonitoringClient as futureJobMonitoringClient, -) +# from DIRAC.WorkloadManagementSystem.FutureClient.JobMonitoringClient import ( +# JobMonitoringClient as futureJobMonitoringClient, +# ) @createClient("WorkloadManagement/JobMonitoring") @@ -15,7 +15,7 @@ def __init__(self, **kwargs): super().__init__(**kwargs) self.setServer("WorkloadManagement/JobMonitoring") - httpsClient = futureJobMonitoringClient + # httpsClient = futureJobMonitoringClient @ignoreEncodeWarning def getJobsStatus(self, jobIDs): diff --git a/src/DIRAC/WorkloadManagementSystem/FutureClient/JobMonitoringClient.py b/src/DIRAC/WorkloadManagementSystem/FutureClient/JobMonitoringClient.py index b85531664bb..79302b21cc1 100644 --- a/src/DIRAC/WorkloadManagementSystem/FutureClient/JobMonitoringClient.py +++ b/src/DIRAC/WorkloadManagementSystem/FutureClient/JobMonitoringClient.py @@ -1,34 +1,34 @@ -from diracx.client import Dirac -from diracx.client.models import JobSearchParams +# from diracx.client import Dirac +# from diracx.client.models import JobSearchParams -from diracx.cli.utils import get_auth_headers +# from diracx.cli.utils import get_auth_headers -from DIRAC.Core.Utilities.ReturnValues import convertToReturnValue +# from DIRAC.Core.Utilities.ReturnValues import convertToReturnValue -def fetch(parameters, jobIDs): - # breakpoint() - with Dirac(endpoint="http://localhost:8000") as api: - jobs = api.jobs.search( - parameters=["JobID"] + parameters, - search=[{"parameter": "JobID", "operator": "in", "values": jobIDs}], - headers=get_auth_headers(), - ) - return {j["JobID"]: {param: j[param] for param in parameters} for j in jobs} +# def fetch(parameters, jobIDs): +# # breakpoint() +# with Dirac(endpoint="http://localhost:8000") as api: +# jobs = api.jobs.search( +# parameters=["JobID"] + parameters, +# search=[{"parameter": "JobID", "operator": "in", "values": jobIDs}], +# headers=get_auth_headers(), +# ) +# return {j["JobID"]: {param: j[param] for param in parameters} for j in jobs} -class JobMonitoringClient: - def __init__(self, *args, **kwargs): - """TODO""" +# class JobMonitoringClient: +# def __init__(self, *args, **kwargs): +# """TODO""" - @convertToReturnValue - def getJobsMinorStatus(self, jobIDs): - return fetch(["MinorStatus"], jobIDs) +# @convertToReturnValue +# def getJobsMinorStatus(self, jobIDs): +# return fetch(["MinorStatus"], jobIDs) - @convertToReturnValue - def getJobsStates(self, jobIDs): - return fetch(["Status", "MinorStatus", "ApplicationStatus"], jobIDs) +# @convertToReturnValue +# def getJobsStates(self, jobIDs): +# return fetch(["Status", "MinorStatus", "ApplicationStatus"], jobIDs) - @convertToReturnValue - def getJobsSites(self, jobIDs): - return fetch(["Site"], jobIDs) +# @convertToReturnValue +# def getJobsSites(self, jobIDs): +# return fetch(["Site"], jobIDs) diff --git a/tests/CI/docker-compose.yml b/tests/CI/docker-compose.yml index b37dc7d3281..3be8cd70f04 100644 --- a/tests/CI/docker-compose.yml +++ b/tests/CI/docker-compose.yml @@ -1,5 +1,9 @@ version: '3.4' +volumes: + cs-store-diracx: + key-store: + services: mysql: image: ${MYSQL_VER} @@ -42,14 +46,6 @@ services: retries: 15 start_period: 60s - diracx: - image: gitlab-registry.cern.ch/chaen/chrissquare-hack-a-ton/diracx - container_name: diracx - environment: - - DIRAC_CS_SOURCE=todo - ports: - - 80:80 - # Mock of an S3 storage s3-direct: image: adobe/s3mock @@ -74,6 +70,9 @@ services: condition: service_healthy ulimits: nofile: 8192 + volumes: + - cs-store-diracx:/cs_store + - key-store:/signing-key dirac-client: image: ${CI_REGISTRY_IMAGE}/${HOST_OS}-dirac @@ -84,3 +83,92 @@ services: - dirac-server ulimits: nofile: 8192 + + + diracx-init-key: + image: gitlab-registry.cern.ch/chaen/chrissquare-hack-a-ton/diracx + container_name: diracx-init-key + environment: + - DIRACX_SERVICE_AUTH_TOKEN_KEY="file:///signing-key/rs256.key" + volumes: + - key-store:/signing-key/ + entrypoint: | + /dockerMicroMambaEntrypoint.sh ssh-keygen -P '' -trsa -b4096 -mPEM -f/signing-key/rs256.key + + # diracx-init-empty-repo: + # image: gitlab-registry.cern.ch/chaen/chrissquare-hack-a-ton/diracx + # container_name: diracx-init-empty-repo + # environment: + # - DIRACX_CONFIG_BACKEND_URL="git+file:///cs_store/initialRepo" + # volumes: + # - cs-store-diracx:/cs_store/ + # entrypoint: /dockerMicroMambaEntrypoint.sh + # command: mkdir -p /cs_store/initialRepo && cd /cs_store/initialRepo && git init + + + + diracx-init-cs: + image: gitlab-registry.cern.ch/chaen/chrissquare-hack-a-ton/diracx + container_name: diracx-init-cs + environment: + - DIRACX_CONFIG_BACKEND_URL="git+file:///cs_store/initialRepo" + - DIRACX_SERVICE_AUTH_TOKEN_KEY="file:///signing-key/rs256.key" + volumes: + - cs-store-diracx:/cs_store/ + - key-store:/signing-key/ + entrypoint: | + /dockerMicroMambaEntrypoint.sh dirac internal generate-cs /cs_store/initialRepo --vo=diracAdmin --user-group=admin --idp-url=http://dsdsd.csds/a/b + + + # diracx-init-cs-user: + # image: gitlab-registry.cern.ch/chaen/chrissquare-hack-a-ton/diracx + # container_name: diracx-init-cs-user + # depends_on: + # # dirac-server: + # # condition: service_healthy + # diracx-init-cs: + # condition: service_completed_successfully # Let the init container create the cs + # environment: + # - DIRACX_CONFIG_BACKEND_URL="git+file:///cs_store/initialRepo" + # - DIRACX_SERVICE_AUTH_TOKEN_KEY="file:///signing-key/rs256.key" + # volumes: + # - cs-store-diracx:/cs_store/ + # - key-store:/signing-key/ + # entrypoint: | + # /dockerMicroMambaEntrypoint.sh dirac internal add-user /cs_store/initialRepo --vo=diracAdmin --user-group=admin --sub=EgVsb2NhbA + + + + + + diracx: + image: gitlab-registry.cern.ch/chaen/chrissquare-hack-a-ton/diracx + container_name: diracx + environment: + - DIRACX_CONFIG_BACKEND_URL=git+file:///cs_store/initialRepo + - "DIRACX_DB_URL_AUTHDB=sqlite+aiosqlite:///:memory:" + - "DIRACX_DB_URL_JOBDB=sqlite+aiosqlite:///:memory:" + - DIRACX_SERVICE_AUTH_TOKEN_KEY=file:///signing-key/rs256.key + - DIRACX_SERVICE_AUTH_ALLOWED_REDIRECTS=["http://pclhcb211:8000/docs/oauth2-redirect"] + ports: + - 8000:8000 + depends_on: + # dirac-server: + # condition: service_healthy + diracx-init-cs: + condition: service_completed_successfully # Let the init container create the cs + # diracx-init-cs-user: + # condition: service_completed_successfully # Let the init container create the cs + diracx-init-key: + condition: service_completed_successfully # Let the init container create the cs + volumes: + - cs-store-diracx:/cs_store/ + - key-store:/signing-key/ + # entrypoint: sleep 1000 + + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/.well-known/openid-configuration"] + interval: 5s + timeout: 2s + retries: 15 + start_period: 60s diff --git a/tests/CI/exportCS.sh b/tests/CI/exportCS.sh new file mode 100755 index 00000000000..b32e847ec28 --- /dev/null +++ b/tests/CI/exportCS.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +source /home/dirac/ServerInstallDIR/bashrc +git config --global user.name "DIRAC Server CI" +git config --global user.email "dirac-server-ci@invalid" +curl -L https://gitlab.cern.ch/chaen/chris-hackaton-cs/-/raw/master/convert-from-legacy.py |DIRAC_COMPAT_ENABLE_CS_CONVERSION=True ~/ServerInstallDIR/diracos/bin/python - ~/ServerInstallDIR/etc/Production.cfg /cs_store/initialRepo/ +git -C /cs_store/initialRepo/ commit -am "export $(date)"