From 7ec0c8a26d599699a8342fece2551288a61770ff Mon Sep 17 00:00:00 2001 From: wyyalt Date: Mon, 9 Oct 2023 10:13:13 +0800 Subject: [PATCH] =?UTF-8?q?feature:=20=20=E6=8F=90=E4=BE=9B=20Agent=20?= =?UTF-8?q?=E5=8C=85=E7=AE=A1=E7=90=86=E5=90=8E=E5=8F=B0=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=20(closed=20#1683)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/backend/agent/artifact_builder/base.py | 68 +++- apps/backend/agent/tools.py | 5 +- .../components/collections/agent_new/base.py | 4 +- apps/backend/subscription/errors.py | 8 + .../steps/agent_adapter/adapter.py | 90 +++++- apps/backend/subscription/views.py | 8 +- .../agent/artifact_builder/test_agent.py | 23 +- .../artifact_builder/test_manage_commands.py | 5 +- .../agent/artifact_builder/test_proxy.py | 1 + .../collections/agent_new/test_install.py | 13 +- .../agent_new/test_push_agent_pkg_to_proxy.py | 1 + .../agent_new/test_push_upgrade_package.py | 1 + .../agent_new/test_run_upgrade_command.py | 1 + .../components/collections/agent_new/utils.py | 15 +- .../agent_adapter/test_adapter.py | 28 +- .../backend/tests/view/test_get_gse_config.py | 14 +- apps/core/tag/constants.py | 8 - apps/core/tag/targets/agent.py | 7 + apps/generic.py | 4 + apps/mock_data/common_unit/subscription.py | 2 +- apps/node_man/constants.py | 14 + apps/node_man/handlers/meta.py | 28 ++ .../0081_gsepackagedesc_gsepackages.py | 97 ++++++ apps/node_man/models.py | 46 +++ apps/node_man/permissions/__init__.py | 10 + apps/node_man/permissions/package_manage.py | 27 ++ apps/node_man/serializers/job.py | 24 +- apps/node_man/serializers/package_manage.py | 121 ++++++++ apps/node_man/urls.py | 5 + apps/node_man/utils/filters.py | 30 ++ apps/node_man/views/meta.py | 1 + apps/node_man/views/package_manage.py | 290 ++++++++++++++++++ common/utils/drf_utils.py | 271 ++++++++++++++++ config/default.py | 2 + 34 files changed, 1197 insertions(+), 75 deletions(-) create mode 100644 apps/node_man/migrations/0081_gsepackagedesc_gsepackages.py create mode 100644 apps/node_man/permissions/__init__.py create mode 100644 apps/node_man/permissions/package_manage.py create mode 100644 apps/node_man/serializers/package_manage.py create mode 100644 apps/node_man/utils/filters.py create mode 100644 apps/node_man/views/package_manage.py create mode 100644 common/utils/drf_utils.py diff --git a/apps/backend/agent/artifact_builder/base.py b/apps/backend/agent/artifact_builder/base.py index 8bcb8e8c7..040e8c409 100644 --- a/apps/backend/agent/artifact_builder/base.py +++ b/apps/backend/agent/artifact_builder/base.py @@ -23,8 +23,9 @@ from apps.backend.agent.config_parser import GseConfigParser from apps.core.files import core_files_constants from apps.core.files.storage import get_storage -from apps.core.tag.constants import AGENT_NAME_TARGET_ID_MAP, TargetType +from apps.core.tag.constants import TargetType from apps.core.tag.handlers import TagHandler +from apps.core.tag.targets import AgentTargetHelper from apps.node_man import constants, models from apps.utils import cache, files @@ -441,13 +442,57 @@ def _get_changelog(self, extract_dir: str) -> str: changelog: str = changelog_fs.read() return changelog - def update_or_create_record(self, artifact_meta_info: typing.Dict[str, typing.Any]): + def generate_location_path(self, upload_path: str, pkg_name: str) -> str: + if settings.STORAGE_TYPE == core_files_constants.StorageType.BLUEKING_ARTIFACTORY.value: + location_path: str = f"{settings.BKREPO_ENDPOINT_URL}/generic/blueking/bknodeman/{upload_path}/{pkg_name}" + else: + location_path: str = f"http://{settings.BKAPP_LAN_IP}/{upload_path}/{pkg_name}" + + return location_path + + def update_or_create_package_records(self, package_infos: typing.List[typing.Dict[str, typing.Any]]): """ - 创建或更新制品记录，待 Agent 包管理完善 - :param artifact_meta_info: + 创建或更新制品记录 + :param package_infos: :return: """ - pass + for package_info in package_infos: + models.GsePackages.objects.update_or_create( + defaults={ + "pkg_size": package_info["package_upload_info"]["pkg_size"], + "pkg_path": package_info["package_upload_info"]["pkg_path"], + "md5": package_info["package_upload_info"]["md5"], + "location": self.generate_location_path( + package_info["package_upload_info"]["pkg_path"], + package_info["package_upload_info"]["pkg_name"], + ), + "version_log": package_info["artifact_meta_info"]["changelog"], + }, + pkg_name=package_info["package_upload_info"]["pkg_name"], + version=package_info["artifact_meta_info"]["version"], + project=package_info["artifact_meta_info"]["name"], + os=package_info["package_dir_info"]["os"], + cpu_arch=package_info["package_dir_info"]["cpu_arch"], + ) + logger.info( + f"[update_or_create_package_record] " + f"package name -> {package_info['package_upload_info']['pkg_name']} success" + ) + + if package_infos: + models.GsePackageDesc.objects.update_or_create( + defaults={ + "description": package_infos[0]["artifact_meta_info"]["changelog"], + }, + project=package_infos[0]["artifact_meta_info"]["name"], + category=constants.CategoryType.official, + ) + + logger.info( + f"[update_or_create_package_record] " + f"package desc -> {package_info['package_upload_info']['pkg_name']}, " + f"project -> {package_infos[0]['artifact_meta_info']['name']} success" + ) def update_or_create_tag(self, artifact_meta_info: typing.Dict[str, typing.Any]): """ @@ -455,11 +500,12 @@ def update_or_create_tag(self, artifact_meta_info: typing.Dict[str, typing.Any]) :param artifact_meta_info: :return: """ + agent_name_target_id_map: typing.Dict[str, int] = AgentTargetHelper.get_agent_name_target_id_map() for tag in self.tags: TagHandler.publish_tag_version( name=tag, target_type=TargetType.AGENT.value, - target_id=AGENT_NAME_TARGET_ID_MAP[self.NAME], + target_id=agent_name_target_id_map[self.NAME], target_version=artifact_meta_info["version"], ) logger.info( @@ -517,14 +563,6 @@ def update_or_create_support_files(self, package_infos: typing.List[typing.Dict] agent_name=self.NAME, ) - def update_or_create_package_records(self, v): - """ - 创建或更新安装包记录，待 Agent 包管理完善 - :param package_infos: - :return: - """ - pass - def get_artifact_meta_info(self, extract_dir: str) -> typing.Dict[str, typing.Any]: """ 获取制品的基础信息、配置文件信息 @@ -591,8 +629,6 @@ def make( artifact_meta_info["operator"] = operator # Agent 包先导入文件源 -> 写配置文件 -> 创建包记录 -> 创建 Tag self.update_or_create_support_files(package_infos) - # TODO update_or_create_record & update_or_create_package_records 似乎是一样的功能？ - self.update_or_create_record(artifact_meta_info) self.update_or_create_package_records(package_infos) self.update_or_create_tag(artifact_meta_info) diff --git a/apps/backend/agent/tools.py b/apps/backend/agent/tools.py index 2ba78491c..f58591452 100644 --- a/apps/backend/agent/tools.py +++ b/apps/backend/agent/tools.py @@ -320,7 +320,7 @@ def check_run_commands(run_commands): def batch_gen_commands( - base_agent_setup_info: AgentSetupInfo, + agent_step_adapter, hosts: List[models.Host], pipeline_id: str, is_uninstall: bool, @@ -336,7 +336,6 @@ def batch_gen_commands( # 批量查出主机的属性并设置为property，避免在循环中进行ORM查询，提高效率 host_id__installation_tool_map = {} bk_host_ids = [host.bk_host_id for host in hosts] - base_agent_setup_info_dict: Dict[str, Any] = asdict(base_agent_setup_info) host_id_identity_map = { identity.bk_host_id: identity for identity in models.IdentityData.objects.filter(bk_host_id__in=bk_host_ids) } @@ -354,7 +353,7 @@ def batch_gen_commands( host_id__installation_tool_map[host.bk_host_id] = gen_commands( agent_setup_info=AgentSetupInfo( **{ - **base_agent_setup_info_dict, + **asdict(agent_step_adapter.get_host_setup_info(host)), "force_update_agent_id": agent_setup_extra_info_dict.get("force_update_agent_id", False), } ), diff --git a/apps/backend/components/collections/agent_new/base.py b/apps/backend/components/collections/agent_new/base.py index 2e4931b33..19de5cd71 100644 --- a/apps/backend/components/collections/agent_new/base.py +++ b/apps/backend/components/collections/agent_new/base.py @@ -135,7 +135,7 @@ def get_agent_pkg_name( package_type = ("client", "proxy")[host.node_type == constants.NodeType.PROXY] agent_step_adapter = common_data.agent_step_adapter if not agent_step_adapter.is_legacy: - setup_info = agent_step_adapter.setup_info + setup_info = agent_step_adapter.get_host_setup_info(host) return f"{setup_info.name}-{setup_info.version}.tgz" # GSE1.0 的升级包是独立的，添加了 _upgrade 后缀 @@ -262,7 +262,7 @@ def get_host_id__installation_tool_map( host for host in hosts_need_gen_commands if host.bk_host_id in host_id__install_channel_map ] host_id__installation_tool_map = batch_gen_commands( - base_agent_setup_info=common_data.agent_step_adapter.setup_info, + agent_step_adapter=common_data.agent_step_adapter, hosts=hosts_need_gen_commands, pipeline_id=self.id, is_uninstall=is_uninstall, diff --git a/apps/backend/subscription/errors.py b/apps/backend/subscription/errors.py index 93f432e0a..b6d814673 100644 --- a/apps/backend/subscription/errors.py +++ b/apps/backend/subscription/errors.py @@ -169,3 +169,11 @@ class SubscriptionIncludeGrayBizError(AppBaseException): ERROR_CODE = 19 MESSAGE = _("订阅任务包含Gse2.0灰度业务，任务将暂缓执行无需重复点击") MESSAGE_TPL = _("订阅任务包含Gse2.0灰度业务，任务将暂缓执行无需重复点击") + + +class AgentPackageValidationError(AppBaseException): + """AgentPackage校验错误""" + + ERROR_CODE = 20 + MESSAGE = _("AgentPackage校验错误") + MESSAGE_TPL = _("{msg}") diff --git a/apps/backend/subscription/steps/agent_adapter/adapter.py b/apps/backend/subscription/steps/agent_adapter/adapter.py index aea7b186b..dfdafe305 100644 --- a/apps/backend/subscription/steps/agent_adapter/adapter.py +++ b/apps/backend/subscription/steps/agent_adapter/adapter.py @@ -18,7 +18,8 @@ from apps.backend.agent.tools import fetch_proxies from apps.backend.constants import ProxyConfigFile -from apps.core.tag.constants import AGENT_NAME_TARGET_ID_MAP, TargetType +from apps.backend.subscription import errors +from apps.core.tag.constants import TargetType from apps.core.tag.targets import get_target_helper from apps.node_man import constants, models from apps.utils import cache @@ -34,11 +35,21 @@ LEGACY = "legacy" +class AgentVersionSerializer(serializers.Serializer): + os_cpu_arch = serializers.CharField(label="系统CPU架构", required=False) + bk_host_id = serializers.IntegerField(label="主机ID", required=False) + version = serializers.CharField(label="Agent Version") + + class AgentStepConfigSerializer(serializers.Serializer): name = serializers.CharField(required=False, label="构件名称") # LEGACY 表示旧版本 Agent，仅做兼容 version = serializers.CharField(required=False, label="构件版本", default=LEGACY) job_type = serializers.ChoiceField(required=True, choices=constants.JOB_TUPLE) + choice_version_type = serializers.ChoiceField( + required=False, choices=constants.AgentVersionType.list_choices(), label="选择Agent Version类型" + ) + version_map_list = AgentVersionSerializer(many=True) @dataclass @@ -57,6 +68,9 @@ class AgentStepAdapter: log_prefix: str = field(init=False) # 配置处理模块缓存 _config_handler_cache: typing.Dict[str, GseConfigHandler] = field(init=False) + _setup_info_cache: typing.Dict[str, base.AgentSetupInfo] = field(init=False) + _target_version_cache: typing.Dict[str, str] = field(init=False) + agent_name: str = field(init=False) def __post_init__(self): self.is_legacy = self.gse_version == GseVersion.V1.value @@ -64,6 +78,9 @@ def __post_init__(self): f"[{self.__class__.__name__}({self.subscription_step.step_id})] | {self.subscription_step} |" ) self._config_handler_cache: typing.Dict[str, GseConfigHandler] = {} + self._setup_info_cache: typing.Dict[str, base.AgentSetupInfo] = {} + self._target_version_cache: typing.Dict[str, str] = {} + self.agent_name = self.config.get("name") def get_config_handler(self, agent_name: str, target_version: str) -> GseConfigHandler: @@ -104,11 +121,12 @@ def _get_config( install_channel: typing.Tuple[typing.Optional[models.Host], typing.Dict[str, typing.List]], target_version: typing.Optional[typing.Dict[int, str]] = None, ) -> str: - agent_setup_info: base.AgentSetupInfo = self.setup_info + agent_setup_info: base.AgentSetupInfo = self.get_host_setup_info(host) # 目标版本优先使用传入版本，传入版本必不会是标签所以可直接使用 config_handler: GseConfigHandler = self.get_config_handler( agent_setup_info.name, target_version or agent_setup_info.version ) + config_tmpl_obj: base.AgentConfigTemplate = config_handler.get_matching_config_tmpl( os_type=host.os_type, cpu_arch=host.cpu_arch, @@ -168,29 +186,65 @@ def get_config( @property @cache.class_member_cache() - def setup_info(self) -> base.AgentSetupInfo: + def bk_host_id_version_map(self) -> typing.Dict[int, str]: + return {versiom_map["bk_host_id"]: versiom_map["version"] for versiom_map in self.config["version_map_list"]} + + def get_host_setup_info(self, host: models.Host) -> base.AgentSetupInfo: """ 获取 Agent 设置信息 - TODO 后续如需支持多版本，该方法改造为 `get_host_setup_info`，根据维度进行缓存，参考 _config_handler_cache :return: """ # 如果版本号匹配到标签名称，取对应标签下的真实版本号，否则取原来的版本号 - agent_name: typing.Optional[str] = self.config.get("name") - if agent_name not in AGENT_NAME_TARGET_ID_MAP: - # 1.0 Install + if self.agent_name is None: + # 1.0 Install 或者 2.0统一版本 target_version = self.config.get("version") + setup_info_cache_key: str = f"agent_name_is_none:version:{target_version}" else: - target_version: str = get_target_helper(TargetType.AGENT.value).get_target_version( - target_id=AGENT_NAME_TARGET_ID_MAP[agent_name], - target_version=self.config.get("version"), - ) + if self.config["choice_version_type"] == constants.AgentVersionType.UNIFIED.value: + agent_version = self.config.get("version") + setup_info_cache_key: str = ( + f"agent_name:{self.agent_name}:" + f"type:{constants.AgentVersionType.UNIFIED.value}:version:{agent_version}" + ) + elif self.config["choice_version_type"] == constants.AgentVersionType.BY_SYSTEM_ARCH.value: + # TODO 按系统架构维度, 当前只支持按系统，后续需求完善按系统架构 + os_cpu_arch_version_list: typing.List[str] = [ + versiom_map["version"] + for versiom_map in self.config["version_map_list"] + if host.os_type.lower() in versiom_map["os_cpu_arch"] + ] + agent_version: str = os_cpu_arch_version_list[0] if os_cpu_arch_version_list else "stable" + setup_info_cache_key: str = ( + f"agent_name:{self.agent_name}:type:{constants.AgentVersionType.BY_SYSTEM_ARCH.value}:" + f"os:{host.os_type.lower()}:version:{agent_version}" + ) + else: + # 按主机维度 + agent_version: str = self.bk_host_id_version_map[host.bk_host_id] + + target_version_cache_key: str = f"agent_desc_id:{self.agent_desc.id}:agent_version:{agent_version}" + target_version: str = self._target_version_cache.get(target_version_cache_key) + if target_version is None: + target_version: str = get_target_helper(TargetType.AGENT.value).get_target_version( + target_id=self.agent_desc.id, + target_version=agent_version, + ) + self._target_version_cache[target_version_cache_key] = target_version - return base.AgentSetupInfo( + if self.config["choice_version_type"] != constants.AgentVersionType.BY_HOST.value: + agent_setup_info: typing.Optional[base.AgentSetupInfo] = self._setup_info_cache.get(setup_info_cache_key) + if agent_setup_info: + return agent_setup_info + + agent_setup_info: base.AgentSetupInfo = base.AgentSetupInfo( is_legacy=self.is_legacy, agent_tools_relative_dir=("agent_tools/agent2", "")[self.is_legacy], name=self.config.get("name"), version=target_version, ) + if self.config["choice_version_type"] != constants.AgentVersionType.BY_HOST.value: + self._setup_info_cache[setup_info_cache_key] = agent_setup_info + return agent_setup_info @staticmethod def validated_data(data, serializer) -> OrderedDict: @@ -204,3 +258,15 @@ def get_os_key(os_type: str, cpu_arch: str) -> str: os_type = os_type or constants.OsType.LINUX cpu_arch = cpu_arch or constants.CpuType.x86_64 return f"{os_type.lower()}-{cpu_arch}" + + @property + def agent_desc(self) -> models.GsePackageDesc: + if hasattr(self, "_agent_desc") and self._agent_desc: + return self._agent_desc + try: + agent_desc = models.GsePackageDesc.objects.get(project=self.agent_name) + except models.GsePackageDesc.DoesNotExist: + raise errors.AgentPackageValidationError(msg="GsePackageDesc [{name}] 不存在".format(name=self.agent_name)) + + setattr(self, "_agent_desc", agent_desc) + return self._agent_desc diff --git a/apps/backend/subscription/views.py b/apps/backend/subscription/views.py index 50ce35d67..dd4206ab8 100644 --- a/apps/backend/subscription/views.py +++ b/apps/backend/subscription/views.py @@ -15,7 +15,7 @@ from collections import defaultdict from dataclasses import asdict from functools import cmp_to_key, reduce -from typing import Any, Dict, List, Set +from typing import Dict, List, Set from django.core.cache import caches from django.db import transaction @@ -691,14 +691,14 @@ def fetch_commands(self, request): ap_id_obj_map: Dict[int, models.AccessPoint] = models.AccessPoint.ap_id_obj_map() host_ap: models.AccessPoint = ap_id_obj_map[host_ap_id] - base_agent_setup_info_dict: Dict[str, Any] = asdict( - AgentStepAdapter(subscription_step=sub_step_obj, gse_version=host_ap.gse_version).setup_info + agent_setup_adapter: AgentStepAdapter = AgentStepAdapter( + subscription_step=sub_step_obj, gse_version=host_ap.gse_version ) agent_setup_extra_info_dict = sub_inst.instance_info["host"].get("agent_setup_extra_info") or {} installation_tool = gen_commands( agent_setup_info=AgentSetupInfo( **{ - **base_agent_setup_info_dict, + **asdict(agent_setup_adapter.get_host_setup_info(host)), "force_update_agent_id": agent_setup_extra_info_dict.get("force_update_agent_id", False), } ), diff --git a/apps/backend/tests/agent/artifact_builder/test_agent.py b/apps/backend/tests/agent/artifact_builder/test_agent.py index b6d2bbf51..5e108eaf9 100644 --- a/apps/backend/tests/agent/artifact_builder/test_agent.py +++ b/apps/backend/tests/agent/artifact_builder/test_agent.py @@ -16,8 +16,9 @@ from apps.backend.subscription.steps.agent_adapter.handlers import GseConfigHandler from apps.backend.tests.agent import template_env, utils -from apps.core.tag.constants import AGENT_NAME_TARGET_ID_MAP, TargetType +from apps.core.tag.constants import TargetType from apps.core.tag.models import Tag +from apps.core.tag.targets.agent import AgentTargetHelper from apps.mock_data import utils as mock_data_utils from apps.node_man import constants, models @@ -49,10 +50,11 @@ def pkg_checker(self, version_str: str): ) self.assertTrue(os.path.exists(package_path)) - def tag_checker(self, target_id: int): + def tag_checker(self): + agent_name_target_id_map: typing.Dict[str, int] = AgentTargetHelper.get_agent_name_target_id_map() agent_target_version = Tag.objects.get( - target_id=target_id, + target_id=agent_name_target_id_map[self.NAME], name=self.OVERWRITE_VERSION, target_type=TargetType.AGENT.value, ).target_version @@ -106,12 +108,24 @@ def template_and_env_checker(self, version_str): self.assertTrue(models.GseConfigTemplate.objects.filter(**filter_kwargs).exists()) + def gse_package_and_desc_records_checker(self, version_str): + for package_os, cpu_arch in self.OS_CPU_CHOICES: + filter_kwargs: dict = { + "project": self.NAME, + "os": package_os, + "cpu_arch": cpu_arch, + "version": version_str, + } + self.assertTrue(models.GsePackages.objects.filter(**filter_kwargs).exists()) + self.assertTrue(models.GsePackageDesc.objects.filter(**{"project": filter_kwargs.pop("project")}).exists()) + def test_make(self): """测试安装包制作""" with self.ARTIFACT_BUILDER_CLASS(initial_artifact_path=self.ARCHIVE_PATH) as builder: builder.make() self.pkg_checker(version_str=utils.VERSION) self.template_and_env_checker(version_str=utils.VERSION) + self.gse_package_and_desc_records_checker(version_str=utils.VERSION) def test_make__overwrite_version(self): """测试版本号覆盖""" @@ -129,7 +143,8 @@ def test_make__overwrite_version(self): self.pkg_checker(version_str=utils.VERSION) self.template_and_env_checker(version_str=utils.VERSION) self.pkg_checker(version_str=self.OVERWRITE_VERSION) - self.tag_checker(target_id=AGENT_NAME_TARGET_ID_MAP[self.NAME]) + self.tag_checker() + self.gse_package_and_desc_records_checker(version_str=utils.VERSION) class BkRepoTestCase(FileSystemTestCase): diff --git a/apps/backend/tests/agent/artifact_builder/test_manage_commands.py b/apps/backend/tests/agent/artifact_builder/test_manage_commands.py index ed2d7bb48..ee6f57429 100644 --- a/apps/backend/tests/agent/artifact_builder/test_manage_commands.py +++ b/apps/backend/tests/agent/artifact_builder/test_manage_commands.py @@ -17,7 +17,6 @@ from django.core.management import call_command from apps.backend.tests.agent import utils -from apps.core.tag.constants import AGENT_NAME_TARGET_ID_MAP from apps.mock_data import utils as mock_data_utils from apps.node_man import models @@ -44,6 +43,7 @@ def test_make(self): self.assertTrue(models.UploadPackage.objects.all().exists()) self.pkg_checker(version_str=utils.VERSION) self.template_and_env_checker(version_str=utils.VERSION) + self.gse_package_and_desc_records_checker(version_str=utils.VERSION) def test_make__overwrite_version(self): """测试版本号覆盖""" @@ -51,7 +51,8 @@ def test_make__overwrite_version(self): self.pkg_checker(version_str=utils.VERSION) self.template_and_env_checker(version_str=utils.VERSION) self.pkg_checker(version_str=self.OVERWRITE_VERSION) - self.tag_checker(target_id=AGENT_NAME_TARGET_ID_MAP[self.NAME]) + self.tag_checker() + self.gse_package_and_desc_records_checker(version_str=utils.VERSION) class BkRepoImportAgentTestCase(FileSystemImportAgentTestCase): diff --git a/apps/backend/tests/agent/artifact_builder/test_proxy.py b/apps/backend/tests/agent/artifact_builder/test_proxy.py index 15a5468ad..d182aabe5 100644 --- a/apps/backend/tests/agent/artifact_builder/test_proxy.py +++ b/apps/backend/tests/agent/artifact_builder/test_proxy.py @@ -34,6 +34,7 @@ def test_make(self): builder.make() self.pkg_checker(version_str=utils.VERSION) self.template_and_env_checker(version_str=utils.VERSION) + self.gse_package_and_desc_records_checker(version_str=utils.VERSION) class BkRepoTestCase(FileSystemTestCase): diff --git a/apps/backend/tests/components/collections/agent_new/test_install.py b/apps/backend/tests/components/collections/agent_new/test_install.py index 877d05755..ea028198e 100644 --- a/apps/backend/tests/components/collections/agent_new/test_install.py +++ b/apps/backend/tests/components/collections/agent_new/test_install.py @@ -161,6 +161,7 @@ def start_patch(self): fs.write("哈哈哈113343ddfd🐒") def setUp(self) -> None: + self.obj_factory.init_gse_package_desc() self.update_callback_url() self.init_mock_clients() self.init_hosts() @@ -389,7 +390,9 @@ async def connect(self): class InstallAgent2WindowsTestCase(InstallWindowsTestCase): def adjust_db(self): sub_step_obj: models.SubscriptionStep = self.obj_factory.sub_step_objs[0] - sub_step_obj.config.update({"name": "gse_agent", "version": "2.0.0"}) + sub_step_obj.config.update( + {"name": "gse_agent", "version": "2.0.0", "version_map_list": [], "choice_version_type": "unified"} + ) sub_step_obj.save(update_fields=["config"]) def structure_common_inputs(self): @@ -405,7 +408,7 @@ def test_batch_solution(self): gse_version=GseVersion.V2.value, ) installation_tool = gen_commands( - agent_step_adapter.setup_info, + agent_step_adapter.get_host_setup_info(host), host, mock_data_utils.JOB_TASK_PIPELINE_ID, is_uninstall=False, @@ -862,7 +865,9 @@ class LinuxAgent2InstallTestCase(InstallBaseTestCase): def adjust_db(self): sub_step_obj: models.SubscriptionStep = self.obj_factory.sub_step_objs[0] - sub_step_obj.config.update({"name": "gse_agent", "version": "2.0.0"}) + sub_step_obj.config.update( + {"name": "gse_agent", "version": "2.0.0", "version_map_list": [], "choice_version_type": "unified"} + ) sub_step_obj.save() def structure_common_inputs(self): @@ -878,7 +883,7 @@ def test_shell_solution(self): gse_version=GseVersion.V2.value, ) installation_tool = gen_commands( - agent_step_adapter.setup_info, + agent_step_adapter.get_host_setup_info(host), host, mock_data_utils.JOB_TASK_PIPELINE_ID, is_uninstall=False, diff --git a/apps/backend/tests/components/collections/agent_new/test_push_agent_pkg_to_proxy.py b/apps/backend/tests/components/collections/agent_new/test_push_agent_pkg_to_proxy.py index ed1922fc0..b29b4eba9 100644 --- a/apps/backend/tests/components/collections/agent_new/test_push_agent_pkg_to_proxy.py +++ b/apps/backend/tests/components/collections/agent_new/test_push_agent_pkg_to_proxy.py @@ -127,6 +127,7 @@ def component_cls(self): @classmethod def setUpTestData(cls): super().setUpTestData() + cls.obj_factory.init_gse_package_desc() sub_step_obj: models.SubscriptionStep = cls.obj_factory.sub_step_objs[0] sub_step_obj.config.update({"name": "gse_agent", "version": "2.0.0"}) sub_step_obj.save() diff --git a/apps/backend/tests/components/collections/agent_new/test_push_upgrade_package.py b/apps/backend/tests/components/collections/agent_new/test_push_upgrade_package.py index 69af15a7e..9f8a56276 100644 --- a/apps/backend/tests/components/collections/agent_new/test_push_upgrade_package.py +++ b/apps/backend/tests/components/collections/agent_new/test_push_upgrade_package.py @@ -53,6 +53,7 @@ def get_default_case_name(cls) -> str: @classmethod def setUpTestData(cls): super().setUpTestData() + cls.obj_factory.init_gse_package_desc() models.Host.objects.filter(bk_host_id__in=cls.obj_factory.bk_host_ids).update(bk_agent_id=get_random_string()) sub_step_obj: models.SubscriptionStep = cls.obj_factory.sub_step_objs[0] sub_step_obj.config.update({"name": "gse_agent", "version": "2.0.0"}) diff --git a/apps/backend/tests/components/collections/agent_new/test_run_upgrade_command.py b/apps/backend/tests/components/collections/agent_new/test_run_upgrade_command.py index f2cba8d4b..5233a94cd 100644 --- a/apps/backend/tests/components/collections/agent_new/test_run_upgrade_command.py +++ b/apps/backend/tests/components/collections/agent_new/test_run_upgrade_command.py @@ -85,6 +85,7 @@ def structure_common_inputs(self): @classmethod def setUpTestData(cls): super().setUpTestData() + cls.obj_factory.init_gse_package_desc() sub_step_obj: models.SubscriptionStep = cls.obj_factory.sub_step_objs[0] sub_step_obj.config.update({"name": "gse_agent", "version": "2.0.0"}) sub_step_obj.save() diff --git a/apps/backend/tests/components/collections/agent_new/utils.py b/apps/backend/tests/components/collections/agent_new/utils.py index cc6c3c9f2..ded153510 100644 --- a/apps/backend/tests/components/collections/agent_new/utils.py +++ b/apps/backend/tests/components/collections/agent_new/utils.py @@ -295,7 +295,11 @@ def structure_sub_step_data_list(self) -> List[Dict[str, Any]]: sub_step_data.update( { "subscription_id": self.sub_obj.id, - "config": {"job_type": constants.JobType.INSTALL_AGENT}, + "config": { + "job_type": constants.JobType.INSTALL_AGENT, + "version_map_list": [], + "choice_version_type": "unified", + }, } ) return [sub_step_data] @@ -371,6 +375,15 @@ def init_host_related_data_in_db(self): self.bulk_create_model(model=models.IdentityData, create_data_list=identity_data_list) self.identity_data_objs = models.IdentityData.objects.filter(bk_host_id__in=self.bk_host_ids) + @classmethod + def init_gse_package_desc(cls): + models.GsePackageDesc.objects.update_or_create( + defaults={"description": ""}, project="gse_agent", category="official" + ) + models.GsePackageDesc.objects.update_or_create( + defaults={"description": ""}, project="gse_proxy", category="official" + ) + def init_db(self): """ 初始化DB测试数据 diff --git a/apps/backend/tests/subscription/agent_adapter/test_adapter.py b/apps/backend/tests/subscription/agent_adapter/test_adapter.py index 62f4b6013..56b5aec76 100644 --- a/apps/backend/tests/subscription/agent_adapter/test_adapter.py +++ b/apps/backend/tests/subscription/agent_adapter/test_adapter.py @@ -13,6 +13,9 @@ from apps.backend.subscription.steps.agent_adapter.adapter import AgentStepAdapter from apps.backend.tests.agent.utils import VERSION, AgentBaseTestCase, ProxyBaseTestCase +from apps.backend.tests.components.collections.agent_new.utils import ( + AgentTestObjFactory, +) from apps.mock_data import common_unit from apps.node_man import constants, models from apps.utils import basic @@ -31,6 +34,7 @@ def setUpTestData(cls): super().setUpTestData() host_model_data = copy.deepcopy(common_unit.host.HOST_MODEL_DATA) cls.host = models.Host.objects.create(**host_model_data) + AgentTestObjFactory.init_gse_package_desc() # 创建订阅相关数据 sub_inst_data = basic.remove_keys_from_dict( @@ -45,7 +49,7 @@ def setUpTestData(cls): **sub_step_data, **{ "subscription_id": cls.sub_inst_record_obj.subscription_id, - "config": {"job_type": constants.JobType.INSTALL_AGENT}, + "config": {"job_type": constants.JobType.INSTALL_AGENT, "version_map_list": []}, }, } ) @@ -85,6 +89,8 @@ def setUpTestData(cls): "job_type": constants.JobType.INSTALL_AGENT, "name": constants.GsePackageCode.PROXY.value, "version": VERSION, + "version_map_list": [], + "choice_version_type": "unified", } cls.sub_step_obj.save() cls.agent_step_adapter = AgentStepAdapter(subscription_step=cls.sub_step_obj, gse_version=GseVersion.V2.value) @@ -93,12 +99,13 @@ def test_get_config(self): self.clear_agent_data() self.host.node_type = "PROXY" self.host.bk_cloud_id = 1 + agent_setup_info = self.agent_step_adapter.get_host_setup_info(self.host) for config_name in constants.GsePackageTemplate.PROXY.value: self.get_config(config_name) self.assertEqual( self.agent_step_adapter.get_config_handler( - agent_name=self.agent_step_adapter.setup_info.name, - target_version=self.agent_step_adapter.setup_info.version, + agent_name=agent_setup_info.name, + target_version=agent_setup_info.version, ) .get_matching_config_tmpl(self.host.os_type, self.host.cpu_arch, config_name) .agent_name_from, @@ -108,8 +115,8 @@ def test_get_config(self): self.get_config(config_name) self.assertEqual( self.agent_step_adapter.get_config_handler( - agent_name=self.agent_step_adapter.setup_info.name, - target_version=self.agent_step_adapter.setup_info.version, + agent_name=agent_setup_info.name, + target_version=agent_setup_info.version, ) .get_matching_config_tmpl(self.host.os_type, self.host.cpu_arch, config_name) .agent_name_from, @@ -128,16 +135,17 @@ def clear_agent_data(cls): pass def test_get_env(self): + agent_setup_info = self.agent_step_adapter.get_host_setup_info(self.host) agent_env = self.agent_step_adapter.get_config_handler( - agent_name=self.agent_step_adapter.setup_info.name, - target_version=self.agent_step_adapter.setup_info.version, + agent_name=agent_setup_info.name, + target_version=agent_setup_info.version, ).get_matching_template_env(self.host.os_type, self.host.cpu_arch, constants.GsePackageCode.AGENT.value) self.assertEqual(agent_env["BK_GSE_HOME_DIR"], "/usr/local/gse/agent") proxy_env = self.agent_step_adapter.get_config_handler( - agent_name=self.agent_step_adapter.setup_info.name, - target_version=self.agent_step_adapter.setup_info.version, + agent_name=agent_setup_info.name, + target_version=agent_setup_info.version, ).get_matching_template_env(self.host.os_type, self.host.cpu_arch, constants.GsePackageCode.PROXY.value) self.assertEqual(proxy_env["BK_GSE_HOME_DIR"], "/usr/local/gse/proxy") @@ -155,6 +163,8 @@ def setUpTestData(cls): "job_type": constants.JobType.INSTALL_AGENT, "name": constants.GsePackageCode.AGENT.value, "version": VERSION, + "version_map_list": [], + "choice_version_type": "unified", } cls.sub_step_obj.save() cls.agent_step_adapter = AgentStepAdapter(subscription_step=cls.sub_step_obj, gse_version=GseVersion.V2.value) diff --git a/apps/backend/tests/view/test_get_gse_config.py b/apps/backend/tests/view/test_get_gse_config.py index 37e29d55a..cc9e98090 100644 --- a/apps/backend/tests/view/test_get_gse_config.py +++ b/apps/backend/tests/view/test_get_gse_config.py @@ -38,7 +38,7 @@ def setUpTestData(cls): host_model_data = copy.deepcopy(common_unit.host.HOST_MODEL_DATA) cls.host = models.Host.objects.create(**host_model_data) models.AccessPoint.objects.all().update(gse_version=cls.GSE_VERSION) - + models.GsePackageDesc.objects.create(project="gse_agent", description="", category="official") # 创建订阅相关数据 sub_inst_data = basic.remove_keys_from_dict( origin_data=common_unit.subscription.SUB_INST_RECORD_MODEL_DATA, keys=["id"] @@ -52,7 +52,7 @@ def setUpTestData(cls): **sub_step_data, **{ "subscription_id": cls.sub_inst_record_obj.subscription_id, - "config": {"job_type": constants.JobType.INSTALL_AGENT}, + "config": {"job_type": constants.JobType.INSTALL_AGENT, "version_map_list": []}, }, } ) @@ -117,13 +117,19 @@ def setUp(self) -> None: @classmethod def setUpTestData(cls): super().setUpTestData() - cls.sub_step_obj.config = {"job_type": constants.JobType.INSTALL_AGENT, "name": "gse_agent", "version": "2.0.0"} + cls.sub_step_obj.config = { + "job_type": constants.JobType.INSTALL_AGENT, + "name": "gse_agent", + "version": "2.0.0", + "version_map_list": [], + "choice_version_type": "unified", + } cls.sub_step_obj.save() cls.redis_agent_conf_key = REDIS_AGENT_CONF_KEY_TPL.format( file_name=cls.agent_step_adapter.get_main_config_filename(), sub_inst_id=cls.sub_inst_record_obj.id ) cls.agent_step_adapter = AgentStepAdapter(subscription_step=cls.sub_step_obj, gse_version=cls.GSE_VERSION) - target_version = cls.agent_step_adapter.setup_info.version + target_version = cls.agent_step_adapter.get_host_setup_info(cls.host).version models.GseConfigEnv.objects.create( agent_name="gse_agent", version=target_version, diff --git a/apps/core/tag/constants.py b/apps/core/tag/constants.py index 8b0c27614..cf922484a 100644 --- a/apps/core/tag/constants.py +++ b/apps/core/tag/constants.py @@ -13,7 +13,6 @@ from django.utils.translation import ugettext_lazy as _ -from apps.node_man.constants import GsePackageCode from apps.utils.enum import EnhanceEnum @@ -37,10 +36,3 @@ class TagChangeAction(EnhanceEnum): @classmethod def _get_member__alias_map(cls) -> Dict[Enum, str]: return {cls.DELETE: _("删除标签"), cls.CREATE: _("新建标签"), cls.UPDATE: _("更新版本"), cls.OVERWRITE: _("同版本覆盖更新")} - - -# TODO: target_id 临时写死 -AGENT_NAME_TARGET_ID_MAP: Dict[str, int] = { - GsePackageCode.AGENT.value: 1, - GsePackageCode.PROXY.value: 2, -} diff --git a/apps/core/tag/targets/agent.py b/apps/core/tag/targets/agent.py index 400acd7ac..9f51fcae0 100644 --- a/apps/core/tag/targets/agent.py +++ b/apps/core/tag/targets/agent.py @@ -11,8 +11,10 @@ import logging +import typing from apps.core.tag.models import Tag +from apps.node_man.models import GsePackageDesc from .. import constants from . import base @@ -35,3 +37,8 @@ def _publish_tag_version(self): def _delete_tag_version(self): return super()._delete_tag_version() + + @classmethod + def get_agent_name_target_id_map(cls) -> typing.Dict[str, int]: + package_descs = GsePackageDesc.objects.values("project", "id") + return {package_desc["project"]: package_desc["id"] for package_desc in package_descs} diff --git a/apps/generic.py b/apps/generic.py index d8e3181d2..a612d3b3e 100644 --- a/apps/generic.py +++ b/apps/generic.py @@ -173,6 +173,10 @@ def get_serializer_class(self, *args, **kwargs): return type(self.serializer_class.__name__, (self.serializer_class,), {"Meta": self.serializer_meta}) +class ApiMixinModelViewSet(ApiMixin, _ModelViewSet): + pagination_class = DataPageNumberPagination + + def custom_exception_handler(exc, context): """ 自定义错误处理方式 diff --git a/apps/mock_data/common_unit/subscription.py b/apps/mock_data/common_unit/subscription.py index 39d003403..3b2725f54 100644 --- a/apps/mock_data/common_unit/subscription.py +++ b/apps/mock_data/common_unit/subscription.py @@ -92,7 +92,7 @@ "subscription_id": DEFAULT_SUBSCRIPTION_ID, "step_id": constants.SubStepType.AGENT.lower(), "type": constants.SubStepType.AGENT, - "config": {"job_type": constants.JobType.INSTALL_AGENT}, + "config": {"job_type": constants.JobType.INSTALL_AGENT, "version_map_list": []}, "params": {"context": {}, "blueking_language": "zh-hans"}, } diff --git a/apps/node_man/constants.py b/apps/node_man/constants.py index 4905c7972..b3cf89689 100644 --- a/apps/node_man/constants.py +++ b/apps/node_man/constants.py @@ -1136,3 +1136,17 @@ class CommonExecutionSolutionStepType(EnhanceEnum): @classmethod def _get_member__alias_map(cls) -> Dict[Enum, str]: return {cls.DEPENDENCIES: _("依赖文件"), cls.COMMANDS: _("命令")} + + +class AgentVersionType(EnhanceEnum): + UNIFIED = "unified" + BY_HOST = "by_host" + BY_SYSTEM_ARCH = "by_system_arch" + + @classmethod + def _get_member__alias_map(cls) -> Dict[Enum, str]: + return { + cls.UNIFIED: _("统一的版本"), + cls.BY_HOST: _("按主机的"), + cls.BY_SYSTEM_ARCH: _("按系统架构"), + } diff --git a/apps/node_man/handlers/meta.py b/apps/node_man/handlers/meta.py index 39b5ea16b..1bdca27de 100644 --- a/apps/node_man/handlers/meta.py +++ b/apps/node_man/handlers/meta.py @@ -511,6 +511,32 @@ def fetch_os_type_children(os_types: Tuple = constants.OsType): os_type_children.append({"id": os_type, "name": constants.OS_CHN.get(os_type, os_type)}) return os_type_children + @staticmethod + def fetch_agent_pkg_manager_children(): + mock_version = [ + {"name": "2.1.8", "id": "2.1.8"}, + {"name": "2.1.7", "id": "2.1.7"}, + ] + mock_tags = [ + {"name": "稳定版本", "id": "stable"}, + {"name": "最新版本", "id": "latest"}, + ] + mock_creator = [ + {"name": "user1", "id": "user1"}, + {"name": "user2", "id": "user2"}, + ] + mock_is_ready = [ + {"name": "启用", "id": True}, + {"name": "停用", "id": False}, + ] + + return [ + {"name": _("版本号"), "id": "version", "children": mock_version}, + {"name": _("标签信息"), "id": "tags", "children": mock_tags}, + {"name": _("上传用户"), "id": "creator", "children": mock_creator}, + {"name": _("状态"), "id": "is_ready", "children": mock_is_ready}, + ] + def filter_condition(self, category, params=None): """ 获取过滤条件 @@ -538,6 +564,8 @@ def filter_condition(self, category, params=None): elif category == "os_type": ret = self.fetch_os_type_children() return ret + elif category == "agent_pkg_manage": + return self.fetch_agent_pkg_manager_children() @staticmethod def install_default_values_formatter(install_default_values: Dict[str, Dict[str, Any]]): diff --git a/apps/node_man/migrations/0081_gsepackagedesc_gsepackages.py b/apps/node_man/migrations/0081_gsepackagedesc_gsepackages.py new file mode 100644 index 000000000..3b743f7db --- /dev/null +++ b/apps/node_man/migrations/0081_gsepackagedesc_gsepackages.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("node_man", "0080_auto_20231122_1552"), + ] + + operations = [ + migrations.CreateModel( + name="GsePackageDesc", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("created_time", models.DateTimeField(auto_now_add=True, verbose_name="创建时间")), + ("created_by", models.CharField(default="", max_length=32, verbose_name="创建者")), + ("updated_time", models.DateTimeField(auto_now=True, null=True, verbose_name="更新时间")), + ("updated_by", models.CharField(blank=True, default="", max_length=32, verbose_name="修改者")), + ("project", models.CharField(db_index=True, max_length=32, unique=True, verbose_name="工程名")), + ("description", models.TextField(verbose_name="安装包描述")), + ("description_en", models.TextField(blank=True, null=True, verbose_name="英文插件描述")), + ( + "category", + models.CharField( + choices=[("official", "official"), ("external", "external"), ("scripts", "scripts")], + max_length=32, + verbose_name="所属范围", + ), + ), + ], + options={ + "verbose_name": "Gse包描述（GsePackageDesc）", + "verbose_name_plural": "Gse包描述（GsePackageDesc）", + }, + ), + migrations.CreateModel( + name="GsePackages", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("created_time", models.DateTimeField(auto_now_add=True, verbose_name="创建时间")), + ("created_by", models.CharField(default="", max_length=32, verbose_name="创建者")), + ("updated_time", models.DateTimeField(auto_now=True, null=True, verbose_name="更新时间")), + ("updated_by", models.CharField(blank=True, default="", max_length=32, verbose_name="修改者")), + ("pkg_name", models.CharField(max_length=128, verbose_name="压缩包名")), + ("version", models.CharField(max_length=128, verbose_name="版本号")), + ("project", models.CharField(db_index=True, max_length=32, verbose_name="工程名")), + ("pkg_size", models.IntegerField(verbose_name="包大小")), + ("pkg_path", models.CharField(max_length=128, verbose_name="包路径")), + ("md5", models.CharField(max_length=32, verbose_name="md5值")), + ("location", models.CharField(max_length=512, verbose_name="安装包链接")), + ( + "os", + models.CharField( + choices=[("windows", "windows"), ("linux", "linux"), ("aix", "aix"), ("solaris", "solaris")], + db_index=True, + default="linux", + max_length=32, + verbose_name="系统类型", + ), + ), + ( + "cpu_arch", + models.CharField( + choices=[ + ("x86", "x86"), + ("x86_64", "x86_64"), + ("powerpc", "powerpc"), + ("aarch64", "aarch64"), + ("sparc", "sparc"), + ], + db_index=True, + default="x86_64", + max_length=32, + verbose_name="CPU类型", + ), + ), + ("is_ready", models.BooleanField(default=True, verbose_name="插件是否可用")), + ("version_log", models.TextField(blank=True, null=True, verbose_name="版本日志")), + ("version_log_en", models.TextField(blank=True, null=True, verbose_name="英文版本日志")), + ], + options={ + "verbose_name": "Gse包（GsePackages）", + "verbose_name_plural": "Gse包（GsePackages）", + }, + ), + ] diff --git a/apps/node_man/models.py b/apps/node_man/models.py index 4d8deae83..37aaf901f 100644 --- a/apps/node_man/models.py +++ b/apps/node_man/models.py @@ -2499,3 +2499,49 @@ class Meta: index_together = [ ["bk_biz_id", "enable"], ] + + +class GsePackages(orm.OperateRecordModel): + pkg_name = models.CharField(_("压缩包名"), max_length=128) + version = models.CharField(_("版本号"), max_length=128) + project = models.CharField(_("工程名"), max_length=32, db_index=True) + pkg_size = models.IntegerField(_("包大小")) + pkg_path = models.CharField(_("包路径"), max_length=128) + md5 = models.CharField(_("md5值"), max_length=32) + location = models.CharField(_("安装包链接"), max_length=512) + os = models.CharField( + _("系统类型"), + max_length=32, + choices=constants.PLUGIN_OS_CHOICES, + default=constants.PluginOsType.linux, + db_index=True, + ) + cpu_arch = models.CharField( + _("CPU类型"), max_length=32, choices=constants.CPU_CHOICES, default=constants.CpuType.x86_64, db_index=True + ) + + # 由于创建记录时，文件可能仍然在传输过程中，因此需要标志位判断是否已经可用 + is_ready = models.BooleanField(_("插件是否可用"), default=True) + + version_log = models.TextField(_("版本日志"), null=True, blank=True) + version_log_en = models.TextField(_("英文版本日志"), null=True, blank=True) + + class Meta: + verbose_name = _("Gse包（GsePackages）") + verbose_name_plural = _("Gse包（GsePackages）") + + +class GsePackageDesc(orm.OperateRecordModel): + """ + Gse包描述表 + """ + + # 安装包名需要全局唯一，防止冲突 + project = models.CharField(_("工程名"), max_length=32, unique=True, db_index=True) + description = models.TextField(_("安装包描述")) + description_en = models.TextField(_("英文插件描述"), null=True, blank=True) + category = models.CharField(_("所属范围"), max_length=32, choices=constants.CATEGORY_CHOICES) + + class Meta: + verbose_name = _("Gse包描述（GsePackageDesc）") + verbose_name_plural = _("Gse包描述（GsePackageDesc）") diff --git a/apps/node_man/permissions/__init__.py b/apps/node_man/permissions/__init__.py new file mode 100644 index 000000000..29ed269e0 --- /dev/null +++ b/apps/node_man/permissions/__init__.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/apps/node_man/permissions/package_manage.py b/apps/node_man/permissions/package_manage.py new file mode 100644 index 000000000..bc8ae81d8 --- /dev/null +++ b/apps/node_man/permissions/package_manage.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from django.utils.translation import ugettext_lazy as _ +from rest_framework import permissions + +from apps.node_man.handlers.iam import IamHandler +from apps.utils.local import get_request_username + + +class PackageManagePermission(permissions.BasePermission): + message = _("您没有该操作的权限") + + def has_permission(self, request, view): + + if IamHandler().is_superuser(get_request_username()): + return True + + return False diff --git a/apps/node_man/serializers/job.py b/apps/node_man/serializers/job.py index 1985fe825..700fa4f0e 100644 --- a/apps/node_man/serializers/job.py +++ b/apps/node_man/serializers/job.py @@ -16,7 +16,10 @@ from django.utils.translation import ugettext_lazy as _ from rest_framework import serializers -from apps.backend.subscription.steps.agent_adapter.adapter import LEGACY +from apps.backend.subscription.steps.agent_adapter.adapter import ( + LEGACY, + AgentVersionSerializer, +) from apps.core.gray.constants import INSTALL_OTHER_AGENT_AP_ID_OFFSET from apps.core.gray.handlers import GrayHandler from apps.core.gray.tools import GrayTools @@ -51,11 +54,18 @@ def set_agent_setup_info_to_attrs(attrs): # 如果开启 DHCP，安装 2.0 Agent，开启 AgentID 特性 # 在执行模块根据主机接入点所属的 GSE 版本决定是否采用下列的 agent_setup_info name = ("gse_agent", "gse_proxy")[attrs["node_type"] == "PROXY"] + # attrs["agent_setup_info"]["name"] = name + # 处理重装类型setup_info结构 + agent_setup_info = attrs.get("agent_setup_info", {}) + global_settings_agent_version = models.GlobalSettings.get_config( + models.GlobalSettings.KeyEnum.GSE_AGENT2_VERSION.value, default="stable" + ) + attrs["agent_setup_info"] = { "name": name, - "version": models.GlobalSettings.get_config( - models.GlobalSettings.KeyEnum.GSE_AGENT2_VERSION.value, default="stable" - ), + "version": agent_setup_info.get("version") or global_settings_agent_version, + "choice_version_type": agent_setup_info.get("choice_version_type") or constants.AgentVersionType.UNIFIED.value, + "version_map_list": agent_setup_info.get("version_map_list", []), } @@ -112,7 +122,6 @@ def backfill_bk_host_id(self, hosts): else: sub_query.children.append(("inner_ipv6", _host["inner_ipv6"])) ip_key = _host["inner_ipv6"] - cloud_ip_host_info_map[f"{_host['bk_cloud_id']}:{ip_key}"] = _host query_params.children.append(sub_query) @@ -257,6 +266,11 @@ class AgentSetupInfoSerializer(serializers.Serializer): # LEGACY 表示旧版本 Agent，仅做兼容 version = serializers.CharField(required=False, label="构件版本", default=LEGACY) + choice_version_type = serializers.ChoiceField( + required=False, choices=constants.AgentVersionType.list_choices(), label=_("选择Agent Version类型") + ) + version_map_list = AgentVersionSerializer(required=False, many=True) + class ScriptHook(serializers.Serializer): name = serializers.CharField(label=_("脚本名称"), min_length=1) diff --git a/apps/node_man/serializers/package_manage.py b/apps/node_man/serializers/package_manage.py new file mode 100644 index 000000000..177f6c644 --- /dev/null +++ b/apps/node_man/serializers/package_manage.py @@ -0,0 +1,121 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from django.utils.translation import ugettext_lazy as _ +from rest_framework import serializers + +from apps.exceptions import ValidationError +from apps.node_man.constants import GsePackageCode + + +class TagsSerializer(serializers.Serializer): + id = serializers.CharField() + name = serializers.CharField() + children = serializers.ListField() + + +class ConditionsSerializer(serializers.Serializer): + key = serializers.ChoiceField(choices=["version", "os_cpu_arch", "tags", "is_ready"]) + values = serializers.ListField() + + +class PackageSerializer(serializers.Serializer): + id = serializers.IntegerField() + pkg_name = serializers.CharField() + version = serializers.CharField() + os = serializers.CharField() + cpu_arch = serializers.CharField() + tags = TagsSerializer(many=True) + creator = serializers.CharField() + pkg_ctime = serializers.DateTimeField() + is_ready = serializers.BooleanField() + + +class PackageDescSerializer(serializers.Serializer): + id = serializers.IntegerField() + version = serializers.CharField() + tags = TagsSerializer(many=True) + packages = PackageSerializer(many=True) + is_ready = serializers.BooleanField() + + +class ListResponseSerializer(serializers.Serializer): + total = serializers.IntegerField() + list = PackageSerializer(many=True) + + +class PackageDescResponseSerialiaer(serializers.Serializer): + total = serializers.IntegerField() + list = PackageDescSerializer(many=True) + + +class OperateSerializer(serializers.Serializer): + is_ready = serializers.BooleanField() + + +class QuickSearchSerializer(serializers.Serializer): + project = serializers.ChoiceField(choices=GsePackageCode.list_choices()) + + +# TODO 与plugin相同可抽取公共Serializer +class UploadSerializer(serializers.Serializer): + class PkgFileField(serializers.FileField): + def to_internal_value(self, data): + data = super().to_internal_value(data) + file_name = data.name + if not (file_name.endswith(".tgz") or file_name.endswith(".tar.gz")): + raise ValidationError(_("仅支持'tgz', 'tar.gz'拓展名的文件")) + return data + + module = serializers.ChoiceField(choices=["gse_agent", "gse_proxy"], required=False, default="gse_agent") + package_file = PkgFileField() + + +class UploadResponseSerializer(serializers.Serializer): + id = serializers.IntegerField() + name = serializers.CharField() + pkg_size = serializers.IntegerField() + + +class ParseSerializer(serializers.Serializer): + file_name = serializers.CharField() + + +class ParseResponseSerializer(serializers.Serializer): + class ParsePackageSerializer(serializers.Serializer): + module = serializers.ChoiceField(choices=["agent", "proxy"]) + pkg_name = serializers.CharField() + pkg_abs_path = serializers.CharField() + version = serializers.CharField() + os = serializers.CharField() + cpu_arch = serializers.CharField() + config_templates = serializers.ListField() + + description = serializers.CharField() + packages = ParsePackageSerializer(many=True) + + +class AgentRegisterSerializer(serializers.Serializer): + class RegisterPackageSerializer(serializers.Serializer): + pkg_abs_path = serializers.CharField() + tags = serializers.ListField() + + is_release = serializers.BooleanField() + packages = RegisterPackageSerializer(many=True) + + +class AgentRegisterTaskSerializer(serializers.Serializer): + job_id = serializers.IntegerField() + + +class AgentRegisterTaskResponseSerializer(serializers.Serializer): + is_finish = serializers.BooleanField() + status = serializers.ChoiceField(choices=["SUCCESS", "FAILED", "RUNNING"]) + message = serializers.CharField() diff --git a/apps/node_man/urls.py b/apps/node_man/urls.py index faa90fc48..d2f0724e2 100644 --- a/apps/node_man/urls.py +++ b/apps/node_man/urls.py @@ -40,6 +40,9 @@ ) from apps.node_man.views.healthz import HealthzViewSet from apps.node_man.views.host_v2 import HostV2ViewSet +from apps.node_man.views.package_manage import ( # AgentPackageDescViewSet, + PackageManageViewSet, +) from apps.node_man.views.plugin import GsePluginViewSet from apps.node_man.views.plugin_v2 import PluginV2ViewSet from apps.node_man.views.sync_task import SyncTaskViewSet @@ -67,6 +70,8 @@ router.register(r"v2/plugin", PluginV2ViewSet, basename="plugin_v2") router.register(r"healthz", HealthzViewSet, basename="healthz") router.register(r"sync_task", SyncTaskViewSet, basename="sync_task") +router.register(r"agent/package", PackageManageViewSet, basename="package_manage") +# router.register(r"agent/package_desc", AgentPackageDescViewSet, basename="package_desc") biz_dispatcher = DjangoBasicResourceApiDispatcher(iam, settings.BK_IAM_SYSTEM_ID) biz_dispatcher.register("biz", BusinessResourceProvider()) diff --git a/apps/node_man/utils/filters.py b/apps/node_man/utils/filters.py new file mode 100644 index 000000000..722985b0a --- /dev/null +++ b/apps/node_man/utils/filters.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import django_filters +from django_filters.rest_framework import FilterSet + +from apps.node_man.models import GsePackages + + +class GsePackageFilter(FilterSet): + tags = django_filters.CharFilter(name="tags", method="filter_tags") + os_cpu_arch = django_filters.CharFilter(name="os_cpu_arch", method="filter_os_cpu_arch") + + def filter_tags(self, queryset, name, value): + pass + + def filter_os_cpu_arch(self, queryset, name, value): + pass + + class Meta: + model = GsePackages + fields = ["os_cpu_arch", "tags", "project", "created_by", "is_ready", "version"] diff --git a/apps/node_man/views/meta.py b/apps/node_man/views/meta.py index 0a9f0c75b..3736d5bbf 100644 --- a/apps/node_man/views/meta.py +++ b/apps/node_man/views/meta.py @@ -29,6 +29,7 @@ class MetaViews(APIViewSet): @swagger_auto_schema( operation_summary="获取过滤条件", + query_serializer=FilterConditionSerializer, tags=META_VIEW_TAGS, methods=["GET", "POST"], ) diff --git a/apps/node_man/views/package_manage.py b/apps/node_man/views/package_manage.py new file mode 100644 index 000000000..b4793495e --- /dev/null +++ b/apps/node_man/views/package_manage.py @@ -0,0 +1,290 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from django_filters.rest_framework import DjangoFilterBackend +from drf_yasg import openapi +from rest_framework import filters +from rest_framework.decorators import action +from rest_framework.response import Response +from rest_framework.status import HTTP_200_OK + +from apps.generic import ApiMixinModelViewSet as ModelViewSet +from apps.node_man import models +from apps.node_man.permissions import package_manage as pkg_permission +from apps.node_man.serializers import package_manage as pkg_manage +from apps.node_man.utils.filters import GsePackageFilter +from common.utils.drf_utils import swagger_auto_schema + +PACKAGE_MANAGE_VIEW_TAGS = ["PKG_Manager"] +PACKAGE_DES_VIEW_TAGS = ["PKG_Desc"] + + +class PackageManageViewSet(ModelViewSet): + queryset = models.GsePackages.objects.all() + # http_method_names = ["get", "post"] + # ordering_fields = ("module",) + serializer_class = pkg_manage.PackageSerializer + permission_classes = (pkg_permission.PackageManagePermission,) + filter_backends = (DjangoFilterBackend, filters.SearchFilter, filters.OrderingFilter) + filter_class = GsePackageFilter + + @swagger_auto_schema( + responses={200: pkg_manage.ListResponseSerializer}, + operation_summary="安装包列表", + tags=PACKAGE_MANAGE_VIEW_TAGS, + ) + def list(self, request, *args, **kwargs): + mock_data = { + "total": 2, + "list": [ + { + "id": 1, + "pkg_name": "pkg_name", + "version": "1.1.1", + "os": "Linux", + "cpu_arch": "x86_64", + "tags": [{"id": "stable", "name": "稳定版本"}], + "creator": "string", + "pkg_ctime": "2019-08-24 14:15:22", + "is_ready": True, + }, + { + "id": 2, + "pkg_name": "pkg_name", + "version": "1.1.2", + "os": "Linux", + "os_cpu_arch": "x86_64", + "tags": [{"id": "stable", "name": "稳定版本"}], + "creator": "string", + "pkg_ctime": "2019-08-24 14:15:22", + "is_ready": True, + }, + ], + } + return Response(mock_data) + # return super().list(request, *args, **kwargs) + + @swagger_auto_schema( + operation_summary="操作类动作：启用/停用", + body_in=pkg_manage.OperateSerializer, + responses={200: pkg_manage.PackageSerializer}, + tags=PACKAGE_MANAGE_VIEW_TAGS, + ) + def update(self, request, validated_data, *args, **kwargs): + mock_data = { + "id": 1, + "pkg_name": "pkg_name", + "version": "1.1.1", + "os": "Linux", + "cpu_arch": "x86_64", + "tags": [{"id": "stable", "name": "稳定版本"}], + "creator": "string", + "pkg_ctime": "2019-08-24 14:15:22", + "is_ready": True, + } + + return Response(mock_data) + + @swagger_auto_schema( + operation_summary="删除安装包", + tags=PACKAGE_MANAGE_VIEW_TAGS, + ) + def destroy(self, request, *args, **kwargs): + + return Response() + + @swagger_auto_schema( + operation_summary="获取快速筛选信息", + manual_parameters=[ + openapi.Parameter( + "project", in_=openapi.TYPE_STRING, description="区分gse_agent, gse_proxy", type=openapi.TYPE_STRING + ) + ], + tags=PACKAGE_MANAGE_VIEW_TAGS, + ) + @action(detail=False, methods=["GET"]) + def quick_search_condition(self, request, *args, **kwargs): + mock_version = [ + {"name": "2.1.8", "id": "2.1.8", "count": 10}, + {"name": "2.1.7", "id": "2.1.7", "count": 10}, + {"name": "ALL", "id": "all", "count": 20}, + ] + + mock_os_cpu_arch = [ + {"name": "Linux_x86_64", "id": "linux_x86_64", "count": 10}, + {"name": "Linux_x86", "id": "linux_x86", "count": 10}, + {"name": "ALL", "id": "all", "count": 20}, + ] + + mock_data = [ + {"name": "操作系统/架构", "id": "os_cpu_arch", "children": mock_os_cpu_arch}, + {"name": "版本号", "id": "version", "children": mock_version}, + ] + + return Response(mock_data) + + @swagger_auto_schema( + operation_summary="Agent包上传", + tags=PACKAGE_MANAGE_VIEW_TAGS, + responses={HTTP_200_OK: pkg_manage.UploadResponseSerializer}, + ) + @action(detail=False, methods=["POST"], serializer_class=pkg_manage.UploadSerializer) + def upload(self, request): + # data = self.validated_data + mock_data = { + "id": 1, + "name": "gse_agent.tgz", + "pkg_size": 100, + } + return Response(mock_data) + + @swagger_auto_schema( + operation_summary="解析Agent包", + tags=PACKAGE_MANAGE_VIEW_TAGS, + responses={HTTP_200_OK: pkg_manage.ParseResponseSerializer}, + ) + @action(detail=False, methods=["POST"], serializer_class=pkg_manage.ParseSerializer) + def parse(self, request): + mock_data = { + "description": "test", + "packages": [ + { + "pkg_abs_path": "xxx/xxxxx", + "pkg_name": "gseagent_2.1.7_linux_x86_64.tgz", + "module": "agent", + "version": "2.1.7", + "config_templates": [], + "os": "x86_64", + }, + { + "pkg_abs_path": "xxx/xxxxx", + "pkg_name": "gseagent_2.1.7_linux_x86.tgz", + "module": "agent", + "version": "2.1.7", + "config_templates": [], + "os": "x86", + }, + ], + } + return Response(mock_data) + + @swagger_auto_schema( + operation_summary="创建Agent包注册任务", + tags=PACKAGE_MANAGE_VIEW_TAGS, + responses={HTTP_200_OK: pkg_manage.AgentRegisterTaskSerializer}, + ) + @action(detail=False, methods=["POST"], serializer_class=pkg_manage.AgentRegisterSerializer) + def create_register_task(self, request): + mock_data = {"job_id": 1} + return Response(mock_data) + + @swagger_auto_schema( + operation_summary="查询Agent包注册任务", + tags=PACKAGE_MANAGE_VIEW_TAGS, + query_in=pkg_manage.AgentRegisterTaskSerializer, + responses={HTTP_200_OK: pkg_manage.AgentRegisterTaskResponseSerializer}, + ) + @action(detail=False, methods=["GET"]) + def query_register_task(self, request, validated_data): + + mock_data = { + "is_finish": True, + "status": "SUCCESS", + "message": "", + } + return Response(mock_data) + + @swagger_auto_schema( + operation_summary="获取Agent包标签", + tags=PACKAGE_MANAGE_VIEW_TAGS, + responses={HTTP_200_OK: pkg_manage.TagsSerializer(many=True)}, + ) + @action(detail=False, methods=["GET"]) + def tags(self, request): + # 由tag handler 实现 + mock_data = [ + { + "id": "builtin", + "name": "内置标签", + "children": [ + {"id": "stable", "name": "稳定版本", "children": []}, + {"id": "latest", "name": "最新版本", "children": []}, + ], + }, + {"id": "custom", "name": "自定义标签", "children": [{"id": "custom", "name": "自定义版本", "children": []}]}, + ] + return Response(mock_data) + + @swagger_auto_schema( + operation_summary="获取Agent包版本", + tags=PACKAGE_MANAGE_VIEW_TAGS, + responses={HTTP_200_OK: pkg_manage.PackageDescResponseSerialiaer}, + ) + @action(detail=False, methods=["GET"]) + def version(self, request): + mock_data = { + "total": 10, + "list": [ + { + "id": 1, + "version": "2.1.2", + "tags": [{"id": "stable", "name": "稳定版本"}], + "is_ready": True, + "description": "我是描述", + "packages": [ + { + "pkg_name": "gseagent-2.1.2.tgz", + "tags": [{"id": "stable", "name": "稳定版本"}, {"id": "latest", "name": "最新版本"}], + } + ], + } + ], + } + return Response(mock_data) + + +# class AgentPackageDescViewSet(ModelViewSet): +# queryset = models.AgentPackageDesc.objects.all() +# # model = models.Packages +# # http_method_names = ["get", "post"] +# # ordering_fields = ("module",) +# # serializer_class = pkg_manage.PackageSerializer +# # filter_backends = (DjangoFilterBackend, filters.SearchFilter, filters.OrderingFilter) + +# # filter_fields = ("module", "creator", "is_ready", "version") + +# @swagger_auto_schema( +# query_in=pkg_manage.PackageDescSearchSerializer, +# responses={200: pkg_manage.PackageDescResponseSerialiaer}, +# operation_summary="Agent版本列表", +# tags=PACKAGE_DES_VIEW_TAGS, +# ) +# def list(self, request, *args, **kwargs): + +# mock_data = { +# "total": 10, +# "list": [ +# { +# "id": 1, +# "version": "2.1.2", +# "tags": [{"id": "stable", "name": "稳定版本"}], +# "is_ready": True, +# "description": "我是描述", +# "packages": [ +# { +# "pkg_name": "gseagent-2.1.2.tgz", +# "tags": [{"id": "stable", "name": "稳定版本"}, {"id": "latest", "name": "最新版本"}], +# } +# ], +# } +# ], +# } +# return Response(mock_data) +# # return super().list(request, *args, **kwargs) diff --git a/common/utils/drf_utils.py b/common/utils/drf_utils.py new file mode 100644 index 000000000..bfb568a01 --- /dev/null +++ b/common/utils/drf_utils.py @@ -0,0 +1,271 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import copy +import functools +from collections import namedtuple +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Type, Union + +from django.conf import settings +from django.http.response import HttpResponseBase +from django.utils.module_loading import import_string +from rest_framework import status +from rest_framework.exceptions import ValidationError +from rest_framework.fields import empty +from rest_framework.serializers import BaseSerializer +from rest_framework.settings import api_settings +from rest_framework.utils.serializer_helpers import ReturnDict, ReturnList + +if TYPE_CHECKING: + from rest_framework.request import Request + + +def stringify_validation_error(error: ValidationError) -> List[str]: + """Transform DRF's ValidationError into a list of error strings + + >>> stringify_validation_error(ValidationError({'foo': ErrorDetail('err')})) + ['foo: err'] + """ + results: List[str] = [] + + def traverse(err_detail: Any, keys: List[str]): + """Traverse error data to collect all error messages""" + + # Dig deeper when structure is list or dict + if isinstance(err_detail, (ReturnList, list, tuple)): + for err in err_detail: + traverse(err, keys) + elif isinstance(err_detail, (ReturnDict, dict)): + for key, err in err_detail.items(): + # Make a copy of keys so the inner loop won't affect outer scope + _keys = copy.copy(keys) + if key != api_settings.NON_FIELD_ERRORS_KEY: + _keys.append(str(key)) + traverse(err, _keys) + else: + if not keys: + results.append(str(err_detail)) + else: + results.append("{}: {}".format(".".join(keys), str(err_detail))) + + traverse(error.detail, []) + return sorted(results) + + +############# +# drf crown # +############# +class WearOptions: + is_unittest = False + skip_swagger_schema = False + + +try: + from drf_yasg.utils import swagger_auto_schema as drf_swagger_auto_schema + +except ImportError: + WearOptions.skip_swagger_schema = True + + +ResponseParams = namedtuple("ResponseParams", "data,params") + + +_DEFAULT_SETTINGS_PREFIX = "DRF_CROWN_" + + +def enable_unittest(): + """Call me when you running testing""" + WearOptions.is_unittest = True + + +@dataclass +class Config: + """Config for Injector, control the process of injecting""" + + return_validated_data: bool = True + remain_request: bool = False + # sometime return raw data instead of serializer + skip_out_cls: bool = False + default_return_status: status = status.HTTP_200_OK + + +@dataclass +class ViewCrown: + """A injector for injecting serializer as dependency""" + + body_in: Optional[Union[Type[BaseSerializer], BaseSerializer]] + query_in: Optional[Union[Type[BaseSerializer], BaseSerializer]] + out: Union[Type[BaseSerializer], BaseSerializer] + config_params: Optional[dict] = field(default_factory=dict) + valid_params: dict = field(default_factory=dict) + + def __post_init__(self): + if self.query_in and self.body_in: + raise ValueError("there should be only one param between in_body & in_query") + + self.valid_params = self.valid_params or {"raise_exception": True} + + # Priority decreases + # 1. config as parameter from decorator + # 2. config from django.settings + # 3. config from Config class(above) + _config = getattr(settings, _DEFAULT_SETTINGS_PREFIX + "DEFAULT_CONFIG", {}).copy() + _config.update(self.config_params or {}) + self.config = Config(**_config) + + # remain an entrance for custom response class + try: + self.resp_cls = import_string(getattr(settings, _DEFAULT_SETTINGS_PREFIX + "RESP_CLS")) + except AttributeError: + self.resp_cls = import_string("rest_framework.response.Response") + + def get_in_serializer_instance(self, request: Optional["Request"] = None) -> "BaseSerializer": + if not self.body_in and not self.query_in: + raise ValueError("should given at least one serializer input") + + _data = empty + if self.body_in: + _in = self.body_in + + if request is not None: + _data = getattr(request, "data") + else: + _in = self.query_in + + if request is not None: + _data = getattr(request, "query_params") + + if isinstance(_in, BaseSerializer): + # 由于传入的是全局对象，会残留上一次请求的结果 + # 这里需要手动清理一下 + if hasattr(_in, "_validated_data"): + delattr(_in, "_validated_data") + + _in.initial_data = _data + slz_obj = _in + elif issubclass(_in, BaseSerializer): + slz_obj = _in(data=_data) + else: + raise ValueError("unknown serializer input") + + return slz_obj + + def get_serializer_instance_by_request(self, request: "Request") -> "BaseSerializer": + """Get in serializer instance""" + slz_obj = self.get_in_serializer_instance(request) + slz_obj.is_valid(**self.valid_params) + return slz_obj + + def get_validated_data(self, request: "Request") -> dict: + """Get validated data via in_serializer""" + return self.get_serializer_instance_by_request(request).validated_data + + def get_in_params(self, request: "Request") -> dict: + """Get extra params before view logic""" + if WearOptions.is_unittest: + return {} + + if self.config.return_validated_data: + return {"validated_data": self.get_validated_data(request)} + else: + return {"serializer_instance": self.get_serializer_instance_by_request(request)} + + def get_response(self, data, out_params: dict) -> Any: + """Get Response data""" + if WearOptions.is_unittest: + return data + + if self.config.skip_out_cls: + return data + + if isinstance(data, (self.resp_cls, HttpResponseBase)): + return data + + if isinstance(self.out, BaseSerializer): + # 由于传入的是全局对象，会残留上一次请求的结果 + # 这里需要手动清理一下 + if hasattr(self.out, "_data"): + delattr(self.out, "_data") + + self.out.instance = data + _data = self.out.data + elif issubclass(self.out, BaseSerializer): + _data = self.out(data, **out_params).data + else: + raise ValueError("unknown serializer output") + + return self.resp_cls(data=_data, status=self.config.default_return_status) + + +def generate_swagger_params(crown: ViewCrown, swagger_params: dict) -> dict: + """ + assemble params for swagger_auto_schema by crown + """ + default_params = {} + if crown.body_in: + default_params = {"request_body": crown.get_in_serializer_instance()} + elif crown.query_in: + default_params = {"query_serializer": crown.get_in_serializer_instance()} + + if crown.out: + default_params.update({"responses": {crown.config.default_return_status: crown.out}}) + + default_params.update(swagger_params or {}) + return default_params + + +def swagger_auto_schema( + body_in: Optional[Union[Type[BaseSerializer], BaseSerializer]] = None, + query_in: Optional[Union[Type[BaseSerializer], BaseSerializer]] = None, + out: Optional[Union[Type[BaseSerializer], BaseSerializer]] = None, + config: Optional[dict] = None, + **swagger_kwargs +): + """ + Sugar for simpling drf serializer specification + :param body_in: input serializer (request body) + :param query_in: input serializer (query) + :param out: output serializer + :param config: initial info of Config + :param swagger_kwargs: pass to swagger_auto_schema of drf-yasg + """ + + def decorator_serializer_inject(func): + crown = ViewCrown(body_in, query_in, out, config) + + if not WearOptions.skip_swagger_schema: + func = drf_swagger_auto_schema(**generate_swagger_params(crown, swagger_kwargs))(func) + + @functools.wraps(func) + def decorated(*args, **kwargs): + new_args = list(args) + in_content: Dict[str, Any] = {} + if body_in or query_in: + in_content.update(**crown.get_in_params(new_args[1])) + + if not crown.config.remain_request: + del new_args[1] + + original_data = func(*new_args, **kwargs, **in_content) + if not out: + return original_data + + # support runtime serializer params, like "context" + params = {} + if isinstance(original_data, ResponseParams): + params = original_data.params + original_data = original_data.data + + return crown.get_response(original_data, params) + + return decorated + + return decorator_serializer_inject diff --git a/config/default.py b/config/default.py index 440cdfcf5..5e3ad33fe 100644 --- a/config/default.py +++ b/config/default.py @@ -816,6 +816,8 @@ def get_standard_redis_mode(cls, config_redis_mode: str, default: Optional[str] if env.BKAPP_MONITOR_REPORTER_ENABLE: monitor_report_config() +DRF_CROWN_DEFAULT_CONFIG = {"remain_request": True} + # remove disabled apps if locals().get("DISABLED_APPS"): INSTALLED_APPS = locals().get("INSTALLED_APPS", [])