From ca21f6960c381f3c5c1dfda2c4ab80892e5e0bcd Mon Sep 17 00:00:00 2001 From: xcwang <1366993017@qq.com> Date: Fri, 8 Dec 2023 18:04:40 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=8E=A5=E5=85=A5=E7=82=B9=E6=94=B9?= =?UTF-8?q?=E9=80=A0=20(closed=20#852)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../test_check_policy_gse_to_proxy.py | 4 +- apps/mock_data/common_unit/host.py | 15 +- .../management/commands/transform_ap_data.py | 86 +++ .../0084_covert_ap_data_20231109_1336.py | 34 + apps/node_man/models.py | 54 +- .../periodic_tasks/gse_svr_discovery.py | 19 +- apps/node_man/serializers/ap.py | 64 +- .../tests/test_pericdic_tasks/mock_data.py | 17 +- .../test_gse_svr_discovery.py | 26 +- .../tests/test_tools/test_ap_transform.py | 639 ++++++++++++++++++ apps/node_man/tests/utils.py | 10 +- apps/node_man/utils/endpoint.py | 127 +++- apps/node_man/views/ap.py | 72 +- 13 files changed, 1060 insertions(+), 107 deletions(-) create mode 100644 apps/node_man/management/commands/transform_ap_data.py create mode 100644 apps/node_man/migrations/0084_covert_ap_data_20231109_1336.py create mode 100644 apps/node_man/tests/test_tools/test_ap_transform.py diff --git a/apps/backend/tests/components/collections/agent_new/test_check_policy_gse_to_proxy.py b/apps/backend/tests/components/collections/agent_new/test_check_policy_gse_to_proxy.py index 7ce0d8d09..cea8e1d2d 100644 --- a/apps/backend/tests/components/collections/agent_new/test_check_policy_gse_to_proxy.py +++ b/apps/backend/tests/components/collections/agent_new/test_check_policy_gse_to_proxy.py @@ -37,7 +37,9 @@ def get_default_case_name(cls) -> str: def setUpTestData(cls): super().setUpTestData() host = models.Host.objects.all()[0] - models.AccessPoint.objects.all().update(btfileserver=[{"inner_ip": host.inner_ip, "outer_ip": host.outer_ip}]) + models.AccessPoint.objects.all().update( + btfileserver={"inner_ip_infos": [{"ip": host.inner_ip}], "outer_ip_infos": [{"ip": host.outer_ip}]} + ) host_data = copy.deepcopy(common_unit.host.HOST_MODEL_DATA) host_data.update({"bk_host_id": cls.obj_factory.RANDOM_BEGIN_HOST_ID + cls.obj_factory.total_host_num}) cls.obj_factory.bulk_create_model(models.Host, [host_data]) diff --git a/apps/mock_data/common_unit/host.py b/apps/mock_data/common_unit/host.py index 33411c20f..80341ca0b 100644 --- a/apps/mock_data/common_unit/host.py +++ b/apps/mock_data/common_unit/host.py @@ -30,9 +30,18 @@ "ap_type": "system", "region_id": "test", "city_id": "test", - "btfileserver": [{"inner_ip": DEFAULT_IP, "outer_ip": DEFAULT_IP}], - "dataserver": [{"inner_ip": DEFAULT_IP, "outer_ip": DEFAULT_IP}], - "taskserver": [{"inner_ip": DEFAULT_IP, "outer_ip": DEFAULT_IP}], + "btfileserver": { + "inner_ip_infos": [{"ip": DEFAULT_IP}], + "outer_ip_infos": [{"ip": DEFAULT_IP}], + }, + "dataserver": { + "inner_ip_infos": [{"ip": DEFAULT_IP}], + "outer_ip_infos": [{"ip": DEFAULT_IP}], + }, + "taskserver": { + "inner_ip_infos": [{"ip": DEFAULT_IP}], + "outer_ip_infos": [{"ip": DEFAULT_IP}], + }, "zk_hosts": [{"zk_ip": DEFAULT_IP, "zk_port": "2181"}], "zk_account": "zk_account", "zk_password": "zk_password", diff --git a/apps/node_man/management/commands/transform_ap_data.py b/apps/node_man/management/commands/transform_ap_data.py new file mode 100644 index 000000000..dc2d78561 --- /dev/null +++ b/apps/node_man/management/commands/transform_ap_data.py @@ -0,0 +1,86 @@ +# coding: utf-8 +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import typing + +from django.core.management.base import BaseCommand, CommandError + +from apps.node_man import models +from apps.node_man.utils.endpoint import EndPointTransform +from common.log import logger + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument( + "-e", + "--transform", + required=False, + help="AP_ID create from V1 AP_ID", + default=False, + action="store_true", + ) + parser.add_argument( + "-l", + "--transform_endpoint_to_legacy", + action="store_true", + default=False, + help="Clean up the original mapping ID", + ) + parser.add_argument( + "-a", + "--all_ap", + action="store_true", + default=False, + help="Transform all the AP_IDs in the database", + ) + parser.add_argument( + "-t", + "--transform_ap_id", + required=False, + help="Transform target AP_ID in the database", + ) + + def handle(self, **options): + transform_endpoint_to_legacy = options.get("transform_endpoint_to_legacy") + transform = options.get("transform") + if not transform_endpoint_to_legacy and not transform: + raise CommandError("Please specify the AP_ID to be transformed") + if transform and transform_endpoint_to_legacy: + raise CommandError("Please specify only one AP_ID to be transformed") + + all_ap_transform = options.get("all_ap") + transform_ap_id = options.get("transform_ap_id") + if all_ap_transform and transform_ap_id: + raise CommandError("Please specify only one AP_ID to be transformed") + if not all_ap_transform and not transform_ap_id: + raise CommandError("Please specify the AP_ID to be transformed") + + if all_ap_transform: + ap_objects: typing.List[models.AccessPoint] = models.AccessPoint.objects.all() + else: + ap_objects: typing.List[models.AccessPoint] = models.AccessPoint.objects.filter(id=transform_ap_id) + + if transform_endpoint_to_legacy: + transform_func: typing.Callable = EndPointTransform().transform_endpoint_to_legacy + elif transform: + transform_func: typing.Callable = EndPointTransform().transform + else: + raise CommandError("Please specify the transformation method") + + for ap_object in ap_objects: + logger.info(f"Transforming AP_ID: {ap_object.id}") + try: + ap_object.taskserver = transform_func(ap_object.taskserver) + ap_object.dataserver = transform_func(ap_object.dataserver) + ap_object.btfileserver = transform_func(ap_object.btfileserver) + ap_object.save() + except Exception as e: + raise CommandError(f"Failed to transform AP_ID: {ap_object.id}, error: {e}") diff --git a/apps/node_man/migrations/0084_covert_ap_data_20231109_1336.py b/apps/node_man/migrations/0084_covert_ap_data_20231109_1336.py new file mode 100644 index 000000000..d432d938b --- /dev/null +++ b/apps/node_man/migrations/0084_covert_ap_data_20231109_1336.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from django.db import migrations, models + +from apps.node_man.utils.endpoint import EndPointTransform + + +def covert_ap_data(apps, schema_editor): + AccessPoint = apps.get_model("node_man", "AccessPoint") + aps = AccessPoint.objects.all() + for ap in aps: + # 转换 gse 地址，从一对一关系，转换为两个列表 + ap.btfileserver = EndPointTransform().transform(legacy_endpoints=ap.btfileserver) + ap.dataserver = EndPointTransform().transform(legacy_endpoints=ap.dataserver) + ap.taskserver = EndPointTransform().transform(legacy_endpoints=ap.taskserver) + ap.save() + + +class Migration(migrations.Migration): + dependencies = [ + ("node_man", "0083_subscription_operate_info"), + ] + + operations = [ + migrations.RunPython(covert_ap_data), + ] diff --git a/apps/node_man/models.py b/apps/node_man/models.py index 7c9a84ddb..0499ae06e 100644 --- a/apps/node_man/models.py +++ b/apps/node_man/models.py @@ -567,15 +567,21 @@ class AccessPoint(models.Model): @property def file_endpoint_info(self) -> EndpointInfo: - return EndpointInfo(inner_server_infos=self.btfileserver, outer_server_infos=self.btfileserver) + return EndpointInfo( + inner_ip_infos=self.btfileserver["inner_ip_infos"], outer_ip_infos=self.btfileserver["outer_ip_infos"] + ) @property def data_endpoint_info(self) -> EndpointInfo: - return EndpointInfo(inner_server_infos=self.dataserver, outer_server_infos=self.dataserver) + return EndpointInfo( + inner_ip_infos=self.dataserver["inner_ip_infos"], outer_ip_infos=self.dataserver["outer_ip_infos"] + ) @property def cluster_endpoint_info(self) -> EndpointInfo: - return EndpointInfo(inner_server_infos=self.taskserver, outer_server_infos=self.taskserver) + return EndpointInfo( + inner_ip_infos=self.taskserver["inner_ip_infos"], outer_ip_infos=self.taskserver["outer_ip_infos"] + ) @classmethod def ap_id_obj_map(cls): @@ -640,12 +646,18 @@ def test(cls, params: dict): 接入点可用性测试 :param params: Dict { - "servers": [ - { - "inner_ip": "127.0.0.1", - "outer_ip": "127.0.0.2" - } - ], + "btfileserver": { + "inner_ip_infos": [{"ip": "127.0.0.1"}], + "outer_ip_infos": [{"ip": "127.0.0.2"}] + }, + "taskserver": { + "inner_ip_infos": [{"ip": "127.0.0.1"}], + "outer_ip_infos": [{"ip": "127.0.0.2"}] + }, + "dataserver": { + "inner_ip_infos": [{"ip": "127.0.0.1"}], + "outer_ip_infos": [{"ip": "127.0.0.2"}] + }, "package_inner_url": "http://127.0.0.1/download/", "package_outer_url": "http://127.0.0.2/download/" } @@ -710,9 +722,10 @@ def _check_callback_url(url: str, _logs: list): test_logs = [] detect_hosts: Set[str] = set() - for server in params.get("btfileserver", []) + params.get("dataserver", []) + params.get("taskserver", []): - detect_hosts.add(server.get("inner_ip") or server.get("inner_ipv6")) + for server_type in ["btfileserver", "dataserver", "taskserver"]: + for ip_info in params[server_type]["inner_ip_infos"]: + detect_hosts.add(ip_info.get("ip")) with ThreadPoolExecutor(max_workers=settings.CONCURRENT_NUMBER) as ex: tasks = [ex.submit(_check_ip, detect_host, test_logs) for detect_host in detect_hosts] tasks.append(ex.submit(_check_package_url, params["package_inner_url"], test_logs)) @@ -909,7 +922,10 @@ class GsePluginDesc(models.Model): scenario_en = models.TextField(_("英文使用场景"), null=True, blank=True) category = models.CharField(_("所属范围"), max_length=32, choices=constants.CATEGORY_CHOICES) launch_node = models.CharField( - _("宿主节点类型要求"), max_length=32, choices=[("agent", "agent"), ("proxy", "proxy"), ("all", "all")], default="all" + _("宿主节点类型要求"), + max_length=32, + choices=[("agent", "agent"), ("proxy", "proxy"), ("all", "all")], + default="all", ) config_file = models.CharField(_("配置文件名称"), max_length=128, null=True, blank=True) @@ -1472,12 +1488,10 @@ class DownloadRecord(models.Model): @property def is_finish(self): - return self.task_status == self.TASK_STATUS_FAILED or self.task_status == self.TASK_STATUS_SUCCESS @property def is_failed(self): - return self.task_status == self.TASK_STATUS_FAILED @property @@ -1973,7 +1987,7 @@ def get_host_id__bk_obj_sub_map( host_id__bk_obj_sub_map[proc_status["bk_host_id"]].append( { "bk_obj_id": proc_status["bk_obj_id"], - "subscription": exist_subscription_id__obj_map.get(int(proc_status["source_id"])) + "subscription": exist_subscription_id__obj_map.get(int(proc_status["source_id"])), # "subscription_id": int(proc_status.source_id), # "name": exist_subscription_id__obj_map.get(int(proc_status.source_id)), } @@ -2229,7 +2243,10 @@ class SubscriptionInstanceRecord(models.Model): is_latest = models.BooleanField(_("是否为实例最新记录"), default=True, db_index=True) status = models.CharField( - _("任务状态"), max_length=45, choices=constants.JobStatusType.get_choices(), default=constants.JobStatusType.PENDING + _("任务状态"), + max_length=45, + choices=constants.JobStatusType.get_choices(), + default=constants.JobStatusType.PENDING, ) @property @@ -2378,7 +2395,10 @@ class SubscriptionInstanceStatusDetail(models.Model): subscription_instance_record_id = models.BigIntegerField(_("订阅实例ID"), db_index=True) node_id = models.CharField(_("Pipeline原子ID"), max_length=50, default="", blank=True, db_index=True) status = models.CharField( - _("任务状态"), max_length=45, choices=constants.JobStatusType.get_choices(), default=constants.JobStatusType.RUNNING + _("任务状态"), + max_length=45, + choices=constants.JobStatusType.get_choices(), + default=constants.JobStatusType.RUNNING, ) log = models.TextField(_("日志内容")) update_time = models.DateTimeField(_("更新时间"), null=True, blank=True, db_index=True) diff --git a/apps/node_man/periodic_tasks/gse_svr_discovery.py b/apps/node_man/periodic_tasks/gse_svr_discovery.py index 5a437ab92..0793674bc 100644 --- a/apps/node_man/periodic_tasks/gse_svr_discovery.py +++ b/apps/node_man/periodic_tasks/gse_svr_discovery.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ from telnetlib import Telnet -from typing import Any, Dict, List, Optional, Tuple +from typing import Dict, List, Optional, Tuple from celery.task import periodic_task from django.conf import settings @@ -21,7 +21,6 @@ def check_ip_ports_reachable(host: str, ports: List[int]) -> bool: - for port in ports: try: with Telnet(host=host, port=port, timeout=2): @@ -33,7 +32,6 @@ def check_ip_ports_reachable(host: str, ports: List[int]) -> bool: class ZkSafeClient: - zk_client: Optional[KazooClient] def __init__(self, hosts: str, auth_data: List[Tuple[str, str]], **kwargs): @@ -110,16 +108,15 @@ def gse_svr_discovery_periodic_task(): continue logger.info(f"zk_node_path -> {zk_node_path}, svr_ips -> {svr_ips}") - inner_ip__outer_ip_map: Dict[str, str] = {} - for svr_info in getattr(ap, ap_field, []): - inner_ip__outer_ip_map[svr_info.get("inner_ip")] = svr_info.get("outer_ip") + outer_ips = getattr(ap, ap_field, {}).get("outer_ip_infos") or {} + inner_ip_infos: List[Dict[str, str]] = [{"ip": inner_ip} for inner_ip in svr_ips] - svr_infos: List[Dict[str, Any]] = [] - for svr_ip in svr_ips: - # svr_ip 通常解析为内网IP，外网IP允许自定义，如果为空再取 svr_ip - outer_ip = inner_ip__outer_ip_map.get(svr_ip) or svr_ip - svr_infos.append({"inner_ip": svr_ip, "outer_ip": outer_ip}) + svr_infos: Dict[str, List[Dict[str, str]]] = { + "inner_ip_infos": inner_ip_infos, + "outer_ip_infos": outer_ips if outer_ips else inner_ip_infos, + } setattr(ap, ap_field, svr_infos) + logger.info(f"update ap -> {ap}, {ap_field} -> {svr_infos}") is_change = True if is_change: ap.save() diff --git a/apps/node_man/serializers/ap.py b/apps/node_man/serializers/ap.py index 9adf3fd8a..f242d2b70 100644 --- a/apps/node_man/serializers/ap.py +++ b/apps/node_man/serializers/ap.py @@ -8,7 +8,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from typing import List +import typing from django.utils.translation import ugettext_lazy as _ from rest_framework import serializers @@ -84,29 +84,53 @@ class UpdateOrCreateSerializer(serializers.ModelSerializer): """ class ServersSerializer(serializers.Serializer): - inner_ip = serializers.CharField(label=_("内网IP"), required=False) - inner_ipv6 = serializers.CharField(label=_("内网IPv6"), required=False) - outer_ip = serializers.CharField(label=_("外网IP"), required=False) - outer_ipv6 = serializers.CharField(label=_("外网IPv6"), required=False) - bk_host_id = serializers.IntegerField(label=_("主机ID"), required=False) + class ServerInfoSerializer(serializers.Serializer): + ip = serializers.CharField(label=_("IP地址"), required=False) + bk_host_id = serializers.IntegerField(label=_("主机ID"), required=False) - def validate(self, attrs): - basic.ipv6_formatter(data=attrs, ipv6_field_names=["inner_ipv6", "outer_ipv6"]) + inner_ip_infos = serializers.ListField(label=_("内网IP信息"), required=False, child=ServerInfoSerializer()) + outer_ip_infos = serializers.ListField(label=_("外网IP信息"), required=False, child=ServerInfoSerializer()) - if not (attrs.get("inner_ip") or attrs.get("inner_ipv6")): - raise ValidationError(_("请求参数 inner_ip 和 inner_ipv6 不能同时为空")) - if not (attrs.get("outer_ip") or attrs.get("outer_ipv6")): - raise ValidationError(_("请求参数 outer_ip 和 outer_ipv6 不能同时为空")) + def validate(self, attrs): + if not attrs.get("inner_ip_infos") and not attrs.get("outer_ip_infos"): + raise ValidationError(_("请求参数 inner_ip_info, outer_ip_infos 不可同时为空")) + + for attr in attrs.keys(): + v4_ips, abnormal_ips, format_ipv6_infos = [], [], [] + for ip_info in attrs.get(attr, []): + ip = ip_info.get("ip", None) + if not ip: + continue + if basic.is_v4(ip): + v4_ips.append(ip_info) + elif basic.is_v6(ip): + ip_info.update({"ip": basic.exploded_ip(ip)}) + format_ipv6_infos.append(ip_info) + else: + # 不是有效的 IP 地址 + abnormal_ips.append(ip_info) + + if v4_ips and format_ipv6_infos: + raise ValidationError(_(f"{attr} 中不能同时包括 ipv4 和 ipv6")) + if abnormal_ips: + raise ValidationError(_(f"{attr} 中存在非法 IP 地址: {abnormal_ips}")) + + if format_ipv6_infos: + attrs[attr] = format_ipv6_infos + + # 去重复 + attrs[attr] = list({frozenset(d.items()): d for d in attrs[attr] if d is not None}.values()) return attrs class ZKSerializer(serializers.Serializer): - zk_ip = serializers.CharField(label=_("ZK IP地址")) - zk_port = serializers.CharField(label=_("ZK 端口")) - - btfileserver = serializers.ListField(child=ServersSerializer()) - dataserver = serializers.ListField(child=ServersSerializer()) - taskserver = serializers.ListField(child=ServersSerializer()) - zk_hosts = serializers.ListField(child=ZKSerializer()) + zk_ip = serializers.CharField(label=_("ZK IP地址"), required=False, allow_blank=True) + zk_port = serializers.CharField(label=_("ZK 端口"), required=False, allow_blank=True) + + btfileserver = ServersSerializer() + dataserver = ServersSerializer() + taskserver = ServersSerializer() + gse_version = serializers.ChoiceField(label=_("GSE版本"), choices=GseVersion.list_choices(), required=False) + zk_hosts = serializers.ListField(child=ZKSerializer(), required=False, default=[]) zk_account = serializers.CharField(label=_("ZK账号"), required=False, allow_blank=True) zk_password = serializers.CharField(label=_("ZK密码"), required=False, allow_blank=True) agent_config = serializers.DictField(label=_("Agent配置")) @@ -119,7 +143,7 @@ class ZKSerializer(serializers.Serializer): callback_url = serializers.CharField(label=_("节点管理内网回调地址"), required=False, allow_blank=True) def validate(self, data): - gse_version_list: List[str] = list(set(AccessPoint.objects.values_list("gse_version", flat=True))) + gse_version_list: typing.List[str] = list(set(AccessPoint.objects.values_list("gse_version", flat=True))) # 存量接入点版本全部为V2新建/更新版本也为V2版本 if GseVersion.V1.value not in gse_version_list: data["gse_version"] = GseVersion.V2.value diff --git a/apps/node_man/tests/test_pericdic_tasks/mock_data.py b/apps/node_man/tests/test_pericdic_tasks/mock_data.py index 0ec2a3c30..f188e5018 100644 --- a/apps/node_man/tests/test_pericdic_tasks/mock_data.py +++ b/apps/node_man/tests/test_pericdic_tasks/mock_data.py @@ -128,8 +128,15 @@ "/gse/config/server/task/all": ["127.0.0.1", "127.0.0.2", "127.0.0.3"], "/gse/config/server/btfiles/all": ["127.0.0.1", "127.0.0.2", "127.0.0.3"], } -MOCK_AP_FIELD_MAP = [ - {"inner_ip": "127.0.0.1", "outer_ip": "127.0.0.1"}, - {"inner_ip": "127.0.0.2", "outer_ip": "127.0.0.2"}, - {"inner_ip": "127.0.0.3", "outer_ip": "127.0.0.3"}, -] +MOCK_AP_FIELD_MAP = { + "inner_ip_infos": [ + {"ip": "127.0.0.1"}, + {"ip": "127.0.0.2"}, + {"ip": "127.0.0.3"}, + ], + "outer_ip_infos": [ + {"ip": "127.0.0.1"}, + {"ip": "127.0.0.2"}, + {"ip": "127.0.0.3"}, + ], +} diff --git a/apps/node_man/tests/test_pericdic_tasks/test_gse_svr_discovery.py b/apps/node_man/tests/test_pericdic_tasks/test_gse_svr_discovery.py index fbefa1de6..8baea6702 100644 --- a/apps/node_man/tests/test_pericdic_tasks/test_gse_svr_discovery.py +++ b/apps/node_man/tests/test_pericdic_tasks/test_gse_svr_discovery.py @@ -38,7 +38,7 @@ def test_gse_svr_discovery(self): # 检查ap_field是否已经更新。注: 如果gse_svr_discovery的ap_field更改了，单测这里也需要同步更改 ap_field_list = ["dataserver", "dataserver", "btfileserver"] for ap_field in ap_field_list: - self.assertEqual(getattr(ap, ap_field, []), MOCK_AP_FIELD_MAP) + self.assertEqual(getattr(ap, ap_field, {}), MOCK_AP_FIELD_MAP) class TestGseSvrDiscoveryEmptyRegionCity(TestGseSvrDiscovery): @@ -47,3 +47,27 @@ def setUpTestData(cls): ap = AccessPoint.objects.all().first() ap.city_id = ap.region_id = None ap.save() + + +class TestExistOuterIpInfoDiscovery(CustomBaseTestCase): + @classmethod + def setUpTestData(cls): + ap = AccessPoint.objects.all().first() + ap.city_id = ap.region_id = "test" + ap.btfileserver = {"inner_ip_infos": [{"ip": "127.0.0.1"}], "outer_ip_infos": [{"ip": "121.0.0.1"}]} + ap.save() + + @patch("apps.node_man.periodic_tasks.gse_svr_discovery.settings.GSE_ENABLE_SVR_DISCOVERY", True) + @patch("apps.node_man.periodic_tasks.gse_svr_discovery.KazooClient", MockKazooClient) + @patch("apps.node_man.periodic_tasks.gse_svr_discovery.check_ip_ports_reachable", check_ip_ports_reachable) + def test_gse_bt_svr_discovery(self): + gse_svr_discovery_periodic_task() + ap = AccessPoint.objects.all().first() + + self.assertEqual( + ap.btfileserver, + { + "inner_ip_infos": [{"ip": "127.0.0.1"}, {"ip": "127.0.0.2"}, {"ip": "127.0.0.3"}], + "outer_ip_infos": [{"ip": "121.0.0.1"}], + }, + ) diff --git a/apps/node_man/tests/test_tools/test_ap_transform.py b/apps/node_man/tests/test_tools/test_ap_transform.py new file mode 100644 index 000000000..e6d5a7450 --- /dev/null +++ b/apps/node_man/tests/test_tools/test_ap_transform.py @@ -0,0 +1,639 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-节点管理(BlueKing-BK-NODEMAN) available. +Copyright (C) 2017-2022 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from copy import deepcopy + +from django.core.management import call_command +from django.core.management.base import CommandError +from django.test import TestCase + +from apps.mock_data.common_unit.host import ( + AP_MODEL_DATA, + DEFAULT_HOST_ID, + DEFAULT_IP, + DEFAULT_IPV6, +) +from apps.node_man import models +from apps.node_man.utils.endpoint import Endpoint, EndPointTransform +from apps.utils.basic import exploded_ip +from apps.utils.unittest.testcase import CustomAPITestCase +from env.constants import GseVersion + + +class TestApTransform(TestCase): + def mock_outer_ipv6(self, ipv6: str): + return DEFAULT_IPV6.replace("6", "A") + + def setUp(self): + test_data_list = [ + { + "name": "公有云接入点", + "ap_type": "system", + "region_id": "2", + "city_id": "30", + "gse_version": "V2", + "btfileserver": [ + {"inner_ip": "127.0.0.120", "outer_ip": "127.0.0.20", "inner_ipv6": None, "outer_ipv6": None}, + { + "inner_ip": "127.0.0.27", + "outer_ip": "127.0.0.27", + "inner_ipv6": None, + "outer_ipv6": None, + }, + ], + "dataserver": [ + {"inner_ip": "127.0.0.120", "outer_ip": "127.0.0.120", "inner_ipv6": None, "outer_ipv6": None}, + { + "inner_ip": "127.0.0.27", + "outer_ip": "127.0.0.27", + "inner_ipv6": None, + "outer_ipv6": None, + }, + ], + "taskserver": [ + {"inner_ip": "127.0.0.120", "outer_ip": "127.0.0.120", "inner_ipv6": None, "outer_ipv6": None}, + { + "inner_ip": "127.0.0.27", + "outer_ip": "127.0.0.27", + "inner_ipv6": None, + "outer_ipv6": None, + }, + ], + "zk_hosts": [{"zk_ip": "127.0.0.190", "zk_port": "2182"}], + "zk_account": "", + "zk_password": "", + "package_inner_url": "http://nodeman.test.com/download/prod", + "package_outer_url": "http://127.0.0.161/download/", + "nginx_path": "/data/bkee/public/bknodeman/download", + "agent_config": { + "linux": { + "dataipc": "/usr/local/gse/agent/data/ipc.state.report", + "log_path": "/var/log/gse", + "run_path": "/var/run/gse", + "data_path": "/var/lib/gse", + "temp_path": "/tmp", + "setup_path": "/usr/local/gse", + "hostid_path": "/var/lib/gse/host/hostid", + }, + "windows": { + "dataipc": 27002, + "log_path": "C:\\gse\\logs", + "run_path": "C:\\gse\\data", + "data_path": "C:\\gse\\data", + "temp_path": "C:\\Temp", + "setup_path": "C:\\gse", + "hostid_path": "C:\\gse\\data\\host\\hostid", + }, + }, + "status": None, + "description": "GSE2_上海外网", + "is_enabled": True, + "is_default": True, + "creator": ["admin"], + "port_config": { + "bt_port": 20020, + "io_port": 28668, + "data_port": 28625, + "proc_port": 50000, + "trunk_port": 48329, + "bt_port_end": 60030, + "tracker_port": 20030, + "bt_port_start": 60020, + "db_proxy_port": 58859, + "file_svr_port": 28925, + "api_server_port": 50002, + "file_svr_port_v1": 58926, + "agent_thrift_port": 48669, + "btsvr_thrift_port": 58931, + "data_prometheus_port": 29402, + "file_metric_bind_port": 29404, + "file_topology_bind_port": 28930, + }, + "proxy_package": ["gse_client-windows-x86_64.tgz", "gse_client-linux-x86_64.tgz"], + "outer_callback_url": "", + "callback_url": "", + }, + { + "name": "公有云接入点_v1", + "ap_type": "system", + "region_id": "test", + "city_id": "test", + "gse_version": "V1", + "btfileserver": [ + { + "inner_ip": "127.0.0.198", + "outer_ip": "127.0.0.69", + "inner_ipv6": None, + "outer_ipv6": None, + }, + { + "inner_ip": "127.0.0.198", + "outer_ip": "127.0.0.76", + "inner_ipv6": None, + "outer_ipv6": None, + }, + ], + "dataserver": [ + { + "inner_ip": "127.0.0.198", + "outer_ip": "127.0.0.69", + "inner_ipv6": None, + "outer_ipv6": None, + }, + { + "inner_ip": "127.0.0.198", + "outer_ip": "127.0.0.76", + "inner_ipv6": None, + "outer_ipv6": None, + }, + ], + "taskserver": [ + { + "inner_ip": "127.0.0.198", + "outer_ip": "127.0.0.69", + "inner_ipv6": None, + "outer_ipv6": None, + }, + { + "inner_ip": "127.0.0.198", + "outer_ip": "127.0.0.76", + "inner_ipv6": None, + "outer_ipv6": None, + }, + ], + "zk_hosts": [{"zk_ip": DEFAULT_IP, "zk_port": "2182"}], + "zk_account": "noneed", + "zk_password": "noneed", + "package_inner_url": "http://nodeman.test.com/download/prod-oa", + "package_outer_url": "http://127.0.0.1/download/prod-oa", + "nginx_path": "/data/bkee/public/bknodeman/download/prod-oa", + "agent_config": { + "linux": { + "dataipc": "/usr/local/gse/agent/data/ipc.state.report", + "log_path": "/var/log/gse", + "run_path": "/var/run/gse", + "data_path": "/var/lib/gse", + "temp_path": "/tmp", + "setup_path": "/usr/local/gse", + "hostid_path": "/var/lib/gse/host/hostid", + }, + "windows": { + "dataipc": "47000", + "log_path": "C:\\gse\\logs", + "run_path": "C:\\gse\\data", + "data_path": "C:\\gse\\data", + "temp_path": "C:\\Temp", + "setup_path": "C:\\gse", + "hostid_path": "C:\\gse\\data\\host\\hostid", + }, + }, + "status": None, + "description": "专用Proxy请勿选择", + "is_enabled": True, + "is_default": False, + "creator": ["admin"], + "port_config": { + "bt_port": 10020, + "io_port": 48668, + "data_port": 58625, + "proc_port": 50000, + "trunk_port": 48329, + "bt_port_end": 60030, + "tracker_port": 10030, + "bt_port_start": 60020, + "db_proxy_port": 58859, + "file_svr_port": 58925, + "api_server_port": 50002, + "agent_thrift_port": 48669, + "btsvr_thrift_port": 58930, + "data_prometheus_port": 59402, + }, + "proxy_package": [ + "gse_client-windows-x86_64.tgz", + "gse_client-linux-x86_64.tgz", + "gse_client-linux-aarch64.tgz", + ], + "outer_callback_url": "", + "callback_url": "", + }, + { + "name": "GSE2_IPV6", + "ap_type": "system", + "region_id": "2", + "city_id": "30", + "gse_version": "V2", + "btfileserver": [ + { + "inner_ip": "", + "outer_ip": "", + "inner_ipv6": DEFAULT_IPV6, + "outer_ipv6": self.mock_outer_ipv6(DEFAULT_IPV6), + }, + { + "inner_ip": "", + "outer_ip": "", + "inner_ipv6": DEFAULT_IPV6, + "outer_ipv6": self.mock_outer_ipv6(DEFAULT_IPV6), + }, + ], + "dataserver": [ + { + "inner_ip": "", + "outer_ip": "", + "inner_ipv6": DEFAULT_IPV6, + "outer_ipv6": self.mock_outer_ipv6(DEFAULT_IPV6), + }, + { + "inner_ip": "", + "outer_ip": "", + "inner_ipv6": DEFAULT_IPV6, + "outer_ipv6": self.mock_outer_ipv6(DEFAULT_IPV6), + }, + ], + "taskserver": [ + { + "inner_ip": "", + "outer_ip": "", + "inner_ipv6": DEFAULT_IPV6, + "outer_ipv6": self.mock_outer_ipv6(DEFAULT_IPV6), + }, + { + "inner_ip": "", + "outer_ip": "", + "inner_ipv6": DEFAULT_IPV6, + "outer_ipv6": self.mock_outer_ipv6(DEFAULT_IPV6), + }, + ], + "zk_hosts": [{"zk_ip": DEFAULT_IP, "zk_port": "2182"}], + "zk_account": "noneed", + "zk_password": "noneed", + "package_inner_url": "http://nodeman.test.com/download/prod-oa", + "package_outer_url": "http://127.0.0.1/download/", + "nginx_path": "/data/bkee/public/bknodeman/download/", + "agent_config": { + "linux": { + "dataipc": "/usr/local/gse/agent/data/ipc.state.report", + "log_path": "/var/log/gse", + "run_path": "/var/run/gse", + "data_path": "/var/lib/gse", + "temp_path": "/tmp", + "setup_path": "/usr/local/gse", + "hostid_path": "/var/lib/gse/host/hostid", + }, + "windows": { + "dataipc": 27002, + "log_path": "C:\\gse\\logs", + "run_path": "C:\\gse\\data", + "data_path": "C:\\gse\\data", + "temp_path": "C:\\Temp", + "setup_path": "C:\\gse", + "hostid_path": "C:\\gse\\data\\host\\hostid", + }, + }, + "status": None, + "description": "专用Proxy请勿选择", + "is_enabled": True, + "is_default": False, + "creator": ["admin"], + "port_config": { + "bt_port": 20020, + "io_port": 28668, + "data_port": 28625, + "proc_port": 50000, + "trunk_port": 48329, + "bt_port_end": 60030, + "tracker_port": 20030, + "bt_port_start": 60020, + "db_proxy_port": 58859, + "file_svr_port": 28925, + "api_server_port": 50002, + "file_svr_port_v1": 58926, + "agent_thrift_port": 48669, + "btsvr_thrift_port": 58931, + "file_metric_bind_port": 29404, + "file_topology_bind_port": 28930, + }, + "proxy_package": ["gse_client-windows-x86_64.tgz", "gse_client-linux-x86_64.tgz"], + "outer_callback_url": "", + "callback_url": "", + }, + { + "name": "内外网相同接入点", + "ap_type": "system", + "region_id": "2", + "city_id": "30", + "gse_version": "V2", + "btfileserver": [ + {"inner_ip": "127.0.0.120", "outer_ip": "127.0.0.120", "inner_ipv6": None, "outer_ipv6": None}, + ], + "dataserver": [ + {"inner_ip": "127.0.0.120", "outer_ip": "127.0.0.120", "inner_ipv6": None, "outer_ipv6": None}, + { + "inner_ip": "127.0.0.27", + "outer_ip": "127.0.0.27", + "inner_ipv6": None, + "outer_ipv6": None, + }, + ], + "taskserver": [ + {"inner_ip": "127.0.0.120", "outer_ip": "127.0.0.120", "inner_ipv6": None, "outer_ipv6": None}, + { + "inner_ip": "127.0.0.27", + "outer_ip": "127.0.0.27", + "inner_ipv6": None, + "outer_ipv6": None, + }, + ], + "zk_hosts": [{"zk_ip": "127.0.0.190", "zk_port": "2182"}], + "zk_account": "", + "zk_password": "", + "package_inner_url": "http://nodeman.test.com/download/prod", + "package_outer_url": "http://127.0.0.161/download/", + "nginx_path": "/data/bkee/public/bknodeman/download", + "agent_config": { + "linux": { + "dataipc": "/usr/local/gse/agent/data/ipc.state.report", + "log_path": "/var/log/gse", + "run_path": "/var/run/gse", + "data_path": "/var/lib/gse", + "temp_path": "/tmp", + "setup_path": "/usr/local/gse", + "hostid_path": "/var/lib/gse/host/hostid", + }, + "windows": { + "dataipc": 27002, + "log_path": "C:\\gse\\logs", + "run_path": "C:\\gse\\data", + "data_path": "C:\\gse\\data", + "temp_path": "C:\\Temp", + "setup_path": "C:\\gse", + "hostid_path": "C:\\gse\\data\\host\\hostid", + }, + }, + }, + ] + + for ap_data in test_data_list: + models.AccessPoint.objects.update_or_create(**ap_data) + + def test_ap_transform(self): + gse_v1_ap = models.AccessPoint.objects.get(gse_version=GseVersion.V1.value, name="公有云接入点_v1") + self.assertEqual( + EndPointTransform().transform(gse_v1_ap.taskserver), + { + "inner_ip_infos": [{"ip": "127.0.0.198"}], + "outer_ip_infos": [{"ip": "127.0.0.69"}, {"ip": "127.0.0.76"}], + }, + ) + + gse_v2_ap = models.AccessPoint.objects.get(gse_version=GseVersion.V2.value, name="公有云接入点") + self.assertEqual( + EndPointTransform().transform(gse_v2_ap.btfileserver), + { + "inner_ip_infos": [{"ip": "127.0.0.120"}, {"ip": "127.0.0.27"}], + "outer_ip_infos": [{"ip": "127.0.0.20"}, {"ip": "127.0.0.27"}], + }, + ) + outer_and_inner_same_ip_ap = models.AccessPoint.objects.get(name="内外网相同接入点") + self.assertEqual( + EndPointTransform().transform(outer_and_inner_same_ip_ap.btfileserver), + { + "inner_ip_infos": [{"ip": "127.0.0.120"}], + "outer_ip_infos": [{"ip": "127.0.0.120"}], + }, + ) + + def test_transfrom_command(self): + # 调用 django command transform_ap_data 进行数据转换 + default_ap_id = models.AccessPoint.objects.get(name="默认接入点").id + # 因为默认的接入点已经经过转换，所以这里需要先把默认接入点的数据转换为旧的格式 + call_command("transform_ap_data", transform_endpoint_to_legacy=True, transform_ap_id=default_ap_id) + default_ap_obj = models.AccessPoint.objects.get(name="默认接入点") + self.assertEqual( + default_ap_obj.taskserver, + [ + { + "inner_ip": "", + "outer_ip": "", + } + ], + ) + # 把所有的接入点都转换一遍， 转换为新的格式 + call_command("transform_ap_data", transform=True, all_ap=True) + for ap in models.AccessPoint.objects.all(): + self.assertTrue(isinstance(ap.taskserver, dict)) + + gse_v1_ap = models.AccessPoint.objects.get(gse_version="V1", name="公有云接入点_v1") + self.assertEqual( + gse_v1_ap.btfileserver, + { + "inner_ip_infos": [{"ip": "127.0.0.198"}], + "outer_ip_infos": [{"ip": "127.0.0.69"}, {"ip": "127.0.0.76"}], + }, + ) + v6_ap_obj = models.AccessPoint.objects.get(name="GSE2_IPV6") + self.assertEqual( + v6_ap_obj.btfileserver, + { + "inner_ip_infos": [{"ip": DEFAULT_IPV6}], + "outer_ip_infos": [{"ip": self.mock_outer_ipv6(DEFAULT_IPV6)}], + }, + ) + + # 转换回旧的数据 + call_command("transform_ap_data", transform_endpoint_to_legacy=True, transform_ap_id=gse_v1_ap.id) + self.assertEqual( + # 转换回旧的数据，并且过滤为 None 的字段 + models.AccessPoint.objects.get(id=gse_v1_ap.id).btfileserver, + [ + { + "inner_ip": "127.0.0.198", + "outer_ip": "127.0.0.69", + }, + { + "inner_ip": "127.0.0.198", + "outer_ip": "127.0.0.76", + }, + ], + ) + call_command("transform_ap_data", transform=True, transform_ap_id=gse_v1_ap.id) + self.assertEqual( + models.AccessPoint.objects.get(id=gse_v1_ap.id).btfileserver, + { + "inner_ip_infos": [{"ip": "127.0.0.198"}], + "outer_ip_infos": [{"ip": "127.0.0.69"}, {"ip": "127.0.0.76"}], + }, + ) + + # 校验参数 + self.assertRaises( + CommandError, call_command, "transform_ap_data", transform_endpoint_to_legacy=True, transform=True + ) + self.assertRaises( + CommandError, call_command, "transform_ap_data", transform_endpoint_to_legacy=True, transform=False + ) + self.assertRaises( + CommandError, + call_command, + "transform_ap_data", + transform_endpoint_to_legacy=True, + transform=False, + all_ap=True, + transform_ap_id=gse_v1_ap.id, + ) + + def test_transform_with_host_id(self): + ap = models.AccessPoint.objects.get(name="公有云接入点_v1") + ap_btfileserver = ap.btfileserver + for server in ap_btfileserver: + server.update(bk_host_id=DEFAULT_HOST_ID) + models.AccessPoint.objects.filter(name="公有云接入点_v1").update(btfileserver=ap_btfileserver) + call_command("transform_ap_data", transform=True, transform_ap_id=ap.id) + self.assertEqual( + models.AccessPoint.objects.get(id=ap.id).btfileserver, + { + "inner_ip_infos": [{"ip": "127.0.0.198", "bk_host_id": DEFAULT_HOST_ID}], + "outer_ip_infos": [ + {"ip": "127.0.0.69", "bk_host_id": DEFAULT_HOST_ID}, + {"ip": "127.0.0.76", "bk_host_id": DEFAULT_HOST_ID}, + ], + }, + ) + + # test ap endpoint + self.assertEqual( + models.AccessPoint.objects.get(id=ap.id).file_endpoint_info.outer_hosts, ["127.0.0.69", "127.0.0.76"] + ) + self.assertEqual(models.AccessPoint.objects.get(id=ap.id).file_endpoint_info.inner_hosts, ["127.0.0.198"]) + self.assertEqual( + models.AccessPoint.objects.get(id=ap.id).file_endpoint_info.inner_endpoints, + [Endpoint(v4="127.0.0.198", v6=None, host_id=DEFAULT_HOST_ID)], + ) + + +class ApViewTransformTestCase(CustomAPITestCase): + TEST_AP_NAME = "CUSTOM_AP" + CREATE_URL = "/api/ap/" + + def setUp(self): + ap_data = deepcopy(AP_MODEL_DATA) + ap_data["name"] = self.TEST_AP_NAME + self.ap_data = ap_data + + def test_ap_create(self): + self.client.post(path=self.CREATE_URL, data=self.ap_data) + ap = models.AccessPoint.objects.get(name=self.TEST_AP_NAME) + self.assertEqual(ap.name, self.TEST_AP_NAME) + self.assertEqual(ap.taskserver, AP_MODEL_DATA["taskserver"]) + + def test_mix_ip_ap_create(self): + self.ap_data["taskserver"] = {"inner_ip_infos": [{"ip": DEFAULT_IP}, {"ip": DEFAULT_IPV6}]} + result = self.client.post(self.CREATE_URL, self.ap_data) + self.assertFalse(result["result"]) + self.assertEqual(result["message"], "inner_ip_infos 中不能同时包括 ipv4 和 ipv6（3800001）") + + # 支持 taskserver v6 & fileserver v4 混合 + mix_server_ap_data = deepcopy(self.ap_data) + mix_server_ap_data["taskserver"] = {"inner_ip_infos": [{"ip": DEFAULT_IPV6}]} + result = self.client.post(self.CREATE_URL, mix_server_ap_data) + self.assertTrue(result["result"]) + + def test_multi_v4_ap_create(self): + # 支持多个 v4 地址 + self.ap_data["taskserver"] = { + "inner_ip_infos": [{"ip": DEFAULT_IP}, {"ip": DEFAULT_IP.replace("1", "2")}], + "outer_ip_infos": [{"ip": DEFAULT_IP}, {"ip": DEFAULT_IP.replace("1", "2")}], + } + result = self.client.post(self.CREATE_URL, self.ap_data) + self.assertTrue(result["result"]) + ap = models.AccessPoint.objects.get(name=self.TEST_AP_NAME) + self.assertEqual( + ap.taskserver, + { + "inner_ip_infos": [{"ip": DEFAULT_IP}, {"ip": DEFAULT_IP.replace("1", "2")}], + "outer_ip_infos": [{"ip": DEFAULT_IP}, {"ip": DEFAULT_IP.replace("1", "2")}], + }, + ) + + def test_multi_v6_ap_create(self): + # 支持多个 v6 地址 + self.ap_data["taskserver"] = {"inner_ip_infos": [{"ip": DEFAULT_IPV6}, {"ip": DEFAULT_IPV6.replace("1", "2")}]} + result = self.client.post(self.CREATE_URL, self.ap_data) + self.assertTrue(result["result"]) + + def test_v4_v6_mix_ap_create(self): + # 支持同一个 server inner & outer v4 v6 混合 + self.ap_data["taskserver"] = { + "inner_ip_infos": [{"ip": DEFAULT_IP}], + "outer_ip_infos": [{"ip": DEFAULT_IPV6}], + } + result = self.client.post(self.CREATE_URL, self.ap_data) + self.assertTrue(result["result"]) + ap = models.AccessPoint.objects.get(name=self.TEST_AP_NAME) + self.assertEqual( + ap.taskserver, + { + "inner_ip_infos": [{"ip": DEFAULT_IP}], + "outer_ip_infos": [{"ip": exploded_ip(DEFAULT_IPV6)}], + }, + ) + + def test_filter_ip_ap_crete(self): + # 支持过滤掉重复 ip + self.ap_data["taskserver"] = {"inner_ip_infos": [{"ip": DEFAULT_IP}, {"ip": DEFAULT_IP}]} + result = self.client.post(self.CREATE_URL, self.ap_data) + self.assertTrue(result["result"]) + self.assertEqual( + result["data"]["taskserver"]["inner_ip_infos"], + [{"ip": DEFAULT_IP}], + ) + + def test_illegal_ip_ap_create(self): + # 支持 v4 / v6 IP格式检测 + self.ap_data["taskserver"] = {"inner_ips": [{"inner_ip": "11"}]} + self.assertFalse(self.client.post(self.CREATE_URL, self.ap_data)["result"]) + self.ap_data["taskserver"] = {"inner_ips": [{"inner_ip": DEFAULT_IP}, {"inner_ip": "11"}]} + self.assertFalse(self.client.post(self.CREATE_URL, self.ap_data)["result"]) + self.ap_data["taskserver"] = {"inner_ips": [{"inner_ipv6": DEFAULT_IP}]} + self.assertFalse(self.client.post(self.CREATE_URL, self.ap_data)["result"]) + + def test_update_ap(self): + update_ap_name = "默认update ap" + self.ap_data["taskserver"] = {"inner_ip_infos": [{"ip": "111.1.1.1"}]} + self.ap_data["name"] = "默认update ap" + ap_id = models.AccessPoint.objects.get(name="默认接入点").id + update_url = f"/api/ap/{ap_id}/" + result = self.client.put(update_url, self.ap_data) + self.assertTrue(result["result"]) + ap_obj = models.AccessPoint.objects.get(name=update_ap_name) + self.assertEqual(ap_obj.taskserver, {"inner_ip_infos": [{"ip": "111.1.1.1"}]}) + + def test_ap_retrieve(self): + self.test_ap_create() + ap_id = models.AccessPoint.objects.get(name=self.TEST_AP_NAME).id + retrieve_url = f"/api/ap/{ap_id}/" + result = self.client.get(retrieve_url) + self.assertTrue(result["result"]) + self.assertEqual(result["data"]["name"], self.TEST_AP_NAME) + self.assertEqual(result["data"]["taskserver"], AP_MODEL_DATA["taskserver"]) + + def test_ap_test(self): + ap_tes_url: str = "/api/ap/test/" + test_ap_data: dict = { + "btfileserver": {"inner_ip_infos": [{"ip": DEFAULT_IP}], "outer_ip_infos": [{"ip": "127.0.0.2"}]}, + "taskserver": {"inner_ip_infos": [{"ip": DEFAULT_IP}], "outer_ip_infos": [{"ip": "127.0.0.2"}]}, + "dataserver": {"inner_ip_infos": [{"ip": DEFAULT_IP}], "outer_ip_infos": [{"ip": "127.0.0.2"}]}, + "package_inner_url": "http://127.0.0.1/download/", + "package_outer_url": "http://127.0.0.2/download/", + } + result = self.client.post(ap_tes_url, test_ap_data) + self.assertFalse(result["data"]["test_result"]) + self.assertEqual(result["data"]["test_logs"][1]["log"], f"Ping {DEFAULT_IP} 正常") diff --git a/apps/node_man/tests/utils.py b/apps/node_man/tests/utils.py index 576f7883d..6583c38f3 100644 --- a/apps/node_man/tests/utils.py +++ b/apps/node_man/tests/utils.py @@ -192,7 +192,6 @@ def create_host( class Subscription: def create_subscription(self, job_type, nodes, *args, **kwargs): - cipher = tools.HostTools.get_asymmetric_cipher() subscription_id = random.randint(100, 1000) task_id = random.randint(10, 1000) @@ -659,9 +658,12 @@ def create_ap(number): "is_enabled": 1, "zk_account": "bkzk", "zk_password": "bkzk", - "btfileserver": [{"inner_ip": random_ip(), "outer_ip": random_ip()}], - "dataserver": [{"inner_ip": random_ip(), "outer_ip": random_ip()}], - "taskserver": [{"inner_ip": random_ip(), "outer_ip": random_ip()}], + "btfileserver": { + "inner_ips": [{"inner_ip": random_ip()}], + "outer_ips": [{"outer_ip": random_ip()}], + }, + "dataserver": {"inner_ips": [{"inner_ip": random_ip()}], "outer_ips": [{"outer_ip": random_ip()}]}, + "taskserver": {"inner_ips": [{"inner_ip": random_ip()}], "outer_ips": [{"outer_ip": random_ip()}]}, "package_inner_url": "http://127.0.0.1:80/download", "package_outer_url": "http://127.0.0.1:80/download", "nginx_path": "/data/bkee/public/bknodeman/download", diff --git a/apps/node_man/utils/endpoint.py b/apps/node_man/utils/endpoint.py index 62e8757f4..b2f8d182f 100644 --- a/apps/node_man/utils/endpoint.py +++ b/apps/node_man/utils/endpoint.py @@ -9,9 +9,15 @@ specific language governing permissions and limitations under the License. """ +import json import typing +from collections import defaultdict from dataclasses import dataclass, field +from apps.utils.basic import filter_values, is_v4, is_v6 +from apps.utils.md5 import _count_md5 +from common.log import logger + @dataclass class Endpoint: @@ -27,8 +33,8 @@ def __post_init__(self): class EndpointInfo: def __init__( self, - inner_server_infos: typing.List[typing.Dict[str, typing.Any]], - outer_server_infos: typing.List[typing.Dict[str, typing.Any]], + inner_ip_infos: typing.List[typing.Dict[str, typing.Union[str, int]]], + outer_ip_infos: typing.List[typing.Dict[str, typing.Union[str, int]]], ): self.inner_endpoints: typing.List[Endpoint] = [] self.outer_endpoints: typing.List[Endpoint] = [] @@ -36,22 +42,101 @@ def __init__( self.outer_hosts: typing.List[str] = [] self.inner_hosts: typing.List[str] = [] - for inner_server_info in inner_server_infos: - endpoint: Endpoint = Endpoint( - v4=inner_server_info.get("inner_ip"), - v6=inner_server_info.get("inner_ipv6"), - host_id=inner_server_info.get("host_id"), - ) - self.inner_endpoints.append(endpoint) - if endpoint.host_str: - self.inner_hosts.append(endpoint.host_str) - - for outer_server_info in outer_server_infos: - endpoint: Endpoint = Endpoint( - v4=outer_server_info.get("outer_ip"), - v6=outer_server_info.get("outer_ipv6"), - host_id=outer_server_info.get("host_id"), - ) - self.outer_endpoints.append(endpoint) - if endpoint.host_str: - self.outer_hosts.append(endpoint.host_str) + def create_endpoint(ip_info: typing.Dict[str, typing.Union[str, int]]) -> Endpoint: + ip = ip_info.get("ip") + host_id = ip_info.get("bk_host_id") + if is_v4(ip): + return Endpoint(v4=ip, v6=None, host_id=host_id) + else: + return Endpoint(v6=ip, v4=None, host_id=host_id) + + self.inner_endpoints = [create_endpoint(ip_info) for ip_info in inner_ip_infos] + self.outer_endpoints = [create_endpoint(ip_info) for ip_info in outer_ip_infos] + + self.inner_hosts = [endpoint.host_str for endpoint in self.inner_endpoints if endpoint.host_str] + self.outer_hosts = [endpoint.host_str for endpoint in self.outer_endpoints if endpoint.host_str] + + +class EndPointTransform(object): + # legacy_endpoint: [{ "inner_ip": "10.0.6.44", "outer_ip": "10.0.6.44"}, {"inner_ip": "xxx", "outer_ip": "xxx"}] + # endpoint: { + # "inner_ip_infos": [{"ip": "", "bk_host_id": x], + # "outer_ip_infos": [{"ip": "", "bk_host_id": x] + # } + + def transform( + self, legacy_endpoints: typing.List[typing.Dict[str, typing.Any]] + ) -> typing.Dict[str, typing.List[typing.Dict[str, typing.Union[str, int]]]]: + endpoints = defaultdict(list) + if not isinstance(legacy_endpoints, list): + raise ValueError("legacy_endpoints must be list") + for legacy_endpoint in legacy_endpoints: + inner_ip = legacy_endpoint.get("inner_ip") or legacy_endpoint.get("inner_ipv6") + outer_ip = legacy_endpoint.get("outer_ip") or legacy_endpoint.get("outer_ipv6") + bk_host_id = legacy_endpoint.get("bk_host_id") + if inner_ip: + endpoints["inner_ip_infos"].append( + { + "bk_host_id": bk_host_id, + "ip": legacy_endpoint.get("inner_ip") or legacy_endpoint.get("inner_ipv6"), + } + ) + if outer_ip: + endpoints["outer_ip_infos"].append( + { + "ip": outer_ip, + "bk_host_id": bk_host_id, + } + ) + # 把 endpoints 的 values 包括的字典中的空字段去掉 并且去重 + unique_endpoints = defaultdict(list) + seen = set() + for endpoint_type in endpoints: + for endpoint in endpoints[endpoint_type]: + hash_value = _count_md5(json.dumps(endpoint)) + if hash_value not in seen: + unique_endpoints[endpoint_type].append(filter_values(endpoint)) + seen.add(hash_value) + logger.info(f"unique_endpoints: {unique_endpoints}, endpoints: {endpoints}") + return dict(unique_endpoints) + + def transform_endpoint_to_legacy( + self, + endpoints: typing.Dict[str, typing.List[typing.Dict[str, typing.Union[str, int]]]], + ) -> typing.List[typing.Dict[str, typing.Any]]: + legacy_endpoints = [] + if not isinstance(endpoints, dict): + raise ValueError("endpoints must be dict") + + # endpoints 的 key 必须为这两个值，无序的 ["inner_ip_infos", "outer_ip_infos"]: + legacy_endpoints = [] + for endpoint_type in endpoints: + if endpoint_type not in ["inner_ip_infos", "outer_ip_infos"]: + raise ValueError("endpoints key must be in inner_ip_infos or outer_ip_infos") + if not isinstance(endpoints[endpoint_type], list): + raise ValueError("endpoints value must be list") + + # 这里保留之前的为空的数据默认字段 inner_ip & outer_ip 的值为空字符串, 其他字段过滤掉 + legacy_endpoints = [ + { + "inner_ip": inner_info.get("ip") if is_v4(inner_info.get("ip")) else "", + "outer_ip": outer_info.get("ip") if is_v4(outer_info.get("ip")) else "", + "inner_ipv6": inner_info.get("ip") if is_v6(inner_info.get("ip")) else None, + "outer_ipv6": outer_info.get("ip") if is_v6(outer_info.get("ip")) else None, + "bk_host_id": inner_info.get("bk_host_id") or outer_info.get("bk_host_id"), + } + for inner_info in endpoints.get("inner_ip_infos", []) + for outer_info in endpoints.get("outer_ip_infos", []) + ] + + # 把 legacy_endpoints 通过字典的 md5 去重 + seen = set() + unique_endpoints = [] + for endpoint in legacy_endpoints: + hash_value = _count_md5(json.dumps(endpoint)) + if hash_value not in seen: + unique_endpoints.append(filter_values(endpoint)) + seen.add(hash_value) + logger.info(f"unique_endpoints: {unique_endpoints}, legacy_endpoints: {legacy_endpoints}") + + return unique_endpoints diff --git a/apps/node_man/views/ap.py b/apps/node_man/views/ap.py index dca9ca9f4..71057c4d4 100644 --- a/apps/node_man/views/ap.py +++ b/apps/node_man/views/ap.py @@ -57,12 +57,18 @@ def list(self, request, *args, **kwargs): ] "zk_user": "username", "zk_password": "zk_password", - "servers": [ - { - "inner_ip": "127.0.0.1", - "outer_ip": "127.0.0.2" - } - ], + "btfileserver": { + "inner_ips": [{"inner_ip": "127.0.0.1"}], + "outer_ips": [{"outer_ip": "127.0.0.2"}] + }, + "taskserver": { + "inner_ips": [{"inner_ip": "127.0.0.1"}], + "outer_ips": [{"outer_ip": "127.0.0.2"}] + }, + "dataserver": { + "inner_ips": [{"inner_ip": "127.0.0.1"}], + "outer_ips": [{"outer_ip": "127.0.0.2"}] + }, "package_inner_url": "http://127.0.0.1/download/", "package_outer_url": "http://127.0.0.2/download/", "agent_config": { @@ -125,12 +131,18 @@ def retrieve(self, request, *args, **kwargs): ] "zk_user": "username", "zk_password": "zk_password", - "servers": [ - { - "inner_ip": "127.0.0.1", - "outer_ip": "127.0.0.2" - } - ], + "btfileserver": { + "inner_ips": [{"inner_ip": "127.0.0.1"}], + "outer_ips": [{"outer_ip": "127.0.0.2"}] + }, + "taskserver": { + "inner_ips": [{"inner_ip": "127.0.0.1"}], + "outer_ips": [{"outer_ip": "127.0.0.2"}] + }, + "dataserver": { + "inner_ips": [{"inner_ip": "127.0.0.1"}], + "outer_ips": [{"outer_ip": "127.0.0.2"}] + }, "package_inner_url": "http://127.0.0.1/download/", "package_outer_url": "http://127.0.0.2/download/", "agent_config": { @@ -306,12 +318,18 @@ def update(self, request, *args, **kwargs): @apiParamExample {Json} 请求参数 { "name": "接入点名称", - "servers": [ - { - "inner_ip": "127.0.0.1", - "outer_ip": "127.0.0.2" - } - ], + "btfileserver": { + "inner_ip_infos": [{"ip": "127.0.0.1"}], + "outer_ip_infos": [{"ip": "127.0.0.2"}] + }, + "taskserver": { + "inner_ip_infos": [{"ip": "127.0.0.1"}], + "outer_ip_infos": [{"ip": "127.0.0.2"}] + }, + "dataserver": { + "inner_ip_infos": [{"ip": "127.0.0.1"}], + "outer_ip_infos": [{"ip": "127.0.0.2"}] + }, "package_inner_url": "http://127.0.0.1/download/", "package_outer_url": "http://127.0.0.2/download/", "agent_config": { @@ -407,12 +425,18 @@ def test(self, request, *args, **kwargs): @apiParam {String} package_outer_url 安装包外网地址 @apiParamExample {Json} 请求参数 { - "servers": [ - { - "inner_ip": "127.0.0.1", - "outer_ip": "127.0.0.2" - } - ], + "btfileserver": { + "inner_ip_infos": [{"ip": "127.0.0.1"}], + "outer_ip_infos": [{"ip": "127.0.0.2"}] + }, + "taskserver": { + "inner_ip_infos": [{"ip": "127.0.0.1"}], + "outer_ip_infos": [{"ip": "127.0.0.2"}] + }, + "dataserver": { + "inner_ip_infos": [{"ip": "127.0.0.1"}], + "outer_ip_infos": [{"ip": "127.0.0.2"}] + }, "package_inner_url": "http://127.0.0.1/download/", "package_outer_url": "http://127.0.0.2/download/" }