From 908053653e8b930d4202d1335da6a3b251b454c6 Mon Sep 17 00:00:00 2001 From: liujun Date: Thu, 23 May 2024 16:03:43 +0800 Subject: [PATCH] =?UTF-8?q?perf:=20=E8=8E=B7=E5=8F=96agent=E7=8A=B6?= =?UTF-8?q?=E6=80=81=E5=81=9A=E5=88=86=E9=A1=B5=E8=8E=B7=E5=8F=96=E4=BC=98?= =?UTF-8?q?=E5=8C=96=20#7440?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pipeline_plugins/cmdb_ip_picker/query.py | 37 ++++++++----------- pipeline_plugins/cmdb_ip_picker/utils.py | 22 +++++++++++ .../collections/sites/open/ip_filter_base.py | 35 +++++++----------- 3 files changed, 52 insertions(+), 42 deletions(-) diff --git a/pipeline_plugins/cmdb_ip_picker/query.py b/pipeline_plugins/cmdb_ip_picker/query.py index 666d776a2d..9438601021 100644 --- a/pipeline_plugins/cmdb_ip_picker/query.py +++ b/pipeline_plugins/cmdb_ip_picker/query.py @@ -25,9 +25,11 @@ from gcloud.utils import cmdb from gcloud.utils.handlers import handle_api_error from gcloud.utils.ip import format_sundry_ip +from pipeline_plugins.components.utils.common import batch_execute_func from .constants import ERROR_CODES, NO_ERROR from .utils import ( + agent_params_pagination, format_agent_data, get_cmdb_topo_tree, get_gse_agent_status_ipv6, @@ -162,27 +164,20 @@ def cmdb_search_host(request, bk_biz_id, bk_supplier_account="", bk_supplier_id= host["agent"] = agent_id_status_map.get(bk_agent_id, -1) else: client = BKNodeManClient(username=request.user.username) - agent_kwargs = { - "all_scope": True, - "host_list": [ - { - "cloud_id": host["bk_cloud_id"], - "ip": host["bk_host_innerip"], - "meta": {"bk_biz_id": bk_biz_id, "scope_type": "biz", "scope_id": bk_biz_id}, - } - for host in data - if host["bk_host_innerip"] != "" - ], - } - agent_result = client.get_ipchooser_host_details(agent_kwargs) - if not agent_result["result"]: - message = handle_api_error( - _("节点管理(nodeman)"), "nodeman.get_ipchooser_host_details", agent_kwargs, agent_result - ) - result = {"result": False, "code": ERROR_CODES.API_GSE_ERROR, "message": message} - return JsonResponse(result) - - agent_data = format_agent_data(agent_result["data"]) + agent_kwargs = agent_params_pagination(data, bk_biz_id) + results = batch_execute_func(client.get_ipchooser_host_details, agent_kwargs, interval_enabled=True) + agent_data = [] + for result in results: + agent_result = result["result"] + if not agent_result["result"]: + message = handle_api_error( + _("节点管理(nodeman)"), "nodeman.get_ipchooser_host_details", agent_kwargs, agent_result + ) + result = {"result": False, "code": ERROR_CODES.API_GSE_ERROR, "message": message} + return JsonResponse(result) + agent_data.extend(agent_result["data"]) + + agent_data = format_agent_data(agent_data) for host in data: # agent在线状态，0为不在线，1为在线，-1为未知 agent_info = agent_data.get( diff --git a/pipeline_plugins/cmdb_ip_picker/utils.py b/pipeline_plugins/cmdb_ip_picker/utils.py index 33bbd463e7..c361355268 100644 --- a/pipeline_plugins/cmdb_ip_picker/utils.py +++ b/pipeline_plugins/cmdb_ip_picker/utils.py @@ -11,6 +11,7 @@ specific language governing permissions and limitations under the License. """ import logging +import math import re import requests @@ -774,3 +775,24 @@ def format_agent_data(agents): val = {"ip": agent["ip"], "bk_cloud_id": agent["cloud_area"]["id"], "bk_agent_alive": agent["alive"]} agent_data[key] = val return agent_data + + +def agent_params_pagination(data, bk_biz_id): + params_list = [] + page_size = 1000 + page = math.ceil(len(data) / page_size) + for i in range(page): + agent_kwargs = { + "all_scope": True, + "host_list": [ + { + "cloud_id": host["bk_cloud_id"], + "ip": host["bk_host_innerip"], + "meta": {"bk_biz_id": bk_biz_id, "scope_type": "biz", "scope_id": bk_biz_id}, + } + for host in data[i * page_size : (i + 1) * page_size] + if host.get("bk_host_innerip", "") != "" + ], + } + params_list.append(agent_kwargs) + return params_list diff --git a/pipeline_plugins/variables/collections/sites/open/ip_filter_base.py b/pipeline_plugins/variables/collections/sites/open/ip_filter_base.py index 83cbd96e39..33facca9ef 100644 --- a/pipeline_plugins/variables/collections/sites/open/ip_filter_base.py +++ b/pipeline_plugins/variables/collections/sites/open/ip_filter_base.py @@ -25,8 +25,9 @@ from gcloud.utils.handlers import handle_api_error from gcloud.utils.ip import IpRegexType, extract_ip_from_ip_str, get_ip_by_regex_type from pipeline_plugins.base.utils.inject import supplier_account_for_business, supplier_id_for_project -from pipeline_plugins.cmdb_ip_picker.utils import format_agent_data, get_gse_agent_status_ipv6 +from pipeline_plugins.cmdb_ip_picker.utils import agent_params_pagination, format_agent_data, get_gse_agent_status_ipv6 from pipeline_plugins.components.collections.sites.open.cc.base import cc_get_host_by_innerip_with_ipv6 +from pipeline_plugins.components.utils.common import batch_execute_func logger = logging.getLogger("root") get_client_by_user = gcloud_settings.ESB_GET_CLIENT_BY_USER @@ -98,26 +99,18 @@ def match_gse_v1(self, gse_agent_status, username, bk_biz_id, bk_supplier_id, or match_ip = origin_ip_list if gse_agent_status in [GseAgentStatus.ONlINE.value, GseAgentStatus.OFFLINE.value]: client = get_nodeman_client_by_user(username=username) - agent_kwargs = { - "all_scope": True, - "host_list": [ - { - "cloud_id": host["bk_cloud_id"], - "ip": host["ip"], - "meta": {"bk_biz_id": bk_biz_id, "scope_type": "biz", "scope_id": bk_biz_id}, - } - for host in origin_ip_list - if host["ip"] != "" - ], - } - agent_result = client.get_ipchooser_host_details(agent_kwargs) - - if not agent_result["result"]: - message = handle_api_error( - _("节点管理(nodeman)"), "nodeman.get_ipchooser_host_details", agent_kwargs, agent_result - ) - raise ApiRequestError(f"ERROR:{message}") - agent_data = format_agent_data(agent_result["data"]) + agent_kwargs = agent_params_pagination(origin_ip_list, bk_biz_id) + results = batch_execute_func(client.get_ipchooser_host_details, agent_kwargs, interval_enabled=True) + agent_data = [] + for result in results: + agent_result = result["result"] + if not agent_result["result"]: + message = handle_api_error( + _("节点管理(nodeman)"), "nodeman.get_ipchooser_host_details", agent_kwargs, agent_result + ) + raise ApiRequestError(f"ERROR:{message}") + agent_data.extend(agent_result["data"]) + agent_data = format_agent_data(agent_data) agent_online_ip_list = [] agent_offline_ip_list = [] for plat_ip, info in agent_data.items():