Skip to content

Commit

Permalink
perf: 获取agent状态做分页获取优化 #7440
Browse files Browse the repository at this point in the history
  • Loading branch information
@lTimej
lTimej committed May 23, 2024
1 parent 5f1e50b commit 9080536
Show file tree
Hide file tree
Showing 3 changed files with 52 additions and 42 deletions.
37 changes: 16 additions & 21 deletions pipeline_plugins/cmdb_ip_picker/query.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,11 @@
from gcloud.utils import cmdb
from gcloud.utils.handlers import handle_api_error
from gcloud.utils.ip import format_sundry_ip
from pipeline_plugins.components.utils.common import batch_execute_func

from .constants import ERROR_CODES, NO_ERROR
from .utils import (
agent_params_pagination,
format_agent_data,
get_cmdb_topo_tree,
get_gse_agent_status_ipv6,
Expand Down Expand Up @@ -162,27 +164,20 @@ def cmdb_search_host(request, bk_biz_id, bk_supplier_account="", bk_supplier_id=
host["agent"] = agent_id_status_map.get(bk_agent_id, -1)
else:
client = BKNodeManClient(username=request.user.username)
agent_kwargs = {
"all_scope": True,
"host_list": [
{
"cloud_id": host["bk_cloud_id"],
"ip": host["bk_host_innerip"],
"meta": {"bk_biz_id": bk_biz_id, "scope_type": "biz", "scope_id": bk_biz_id},
}
for host in data
if host["bk_host_innerip"] != ""
],
}
agent_result = client.get_ipchooser_host_details(agent_kwargs)
if not agent_result["result"]:
message = handle_api_error(
_("节点管理(nodeman)"), "nodeman.get_ipchooser_host_details", agent_kwargs, agent_result
)
result = {"result": False, "code": ERROR_CODES.API_GSE_ERROR, "message": message}
return JsonResponse(result)

agent_data = format_agent_data(agent_result["data"])
agent_kwargs = agent_params_pagination(data, bk_biz_id)
results = batch_execute_func(client.get_ipchooser_host_details, agent_kwargs, interval_enabled=True)
agent_data = []
for result in results:
agent_result = result["result"]
if not agent_result["result"]:
message = handle_api_error(
_("节点管理(nodeman)"), "nodeman.get_ipchooser_host_details", agent_kwargs, agent_result
)
result = {"result": False, "code": ERROR_CODES.API_GSE_ERROR, "message": message}
return JsonResponse(result)
agent_data.extend(agent_result["data"])

agent_data = format_agent_data(agent_data)
for host in data:
# agent在线状态，0为不在线，1为在线，-1为未知
agent_info = agent_data.get(
Expand Down
22 changes: 22 additions & 0 deletions pipeline_plugins/cmdb_ip_picker/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
specific language governing permissions and limitations under the License.
"""
import logging
import math
import re

import requests
Expand Down Expand Up @@ -774,3 +775,24 @@ def format_agent_data(agents):
val = {"ip": agent["ip"], "bk_cloud_id": agent["cloud_area"]["id"], "bk_agent_alive": agent["alive"]}
agent_data[key] = val
return agent_data


def agent_params_pagination(data, bk_biz_id):
params_list = []
page_size = 1000
page = math.ceil(len(data) / page_size)
for i in range(page):
agent_kwargs = {
"all_scope": True,
"host_list": [
{
"cloud_id": host["bk_cloud_id"],
"ip": host["bk_host_innerip"],
"meta": {"bk_biz_id": bk_biz_id, "scope_type": "biz", "scope_id": bk_biz_id},
}
for host in data[i * page_size : (i + 1) * page_size]
if host.get("bk_host_innerip", "") != ""
],
}
params_list.append(agent_kwargs)
return params_list
35 changes: 14 additions & 21 deletions pipeline_plugins/variables/collections/sites/open/ip_filter_base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,9 @@
from gcloud.utils.handlers import handle_api_error
from gcloud.utils.ip import IpRegexType, extract_ip_from_ip_str, get_ip_by_regex_type
from pipeline_plugins.base.utils.inject import supplier_account_for_business, supplier_id_for_project
from pipeline_plugins.cmdb_ip_picker.utils import format_agent_data, get_gse_agent_status_ipv6
from pipeline_plugins.cmdb_ip_picker.utils import agent_params_pagination, format_agent_data, get_gse_agent_status_ipv6
from pipeline_plugins.components.collections.sites.open.cc.base import cc_get_host_by_innerip_with_ipv6
from pipeline_plugins.components.utils.common import batch_execute_func

logger = logging.getLogger("root")
get_client_by_user = gcloud_settings.ESB_GET_CLIENT_BY_USER
Expand Down Expand Up @@ -98,26 +99,18 @@ def match_gse_v1(self, gse_agent_status, username, bk_biz_id, bk_supplier_id, or
match_ip = origin_ip_list
if gse_agent_status in [GseAgentStatus.ONlINE.value, GseAgentStatus.OFFLINE.value]:
client = get_nodeman_client_by_user(username=username)
agent_kwargs = {
"all_scope": True,
"host_list": [
{
"cloud_id": host["bk_cloud_id"],
"ip": host["ip"],
"meta": {"bk_biz_id": bk_biz_id, "scope_type": "biz", "scope_id": bk_biz_id},
}
for host in origin_ip_list
if host["ip"] != ""
],
}
agent_result = client.get_ipchooser_host_details(agent_kwargs)

if not agent_result["result"]:
message = handle_api_error(
_("节点管理(nodeman)"), "nodeman.get_ipchooser_host_details", agent_kwargs, agent_result
)
raise ApiRequestError(f"ERROR:{message}")
agent_data = format_agent_data(agent_result["data"])
agent_kwargs = agent_params_pagination(origin_ip_list, bk_biz_id)
results = batch_execute_func(client.get_ipchooser_host_details, agent_kwargs, interval_enabled=True)
agent_data = []
for result in results:
agent_result = result["result"]
if not agent_result["result"]:
message = handle_api_error(
_("节点管理(nodeman)"), "nodeman.get_ipchooser_host_details", agent_kwargs, agent_result
)
raise ApiRequestError(f"ERROR:{message}")
agent_data.extend(agent_result["data"])
agent_data = format_agent_data(agent_data)
agent_online_ip_list = []
agent_offline_ip_list = []
for plat_ip, info in agent_data.items():
Expand Down

0 comments on commit 9080536

Please sign in to comment.