[apache#1924] feat(dashboard): Show Thread Dump, Conf and Metrics in …

…DashBoard (apache#1927)

### What changes were proposed in this pull request?

Add jetty_port in message ShuffleServerId.
Modify dashboard, add some link in dashboard
<img width="1086" alt="企业微信截图_ede78628-56ac-4b7e-86ab-84b224da7ce4" src="https://github.com/user-attachments/assets/a263871f-0a9a-4b1d-9ba6-0341708e0a2d">
<img width="1775" alt="企业微信截图_88b2c855-68e4-4054-94a3-d730f074a4b9" src="https://github.com/user-attachments/assets/480352d6-bf29-48bc-af73-a41fa1fd8248">

### Why are the changes needed?

Enhance dashboard capabilities

Fix: apache#1924

### Does this PR introduce _any_ user-facing change?

No.

common/src/main/java/org/apache/uniffle/common/util/ThreadUtils.java

@@ -17,6 +17,9 @@
 
 package org.apache.uniffle.common.util;
 
+import java.lang.management.ManagementFactory;
+import java.lang.management.ThreadInfo;
+import java.lang.management.ThreadMXBean;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
@@ -39,6 +42,7 @@
 
 public class ThreadUtils {
  private static final Logger LOGGER = LoggerFactory.getLogger(ThreadUtils.class);
+ private static final ThreadMXBean THREAD_BEAN = ManagementFactory.getThreadMXBean();
 
  /** Provide a general method to create a thread factory to make the code more standardized */
  public static ThreadFactory getThreadFactory(String factoryName) {
@@ -183,4 +187,18 @@ public static <T, R> List<R> executeTasks(
  String taskMsg) {
  return executeTasks(executorService, items, task, timeoutMs, taskMsg, future -> null);
  }
+
+ public static synchronized void printThreadInfo(StringBuilder builder, String title) {
+ builder.append("Process Thread Dump: " + title + "\n");
+ builder.append(THREAD_BEAN.getThreadCount() + " active threads\n");
+ long[] threadIds = THREAD_BEAN.getAllThreadIds();
+ for (long id : threadIds) {
+ ThreadInfo info = THREAD_BEAN.getThreadInfo(id, Integer.MAX_VALUE);
+ if (info == null) {
+ // The thread is no longer active, ignore
+ continue;
+ }
+ builder.append(info + "\n");
+ }
+ }
 }

common/src/main/java/org/apache/uniffle/common/web/JettyServer.java

@@ -175,4 +175,8 @@ public void start() throws Exception {
  public void stop() throws Exception {
  server.stop();
  }
+
+ public int getHttpPort() {
+ return httpPort;
+ }
 }

coordinator/src/main/java/org/apache/uniffle/coordinator/CoordinatorGrpcService.java

@@ -433,6 +433,7 @@ private ServerNode toServerNode(ShuffleServerHeartBeatRequest request) {
  Sets.newHashSet(request.getTagsList()),
  serverStatus,
  StorageInfoUtils.fromProto(request.getStorageInfoMap()),
- request.getServerId().getNettyPort());
+ request.getServerId().getNettyPort(),
+ request.getServerId().getJettyPort());
  }
 }

coordinator/src/main/java/org/apache/uniffle/coordinator/ServerNode.java

@@ -42,6 +42,7 @@ public class ServerNode implements Comparable<ServerNode> {
  private ServerStatus status;
  private Map<String, StorageInfo> storageInfo;
  private int nettyPort = -1;
+ private int jettyPort = -1;
 
  public ServerNode(String id) {
  this(id, "", 0, 0, 0, 0, 0, Sets.newHashSet(), ServerStatus.EXCLUDED);
@@ -115,6 +116,7 @@ public ServerNode(
  tags,
  status,
  storageInfoMap,
+ -1,
  -1);
  }
 
@@ -130,6 +132,34 @@ public ServerNode(
  ServerStatus status,
  Map<String, StorageInfo> storageInfoMap,
  int nettyPort) {
+ this(
+ id,
+ ip,
+ grpcPort,
+ usedMemory,
+ preAllocatedMemory,
+ availableMemory,
+ eventNumInFlush,
+ tags,
+ status,
+ storageInfoMap,
+ nettyPort,
+ -1);
+ }
+
+ public ServerNode(
+ String id,
+ String ip,
+ int grpcPort,
+ long usedMemory,
+ long preAllocatedMemory,
+ long availableMemory,
+ int eventNumInFlush,
+ Set<String> tags,
+ ServerStatus status,
+ Map<String, StorageInfo> storageInfoMap,
+ int nettyPort,
+ int jettyPort) {
  this.id = id;
  this.ip = ip;
  this.grpcPort = grpcPort;
@@ -145,6 +175,9 @@ public ServerNode(
  if (nettyPort > 0) {
  this.nettyPort = nettyPort;
  }
+ if (jettyPort > 0) {
+ this.jettyPort = jettyPort;
+ }
  }
 
  public ShuffleServerId convertToGrpcProto() {
@@ -153,6 +186,7 @@ public ShuffleServerId convertToGrpcProto() {
  .setIp(ip)
  .setPort(grpcPort)
  .setNettyPort(nettyPort)
+ .setJettyPort(jettyPort)
  .build();
  }
 
@@ -214,6 +248,8 @@ public String toString() {
  + grpcPort
  + "], netty port["
  + nettyPort
+ + "], jettyPort["
+ + jettyPort
  + "], usedMemory["
  + usedMemory
  + "], preAllocatedMemory["
@@ -277,4 +313,8 @@ public long getTotalMemory() {
  public int getNettyPort() {
  return nettyPort;
  }
+
+ public int getJettyPort() {
+ return jettyPort;
+ }
 }

coordinator/src/main/java/org/apache/uniffle/coordinator/web/resource/CoordinatorServerResource.java

@@ -31,7 +31,10 @@
 import org.apache.hbase.thirdparty.javax.ws.rs.core.MediaType;
 
 import org.apache.uniffle.common.util.RssUtils;
+import org.apache.uniffle.common.util.ThreadUtils;
 import org.apache.uniffle.common.web.resource.BaseResource;
+import org.apache.uniffle.common.web.resource.MetricResource;
+import org.apache.uniffle.common.web.resource.PrometheusMetricResource;
 import org.apache.uniffle.common.web.resource.Response;
 import org.apache.uniffle.coordinator.CoordinatorConf;
 import org.apache.uniffle.coordinator.CoordinatorServer;
@@ -85,4 +88,22 @@ private CoordinatorServer getCoordinatorServer() {
  return (CoordinatorServer)
  servletContext.getAttribute(CoordinatorServer.class.getCanonicalName());
  }
+
+ @Path("/metrics")
+ public Class<MetricResource> getMetricResource() {
+ return MetricResource.class;
+ }
+
+ @Path("/prometheus/metrics")
+ public Class<PrometheusMetricResource> getPrometheusMetricResource() {
+ return PrometheusMetricResource.class;
+ }
+
+ @GET
+ @Path("/stacks")
+ public String getCoordinatorStacks() {
+ StringBuilder builder = new StringBuilder();
+ ThreadUtils.printThreadInfo(builder, "");
+ return builder.toString();
+ }
 }

dashboard/src/main/java/org/apache/uniffle/dashboard/web/proxy/WebProxyServlet.java

@@ -45,11 +45,15 @@ protected String rewriteTarget(HttpServletRequest clientRequest) {
  if (!validateDestination(clientRequest.getServerName(), clientRequest.getServerPort())) {
  return null;
  }
- String targetAddress =
- coordinatorServerAddressesMap.get(clientRequest.getHeader("targetAddress"));
- if (targetAddress == null) {
- // Get random one from coordinatorServerAddressesMap
- targetAddress = coordinatorServerAddressesMap.values().iterator().next();
+ String targetAddress;
+ if (clientRequest.getHeader("serverType").equals("coordinator")) {
+ targetAddress = coordinatorServerAddressesMap.get(clientRequest.getHeader("targetAddress"));
+ if (targetAddress == null) {
+ // Get random one from coordinatorServerAddressesMap
+ targetAddress = coordinatorServerAddressesMap.values().iterator().next();
+ }
+ } else {
+ targetAddress = clientRequest.getHeader("targetAddress");
  }
  StringBuilder target = new StringBuilder();
  if (targetAddress.endsWith("/")) {

dashboard/src/main/webapp/src/api/api.js

@@ -26,6 +26,84 @@ export function getCoordinatorConf(params, headers) {
  return http.get('/coordinator/conf', params, headers, 0)
 }
 
+export async function getShuffleServerConf(address, params, headers) {
+ if (typeof headers === 'undefined') {
+ headers = {};
+ }
+ headers.targetAddress = address;
+ const response = await http.get('/shuffleServer/conf', params, headers, 0);
+ const newWindow = window.open('', '_blank');
+ let tableHTML = `
+ <style>
+ table {
+ width: 100%;
+ }
+ th, td {
+ padding: 0 20px;
+ text-align: left;
+ }
+ </style>
+ <table>
+ <tr>
+ <th>Key</th>
+ <th>Value</th>
+ </tr>
+ `;
+ for (const item of response.data.data) {
+ tableHTML += `<tr><td>${item.argumentKey}</td><td>${item.argumentValue}</td></tr>`;
+ }
+ tableHTML += '</table>';
+ newWindow.document.write(tableHTML);
+}
+
+export async function getCoordinatorMetrics(params, headers) {
+ const response = await http.get('/coordinator/metrics', params, headers, 0)
+ const newWindow = window.open('', '_blank');
+ newWindow.document.write('<pre>' + JSON.stringify(response.data, null, 2) + '</pre>');
+}
+
+export async function getShuffleServerMetrics(address, params, headers) {
+ if (typeof headers === 'undefined') {
+ headers = {}
+ }
+ headers.targetAddress = address
+ const response = await http.get('/shuffleServer/metrics', params, headers, 0)
+ const newWindow = window.open('', '_blank');
+ newWindow.document.write('<pre>' + JSON.stringify(response.data, null, 2) + '</pre>');
+}
+
+export async function getCoordinatorPrometheusMetrics(params, headers) {
+ const response = await http.get('/coordinator/prometheus/metrics/all', params, headers, 0)
+ const newWindow = window.open('', '_blank');
+ newWindow.document.write('<pre>' + response.data + '</pre>');
+}
+
+export async function getShuffleServerPrometheusMetrics(address, params, headers) {
+ if (typeof headers === 'undefined') {
+ headers = {}
+ }
+ headers.targetAddress = address
+ const response = await http.get('/shuffleServer/prometheus/metrics/all', params, headers, 0)
+ const newWindow = window.open('', '_blank');
+ newWindow.document.write('<pre>' + response.data + '</pre>');
+}
+
+export async function getCoordinatorStacks(params, headers) {
+ const response = await http.get('/coordinator/stacks', params, headers, 0)
+ const newWindow = window.open('', '_blank');
+ newWindow.document.write('<pre>' + response.data + '</pre>');
+}
+
+export async function getShuffleServerStacks(address, params, headers) {
+ if (typeof headers === 'undefined') {
+ headers = {}
+ }
+ headers.targetAddress = address
+ const response = await http.get('/shuffleServer/stacks', params, headers, 0)
+ const newWindow = window.open('', '_blank');
+ newWindow.document.write('<pre>' + response.data + '</pre>');
+}
+
 // Create an interface for the total number of nodes
 export function getShufflegetStatusTotal(params, headers) {
  return http.get('/server/nodes/summary', params, headers, 0)

dashboard/src/main/webapp/src/pages/CoordinatorServerPage.vue

@@ -74,19 +74,50 @@
  <el-table-column prop="argumentValue" label="Value" min-width="380" />
  </el-table>
  </el-collapse-item>
+ <el-collapse-item title="Coordinator Metrics" name="3">
+ <el-link @click="getCoordinatorMetrics" target="_blank">
+ <el-icon :style="iconStyle">
+ <Link />
+ </el-icon>
+ metrics
+ </el-link>
+ </el-collapse-item>
+ <el-collapse-item title="Coordinator Prometheus Metrics" name="4">
+ <el-link @click="getCoordinatorPrometheusMetrics" target="_blank">
+ <el-icon :style="iconStyle">
+ <Link />
+ </el-icon>
+ prometheus metrics
+ </el-link>
+ </el-collapse-item>
+ <el-collapse-item title="Coordinator Stacks" name="5">
+ <el-link @click="getCoordinatorStacks" target="_blank">
+ <el-icon :style="iconStyle">
+ <Link />
+ </el-icon>
+ stacks
+ </el-link>
+ </el-collapse-item>
  </el-collapse>
  </div>
 </template>
 
 <script>
 import { ref, reactive, computed, onMounted } from 'vue'
-import { getCoordinatorConf, getCoordinatorServerInfo } from '@/api/api'
+import {
+ getCoordinatorConf,
+ getCoordinatorMetrics,
+ getCoordinatorPrometheusMetrics,
+ getCoordinatorServerInfo,
+ getCoordinatorStacks
+} from '@/api/api'
 import { useCurrentServerStore } from '@/store/useCurrentServerStore'
 
 export default {
+ methods: {getCoordinatorMetrics, getCoordinatorPrometheusMetrics, getCoordinatorStacks},
  setup() {
  const pageData = reactive({
- activeNames: ['1', '2'],
+ activeNames: ['1', '2', '3', '4', '5'],
  tableData: [],
  serverInfo: {}
  })