From bf8fb55ba61b9a41d1b677c33daf72ed130850f3 Mon Sep 17 00:00:00 2001
From: Wang Shilong <shilong.wang@intel.com>
Date: Mon, 6 May 2024 10:35:12 +0800
Subject: [PATCH] DAOS-15076 engine: Restricting Heap Usage to Sorted Resent
 RPCs (#14268)

In certain rare cases, RPC throttling may occur.
The initial motivation behind this change is to utilize the heap
exclusively for sorted resent RPCs and the FIFO list for regular
fetch and update requests. This strategic allocation aims to avoid
potential performance impacts that could result from maintaining a heap
in the critical hot path.

Signed-off-by: Wang Shilong <shilong.wang@intel.com>
---
 src/engine/sched.c                 | 15 +++++++++++----
 src/include/daos_srv/daos_engine.h |  1 +
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/src/engine/sched.c b/src/engine/sched.c
index 763bc0b64be..007b617cbe8 100644
--- a/src/engine/sched.c
+++ b/src/engine/sched.c
@@ -1,5 +1,5 @@
 /**
- * (C) Copyright 2016-2023 Intel Corporation.
+ * (C) Copyright 2016-2024 Intel Corporation.
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -1188,11 +1188,16 @@ policy_fifo_enqueue(struct dss_xstream *dx, struct sched_request *req,
 
 	D_ASSERT(attr->sra_type < SCHED_REQ_TYPE_MAX);
 	/*
-	 * If @sra_enqueue_id is not zero, this is a resent RPC, it will
-	 * be inserted to sorted heap instead of fifo list.
+	 * The initial motivation behind this change is to utilize the heap
+	 * exclusively for sorted resent RPCs and the FIFO list for regular
+	 * fetch and update requests. This strategic allocation aims to avoid
+	 * potential performance impacts that could result from maintaining a
+	 * heap in the critical hot path.
 	 */
-	if (attr->sra_enqueue_id)
+	if (attr->sra_flags & SCHED_REQ_FL_RESENT) {
+		D_ASSERT(attr->sra_enqueue_id > 0);
 		return d_binheap_insert(&info->si_heap, &req->sr_node);
+	}
 
 	d_list_add_tail(&req->sr_link, &info->si_fifo_list);
 
@@ -1389,6 +1394,8 @@ sched_req_enqueue(struct dss_xstream *dx, struct sched_req_attr *attr,
 
 	if (attr->sra_enqueue_id == 0)
 		attr->sra_enqueue_id = ++info->si_cur_id;
+	else
+		attr->sra_flags |= SCHED_REQ_FL_RESENT;
 
 	/*
 	 * A RPC flow control mechanism is introduced to avoid RPC timeout when the
diff --git a/src/include/daos_srv/daos_engine.h b/src/include/daos_srv/daos_engine.h
index b3a82808b3b..4f988fe0da9 100644
--- a/src/include/daos_srv/daos_engine.h
+++ b/src/include/daos_srv/daos_engine.h
@@ -156,6 +156,7 @@ enum {
 	SCHED_REQ_FL_NO_DELAY	= (1 << 0),
 	SCHED_REQ_FL_PERIODIC	= (1 << 1),
 	SCHED_REQ_FL_NO_REJECT	= (1 << 2),
+	SCHED_REQ_FL_RESENT	= (1 << 3),
 };
 
 struct sched_req_attr {