Track application restart generations

CHANGELOG

@@ -1,6 +1,7 @@
 Release 6.0.24 (Not yet released)
 -------------
- *
+ * Update the order Passenger routes requests to app processes. Processes are now chosen based on being in the latest generation (Enterprise), then by newest process, then by oldest, then by
+busyness. Closes GH-2551.
 
 
 Release 6.0.23

src/agent/Core/ApplicationPool/Group.h

@@ -223,9 +223,9 @@ class Group: public boost::enable_shared_from_this<Group> {
  /****** Process list management ******/
 
  Process *findProcessWithStickySessionId(unsigned int id) const;
- Process *findProcessWithStickySessionIdOrLowestBusyness(unsigned int id) const;
- Process *findProcessWithLowestBusyness(const ProcessList &processes) const;
- Process *findEnabledProcessWithLowestBusyness() const;
+ Process *findBestProcessPreferringStickySessionId(unsigned int id) const;
+ Process *findBestProcess(const ProcessList &processes) const;
+ Process *findBestEnabledProcess() const;
 
  void addProcessToList(const ProcessPtr &process, ProcessList &destination);
  void removeProcessFromList(const ProcessPtr &process, ProcessList &source);

src/agent/Core/ApplicationPool/Group/InternalUtils.cpp

@@ -178,7 +178,7 @@ Group::createNullProcessObject() {
  LockGuard l(context->memoryManagementSyncher);
  Process *process = context->processObjectPool.malloc();
  Guard guard(context, process);
- process = new (process) Process(&info, args);
+ process = new (process) Process(&info, info.group->restartsInitiated, args);
  process->shutdownNotRequired();
  guard.clear();
  return ProcessPtr(process, false);
@@ -215,7 +215,7 @@ Group::createProcessObject(const SpawningKit::Spawner &spawner,
  LockGuard l(context->memoryManagementSyncher);
  Process *process = context->processObjectPool.malloc();
  Guard guard(context, process);
- process = new (process) Process(&info, spawnResult, args);
+ process = new (process) Process(&info, info.group->restartsInitiated, spawnResult, args);
  guard.clear();
  return ProcessPtr(process, false);
 }

src/agent/Core/ApplicationPool/Group/ProcessListManagement.cpp

@@ -58,71 +58,81 @@ Group::findProcessWithStickySessionId(unsigned int id) const {
 }
 
 Process *
-Group::findProcessWithStickySessionIdOrLowestBusyness(unsigned int id) const {
- int leastBusyProcessIndex = -1;
- int lowestBusyness = 0;
- unsigned int i, size = enabledProcessBusynessLevels.size();
- const int *enabledProcessBusynessLevels = &this->enabledProcessBusynessLevels[0];
-
- for (i = 0; i < size; i++) {
- Process *process = enabledProcesses[i].get();
+Group::findBestProcessPreferringStickySessionId(unsigned int id) const {
+ Process *bestProcess = nullptr;
+ ProcessList::const_iterator it;
+ ProcessList::const_iterator end = enabledProcesses.end();
+ for (it = enabledProcesses.begin(); it != end; it++) {
+ Process *process = (*it).get();
  if (process->getStickySessionId() == id) {
  return process;
- } else if (leastBusyProcessIndex == -1 || enabledProcessBusynessLevels[i] < lowestBusyness) {
- leastBusyProcessIndex = i;
- lowestBusyness = enabledProcessBusynessLevels[i];
+ } else if (bestProcess == nullptr ||
+ process->generation > bestProcess->generation ||
+ (process->generation == bestProcess->generation && process->spawnerCreationTime < bestProcess->spawnerCreationTime) ||
+ (process->generation == bestProcess->generation && process->spawnerCreationTime == bestProcess->spawnerCreationTime && process->busyness() < bestProcess->busyness())
+ ) {
+ bestProcess = process;
  }
  }
-
- if (leastBusyProcessIndex == -1) {
- return NULL;
- } else {
- return enabledProcesses[leastBusyProcessIndex].get();
- }
+ return bestProcess;
 }
 
 Process *
-Group::findProcessWithLowestBusyness(const ProcessList &processes) const {
+Group::findBestProcess(const ProcessList &processes) const {
  if (processes.empty()) {
- return NULL;
+ return nullptr;
  }
 
- int lowestBusyness = -1;
- Process *leastBusyProcess = NULL;
+ Process *bestProcess = nullptr;
  ProcessList::const_iterator it;
  ProcessList::const_iterator end = processes.end();
  for (it = processes.begin(); it != end; it++) {
  Process *process = (*it).get();
- int busyness = process->busyness();
- if (lowestBusyness == -1 || lowestBusyness > busyness) {
- lowestBusyness = busyness;
- leastBusyProcess = process;
+
+ if (bestProcess == nullptr ||
+ process->generation > bestProcess->generation ||
+ (process->generation == bestProcess->generation && process->spawnerCreationTime < bestProcess->spawnerCreationTime) ||
+ (process->generation == bestProcess->generation && process->spawnerCreationTime == bestProcess->spawnerCreationTime && process->busyness() < bestProcess->busyness())
+ ) {
+ bestProcess = process;
  }
  }
- return leastBusyProcess;
+ return bestProcess;
 }
 
 /**
- * Cache-optimized version of findProcessWithLowestBusyness() for the common case.
+ * Cache-optimized version of findBestProcess() for the common case.
  */
 Process *
-Group::findEnabledProcessWithLowestBusyness() const {
+Group::findBestEnabledProcess() const {
  if (enabledProcesses.empty()) {
- return NULL;
+ return nullptr;
  }
 
- int leastBusyProcessIndex = -1;
- int lowestBusyness = 0;
- unsigned int i, size = enabledProcessBusynessLevels.size();
+ Process* bestProcess = nullptr;
+ unsigned long long bestProcessStartTime = 0;
+ unsigned int bestProcessGen = 0;
+ int bestProcessBusyness = 0;
+ unsigned int size = enabledProcessBusynessLevels.size();
  const int *enabledProcessBusynessLevels = &this->enabledProcessBusynessLevels[0];
 
- for (i = 0; i < size; i++) {
- if (leastBusyProcessIndex == -1 || enabledProcessBusynessLevels[i] < lowestBusyness) {
- leastBusyProcessIndex = i;
- lowestBusyness = enabledProcessBusynessLevels[i];
+ for (unsigned int i = 0; i < size; i++) {
+ Process *process = enabledProcesses.at(i).get();
+ unsigned int gen = process->generation;
+ unsigned long long startTime = process->spawnerCreationTime;
+ int busyness = enabledProcessBusynessLevels[i];
+ if (bestProcess == nullptr ||
+ gen > bestProcess->generation ||
+ (gen == bestProcessGen && startTime < bestProcessStartTime) ||
+ (gen == bestProcessGen && startTime == bestProcessStartTime && busyness < bestProcessBusyness)
+ ) {
+ bestProcess = process;
+ bestProcessGen = gen;
+ bestProcessBusyness = busyness;
+ bestProcessStartTime = startTime;
  }
  }
- return enabledProcesses[leastBusyProcessIndex].get();
+ return bestProcess;
 }
 
 /**

src/agent/Core/ApplicationPool/Group/SessionManagement.cpp

@@ -58,14 +58,14 @@ Group::RouteResult
 Group::route(const Options &options) const {
  if (OXT_LIKELY(enabledCount > 0)) {
  if (options.stickySessionId == 0) {
- Process *process = findEnabledProcessWithLowestBusyness();
+ Process *process = findBestEnabledProcess();
  if (process->canBeRoutedTo()) {
  return RouteResult(process);
  } else {
  return RouteResult(NULL, true);
  }
  } else {
- Process *process = findProcessWithStickySessionIdOrLowestBusyness(
+ Process *process = findBestProcessPreferringStickySessionId(
  options.stickySessionId);
  if (process != NULL) {
  if (process->canBeRoutedTo()) {
@@ -78,7 +78,7 @@ Group::route(const Options &options) const {
  }
  }
  } else {
- Process *process = findProcessWithLowestBusyness(disablingProcesses);
+ Process *process = findBestProcess(disablingProcesses);
  if (process->canBeRoutedTo()) {
  return RouteResult(process);
  } else {
@@ -304,7 +304,7 @@ Group::get(const Options &newOptions, const GetCallback &callback,
  assert(m_spawning || restarting() || poolAtFullCapacity());
 
  if (disablingCount > 0 && !restarting()) {
- Process *process = findProcessWithLowestBusyness(disablingProcesses);
+ Process *process = findBestProcess(disablingProcesses);
  assert(process != NULL);
  if (!process->isTotallyBusy()) {
  return newSession(process, newOptions.currentTime);

src/agent/Core/ApplicationPool/Process.h

@@ -28,7 +28,6 @@
 
 #include <string>
 #include <vector>
-#include <algorithm>
 #include <boost/intrusive_ptr.hpp>
 #include <boost/move/core.hpp>
 #include <boost/container/vector.hpp>
@@ -102,6 +101,12 @@ class Process {
 public:
  static const unsigned int MAX_SOCKETS_ACCEPTING_HTTP_REQUESTS = 3;
 
+ /**
+ * Time at which the Spawner that created this process was created.
+ * Microseconds resolution.
+ */
+ unsigned long long spawnerCreationTime;
+
 private:
  /*************************************************************
  * Read-only fields, set once during initialization and never
@@ -141,12 +146,6 @@ class Process {
  */
  StaticString codeRevision;
 
- /**
- * Time at which the Spawner that created this process was created.
- * Microseconds resolution.
- */
- unsigned long long spawnerCreationTime;
-
  /** Time at which we started spawning this process. Microseconds resolution. */
  unsigned long long spawnStartTime;
 
@@ -384,6 +383,10 @@ class Process {
 
  /** Last time when a session was opened for this Process. */
  unsigned long long lastUsed;
+ /** Which gereration of app processes this one belongs to,
+ inherited from the app group, incremented when a restart
+ is initiated*/
+ const unsigned int generation;
  /** Number of sessions currently open.
  * @invariant session >= 0
  */
@@ -446,18 +449,18 @@ class Process {
  /** Collected by Pool::collectAnalytics(). */
  ProcessMetrics metrics;
 
-
- Process(const BasicGroupInfo *groupInfo, const Json::Value &args)
- : info(this, groupInfo, args),
+ Process(const BasicGroupInfo *groupInfo, const unsigned int gen, const Json::Value &args)
+  : spawnerCreationTime(getJsonUint64Field(args, "spawner_creation_time")),
+   info(this, groupInfo, args),
  socketsAcceptingHttpRequestsCount(0),
- spawnerCreationTime(getJsonUint64Field(args, "spawner_creation_time")),
  spawnStartTime(getJsonUint64Field(args, "spawn_start_time")),
  spawnEndTime(SystemTime::getUsec()),
  type(args["type"] == "dummy" ? SpawningKit::Result::DUMMY : SpawningKit::Result::UNKNOWN),
  requiresShutdown(false),
  refcount(1),
  index(-1),
  lastUsed(spawnEndTime),
+ generation(gen),
  sessions(0),
  processed(0),
  lifeStatus(ALIVE),
@@ -471,18 +474,19 @@ class Process {
  indexSocketsAcceptingHttpRequests();
  }
 
- Process(const BasicGroupInfo *groupInfo, const SpawningKit::Result &skResult,
+ Process(const BasicGroupInfo *groupInfo, const unsigned int gen, const SpawningKit::Result &skResult,
  const Json::Value &args)
- : info(this, groupInfo, skResult),
+ : spawnerCreationTime(getJsonUint64Field(args, "spawner_creation_time")),
+ info(this, groupInfo, skResult),
  socketsAcceptingHttpRequestsCount(0),
- spawnerCreationTime(getJsonUint64Field(args, "spawner_creation_time")),
  spawnStartTime(skResult.spawnStartTime),
  spawnEndTime(skResult.spawnEndTime),
  type(skResult.type),
  requiresShutdown(false),
  refcount(1),
  index(-1),
  lastUsed(spawnEndTime),
+ generation(gen),
  sessions(0),
  processed(0),
  lifeStatus(ALIVE),

test/cxx/Core/ApplicationPool/PoolTest.cpp

@@ -5,7 +5,6 @@
 #include <FileTools/FileManip.h>
 #include <StrIntTools/StrIntUtils.h>
 #include <IOTools/MessageSerialization.h>
-#include <map>
 #include <vector>
 #include <cerrno>
 #include <signal.h>
@@ -632,6 +631,66 @@ namespace tut {
  ensure_equals(pool->getGroupCount(), 0u);
  }
 
+ TEST_METHOD(15) {
+ // Test that the process generation increments when the group restarts
+ Options options = createOptions();
+
+ // Spawn a process and opens a session with it.
+ pool->setMax(1);
+ pool->asyncGet(options, callback);
+ EVENTUALLY(5,
+ result = number == 1;
+ );
+
+ // Close the session so that the process is now idle.
+ ProcessPtr process = currentSession->getProcess()->shared_from_this();
+ currentSession.reset();
+ unsigned int gen1 = process->generation;
+
+ ensure(pool->restartGroupByName(options.appRoot));
+ EVENTUALLY(5,
+ result = pool->getProcessCount() == 1;
+ );
+ pool->asyncGet(options, callback);
+ EVENTUALLY(5,
+ result = number == 2;
+ );
+
+ process = currentSession->getProcess()->shared_from_this();
+ currentSession.reset();
+ unsigned int gen2 = process->generation;
+ ensure_equals(gen1+1,gen2);
+ }
+
+ TEST_METHOD(16) {
+ // Test that the correct process from the pool is routed
+ Options options = createOptions();
+ ensureMinProcesses(2);
+
+ // async restart the group
+ ensure(pool->restartGroupByName(options.appRoot));
+ ensureMinProcesses(1);
+
+ /*
+ Imagine we have these processes (ordered from oldest to newest):
+
+ #. PID 1 (generation A, busyness 5)
+ #. PID 2 (generation A, busyness 3)
+ #. PID 3 (generation B, busyness 1)
+
+ The algorithm should select PID 3
+ */
+
+ /*
+ Imagine we have these processes (ordered from oldest to newest):
+
+ #. PID 1 (generation A, busyness 1)
+ #. PID 2 (generation B, busyness 5)
+
+ The algorithm should select PID 1
+ */
+ }
+
  TEST_METHOD(17) {
  // Test that restartGroupByName() spawns more processes to ensure
  // that minProcesses and other constraints are met.