OD sigterm race condition fix (#544)

docker-compose-test.yml

@@ -2,6 +2,11 @@ version: '3'
 
 services:
   odyssey:
+    ulimits:
+      core:
+        soft: -1
+        hard: -1
+    privileged: true
     build:
       dockerfile: ./docker/Dockerfile
       context: .

docker/Dockerfile

@@ -8,10 +8,10 @@ COPY ./docker/prep_stmts /prep_stmts
 COPY ./docker/config-validation /config-validation
 
 WORKDIR /ody-integration-test
-RUN go mod download && cd pkg && go build -o ody-integration-test
+RUN go mod download && cd pkg && CGO_ENABLED=0 go build -o ody-integration-test
 
 WORKDIR /prep_stmts
-RUN go mod download && cd pkg && go build -o pstmts-test
+RUN go mod download && cd pkg && CGO_ENABLED=0 go build -o pstmts-test
 
 WORKDIR /config-validation
 RUN go mod download && cd pkg && go build -o config-validation

docker/bin/setup

@@ -59,6 +59,11 @@ for database_name in db scram_db ldap_db auth_query_db db1 hba_db tsa_db; do
     }
 done
 
+# pgbench initialization
+mkdir /var/cores
+sudo sysctl -w kernel.core_pattern=/var/cores/core.%p.%e
+pgbench -i -h localhost -p 5432 -U postgres postgres
+
 # Create users
 psql -h localhost -p 5432 -U postgres -c "set password_encryption = 'scram-sha-256'; create user scram_user password 'scram_user_password';" -d scram_db >> $SETUP_LOG 2>&1 || {
     echo "ERROR: users creation failed, examine the log"

docker/entrypoint.sh

@@ -19,21 +19,21 @@ ody-stop
 
 #ldap
 /ldap/test_ldap.sh
-if [ $? -eq 1 ] 
+if [ $? -eq 1 ]
 then
 	exit 1
 fi
 
 # scram
 /scram/test_scram.sh
-if [ $? -eq 1 ] 
+if [ $? -eq 1 ]
 then
 	exit 1
 fi
 
 # auth query
 /auth_query/test_auth_query.sh
-if [ $? -eq 1 ] 
+if [ $? -eq 1 ]
 then
 	exit 1
 fi
@@ -52,9 +52,9 @@ sleep 1
 
 ody-stop
 
-# lag polling 
+# lag polling
 /lagpolling/test-lag.sh
-if [ $? -eq 1 ] 
+if [ $? -eq 1 ]
 then
 	exit 1
 fi
@@ -74,4 +74,3 @@ ody-start
 ody-stop
 
 teardown
-

docker/ody-integration-test/pkg/cores.go

@@ -0,0 +1,82 @@
+package main
+
+import (
+	_ "bufio"
+	_ "bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	_ "os"
+	"os/exec"
+	"strconv"
+	"syscall"
+	"time"
+)
+
+const benchTimeSec = 10
+const timeSleep = 5
+const procName = "odyssey"
+const signal = syscall.SIGTERM
+const testCount = 100
+
+func bunchProcess(ctx context.Context) {
+	_, err := exec.CommandContext(ctx, "pgbench",
+		"--builtin", "select-only",
+		"-c", "40",
+		"-T", strconv.Itoa(benchTimeSec),
+		"-j", "20",
+		"-n",
+		"-h", "localhost",
+		"-p", "6432",
+		"-U", "postgres",
+		"postgres",
+		"-P", "1").Output()
+
+	if err != nil {
+		fmt.Printf("pgbench error: %v\n", err)
+	}
+}
+
+func SigTermAfterHighLoad(ctx context.Context) error {
+	for i := 0; i < testCount; i++ {
+		fmt.Printf("Test number: %d\n", i+1)
+
+		if err := ensurePostgresqlRunning(ctx); err != nil {
+			return err
+		}
+
+		if err := ensureOdysseyRunning(ctx); err != nil {
+			return err
+		}
+
+		go bunchProcess(ctx)
+
+		time.Sleep(timeSleep * time.Second)
+
+		if _, err := signalToProc(signal, procName); err != nil {
+			fmt.Println(err.Error())
+		}
+	}
+
+	files, err := ioutil.ReadDir("/var/cores")
+	if err != nil {
+		fmt.Println(err)
+	}
+	countCores := len(files)
+	coresPercent := (float64(countCores) / float64(testCount)) * 100
+	fmt.Printf("Cores count: %d out of %d (%.2f %%)\n", countCores, testCount, coresPercent)
+
+	return nil
+}
+
+func odyCoresTestSet(ctx context.Context) error {
+	if err := SigTermAfterHighLoad(ctx); err != nil {
+		err = fmt.Errorf("odyCoresTestSet failed: %w", err)
+		fmt.Println(err)
+		return err
+	}
+
+	fmt.Println("odyCoresTestSet: Ok")
+
+	return nil
+}

docker/ody-integration-test/pkg/main.go

@@ -16,6 +16,7 @@ func main() {
 		odyPkgSyncTestSet,
 		odyShowErrsTestSet,
 		odySignalsTestSet,
+		odyCoresTestSet,
 	} {
 		err := f(ctx)
 		if err != nil {

sources/sighandler.c

@@ -55,16 +55,19 @@ od_attribute_noreturn() void od_system_shutdown(od_system_t *system,
 
 	od_worker_pool_stop(worker_pool);
 
-	od_router_free(system->global->router);
 	/* Prevent OpenSSL usage during deinitialization */
 	od_worker_pool_wait();
 
+#ifdef OD_SYSTEM_SHUTDOWN_CLEANUP
+	od_router_free(system->global->router);
+
 	od_extention_free(&instance->logger, system->global->extentions);
 
 	od_system_cleanup(system);
 
 	/* stop machinaruim and free */
 	od_instance_free(instance);
+#endif
 	exit(0);
 }
 
@@ -135,4 +138,4 @@ void od_system_signal_handler(void *arg)
 			break;
 		}
 	}
-}
+}