*: add -e and --disable-stats flags for testing external network

With the `-e` flag bench without the docker container will be started.
The `-a` flag should be used with the `-e` flag for specifying the
external RPC address. To disable collection of docker container
statistics `--disable-stats` have been added.

Close #160

Signed-off-by: Ekaterina Pavlova <[email protected]>

Makefile

@@ -24,7 +24,7 @@ help:
 	@echo ''
 	@awk '/^#/{ comment = substr($$0,3) } comment && /^[a-zA-Z][a-zA-Z0-9_-]+ ?:/{ print "   ", $$1, comment }' $(MAKEFILE_LIST) | column -t -s ':' | grep -v 'IGNORE' | sort | uniq
 
-.PHONY: build prepare push gen build.node.go build.node.sharp stop start config \
+.PHONY: build prepare push gen build.node.go build.node.sharp binary-bench stop start config \
 	start.GoSingle10wrk start.GoSingle30wrk start.GoSingle100wrk \
 	start.GoSingle25rate start.GoSingle50rate start.GoSingle60rate start.GoSingle300rate start.GoSingle1000rate \
 	start.GoFourNodes10wrk start.GoFourNodes30wrk start.GoFourNodes100wrk \
@@ -39,6 +39,15 @@ help:
 # Build all images
 build: gen build.node.bench build.node.go build.node.sharp
 
+# Build Benchmark binary file
+binary-bench:
+	@cd /cmd/bench && \
+		GOGC=off \
+		CGO_ENABLED=0 \
+		GO111MODULE=on \
+		go build -v -o neo-bench -trimpath
+	@mv /cmd/bench/neo-bench ../neo-bench
+
 # Push all images to registry
 push:
 	docker push $(HUB)-bench:$(TAG)

README.md

@@ -195,6 +195,7 @@ $ ./runner.sh --nodes mixed -d "MixedGoRPC4x1" -m rate -q 50 -z 5m -t 30s
   -i, --in                         Path to input file to load transactions.
                                    Example: -i ./dump.txs --in /path/to/import/transactions
       --vote                       Vote before the bench.
+      --disable-stats              Disable memory and CPU usage statistics collection.
 ````
 
 ## Makefile usage
@@ -300,6 +301,7 @@ The following default configurations are available:
        --msPerBlock                 Protocol setting specifying the minimal (and targeted for) time interval between blocks. Must be an integer number of milliseconds.
                                     The default value is set in configuration templates and is 1s and 5s for single node and multinode setup respectively.
                                     Example: --msPerBlock 3000
+   -e, --external                   Use external network for benchmarking. Default is false.
 
 ```
 

cmd/bench/main.go

@@ -33,15 +33,16 @@ func main() {
 	defer cancel()
 
 	var (
-		workers    int
-		rate       int
-		msPerBlock int
-		threshold  time.Duration
-		dump       *internal.Dump
-		desc       = v.GetString("desc")
-		timeLimit  = v.GetDuration("timeLimit")
-		mode       = internal.BenchMode(v.GetString("mode"))
-		client     *internal.RPCClient
+		workers      int
+		rate         int
+		msPerBlock   int
+		threshold    time.Duration
+		dump         *internal.Dump
+		desc         = v.GetString("desc")
+		timeLimit    = v.GetDuration("timeLimit")
+		mode         = internal.BenchMode(v.GetString("mode"))
+		client       *internal.RPCClient
+		disableStats = v.GetBool("disable-stats")
 	)
 
 	switch mode {
@@ -92,32 +93,33 @@ func main() {
 			log.Fatalf("could not close report: %v", err)
 		}
 	}()
+	if !disableStats {
+		statsPeriod := time.Second
 
-	statsPeriod := time.Second
+		ds, err := internal.NewStats(ctx,
+			internal.StatEnableLogger(),
+			internal.StatPeriod(statsPeriod),
+			internal.StatCriteria([]string{"stats"}),
+			internal.StatListVerifier(func(list []types.Container) error {
+				if len(list) == 0 {
+					return errors.New("containers not found by criteria")
+				}
 
-	ds, err := internal.NewStats(ctx,
-		internal.StatEnableLogger(),
-		internal.StatPeriod(statsPeriod),
-		internal.StatCriteria([]string{"stats"}),
-		internal.StatListVerifier(func(list []types.Container) error {
-			if len(list) == 0 {
-				return errors.New("containers not found by criteria")
-			}
+				return nil
+			}))
 
-			return nil
-		}))
+		if err != nil {
+			log.Fatalf("could not create docker stats grabber: %v", err)
+		}
 
-	if err != nil {
-		log.Fatalf("could not create docker stats grabber: %v", err)
+		statsStart := time.Now()
+		// Run stats worker:
+		go ds.Run(ctx, func(cpu, mem float64) {
+			rep.UpdateRes(statsStart, cpu, mem)
+			log.Printf("CPU: %0.3f%%, Mem: %0.3fMB", cpu, mem)
+		})
 	}
 
-	statsStart := time.Now()
-	// Run stats worker:
-	go ds.Run(ctx, func(cpu, mem float64) {
-		rep.UpdateRes(statsStart, cpu, mem)
-		log.Printf("CPU: %0.3f%%, Mem: %0.3fMB", cpu, mem)
-	})
-
 	if in := v.GetString("in"); in != "" {
 		dump = internal.ReadDump(in)
 	} else {

cmd/internal/settings.go

@@ -95,6 +95,7 @@ func InitSettings() *viper.Viper {
 			"Example: -i ./dump.txs --in /path/to/import/transactions")
 
 	flags.BoolP("vote", "", false, "Vote before the bench.")
+	flags.BoolP("disable-stats", "", false, "Disable memory and CPU usage statistics collection.")
 
 	if err := v.BindPFlags(flags); err != nil {
 		panic(err)

runner.sh

@@ -3,13 +3,14 @@
 source .env
 
 OUTPUT=""
-ARGS=(-i "/dump.txs")
+ARGS=()
 FILES=()
 MODE=""
 COUNT=""
 IR_TYPE=go
 RPC_TYPE=
 RPC_ADDR=()
+EXTERNAL_NETWORK=false
 export NEOBENCH_LOGGER=${NEOBENCH_LOGGER:-none}
 export NEOBENCH_TYPE=${NEOBENCH_TYPE:-NEO}
 export NEOBENCH_FROM_COUNT=${NEOBENCH_FROM_COUNT:-1}
@@ -54,6 +55,7 @@ show_help() {
 	echo "       --msPerBlock                 Protocol setting specifying the minimal (and targeted for) time interval between blocks. Must be an integer number of milliseconds."
 	echo "                                    The default value is set in configuration templates and is 1s and 5s for single node and multinode setup respectively."
 	echo "                                    Example: --msPerBlock 1000"
+	echo "   -e, --external                   Use external network for benchmarking. Default is false."
 	exit 0
 }
 
@@ -72,6 +74,9 @@ while test $# -gt 0; do
 
 	case $_opt in
 	-h | --help) show_help ;;
+	-e|--external)
+        EXTERNAL_NETWORK=true
+        ;;
 	-l | --log)
 		if [[ $# -gt 0 && ${1:0:1} != "-" ]]; then
 			case "$1" in
@@ -273,7 +278,18 @@ if [ -n "$NEOBENCH_VOTE" ]; then
 fi
 
 make prepare
+if [ "$EXTERNAL_NETWORK" ]; then
+      ARGS+=(-i ".docker/build/dump.$NEOBENCH_TYPE.$NEOBENCH_FROM_COUNT.$NEOBENCH_TO_COUNT.txs" --disable-stats)
 
-docker compose "${FILES[@]}" run bench neo-bench -o "$OUTPUT" "${ARGS[@]}"
+      make binary-bench
 
-make stop
+      ./neo-bench -o "$OUTPUT" "${ARGS[@]}"
+
+else
+    ARGS+=(-i "/dump.txs")
+    make prepare
+
+    docker compose "${FILES[@]}" run bench neo-bench -o "$OUTPUT" "${ARGS[@]}"
+
+    make stop
+fi