Merge pull request #820 from 0xPolygonHermez/release_v5.0.1

Release v5.0.1

.github/workflows/prover_e2e.yml

@@ -3,12 +3,12 @@ name: Prover e2e test
 on:
   workflow_dispatch:
   push:
-    branches: [ "main","develop", "release_*" ]
+    branches: [ "main","develop", "release_*", "fork_8", "fork_8_2" ]
   pull_request:
-    branches: [ "main","develop", "release_*" ]
+    branches: [ "main","develop", "release_*", "fork_8", "fork_8_2" ]
 
 jobs:
-  build:
+  e2e-collection-tests:
 
     runs-on: [self-hosted, linux, X64, hc, hm]
 
@@ -22,14 +22,18 @@ jobs:
     - name: Install snarkjs
       run: npm -g install snarkjs
     - name: make
-      run: cd src/grpc && make && cd ../.. && make -j
+      run: cd src/grpc && make && cd ../.. && make generate && make -j
     - name: remove all config 
       run: rm -rf config
     - name: remove old proofs
       run: rm -rf runtime/output/*
     - name: link files
-      run: ln -s /home/gha/v4.0.0-rc.6-fork.7/config/ config
+      run: ln -s /home/gha/v6.0.0-rc.1-fork.9/config/ config
+    - name: run collection native tests
+      run: ./build/zkProver -c testvectors/config_collection_native.json
+    - name: run collection generated tests
+      run: ./build/zkProver -c testvectors/config_collection_generated.json
     - name: run e2e tests
       run: ./build/zkProver -c testvectors/config_runFile_e2e.json 
     - name: run snarkjs verification
-      run: snarkjs ffv config/final/final.fflonk.verkey.json $(ls -t runtime/output/*.gen_final_proof_public.json | head -n1) $(ls -t runtime/output/*.final_proof.proof.json | head -n1)
+      run: snarkjs ffv config/final/final.fflonk.verkey.json $(ls -t runtime/output/*.gen_final_proof_public.json | head -n1) $(ls -t runtime/output/*.final_proof.proof.json | head -n1)

.github/workflows/push-docker.yaml

@@ -2,7 +2,7 @@ name: Build docker image and run node tests
 
 on:
   push:
-    branches: [ develop, main, fork_7 ]
+    branches: [ develop, main, fork_8 ]
 
 jobs:
   build-docker:

.gitignore

@@ -1,4 +1,5 @@
 build
+src/main_sm/fork_*/main_exec_generated/*
 testvectors/commit.bin
 testvectors/verifier.dat
 testvectors/constantstree.bin

DEBUG.md

@@ -0,0 +1,118 @@
+# zkEVM Prover Debug
+
+This file explains the zkEVM Prover debug tips & tricks.
+Please read README.md first.
+All configuration parameters are documented in ./src/config/README.md .
+Although the config file can be specified as an argument, we will assume for simplicity that it is named config.json, the default config file name.
+
+## Logs
+
+The first place where to find debugging information is the process logs.  
+You can redirect the error channel to the output channel and save all logs to a file named 'logs' by calling the zkEVM Prover with the following command:
+
+```sh
+./build/zkProver 2>&1 | tee logs
+```
+
+## Native vs. Generated main executor
+
+The main state machine is implemented both in native C code and in generated C code.  
+The native code is slower and easier to debug, while the generated code is faster and more difficult to debug.
+The result of both native and generated code should be identical, apart from the performance.
+You can switch from one to the other by editing the config.json file and rebooting the zkEVM Prover:
+
+```sh
+"useMainExecGenerated": true,
+```
+
+'true' will configure the zkEVM Prover to use generated code, and it is the default value for performance reasons.
+'false' will configure the zkEVM Prover to use native code.
+
+## ROM instruction logs
+
+When the zkEVM Prover is configured to use native code, you can get a log describing what every ROM instruction does.
+This mechanism can be useful to understand the context under which a certain situation happens, e.g. an error.
+Warning: this mechanism can generate a lot of traces, so use it with caution.
+You can enable this mechanism by editing the config.json file and rebooting the zkEVM Prover:
+
+```sh
+"executorROMLineTraces": true,
+```
+
+## Save input to JSON file
+
+The zkEVM Prover can be configured to save to file the input or the output, in different formats.
+Files are generated in the directory './runtime/output' by default.
+You can change the output directory by editing the config.json file and rebooting the zkEVM Prover:
+
+```sh
+"outputPath": "runtime/output",
+```
+
+The most used format is JSON.
+
+You can save executor GRPC input to a JSON file by editing the config.json file and rebooting the zkEVM Prover:
+
+```sh
+"saveInputToFile": true,
+```
+
+With this input file you can re-submit the same batch later, as long as you have access to the same database, by editing the config.json file and rebooting the zkEVM Prover:
+
+```sh
+"runExecutorClient": true,
+"inputFile": "./runtime/output/<filename>.json",
+```
+
+If you don't have access to the database when debugging, the execution will fail with a 'key not found" error.
+If you don't want to depend on the database access, you must save all DB reads to the same JSON file, by editing the config.json file and rebooting the zkEVM Prover:
+
+```sh
+"saveDbReadsToFile": true,
+```
+
+This configuration will save not only the input parameters, but also the DB read data.
+However, this file is saved after the batch has been processed.
+If the process is killed before you reach the point where this file is saved, you can save it after every DB read.
+This slows down the processing, but ensures having the required DB data to reach the problematic point of the execution.
+If you want to get the input file continously updated, you can enable this feature by editing the config.json file and rebooting the zkEVM Prover:
+
+```sh
+"saveDbReadsToFileOnChange": true,
+```
+
+## Save input request to text file
+
+You can also save the GRPC input request to a text file, by editing the config.json file and rebooting the zkEVM Prover:
+
+```sh
+"saveRequestToFile": true,
+```
+
+The content of this file is directly generated by the GRPC library, which can be useful if you are familiar with this format.
+The numeric values that are zero are not logged, since it is their default value.
+The binary non-textual values are logged in octal escaped format.
+This format is useful to ensure that some data sent by the service client actually reached the zkEVM Prover.
+
+## Save output to JSON file
+
+You can save executor GRPC output data to a JSON file by editing the config.json file and rebooting the zkEVM Prover:
+
+```sh
+"saveOutputToFile": true,
+```
+
+## Save output response to text file
+
+You can save executor GRPC output response to a text file by editing the config.json file and rebooting the zkEVM Prover:
+
+```sh
+"saveResponseToFile": true,
+```
+
+## Caution with files generation
+
+Some of the described configuration option generate one file per batch.
+If you have several of this configuration options enabled, you can get several files per batch.
+This can represent a high number of files after some time of operation.
+Once you have completed your debugging, consider disabling these options to avoid storage issues and low execution performance.

Dockerfile

@@ -9,6 +9,7 @@ COPY ./test ./test
 COPY ./tools ./tools
 COPY ./config ./config
 COPY Makefile .
+RUN make generate
 RUN make -j
 
 FROM ubuntu:22.04
@@ -26,6 +27,10 @@ COPY ./src/main_sm/fork_2/scripts/rom.json ./src/main_sm/fork_2/scripts/rom.json
 COPY ./src/main_sm/fork_3/scripts/rom.json ./src/main_sm/fork_3/scripts/rom.json
 COPY ./src/main_sm/fork_4/scripts/rom.json ./src/main_sm/fork_4/scripts/rom.json
 COPY ./src/main_sm/fork_5/scripts/rom.json ./src/main_sm/fork_5/scripts/rom.json
+COPY ./src/main_sm/fork_6/scripts/rom.json ./src/main_sm/fork_6/scripts/rom.json
+COPY ./src/main_sm/fork_7/scripts/rom.json ./src/main_sm/fork_7/scripts/rom.json
+COPY ./src/main_sm/fork_8/scripts/rom.json ./src/main_sm/fork_8/scripts/rom.json
+COPY ./src/main_sm/fork_9/scripts/rom.json ./src/main_sm/fork_9/scripts/rom.json
 
 ENTRYPOINT [ "zkProver" ]
 

Dockerfile-GHA

@@ -13,6 +13,7 @@ COPY ./src ./src
 COPY ./test ./test
 COPY ./tools ./tools
 COPY Makefile .
+RUN make generate
 RUN make -j
 
 
@@ -28,6 +29,8 @@ COPY ./src/main_sm/fork_4/scripts/rom.json ./src/main_sm/fork_4/scripts/rom.json
 COPY ./src/main_sm/fork_5/scripts/rom.json ./src/main_sm/fork_5/scripts/rom.json
 COPY ./src/main_sm/fork_6/scripts/rom.json ./src/main_sm/fork_6/scripts/rom.json
 COPY ./src/main_sm/fork_7/scripts/rom.json ./src/main_sm/fork_7/scripts/rom.json
+COPY ./src/main_sm/fork_8/scripts/rom.json ./src/main_sm/fork_8/scripts/rom.json
+COPY ./src/main_sm/fork_9/scripts/rom.json ./src/main_sm/fork_9/scripts/rom.json
 
 RUN DEBIAN_FRONTEND=noninteractive apt update && \
     DEBIAN_FRONTEND=noninteractive apt install -y libgmp-dev \

Makefile

@@ -94,6 +94,9 @@ $(BUILD_DIR)/$(TARGET_MNG): ./src/main_generator/main_generator.cpp
 	$(MKDIR_P) $(BUILD_DIR)
 	g++ -g ./src/main_generator/main_generator.cpp -o $@ -lgmp
 
+generate: main_generator
+	$(BUILD_DIR)/$(TARGET_MNG) all
+
 pols_generator: $(BUILD_DIR)/$(TARGET_PLG)
 
 $(BUILD_DIR)/$(TARGET_PLG): ./src/pols_generator/pols_generator.cpp
@@ -110,6 +113,7 @@ $(BUILD_DIR)/$(TARGET_PLD): ./src/pols_diff/pols_diff.cpp
 
 clean:
 	$(RM) -r $(BUILD_DIR)
+	find . -name main_exec_generated*pp -delete
 
 -include $(DEPS_ZKP)
 -include $(DEPS_BCT)

README.md

@@ -97,6 +97,7 @@ Run `make` to compile the main project:
 
 ```sh
 make clean
+make generate
 make -j
 ```
 
@@ -108,9 +109,9 @@ To compile in debug mode, run `make -j dbg=1`.
 ./build/zkProver -c testvectors/config_runFile_BatchProof.json
 ```
 
-## StateDB service database
+## HashDB service database
 
-To use persistence in the StateDB (Merkle-tree) service you must create the database objects needed by the service. To do this run the shell script:
+To use persistence in the HashDB (Merkle-tree) service you must create the database objects needed by the service. To do this run the shell script:
 
 ```sh
 ./tools/statedb/create_db.sh <database> <user> <password>

src/config/README.md

@@ -85,6 +85,7 @@ The configuration parameters can be of different uses:
 |`aggregatorClientMockTimeout`|test|u64|Aggregator client mock timeout, in microseconds|60000000 (60 seconds)|AGGREGATOR_CLIENT_MOCK_TIMEOUT|
 |**`aggregatorClientWatchdogTimeout`**|production|u64|Aggregator client watchdog timeout, in microseconds|60000000 (60 seconds)|AGGREGATOR_CLIENT_WATCHDOG_TIMEOUT|
 |`aggregatorClientMaxStreams`|test|u64|Max number of aggregator client streams, used to limit E2E test execution; if 0 then there is no limit|0|AGGREGATOR_CLIENT_MAX_STREAMS|
+|`aggregatorClientMaxRecvMsgSize`|test|u64|Max size of aggregator client received messages; if 0 then there is no limit|1024*1024*1024|AGGREGATOR_CLIENT_MAX_RECV_MSG_SIZE|
 |`executorROMLineTraces`|test|boolean|If true, the main state machine executor will log the content of every executed ROM program line; it only works with native main executor, not with generated code executor|false|EXECUTOR_ROM_LINE_TRACES|
 |`executorTimeStatistics`|test|boolean|If true, the main state machine executor will log the time metrics statistics of external calls|false|EXECUTOR_TIME_STATISTICS|
 |`opcodeTracer`|test|boolean|Generate main state machine executor opcode statistics|false|OPCODE_TRACER|

src/config/config.cpp

@@ -169,7 +169,8 @@ void Config::load(json &config)
     ParseBool(config, "saveFilesInSubfolders", "SAVE_FILES_IN_SUBFOLDERS", saveFilesInSubfolders, false);
 
     // Load DB to mem cache TODO: Discontinue this functionality
-    ParseBool(config, "loadDBToMemCache", "LOAD_DB_TO_MEM_CACHE", loadDBToMemCache, false);
+    //ParseBool(config, "loadDBToMemCache", "LOAD_DB_TO_MEM_CACHE", loadDBToMemCache, false);
+    loadDBToMemCache = false;
     ParseBool(config, "loadDBToMemCacheInParallel", "LOAD_DB_TO_MEM_CACHE_IN_PARALLEL", loadDBToMemCacheInParallel, false);
     ParseU64(config, "loadDBToMemTimeout", "LOAD_DB_TO_MEM_TIMEOUT", loadDBToMemTimeout, 30*1000*1000); // Default = 30 seconds
 
@@ -210,6 +211,7 @@ void Config::load(json &config)
     ParseU64(config, "aggregatorClientMockTimeout", "AGGREGATOR_CLIENT_MOCK_TIMEOUT", aggregatorClientMockTimeout, 60 * 1000 * 1000);
     ParseU64(config, "aggregatorClientWatchdogTimeout", "AGGREGATOR_CLIENT_WATCHDOG_TIMEOUT", aggregatorClientWatchdogTimeout, 60 * 1000 * 1000);
     ParseU64(config, "aggregatorClientMaxStreams", "AGGREGATOR_CLIENT_MAX_STREAMS", aggregatorClientMaxStreams, 0);
+    ParseU64(config, "aggregatorClientMaxRecvMsgSize", "AGGREGATOR_CLIENT_MAX_RECV_MSG_SIZE", aggregatorClientMaxRecvMsgSize, 1024*1024*1024);
 
     // Logs
     ParseBool(config, "executorROMLineTraces", "EXECUTOR_ROM_LINE_TRACES", executorROMLineTraces, false);
@@ -452,6 +454,7 @@ void Config::print(void)
     zklog.info("    aggregatorClientMockTimeout=" + to_string(aggregatorClientMockTimeout));
     zklog.info("    aggregatorClientWatchdogTimeout=" + to_string(aggregatorClientWatchdogTimeout));
     zklog.info("    aggregatorClientMaxStreams=" + to_string(aggregatorClientMaxStreams));
+    zklog.info("    aggregatorClientMaxRecvMsgSize=" + to_string(aggregatorClientMaxRecvMsgSize));
 
     zklog.info("    inputFile=" + inputFile);
     zklog.info("    inputFile2=" + inputFile2);

src/config/config.hpp

@@ -100,6 +100,7 @@ class Config
     uint64_t aggregatorClientMockTimeout;
     uint64_t aggregatorClientWatchdogTimeout;
     uint64_t aggregatorClientMaxStreams; // Max number of streams, used to limit E2E test execution; if 0 then there is no limit
+    uint64_t aggregatorClientMaxRecvMsgSize; // Max received message size, in bytes
 
     // Executor debugging
     bool executorROMLineTraces;

src/config/definitions.hpp

@@ -1,10 +1,10 @@
 #ifndef DEFINITIONS_HPP
 #define DEFINITIONS_HPP
 
-#define PROVER_FORK_ID 7
-#define PROVER_FORK_NAMESPACE fork_7
-#define PROVER_FORK_NAMESPACE_STRING "fork_7"
-#define USING_PROVER_FORK_NAMESPACE using namespace fork_7
+#define PROVER_FORK_ID 9
+#define PROVER_FORK_NAMESPACE fork_9
+#define PROVER_FORK_NAMESPACE_STRING "fork_9"
+#define USING_PROVER_FORK_NAMESPACE using namespace fork_9
 
 /* Log traces selector: uncomment to enable the corresponding trace */
 //#define LOG_START_STEPS

src/config/version.hpp

@@ -1,6 +1,6 @@
 #ifndef ZKEVM_PROVER_VERSION_HPP
 #define ZKEVM_PROVER_VERSION_HPP
 
-#define ZKEVM_PROVER_VERSION "v4.0.19"
+#define ZKEVM_PROVER_VERSION "v6.0.0"
 
 #endif

src/config/zkresult.cpp

@@ -110,9 +110,8 @@ struct
     { ZKR_SM_MAIN_INVALID_WITNESS, "ZKR_SM_MAIN_INVALID_WITNESS" },
     { ZKR_CBOR_INVALID_DATA, "ZKR_CBOR_INVALID_DATA" },
     { ZKR_DATA_STREAM_INVALID_DATA, "ZKR_DATA_STREAM_INVALID_DATA" },
-    { ZKR_SM_MAIN_UNSUPPORTED_PRECOMPILED, "ZKR_SM_MAIN_UNSUPPORTED_PRECOMPILED" },
-    { ZKR_SM_MAIN_OOG_2, "ZKR_SM_MAIN_OOG_2" },
-    { ZKR_SM_MAIN_CLOSE_BATCH, "ZKR_SM_MAIN_CLOSE_BATCH" }
+
+    { ZKR_SM_MAIN_INVALID_TX_STATUS_ERROR, "ZKR_SM_MAIN_INVALID_TX_STATUS_ERROR" }
 };
 
 string zkresult2string (int code)

src/config/zkresult.hpp

@@ -110,9 +110,8 @@ typedef enum : int
     ZKR_SM_MAIN_INVALID_WITNESS = 96, // Main state machine input witness is invalid or corrupt
     ZKR_CBOR_INVALID_DATA = 97, // CBOR data is invalid
     ZKR_DATA_STREAM_INVALID_DATA = 98, // Data stream data is invalid
-    ZKR_SM_MAIN_UNSUPPORTED_PRECOMPILED = 99, // Main state machine tried to use an unsupported precompiled
-    ZKR_SM_MAIN_OOG_2 = 100, // Main state machine found an out of gase
-    ZKR_SM_MAIN_CLOSE_BATCH = 101, // Main state machine completes but batch must be closed
+
+    ZKR_SM_MAIN_INVALID_TX_STATUS_ERROR = 99, // Invalid TX status-error combination
 
 } zkresult;