From d3dfb17794a4e9e5eac5702ab59e5d9f1c9d3532 Mon Sep 17 00:00:00 2001 From: Aidar Sabirov Date: Wed, 27 Feb 2019 16:28:29 +0300 Subject: [PATCH 01/40] Iroha installation security tips --- docs/source/guides/index.rst | 1 + docs/source/guides/sec-install.rst | 63 ++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 docs/source/guides/sec-install.rst diff --git a/docs/source/guides/index.rst b/docs/source/guides/index.rst index 36871e8aab..f484552b44 100644 --- a/docs/source/guides/index.rst +++ b/docs/source/guides/index.rst @@ -11,3 +11,4 @@ Guides and how-tos libraries.rst dependencies.rst k8s-deployment.rst + sec-install.rst diff --git a/docs/source/guides/sec-install.rst b/docs/source/guides/sec-install.rst new file mode 100644 index 0000000000..3c5dcf94ac --- /dev/null +++ b/docs/source/guides/sec-install.rst @@ -0,0 +1,63 @@ +Iroha installation security tips +================================ +This guide is intended to secure Iroha installation. Most of the steps from this guide may seem obvious but it helps to avoid possible security problems in the future. + +Physical security +^^^^^^^^^^^^^^^^^ +In case the servers are located locally (physically accessible), a number of security measures have to be applied. Skip these steps if cloud hosting is used. + +Establish organisational policy and/or access control system such that only authorized personnel has access to the server room. +Next, set BIOS/firmware password and configure boot order to prevent unauthorized booting from alternate media. +Make sure the bootloader is password protected if there is such a functionality. Also, it is good to have a CCTV monitoring in place. + +Deployment +^^^^^^^^^^ +First, verify that official repository is used for downloading `source code `__ and `Docker images `__. +Change any default passwords that are used during installation, e.g., password for connecting to postgres. +Iroha repository contains examples of private and public keys - never use it in production. +Moreover, verify that new keypairs are generated in a safe environment and only administrator has access to those keypairs (or at least minimise the number of people). +After deploying keys to Iroha peers delete private keys from the host that was used to perform deployment, i.e. private keys should reside only inside Iroha peers. +Create an encrypted backup of private keys before deleting them and limit the access to it. + +Network configuration +^^^^^^^^^^^^^^^^^^^^^ +Iroha listens on ports 50051 and 10001. +Firewall settings must allow incoming/outcoming connections to/from these ports. +If possible, disable or remove any other network services with listening ports (FTP, DNS, LDAP, SMB, DHCP, NFS, SNMP, etc). +Ideally, Iroha should be as much isolated as possible in terms of networking. + +Currently, there is no traffic encryption in Iroha, we strongly recommend using VPN or Calico for setting up Docker overlay network, i.e. any mechanism that allows encrypting communication between peers. +Docker swarm encrypts communications by default, but remember to open necessary ports in the firewall configuration. +In case VPN is used, verify that VPN key is unavailable to other users. + +If SSH is used, disable root login. +Apart from that, disable password authentication and use only keys. +It might be helpful to set up SSH log level to INFO as well. + +If IPv6 is not used, it might be a good idea to disable it. + +Updates +^^^^^^^ +Install latest operating system security patches and update it regularly. +If Iroha is running in Docker containers, update Docker regularly. +While being optional, it is considered a good practice to test updates on a separate server before installing to production. + +Logging and monitoring +^^^^^^^^^^^^^^^^^^^^^^ +- Collect and ship logs to a dedicated machine using an agent (e.g., Filebeat). +- Collect logs from all Iroha peers in a central point (e.g., Logstash). +- Enable docker healthcheck. +- Transfer docker healthcheck status to a monitoring system (e.g., Prometheus). +- Transfer logging and monitoring information via an encrypted channel (e.g., https). +- Set up an authentication mechanism to prevent third parties from accessing logs. +- Set up an authentication mechanism to prevent third parties from submitting logs. +- Log all administrator access. + +OS hardening +^^^^^^^^^^^^ +The following steps assume Docker is used for running Iroha. + +- Enable and configure Docker Content Trust. +- Allow only trusted users to control Docker daemon. +- Set up a limit for Docker container resources. + From e7db84d8e4ac6a2f5f850ee420d447e8489fa1c1 Mon Sep 17 00:00:00 2001 From: Artyom Bakhtin Date: Thu, 28 Feb 2019 09:56:27 +0300 Subject: [PATCH 02/40] Fix typo docs/source/guides/sec-install.rst Co-Authored-By: Baydarich --- docs/source/guides/sec-install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/guides/sec-install.rst b/docs/source/guides/sec-install.rst index 3c5dcf94ac..04fb215fc8 100644 --- a/docs/source/guides/sec-install.rst +++ b/docs/source/guides/sec-install.rst @@ -22,7 +22,7 @@ Create an encrypted backup of private keys before deleting them and limit the ac Network configuration ^^^^^^^^^^^^^^^^^^^^^ Iroha listens on ports 50051 and 10001. -Firewall settings must allow incoming/outcoming connections to/from these ports. +Firewall settings must allow incoming/outgoing connections to/from these ports. If possible, disable or remove any other network services with listening ports (FTP, DNS, LDAP, SMB, DHCP, NFS, SNMP, etc). Ideally, Iroha should be as much isolated as possible in terms of networking. From 3e4155e9b1cd3e3863adf2731dd29fb49c0db87f Mon Sep 17 00:00:00 2001 From: Aidar Sabirov Date: Thu, 28 Feb 2019 10:34:57 +0300 Subject: [PATCH 03/40] Deleted healthcheck mentioning --- docs/source/guides/sec-install.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/source/guides/sec-install.rst b/docs/source/guides/sec-install.rst index 04fb215fc8..2f73f894a3 100644 --- a/docs/source/guides/sec-install.rst +++ b/docs/source/guides/sec-install.rst @@ -46,8 +46,6 @@ Logging and monitoring ^^^^^^^^^^^^^^^^^^^^^^ - Collect and ship logs to a dedicated machine using an agent (e.g., Filebeat). - Collect logs from all Iroha peers in a central point (e.g., Logstash). -- Enable docker healthcheck. -- Transfer docker healthcheck status to a monitoring system (e.g., Prometheus). - Transfer logging and monitoring information via an encrypted channel (e.g., https). - Set up an authentication mechanism to prevent third parties from accessing logs. - Set up an authentication mechanism to prevent third parties from submitting logs. From cc2bf61f4bd1efaa49f2870301b386823e92c613 Mon Sep 17 00:00:00 2001 From: Mikhail Boldyrev Date: Mon, 4 Mar 2019 17:04:51 +0300 Subject: [PATCH 04/40] Pluggable consensus (#2125) * Pluggable consensus consistency model (#2091) * Fixes to it Signed-off-by: Mikhail Boldyrev --- irohad/consensus/yac/CMakeLists.txt | 26 +-- irohad/consensus/yac/consistency_model.hpp | 22 ++ .../yac/impl/supermajority_checker_bft.cpp | 34 +++ .../yac/impl/supermajority_checker_bft.hpp | 33 +++ .../yac/impl/supermajority_checker_cft.cpp | 34 +++ .../yac/impl/supermajority_checker_cft.hpp | 33 +++ .../yac/impl/supermajority_checker_getter.cpp | 29 +++ .../yac/impl/supermajority_checker_impl.cpp | 67 ------ .../yac/impl/supermajority_checker_impl.hpp | 60 ----- .../yac/impl/supermajority_checker_kf1.hpp | 41 ++++ .../yac/storage/impl/yac_block_storage.cpp | 4 +- .../yac/storage/impl/yac_proposal_storage.cpp | 27 +-- .../yac/storage/impl/yac_vote_storage.cpp | 16 +- .../yac/storage/yac_block_storage.hpp | 21 +- .../yac/storage/yac_proposal_storage.hpp | 5 +- .../yac/storage/yac_vote_storage.hpp | 9 +- .../consensus/yac/supermajority_checker.hpp | 55 ++--- irohad/main/application.cpp | 13 +- irohad/main/impl/consensus_init.cpp | 114 +++++----- irohad/main/impl/consensus_init.hpp | 54 ++--- irohad/validation/CMakeLists.txt | 2 +- .../validation/impl/chain_validator_impl.cpp | 6 +- irohad/validation/utils.hpp | 19 ++ .../consensus/consensus_sunny_day.cpp | 16 +- .../chain_validator_storage_test.cpp | 30 ++- .../yac/mock_yac_supermajority_checker.hpp | 19 +- .../yac/supermajority_checker_test.cpp | 215 ++++++++++++------ .../consensus/yac/yac_block_storage_test.cpp | 68 +++--- .../irohad/consensus/yac/yac_fixture.hpp | 10 +- .../yac/yac_proposal_storage_test.cpp | 111 +++++---- .../consensus/yac/yac_rainy_day_test.cpp | 45 ++-- .../yac/yac_simple_cold_case_test.cpp | 17 +- .../consensus/yac/yac_sunny_day_test.cpp | 14 ++ .../validation/chain_validation_test.cpp | 15 +- 34 files changed, 747 insertions(+), 537 deletions(-) create mode 100644 irohad/consensus/yac/consistency_model.hpp create mode 100644 irohad/consensus/yac/impl/supermajority_checker_bft.cpp create mode 100644 irohad/consensus/yac/impl/supermajority_checker_bft.hpp create mode 100644 irohad/consensus/yac/impl/supermajority_checker_cft.cpp create mode 100644 irohad/consensus/yac/impl/supermajority_checker_cft.hpp create mode 100644 irohad/consensus/yac/impl/supermajority_checker_getter.cpp delete mode 100644 irohad/consensus/yac/impl/supermajority_checker_impl.cpp delete mode 100644 irohad/consensus/yac/impl/supermajority_checker_impl.hpp create mode 100644 irohad/consensus/yac/impl/supermajority_checker_kf1.hpp diff --git a/irohad/consensus/yac/CMakeLists.txt b/irohad/consensus/yac/CMakeLists.txt index 473035b247..9c7949cb5e 100644 --- a/irohad/consensus/yac/CMakeLists.txt +++ b/irohad/consensus/yac/CMakeLists.txt @@ -1,24 +1,14 @@ # -# Copyright Soramitsu Co., Ltd. 2017 All Rights Reserved. -# http://soramitsu.co.jp -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Copyright Soramitsu Co., Ltd. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 # -add_library(supermajority_check - impl/supermajority_checker_impl.cpp +add_library(supermajority_checker + impl/supermajority_checker_cft.cpp + impl/supermajority_checker_bft.cpp + impl/supermajority_checker_getter.cpp ) -target_link_libraries(supermajority_check +target_link_libraries(supermajority_checker shared_model_interfaces ) @@ -37,7 +27,7 @@ add_library(yac storage/impl/buffered_cleanup_strategy.cpp ) target_link_libraries(yac - supermajority_check + supermajority_checker common rxcpp logger diff --git a/irohad/consensus/yac/consistency_model.hpp b/irohad/consensus/yac/consistency_model.hpp new file mode 100644 index 0000000000..107820b77c --- /dev/null +++ b/irohad/consensus/yac/consistency_model.hpp @@ -0,0 +1,22 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef IROHA_CONSENSUS_CONSISTENCY_MODEL_HPP +#define IROHA_CONSENSUS_CONSISTENCY_MODEL_HPP + +namespace iroha { + namespace consensus { + namespace yac { + + enum class ConsistencyModel { + kBft, ///< BFT consistency + kCft, ///< CFT consistency + }; + + } // namespace yac + } // namespace consensus +} // namespace iroha + +#endif // IROHA_CONSENSUS_CONSISTENCY_MODEL_HPP diff --git a/irohad/consensus/yac/impl/supermajority_checker_bft.cpp b/irohad/consensus/yac/impl/supermajority_checker_bft.cpp new file mode 100644 index 0000000000..f46b2126bd --- /dev/null +++ b/irohad/consensus/yac/impl/supermajority_checker_bft.cpp @@ -0,0 +1,34 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "consensus/yac/impl/supermajority_checker_bft.hpp" + +#include +#include +#include "consensus/yac/impl/supermajority_checker_kf1.hpp" + +namespace iroha { + namespace consensus { + namespace yac { + + bool SupermajorityCheckerBft::hasSupermajority( + PeersNumberType agreed, PeersNumberType all) const { + return checkKfPlus1Supermajority( + agreed, all, detail::kSupermajorityCheckerKfPlus1Bft); + } + + bool SupermajorityCheckerBft::canHaveSupermajority( + const VoteGroups &votes, PeersNumberType all) const { + const PeersNumberType largest_group = + boost::empty(votes) ? 0 : *boost::max_element(votes); + const PeersNumberType voted = boost::accumulate(votes, 0); + const PeersNumberType not_voted = all - voted; + + return hasSupermajority(largest_group + not_voted, all); + } + + } // namespace yac + } // namespace consensus +} // namespace iroha diff --git a/irohad/consensus/yac/impl/supermajority_checker_bft.hpp b/irohad/consensus/yac/impl/supermajority_checker_bft.hpp new file mode 100644 index 0000000000..fbba1acb28 --- /dev/null +++ b/irohad/consensus/yac/impl/supermajority_checker_bft.hpp @@ -0,0 +1,33 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef IROHA_SUPERMAJORITY_CHECKER_BFT_HPP +#define IROHA_SUPERMAJORITY_CHECKER_BFT_HPP + +#include "consensus/yac/supermajority_checker.hpp" + +namespace iroha { + namespace consensus { + namespace yac { + + namespace detail { + /// The free parameter of Kf+1 consistency model for BFT. + constexpr unsigned int kSupermajorityCheckerKfPlus1Bft = 3; + } + + /// An implementation of BFT supermajority checker. + class SupermajorityCheckerBft : public SupermajorityChecker { + public: + bool hasSupermajority(PeersNumberType current, + PeersNumberType all) const override; + + bool canHaveSupermajority(const VoteGroups &votes, + PeersNumberType all) const override; + }; + } // namespace yac + } // namespace consensus +} // namespace iroha + +#endif // IROHA_SUPERMAJORITY_CHECKER_BFT_HPP diff --git a/irohad/consensus/yac/impl/supermajority_checker_cft.cpp b/irohad/consensus/yac/impl/supermajority_checker_cft.cpp new file mode 100644 index 0000000000..4af1123a9e --- /dev/null +++ b/irohad/consensus/yac/impl/supermajority_checker_cft.cpp @@ -0,0 +1,34 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "consensus/yac/impl/supermajority_checker_cft.hpp" + +#include +#include +#include "consensus/yac/impl/supermajority_checker_kf1.hpp" + +namespace iroha { + namespace consensus { + namespace yac { + + bool SupermajorityCheckerCft::hasSupermajority( + PeersNumberType agreed, PeersNumberType all) const { + return checkKfPlus1Supermajority( + agreed, all, detail::kSupermajorityCheckerKfPlus1Cft); + } + + bool SupermajorityCheckerCft::canHaveSupermajority( + const VoteGroups &votes, PeersNumberType all) const { + const PeersNumberType largest_group = + boost::empty(votes) ? 0 : *boost::max_element(votes); + const PeersNumberType voted = boost::accumulate(votes, 0); + const PeersNumberType not_voted = all - voted; + + return hasSupermajority(largest_group + not_voted, all); + } + + } // namespace yac + } // namespace consensus +} // namespace iroha diff --git a/irohad/consensus/yac/impl/supermajority_checker_cft.hpp b/irohad/consensus/yac/impl/supermajority_checker_cft.hpp new file mode 100644 index 0000000000..f9424411ca --- /dev/null +++ b/irohad/consensus/yac/impl/supermajority_checker_cft.hpp @@ -0,0 +1,33 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef IROHA_SUPERMAJORITY_CHECKER_CFT_HPP +#define IROHA_SUPERMAJORITY_CHECKER_CFT_HPP + +#include "consensus/yac/supermajority_checker.hpp" + +namespace iroha { + namespace consensus { + namespace yac { + + namespace detail { + /// The free parameter of Kf+1 consistency model for CFT. + constexpr unsigned int kSupermajorityCheckerKfPlus1Cft = 2; + } + + /// An implementation of CFT supermajority checker. + class SupermajorityCheckerCft : public SupermajorityChecker { + public: + bool hasSupermajority(PeersNumberType current, + PeersNumberType all) const override; + + bool canHaveSupermajority(const VoteGroups &votes, + PeersNumberType all) const override; + }; + } // namespace yac + } // namespace consensus +} // namespace iroha + +#endif // IROHA_SUPERMAJORITY_CHECKER_CFT_HPP diff --git a/irohad/consensus/yac/impl/supermajority_checker_getter.cpp b/irohad/consensus/yac/impl/supermajority_checker_getter.cpp new file mode 100644 index 0000000000..d4c0d31c22 --- /dev/null +++ b/irohad/consensus/yac/impl/supermajority_checker_getter.cpp @@ -0,0 +1,29 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "consensus/yac/supermajority_checker.hpp" + +#include "consensus/yac/impl/supermajority_checker_bft.hpp" +#include "consensus/yac/impl/supermajority_checker_cft.hpp" + +namespace iroha { + namespace consensus { + namespace yac { + + std::unique_ptr getSupermajorityChecker( + ConsistencyModel c) { + switch (c) { + case ConsistencyModel::kCft: + return std::make_unique(); + case ConsistencyModel::kBft: + return std::make_unique(); + default: + throw(std::runtime_error("Unknown consistency model requested!")); + } + } + + } // namespace yac + } // namespace consensus +} // namespace iroha diff --git a/irohad/consensus/yac/impl/supermajority_checker_impl.cpp b/irohad/consensus/yac/impl/supermajority_checker_impl.cpp deleted file mode 100644 index 0c6d47c7ff..0000000000 --- a/irohad/consensus/yac/impl/supermajority_checker_impl.cpp +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Copyright Soramitsu Co., Ltd. 2018 All Rights Reserved. - * http://soramitsu.co.jp - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "consensus/yac/impl/supermajority_checker_impl.hpp" -#include -#include "interfaces/common_objects/peer.hpp" -#include "interfaces/common_objects/signature.hpp" -#include "validation/utils.hpp" - -namespace iroha { - namespace consensus { - namespace yac { - - bool SupermajorityCheckerImpl::hasSupermajority( - const shared_model::interface::types::SignatureRangeType &signatures, - const std::vector> - &peers) const { - return checkSize(boost::size(signatures), peers.size()) - and peersSubset(signatures, peers); - } - - bool SupermajorityCheckerImpl::checkSize(PeersNumberType current, - PeersNumberType all) const { - if (current > all) { - return false; - } - auto f = (all - 1) / 3.0; - return current >= 2 * f + 1; - } - - bool SupermajorityCheckerImpl::peersSubset( - const shared_model::interface::types::SignatureRangeType &signatures, - const std::vector> - &peers) const { - return validation::signaturesSubset( - signatures, - peers - | boost::adaptors::transformed( - [](const auto &p) -> decltype(auto) { - return p->pubkey(); - })); - } - - bool SupermajorityCheckerImpl::hasReject(PeersNumberType frequent, - PeersNumberType voted, - PeersNumberType all) const { - auto not_voted = all - voted; - return not checkSize(frequent + not_voted, all); - } - - } // namespace yac - } // namespace consensus -} // namespace iroha diff --git a/irohad/consensus/yac/impl/supermajority_checker_impl.hpp b/irohad/consensus/yac/impl/supermajority_checker_impl.hpp deleted file mode 100644 index d86a37f80f..0000000000 --- a/irohad/consensus/yac/impl/supermajority_checker_impl.hpp +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Copyright Soramitsu Co., Ltd. 2018 All Rights Reserved. - * http://soramitsu.co.jp - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef IROHA_SUPERMAJORITY_CHECKER_IMPL_HPP -#define IROHA_SUPERMAJORITY_CHECKER_IMPL_HPP - -#include "consensus/yac/supermajority_checker.hpp" - -namespace iroha { - namespace consensus { - namespace yac { - class SupermajorityCheckerImpl : public SupermajorityChecker { - public: - virtual ~SupermajorityCheckerImpl() = default; - - /** - * Checks if size of sets allows to have supermajority, - * then checks subset of signatures - * @param signatures set of votes - * @param peers set of all peers - * @return true on supermajority is achieved or false otherwise - */ - virtual bool hasSupermajority( - const shared_model::interface::types::SignatureRangeType - &signatures, - const std::vector> - &peers) const override; - - virtual bool checkSize(PeersNumberType current, - PeersNumberType all) const override; - - virtual bool peersSubset( - const shared_model::interface::types::SignatureRangeType - &signatures, - const std::vector> - &peers) const override; - - virtual bool hasReject(PeersNumberType frequent, - PeersNumberType voted, - PeersNumberType all) const override; - }; - } // namespace yac - } // namespace consensus -} // namespace iroha - -#endif // IROHA_SUPERMAJORITY_CHECKER_IMPL_HPP diff --git a/irohad/consensus/yac/impl/supermajority_checker_kf1.hpp b/irohad/consensus/yac/impl/supermajority_checker_kf1.hpp new file mode 100644 index 0000000000..ef82b3731d --- /dev/null +++ b/irohad/consensus/yac/impl/supermajority_checker_kf1.hpp @@ -0,0 +1,41 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef IROHA_SUPERMAJORITY_CHECKER_KF1_HPP +#define IROHA_SUPERMAJORITY_CHECKER_KF1_HPP + +#include "consensus/yac/supermajority_checker.hpp" + +namespace iroha { + namespace consensus { + namespace yac { + + /** + * A generic implementation of N = K * f + 1 supermajority checker. + * N is the amount of peers in the network, f is the number of tolerated + * faulty peers, and K is a free parameter. Supermajority is achieved when + * at least N - f peers agree. For the networks of arbitrary peers amount + * Na the tolerated number of faulty peers is (Na - 1) % K. + * + * @param agreed - the number of peers agreed on the state + * @param all - the total number of peers in the network + * @param k - the free parameter of the model + * + * @return whether supermajority is achieved by the agreed peers + */ + inline bool checkKfPlus1Supermajority(PeersNumberType agreed, + PeersNumberType all, + unsigned int k) { + if (agreed > all) { + return false; + } + return agreed * k >= (k - 1) * (all - 1) + k; + } + + } // namespace yac + } // namespace consensus +} // namespace iroha + +#endif // IROHA_SUPERMAJORITY_CHECKER_KF1_HPP diff --git a/irohad/consensus/yac/storage/impl/yac_block_storage.cpp b/irohad/consensus/yac/storage/impl/yac_block_storage.cpp index 78aa26f0f1..65583eb78f 100644 --- a/irohad/consensus/yac/storage/impl/yac_block_storage.cpp +++ b/irohad/consensus/yac/storage/impl/yac_block_storage.cpp @@ -56,8 +56,8 @@ namespace iroha { } boost::optional YacBlockStorage::getState() { - auto supermajority = - supermajority_checker_->checkSize(votes_.size(), peers_in_round_); + auto supermajority = supermajority_checker_->hasSupermajority( + votes_.size(), peers_in_round_); if (supermajority) { return Answer(CommitMessage(votes_)); } diff --git a/irohad/consensus/yac/storage/impl/yac_proposal_storage.cpp b/irohad/consensus/yac/storage/impl/yac_proposal_storage.cpp index 3dd5f9b15e..e96a9e4005 100644 --- a/irohad/consensus/yac/storage/impl/yac_proposal_storage.cpp +++ b/irohad/consensus/yac/storage/impl/yac_proposal_storage.cpp @@ -5,6 +5,8 @@ #include "consensus/yac/storage/yac_proposal_storage.hpp" +#include + using namespace logger; namespace iroha { @@ -116,28 +118,15 @@ namespace iroha { } boost::optional YacProposalStorage::findRejectProof() { - auto max_vote = std::max_element(block_storages_.begin(), - block_storages_.end(), - [](auto &left, auto &right) { - return left.getNumberOfVotes() - < right.getNumberOfVotes(); - }) - ->getNumberOfVotes(); - - auto all_votes = - std::accumulate(block_storages_.begin(), - block_storages_.end(), - 0ull, - [](auto &acc, auto &storage) { - return acc + storage.getNumberOfVotes(); - }); - - auto is_reject = supermajority_checker_->hasReject( - max_vote, all_votes, peers_in_round_); + auto is_reject = not supermajority_checker_->canHaveSupermajority( + block_storages_ + | boost::adaptors::transformed([](const auto &storage) { + return storage.getNumberOfVotes(); + }), + peers_in_round_); if (is_reject) { std::vector result; - result.reserve(all_votes); std::for_each(block_storages_.begin(), block_storages_.end(), [&result](auto &storage) { diff --git a/irohad/consensus/yac/storage/impl/yac_vote_storage.cpp b/irohad/consensus/yac/storage/impl/yac_vote_storage.cpp index e03543c491..1bf6f3ec67 100644 --- a/irohad/consensus/yac/storage/impl/yac_vote_storage.cpp +++ b/irohad/consensus/yac/storage/impl/yac_vote_storage.cpp @@ -9,6 +9,7 @@ #include #include "common/bind.hpp" +#include "consensus/yac/consistency_model.hpp" #include "consensus/yac/storage/yac_proposal_storage.hpp" namespace iroha { @@ -34,11 +35,10 @@ namespace iroha { return val; } if (strategy_->shouldCreateRound(round)) { - return proposal_storages_.emplace( - proposal_storages_.end(), - msg.hash.vote_round, - peers_in_round, - std::make_shared()); + return proposal_storages_.emplace(proposal_storages_.end(), + msg.hash.vote_round, + peers_in_round, + supermajority_checker_); } else { return boost::none; } @@ -58,8 +58,10 @@ namespace iroha { // --------| public api |-------- YacVoteStorage::YacVoteStorage( - std::shared_ptr cleanup_strategy) - : strategy_(std::move(cleanup_strategy)) {} + std::shared_ptr cleanup_strategy, + std::unique_ptr supermajority_checker) + : strategy_(std::move(cleanup_strategy)), + supermajority_checker_(std::move(supermajority_checker)) {} boost::optional YacVoteStorage::store( std::vector state, PeersNumberType peers_in_round) { diff --git a/irohad/consensus/yac/storage/yac_block_storage.hpp b/irohad/consensus/yac/storage/yac_block_storage.hpp index 169ecc7f05..607065a637 100644 --- a/irohad/consensus/yac/storage/yac_block_storage.hpp +++ b/irohad/consensus/yac/storage/yac_block_storage.hpp @@ -1,18 +1,6 @@ /** - * Copyright Soramitsu Co., Ltd. 2018 All Rights Reserved. - * http://soramitsu.co.jp - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 */ #ifndef IROHA_YAC_BLOCK_VOTE_STORAGE_HPP @@ -22,9 +10,9 @@ #include #include -#include "consensus/yac/impl/supermajority_checker_impl.hpp" #include "consensus/yac/outcome_messages.hpp" #include "consensus/yac/storage/storage_result.hpp" +#include "consensus/yac/supermajority_checker.hpp" #include "consensus/yac/yac_types.hpp" #include "logger/logger.hpp" @@ -47,8 +35,7 @@ namespace iroha { YacBlockStorage( YacHash hash, PeersNumberType peers_in_round, - std::shared_ptr supermajority_checker = - std::make_shared()); + std::shared_ptr supermajority_checker); /** * Try to insert vote to storage diff --git a/irohad/consensus/yac/storage/yac_proposal_storage.hpp b/irohad/consensus/yac/storage/yac_proposal_storage.hpp index 29deb56386..e2431a73ae 100644 --- a/irohad/consensus/yac/storage/yac_proposal_storage.hpp +++ b/irohad/consensus/yac/storage/yac_proposal_storage.hpp @@ -10,10 +10,10 @@ #include #include -#include "consensus/yac/impl/supermajority_checker_impl.hpp" #include "consensus/yac/storage/storage_result.hpp" #include "consensus/yac/storage/yac_block_storage.hpp" #include "consensus/yac/storage/yac_common.hpp" +#include "consensus/yac/supermajority_checker.hpp" #include "consensus/yac/yac_types.hpp" #include "logger/logger.hpp" @@ -45,8 +45,7 @@ namespace iroha { YacProposalStorage( Round store_round, PeersNumberType peers_in_round, - std::shared_ptr supermajority_checker = - std::make_shared()); + std::shared_ptr supermajority_checker); /** * Try to insert vote to storage diff --git a/irohad/consensus/yac/storage/yac_vote_storage.hpp b/irohad/consensus/yac/storage/yac_vote_storage.hpp index 5e9132b409..5a3dbbc535 100644 --- a/irohad/consensus/yac/storage/yac_vote_storage.hpp +++ b/irohad/consensus/yac/storage/yac_vote_storage.hpp @@ -11,11 +11,13 @@ #include #include +#include "consensus/yac/consistency_model.hpp" #include "consensus/yac/outcome_messages.hpp" // because messages passed by value #include "consensus/yac/storage/cleanup_strategy.hpp" #include "consensus/yac/storage/storage_result.hpp" // for Answer #include "consensus/yac/storage/yac_common.hpp" // for ProposalHash #include "consensus/yac/storage/yac_proposal_storage.hpp" +#include "consensus/yac/supermajority_checker.hpp" #include "consensus/yac/yac_types.hpp" namespace iroha { @@ -90,8 +92,11 @@ namespace iroha { /** * @param cleanup_strategy - strategy for removing elements from storage + * @param consistency_model - consensus consistency model (CFT, BFT). */ - YacVoteStorage(std::shared_ptr cleanup_strategy); + YacVoteStorage( + std::shared_ptr cleanup_strategy, + std::unique_ptr supermajority_checker); /** * Insert votes in storage @@ -152,6 +157,8 @@ namespace iroha { * storage */ std::shared_ptr strategy_; + + std::shared_ptr supermajority_checker_; }; } // namespace yac diff --git a/irohad/consensus/yac/supermajority_checker.hpp b/irohad/consensus/yac/supermajority_checker.hpp index 387930de8f..d3acd904b7 100644 --- a/irohad/consensus/yac/supermajority_checker.hpp +++ b/irohad/consensus/yac/supermajority_checker.hpp @@ -9,6 +9,8 @@ #include #include +#include +#include "consensus/yac/consistency_model.hpp" #include "consensus/yac/yac_types.hpp" #include "interfaces/common_objects/range_types.hpp" #include "interfaces/common_objects/types.hpp" @@ -20,7 +22,6 @@ namespace shared_model { } // namespace shared_model namespace iroha { - namespace consensus { namespace yac { @@ -29,55 +30,39 @@ namespace iroha { */ class SupermajorityChecker { public: + using VoteGroups = boost::any_range; + virtual ~SupermajorityChecker() = default; /** * Check if supermajority is achieved - * @param signatures set of signatures to check - * @param peers set of peers with signatures - * @return true on supermajority is achieved or false otherwise - */ - virtual bool hasSupermajority( - const shared_model::interface::types::SignatureRangeType - &signatures, - const std::vector> - &peers) const = 0; - - /** - * Check if supermajority is possible * @param current actual number of signatures * @param all number of peers * @return true if supermajority is possible or false otherwise */ - virtual bool checkSize(PeersNumberType current, - PeersNumberType all) const = 0; + virtual bool hasSupermajority(PeersNumberType current, + PeersNumberType all) const = 0; /** - * Checks if signatures is a subset of signatures of peers - * @param signatures to check - * @param peers with signatures - * @return true if is subset or false otherwise - */ - virtual bool peersSubset( - const shared_model::interface::types::SignatureRangeType - &signatures, - const std::vector> - &peers) const = 0; - - /** - * Check if there is available reject proof. - * Reject proof is proof that in current round - * no one hash doesn't achieve supermajority. - * @param frequent - number of times, that appears most frequent element - * @param voted - all number of voted peers + * Check if supermajority is possible + * @param voted - numbers of peers voted for each option * @param all - number of peers in round * @return true, if reject */ - virtual bool hasReject(PeersNumberType frequent, - PeersNumberType voted, - PeersNumberType all) const = 0; + virtual bool canHaveSupermajority(const VoteGroups &votes, + PeersNumberType all) const = 0; }; + /// Get a SupermajorityChecker for the given consistency model. + std::unique_ptr getSupermajorityChecker( + ConsistencyModel c); + } // namespace yac } // namespace consensus } // namespace iroha diff --git a/irohad/main/application.cpp b/irohad/main/application.cpp index 5f7fe77061..7ee51b6192 100644 --- a/irohad/main/application.cpp +++ b/irohad/main/application.cpp @@ -16,7 +16,7 @@ #include "backend/protobuf/proto_transport_factory.hpp" #include "backend/protobuf/proto_tx_status_factory.hpp" #include "common/bind.hpp" -#include "consensus/yac/impl/supermajority_checker_impl.hpp" +#include "consensus/yac/consistency_model.hpp" #include "cryptography/crypto_provider/crypto_model_signer.hpp" #include "interfaces/iroha_internal/transaction_batch_factory_impl.hpp" #include "interfaces/iroha_internal/transaction_batch_parser_impl.hpp" @@ -61,6 +61,10 @@ using namespace iroha::consensus::yac; using namespace std::chrono_literals; +/// Consensus consistency model type. +static constexpr iroha::consensus::yac::ConsistencyModel + kConsensusConsistencyModel = iroha::consensus::yac::ConsistencyModel::kBft; + /** * Configuring iroha daemon */ @@ -200,7 +204,7 @@ void Irohad::initValidators() { stateful_validator = std::make_shared(std::move(factory), batch_parser); chain_validator = std::make_shared( - std::make_shared()); + getSupermajorityChecker(kConsensusConsistencyModel)); log_->info("[Init] => validators"); } @@ -406,7 +410,8 @@ void Irohad::initConsensusGate() { consensus_result_cache_, vote_delay_, async_call_, - common_objects_factory_); + common_objects_factory_, + kConsensusConsistencyModel); consensus_gate->onOutcome().subscribe( consensus_gate_events_subscription, consensus_gate_objects.get_subscriber()); @@ -574,7 +579,7 @@ Irohad::RunResult Irohad::run() { } // Run internal server return internal_server->append(ordering_init.service) - .append(yac_init.consensus_network) + .append(yac_init.getConsensusNetwork()) .append(loader_init.service) .run(); }) diff --git a/irohad/main/impl/consensus_init.cpp b/irohad/main/impl/consensus_init.cpp index 0f05c4918c..752591469c 100644 --- a/irohad/main/impl/consensus_init.cpp +++ b/irohad/main/impl/consensus_init.cpp @@ -4,6 +4,8 @@ */ #include "main/impl/consensus_init.hpp" + +#include "consensus/yac/consistency_model.hpp" #include "consensus/yac/impl/peer_orderer_impl.hpp" #include "consensus/yac/impl/timer_impl.hpp" #include "consensus/yac/impl/yac_crypto_provider_impl.hpp" @@ -13,31 +15,57 @@ #include "consensus/yac/storage/yac_proposal_storage.hpp" #include "consensus/yac/transport/impl/network_impl.hpp" -namespace iroha { - namespace consensus { - namespace yac { +using namespace iroha::consensus::yac; - auto YacInit::createPeerOrderer( - std::shared_ptr peer_query_factory) { - return std::make_shared(peer_query_factory); - } +namespace { + auto createPeerOrderer( + std::shared_ptr peer_query_factory) { + return std::make_shared(peer_query_factory); + } - auto YacInit::createNetwork( - std::shared_ptr< - iroha::network::AsyncGrpcClient> - async_call) { - consensus_network = std::make_shared(async_call); - return consensus_network; - } + auto createCryptoProvider( + const shared_model::crypto::Keypair &keypair, + std::shared_ptr + common_objects_factory) { + auto crypto = std::make_shared( + keypair, std::move(common_objects_factory)); - auto YacInit::createCryptoProvider( - const shared_model::crypto::Keypair &keypair, - std::shared_ptr - common_objects_factory) { - auto crypto = std::make_shared( - keypair, std::move(common_objects_factory)); + return crypto; + } + + auto createHashProvider() { + return std::make_shared(); + } + + std::shared_ptr createYac( + ClusterOrdering initial_order, + const shared_model::crypto::Keypair &keypair, + std::shared_ptr timer, + std::shared_ptr network, + std::shared_ptr + common_objects_factory, + ConsistencyModel consistency_model) { + std::shared_ptr cleanup_strategy = + std::make_shared(); + return Yac::create( + YacVoteStorage(cleanup_strategy, + getSupermajorityChecker(consistency_model)), + std::move(network), + createCryptoProvider(keypair, std::move(common_objects_factory)), + std::move(timer), + initial_order); + } +} // namespace + +namespace iroha { + namespace consensus { + namespace yac { - return crypto; + std::shared_ptr YacInit::getConsensusNetwork() const { + BOOST_ASSERT_MSG(initialized_, + "YacInit::initConsensusGate(...) must be called prior " + "to YacInit::getConsensusNetwork()!"); + return consensus_network_; } auto YacInit::createTimer(std::chrono::milliseconds delay_milliseconds) { @@ -57,32 +85,9 @@ namespace iroha { }); } - auto YacInit::createHashProvider() { - return std::make_shared(); - } - - std::shared_ptr YacInit::createYac( - ClusterOrdering initial_order, - const shared_model::crypto::Keypair &keypair, - std::chrono::milliseconds delay_milliseconds, - std::shared_ptr< - iroha::network::AsyncGrpcClient> - async_call, - std::shared_ptr - common_objects_factory) { - std::shared_ptr - cleanup_strategy = std::make_shared< - iroha::consensus::yac::BufferedCleanupStrategy>(); - return Yac::create( - YacVoteStorage(cleanup_strategy), - createNetwork(std::move(async_call)), - createCryptoProvider(keypair, std::move(common_objects_factory)), - createTimer(delay_milliseconds), - initial_order); - } - std::shared_ptr YacInit::initConsensusGate( - std::shared_ptr peer_query_factory, + std::shared_ptr + peer_query_factory, std::shared_ptr block_creator, std::shared_ptr block_loader, const shared_model::crypto::Keypair &keypair, @@ -93,17 +98,24 @@ namespace iroha { iroha::network::AsyncGrpcClient> async_call, std::shared_ptr - common_objects_factory) { + common_objects_factory, + ConsistencyModel consistency_model) { auto peer_orderer = createPeerOrderer(peer_query_factory); + consensus_network_ = std::make_shared(async_call); + auto yac = createYac(peer_orderer->getInitialOrdering().value(), keypair, - vote_delay_milliseconds, - std::move(async_call), - std::move(common_objects_factory)); - consensus_network->subscribe(yac); + createTimer(vote_delay_milliseconds), + consensus_network_, + std::move(common_objects_factory), + consistency_model); + consensus_network_->subscribe(yac); auto hash_provider = createHashProvider(); + + initialized_ = true; + return std::make_shared(std::move(yac), std::move(peer_orderer), hash_provider, diff --git a/irohad/main/impl/consensus_init.hpp b/irohad/main/impl/consensus_init.hpp index fdd4201f22..dfe67ce484 100644 --- a/irohad/main/impl/consensus_init.hpp +++ b/irohad/main/impl/consensus_init.hpp @@ -22,6 +22,7 @@ #include "cryptography/keypair.hpp" #include "interfaces/common_objects/common_objects_factory.hpp" #include "network/block_loader.hpp" +#include "network/impl/async_grpc_client.hpp" #include "simulator/block_creator.hpp" namespace iroha { @@ -29,33 +30,25 @@ namespace iroha { namespace yac { class YacInit { - private: - // ----------| Yac dependencies |---------- - - auto createPeerOrderer( - std::shared_ptr peer_query_factory); - - auto createNetwork(std::shared_ptr> async_call); - - auto createCryptoProvider( - const shared_model::crypto::Keypair &keypair, - std::shared_ptr - common_objects_factory); - - auto createTimer(std::chrono::milliseconds delay_milliseconds); - - auto createHashProvider(); - - std::shared_ptr createYac( - ClusterOrdering initial_order, + public: + std::shared_ptr initConsensusGate( + std::shared_ptr peer_query_factory, + std::shared_ptr block_creator, + std::shared_ptr block_loader, const shared_model::crypto::Keypair &keypair, - std::chrono::milliseconds delay_milliseconds, + std::shared_ptr block_cache, + std::chrono::milliseconds vote_delay_milliseconds, std::shared_ptr< iroha::network::AsyncGrpcClient> async_call, std::shared_ptr - common_objects_factory); + common_objects_factory, + ConsistencyModel consistency_model); + + std::shared_ptr getConsensusNetwork() const; + + private: + auto createTimer(std::chrono::milliseconds delay_milliseconds); // coordinator has a worker, and a factory for coordinated // observables, subscribers and schedulable functions. @@ -65,21 +58,8 @@ namespace iroha { rxcpp::observe_on_one_worker coordination_{ rxcpp::observe_on_new_thread()}; - public: - std::shared_ptr initConsensusGate( - std::shared_ptr peer_query_factory, - std::shared_ptr block_creator, - std::shared_ptr block_loader, - const shared_model::crypto::Keypair &keypair, - std::shared_ptr block_cache, - std::chrono::milliseconds vote_delay_milliseconds, - std::shared_ptr< - iroha::network::AsyncGrpcClient> - async_call, - std::shared_ptr - common_objects_factory); - - std::shared_ptr consensus_network; + bool initialized_{false}; + std::shared_ptr consensus_network_; }; } // namespace yac } // namespace consensus diff --git a/irohad/validation/CMakeLists.txt b/irohad/validation/CMakeLists.txt index ad0d76602f..7deac2d865 100644 --- a/irohad/validation/CMakeLists.txt +++ b/irohad/validation/CMakeLists.txt @@ -21,5 +21,5 @@ target_link_libraries(chain_validator rxcpp shared_model_interfaces logger - supermajority_check + supermajority_checker ) diff --git a/irohad/validation/impl/chain_validator_impl.cpp b/irohad/validation/impl/chain_validator_impl.cpp index 2dce053c0b..dc07ad5b80 100644 --- a/irohad/validation/impl/chain_validator_impl.cpp +++ b/irohad/validation/impl/chain_validator_impl.cpp @@ -11,6 +11,7 @@ #include "cryptography/public_key.hpp" #include "interfaces/common_objects/peer.hpp" #include "interfaces/iroha_internal/block.hpp" +#include "validation/utils.hpp" namespace iroha { namespace validation { @@ -54,8 +55,9 @@ namespace iroha { const std::vector> &peers) const { const auto &signatures = block.signatures(); - auto has_supermajority = - supermajority_checker_->hasSupermajority(signatures, peers); + auto has_supermajority = supermajority_checker_->hasSupermajority( + boost::size(signatures), peers.size()) + and peersSubset(signatures, peers); if (not has_supermajority) { log_->info( diff --git a/irohad/validation/utils.hpp b/irohad/validation/utils.hpp index 7ee6bca64b..5df78f96ae 100644 --- a/irohad/validation/utils.hpp +++ b/irohad/validation/utils.hpp @@ -9,6 +9,7 @@ #include #include +#include #include #include "cryptography/public_key.hpp" @@ -40,6 +41,24 @@ namespace iroha { }); } + /** + * Checks if `signatures' is a subset of signatures of `peers' + * @param signatures to check + * @param peers with signatures + * @return true if is a subset or false otherwise + */ + template + inline bool peersSubset( + const shared_model::interface::types::SignatureRangeType &signatures, + const Peers &peers) { + return signaturesSubset(signatures, + peers + | boost::adaptors::transformed( + [](const auto &p) -> decltype(auto) { + return p->pubkey(); + })); + } + } // namespace validation } // namespace iroha diff --git a/test/integration/consensus/consensus_sunny_day.cpp b/test/integration/consensus/consensus_sunny_day.cpp index 6a22a53070..a92458e03b 100644 --- a/test/integration/consensus/consensus_sunny_day.cpp +++ b/test/integration/consensus/consensus_sunny_day.cpp @@ -28,6 +28,10 @@ using ::testing::Return; using namespace iroha::consensus::yac; using namespace framework::test_subscriber; +// TODO mboldyrev 14.02.2019 IR-324 Use supermajority checker mock +static const iroha::consensus::yac::ConsistencyModel kConsistencyModel = + iroha::consensus::yac::ConsistencyModel::kBft; + static size_t num_peers = 1, my_num = 0; auto mk_local_peer(uint64_t num) { @@ -103,11 +107,13 @@ class ConsensusSunnyDayTest : public ::testing::Test { auto order = ClusterOrdering::create(default_peers); ASSERT_TRUE(order); - yac = Yac::create(YacVoteStorage(cleanup_strategy), - network, - crypto, - timer, - order.value()); + yac = + Yac::create(YacVoteStorage(cleanup_strategy, + getSupermajorityChecker(kConsistencyModel)), + network, + crypto, + timer, + order.value()); network->subscribe(yac); grpc::ServerBuilder builder; diff --git a/test/integration/validation/chain_validator_storage_test.cpp b/test/integration/validation/chain_validator_storage_test.cpp index 88e5649c98..c2232c86b1 100644 --- a/test/integration/validation/chain_validator_storage_test.cpp +++ b/test/integration/validation/chain_validator_storage_test.cpp @@ -3,17 +3,21 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include "consensus/yac/impl/supermajority_checker_impl.hpp" #include "module/irohad/ametsuchi/ametsuchi_fixture.hpp" #include "validation/impl/chain_validator_impl.hpp" #include "ametsuchi/mutable_storage.hpp" #include "builders/protobuf/transaction.hpp" +#include "consensus/yac/supermajority_checker.hpp" #include "cryptography/crypto_provider/crypto_defaults.hpp" #include "cryptography/default_hash_provider.hpp" #include "cryptography/keypair.hpp" #include "module/shared_model/builders/protobuf/block.hpp" +// TODO mboldyrev 14.02.2019 IR-324 Use supermajority checker mock +static const iroha::consensus::yac::ConsistencyModel kConsistencyModel = + iroha::consensus::yac::ConsistencyModel::kBft; + namespace iroha { class ChainValidatorStorageTest : public ametsuchi::AmetsuchiTest { @@ -21,7 +25,7 @@ namespace iroha { void SetUp() override { ametsuchi::AmetsuchiTest::SetUp(); validator = std::make_shared( - std::make_shared()); + supermajority_checker); for (size_t i = 0; i < 5; ++i) { keys.push_back(shared_model::crypto::DefaultCryptoAlgorithmType:: @@ -108,14 +112,19 @@ namespace iroha { std::shared_ptr validator; std::vector keys; + std::shared_ptr + supermajority_checker = consensus::yac::getSupermajorityChecker( + kConsistencyModel // TODO mboldyrev 13.12.2018 IR- + // Parametrize the tests with + // consistency models + ); }; /** * @given initialized storage * block 1 - initial block with 4 peers - * block 2 - new peer added. signed by supermajority of ledger peers - * block 3 - signed by supermajority of ledger peers, contains signature of - * new peer + * block 2 - new peer added. signed by all ledger peers + * block 3 - signed by all ledger peers, contains signature of new peer * @when blocks 2 and 3 are validated * @then result is successful */ @@ -142,8 +151,8 @@ namespace iroha { /** * @given initialized storage with 4 peers * block 1 - initial block with 4 peers - * block 2 - signed by supermajority of ledger peers - * block 3 - signed by supermajority of ledger peers + * block 2 - signed by all ledger peers + * block 3 - signed by all ledger peers * @when blocks 2 and 3 are validated * @then result is successful */ @@ -167,7 +176,7 @@ namespace iroha { /** * @given initialized storage * block 1 - initial block with 4 peers - * block 2 - invalid previous hash, signed by supermajority + * block 2 - invalid previous hash, signed by all peers * @when block 2 is validated * @then result is not successful */ @@ -181,7 +190,8 @@ namespace iroha { shared_model::crypto::Blob("bad_hash"))) .signAndAddSignature(keys.at(0)) .signAndAddSignature(keys.at(1)) - .signAndAddSignature(keys.at(2))); + .signAndAddSignature(keys.at(2)) + .signAndAddSignature(keys.at(3))); ASSERT_FALSE(createAndValidateChain({clone(block2)})); } @@ -196,6 +206,8 @@ namespace iroha { TEST_F(ChainValidatorStorageTest, NoSupermajority) { auto block1 = generateAndApplyFirstBlock(); + ASSERT_FALSE(supermajority_checker->hasSupermajority(2, 4)) + << "This test assumes that 2 out of 4 peers do not have supermajority!"; auto block2 = completeBlock(baseBlock({dummyTx(2)}, 2, block1.hash()) .signAndAddSignature(keys.at(0)) .signAndAddSignature(keys.at(1))); diff --git a/test/module/irohad/consensus/yac/mock_yac_supermajority_checker.hpp b/test/module/irohad/consensus/yac/mock_yac_supermajority_checker.hpp index 840db36210..42ab2a57f4 100644 --- a/test/module/irohad/consensus/yac/mock_yac_supermajority_checker.hpp +++ b/test/module/irohad/consensus/yac/mock_yac_supermajority_checker.hpp @@ -16,22 +16,9 @@ namespace iroha { class MockSupermajorityChecker : public SupermajorityChecker { public: - MOCK_CONST_METHOD2( - hasSupermajority, - bool(const shared_model::interface::types::SignatureRangeType &, - const std::vector< - std::shared_ptr> &)); - - MOCK_CONST_METHOD2(checkSize, bool(PeersNumberType, PeersNumberType)); - - MOCK_CONST_METHOD2( - peersSubset, - bool(const shared_model::interface::types::SignatureRangeType &, - const std::vector< - std::shared_ptr> &)); - - MOCK_CONST_METHOD3( - hasReject, bool(PeersNumberType, PeersNumberType, PeersNumberType)); + MOCK_CONST_METHOD2(hasSupermajority, bool(PeersNumberType, PeersNumberType)); + MOCK_CONST_METHOD2(canHaveSupermajority, + bool(const VoteGroups &, PeersNumberType)); }; } // namespace yac diff --git a/test/module/irohad/consensus/yac/supermajority_checker_test.cpp b/test/module/irohad/consensus/yac/supermajority_checker_test.cpp index 4a2e95f2a3..f5a4aa66c0 100644 --- a/test/module/irohad/consensus/yac/supermajority_checker_test.cpp +++ b/test/module/irohad/consensus/yac/supermajority_checker_test.cpp @@ -3,11 +3,15 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include "consensus/yac/impl/supermajority_checker_impl.hpp" +#include "consensus/yac/supermajority_checker.hpp" +#include "consensus/yac/impl/supermajority_checker_bft.hpp" +#include "consensus/yac/impl/supermajority_checker_cft.hpp" #include -#include +#include +#include "boost/tuple/tuple.hpp" +#include "consensus/yac/supermajority_checker.hpp" #include "logger/logger.hpp" #include "module/shared_model/interface_mocks.hpp" @@ -18,25 +22,85 @@ using ::testing::ReturnRefOfCopy; static logger::Logger log_ = logger::testLog("YacCommon"); -class SupermajorityCheckerTest : public ::testing::Test, - public SupermajorityCheckerImpl { +static const std::map kf1_param{ + {ConsistencyModel::kCft, detail::kSupermajorityCheckerKfPlus1Cft}, + {ConsistencyModel::kBft, detail::kSupermajorityCheckerKfPlus1Bft}}; + +class SupermajorityCheckerTest + : public ::testing::TestWithParam, + public SupermajorityChecker { public: void SetUp() override {} + + unsigned int getK() const { + return kf1_param.at(GetParam()); + } + + size_t getAllowedFaultyPeers(size_t total_peers) const { + return (total_peers - 1) / getK(); + } + + size_t getSupermajority(size_t total_peers) const { + return total_peers - getAllowedFaultyPeers(total_peers); + } + + std::string modelToString() const { + return "`" + std::to_string(getK()) + " * f + 1' " + + (GetParam() == ConsistencyModel::kBft ? "BFT" : "CFT") + " model"; + } + + bool hasSupermajority(PeersNumberType current, + PeersNumberType all) const override { + return checker->hasSupermajority(current, all); + } + + bool canHaveSupermajority(const VoteGroups &votes, + PeersNumberType all) const override { + return checker->canHaveSupermajority(votes, all); + } + + std::unique_ptr checker{ + getSupermajorityChecker(GetParam())}; }; +using CftAndBftSupermajorityCheckerTest = SupermajorityCheckerTest; +using BftSupermajorityCheckerTest = SupermajorityCheckerTest; + +INSTANTIATE_TEST_CASE_P(Instance, + CftAndBftSupermajorityCheckerTest, + ::testing::Values(ConsistencyModel::kCft, + ConsistencyModel::kBft), + // empty argument for the macro + ); + +INSTANTIATE_TEST_CASE_P(Instance, + BftSupermajorityCheckerTest, + ::testing::Values(ConsistencyModel::kBft), + // empty argument for the macro + ); + /** * @given 2 participants * @when check range of voted participants * @then correct result */ -TEST_F(SupermajorityCheckerTest, SuperMajorityCheckWithSize2) { - log_->info("-----------| F(x, 2), x in {0..3} -----------"); - - int N = 2; - ASSERT_FALSE(checkSize(0, N)); - ASSERT_FALSE(checkSize(1, N)); - ASSERT_TRUE(checkSize(2, N)); - ASSERT_FALSE(checkSize(3, N)); +TEST_P(CftAndBftSupermajorityCheckerTest, SuperMajorityCheckWithSize2) { + log_->info("-----------| F(x, 2), x in [0..3] |-----------"); + + size_t A = 2; // number of all peers + for (size_t i = 0; i < 4; ++i) { + if (i >= getSupermajority(A) // enough votes + and i <= A // not more than total peers amount + ) { + ASSERT_TRUE(hasSupermajority(i, A)) + << i << " votes out of " << A << " are supermajority in " + << modelToString(); + } else { + ASSERT_FALSE(hasSupermajority(i, A)) + << i << " votes out of " << A << " are not a supermajority in " + << modelToString(); + } + } } /** @@ -44,70 +108,79 @@ TEST_F(SupermajorityCheckerTest, SuperMajorityCheckWithSize2) { * @when check range of voted participants * @then correct result */ -TEST_F(SupermajorityCheckerTest, SuperMajorityCheckWithSize4) { - log_->info("-----------| F(x, 4), x in {0..5} |-----------"); - - int N = 4; - ASSERT_FALSE(checkSize(0, N)); - ASSERT_FALSE(checkSize(1, N)); - ASSERT_FALSE(checkSize(2, N)); - ASSERT_TRUE(checkSize(3, N)); - ASSERT_TRUE(checkSize(4, N)); - ASSERT_FALSE(checkSize(5, N)); +TEST_P(SupermajorityCheckerTest, SuperMajorityCheckWithSize4) { + log_->info("-----------| F(x, 4), x in [0..5] |-----------"); + + size_t A = 6; // number of all peers + for (size_t i = 0; i < 5; ++i) { + if (i >= getSupermajority(A) // enough votes + and i <= A // not more than total peers amount + ) { + ASSERT_TRUE(hasSupermajority(i, A)) + << i << " votes out of " << A << " are supermajority in " + << modelToString(); + } else { + ASSERT_FALSE(hasSupermajority(i, A)) + << i << " votes out of " << A << " are not a supermajority in " + << modelToString(); + } + } } /** - * @given 7 participants, 6 voted - * @when check range of frequent elements - * @then correct result + * \attention this test does not address possible supermajority on other peers + * due to malicious peers sending then votes for other hashes + * + * @given some peers vote all for one option, others vote each for own option, + * and the rest do not vote + * @when different amounts of described peer kinds + * @then correct decision on supermajority possibility */ -TEST_F(SupermajorityCheckerTest, RejectProofSuccessfulCase) { - log_->info("-----------| RejectProof(x, 6, 7) in {1..3} |-----------"); - - ASSERT_TRUE(hasReject(1, 6, 7)); - ASSERT_TRUE(hasReject(2, 6, 7)); - ASSERT_TRUE(hasReject(3, 6, 7)); -} - -/** - * @given 7 participants, 6 voted - * @when check range of frequent elements - * @then correct result - */ -TEST_F(SupermajorityCheckerTest, RejectProofNegativeCase) { - log_->info("-----------| RejectProof(x, 6, 7) in {4..6}|-----------"); - - ASSERT_FALSE(hasReject(4, 6, 7)); - ASSERT_FALSE(hasReject(5, 6, 7)); - ASSERT_FALSE(hasReject(6, 6, 7)); -} - -/** - * @given a pair of peers and a pair different signatures by the first peer - * @when hasSupermajority is called - * @then it return false - */ -TEST_F(SupermajorityCheckerTest, PublicKeyUniqueness) { - using namespace shared_model::crypto; - using namespace std::string_literals; - std::vector> peers; - auto make_peer_key = [&peers](const std::string &key) { - PublicKey pub_key(key); - auto peer = std::make_shared(); - EXPECT_CALL(*peer, pubkey()).WillRepeatedly(ReturnRefOfCopy(pub_key)); - - peers.push_back(peer); - return pub_key; +TEST_P(CftAndBftSupermajorityCheckerTest, OneLargeAndManySingleVoteGroups) { + /** + * Make vote groups out of given votes amount such that the largest one has + * the given amount of votes. + */ + auto makeVoteGroups = [](size_t largest_group, size_t voted_peers) { + BOOST_ASSERT_MSG( + largest_group <= voted_peers, + "A votes group cannot have more votes than the amount of voted peers!"); + const size_t num_groups = voted_peers - largest_group + 1; + std::vector vote_groups(num_groups, 1); + vote_groups[0] = largest_group; + return vote_groups; }; - auto peer_key = make_peer_key(std::string(32, '0')); - make_peer_key(std::string(32, '1')); - - auto sig = std::make_shared(); - EXPECT_CALL(*sig, publicKey()).WillRepeatedly(ReturnRefOfCopy(peer_key)); + struct Case { + size_t V; // Voted peers + size_t A; // All peers + }; - // previous version of the test relied on Block interface, which stored a set - // of signatures by public key - std::vector sigs{1, sig}; - ASSERT_FALSE(hasSupermajority(sigs | boost::adaptors::indirected, peers)); + for (const auto &c : std::initializer_list({{6, 7}, {8, 12}})) { + log_->info("--------| ProofOfReject(x, {0}, {1}), x in [1..{0}] |---------", + c.V, + c.A); + + size_t L; // number of votes for the Leading option + for (L = 1; L <= c.V; ++L) { + const auto vote_groups = makeVoteGroups(L, c.V); + const size_t N = c.A - c.V; // not yet voted + const size_t S = getSupermajority(c.A); // supermajority + const size_t Lp = // max possible votes amount for the largest group + L // the votes we know + + N; // the peers that we have no votes from + // Check if any peer on the network can get supermajority: + if (Lp >= S) { + EXPECT_TRUE(canHaveSupermajority(vote_groups, c.A)) + << "if " << N << " not yet voted peers " + << "vote for option already having " << L << " votes, " + << "it will reach supermajority in " << modelToString(); + } else { + EXPECT_FALSE(canHaveSupermajority(vote_groups, c.A)) + << "even if all " << N << " not yet voted peers " + << "vote for option already having " << L << " votes, " + << "it will not reach supermajority in " << modelToString(); + } + } + } } diff --git a/test/module/irohad/consensus/yac/yac_block_storage_test.cpp b/test/module/irohad/consensus/yac/yac_block_storage_test.cpp index 61ecd320d5..4ef97c5de0 100644 --- a/test/module/irohad/consensus/yac/yac_block_storage_test.cpp +++ b/test/module/irohad/consensus/yac/yac_block_storage_test.cpp @@ -7,6 +7,7 @@ #include +#include "consensus/yac/impl/supermajority_checker_bft.hpp" #include "consensus/yac/storage/yac_proposal_storage.hpp" #include "logger/logger.hpp" @@ -14,43 +15,55 @@ using namespace iroha::consensus::yac; +// TODO mboldyrev 14.02.2019 IR-324 Use supermajority checker mock +static const iroha::consensus::yac::ConsistencyModel kConsistencyModel = + iroha::consensus::yac::ConsistencyModel::kBft; + static logger::Logger log_ = logger::testLog("YacBlockStorage"); class YacBlockStorageTest : public ::testing::Test { public: - YacHash hash; - PeersNumberType number_of_peers; - YacBlockStorage storage = YacBlockStorage( - YacHash(iroha::consensus::Round{1, 1}, "proposal", "commit"), 4); + const PeersNumberType number_of_peers = 4; + const PeersNumberType supermajority = number_of_peers + - (number_of_peers - 1) + / detail::kSupermajorityCheckerKfPlus1Bft; // `kf+1' consistency + // model + const YacHash hash = + YacHash(iroha::consensus::Round{1, 1}, "proposal", "commit"); + YacBlockStorage storage = + YacBlockStorage(hash, + number_of_peers, + // todo mboldyrev 13.12.2018 IR-324 use mock super checker + getSupermajorityChecker(kConsistencyModel)); std::vector valid_votes; void SetUp() override { - hash = YacHash(iroha::consensus::Round{1, 1}, "proposal", "commit"); - number_of_peers = 4; - storage = YacBlockStorage(hash, number_of_peers); - valid_votes = {createVote(hash, "one"), - createVote(hash, "two"), - createVote(hash, "three"), - createVote(hash, "four")}; + valid_votes.reserve(number_of_peers); + std::generate_n(std::back_inserter(valid_votes), number_of_peers, [this] { + return createVote(this->hash, std::to_string(this->valid_votes.size())); + }); } }; TEST_F(YacBlockStorageTest, YacBlockStorageWhenNormalDataInput) { - log_->info("-----------| Sequentially insertion of votes |-----------"); - - auto insert_1 = storage.insert(valid_votes.at(0)); - ASSERT_EQ(boost::none, insert_1); - - auto insert_2 = storage.insert(valid_votes.at(1)); - ASSERT_EQ(boost::none, insert_2); - - auto insert_3 = storage.insert(valid_votes.at(2)); - ASSERT_NE(boost::none, insert_3); - ASSERT_EQ(3, boost::get(*insert_3).votes.size()); - - auto insert_4 = storage.insert(valid_votes.at(3)); - ASSERT_NE(boost::none, insert_4); - ASSERT_EQ(4, boost::get(*insert_4).votes.size()); + log_->info("------------| Sequential insertion of votes |------------"); + + for (size_t num_inserted = 0; num_inserted < number_of_peers;) { + auto insert_result = storage.insert(valid_votes.at(num_inserted++)); + if (num_inserted < supermajority) { + EXPECT_EQ(boost::none, insert_result) + << "Got an Answer after inserting " << num_inserted + << " votes, before the supermajority reached at " << supermajority; + } else { + // after supermajority reached we have a CommitMessage + ASSERT_NE(boost::none, insert_result) + << "Did not get an Answer after inserting " << num_inserted + << " votes and the supermajority reached at " << supermajority; + ASSERT_EQ(num_inserted, + boost::get(*insert_result).votes.size()) + << " The commit message must have all the previously inserted votes."; + } + } } TEST_F(YacBlockStorageTest, YacBlockStorageWhenNotCommittedAndCommitAcheive) { @@ -62,7 +75,8 @@ TEST_F(YacBlockStorageTest, YacBlockStorageWhenNotCommittedAndCommitAcheive) { decltype(YacBlockStorageTest::valid_votes) for_insert(valid_votes.begin() + 1, valid_votes.end()); auto insert_commit = storage.insert(for_insert); - ASSERT_EQ(4, boost::get(*insert_commit).votes.size()); + ASSERT_EQ(number_of_peers, + boost::get(*insert_commit).votes.size()); } TEST_F(YacBlockStorageTest, YacBlockStorageWhenGetVotes) { diff --git a/test/module/irohad/consensus/yac/yac_fixture.hpp b/test/module/irohad/consensus/yac/yac_fixture.hpp index 503a88cb00..0a5be760f1 100644 --- a/test/module/irohad/consensus/yac/yac_fixture.hpp +++ b/test/module/irohad/consensus/yac/yac_fixture.hpp @@ -17,6 +17,10 @@ #include "module/irohad/consensus/yac/mock_yac_timer.hpp" #include "module/irohad/consensus/yac/yac_test_util.hpp" +// TODO mboldyrev 14.02.2019 IR-324 Use supermajority checker mock +static const iroha::consensus::yac::ConsistencyModel kConsistencyModel = + iroha::consensus::yac::ConsistencyModel::kBft; + namespace iroha { namespace consensus { namespace yac { @@ -55,8 +59,10 @@ namespace iroha { void initYac(ClusterOrdering ordering) { yac = Yac::create( - YacVoteStorage(std::make_shared< - iroha::consensus::yac::BufferedCleanupStrategy>()), + YacVoteStorage( + std::make_shared< + iroha::consensus::yac::BufferedCleanupStrategy>(), + getSupermajorityChecker(kConsistencyModel)), network, crypto, timer, diff --git a/test/module/irohad/consensus/yac/yac_proposal_storage_test.cpp b/test/module/irohad/consensus/yac/yac_proposal_storage_test.cpp index a5629df948..894dba0b3f 100644 --- a/test/module/irohad/consensus/yac/yac_proposal_storage_test.cpp +++ b/test/module/irohad/consensus/yac/yac_proposal_storage_test.cpp @@ -7,35 +7,43 @@ #include +#include "consensus/yac/impl/supermajority_checker_bft.hpp" // for detail::kSupermajorityCheckerKfPlus1Bft #include "consensus/yac/storage/yac_common.hpp" #include "logger/logger.hpp" +#include "module/irohad/consensus/yac/mock_yac_supermajority_checker.hpp" #include "module/irohad/consensus/yac/yac_test_util.hpp" using namespace iroha::consensus::yac; +using namespace ::testing; static logger::Logger log_ = logger::testLog("YacProposalStorage"); class YacProposalStorageTest : public ::testing::Test { public: - YacHash hash; - PeersNumberType number_of_peers; - YacProposalStorage storage = - YacProposalStorage(iroha::consensus::Round{1, 1}, 4); + const PeersNumberType number_of_peers = 7; + const PeersNumberType supermajority = number_of_peers + - (number_of_peers - 1) + / detail::kSupermajorityCheckerKfPlus1Bft; // `kf+1' consistency + // model + const YacHash hash = + YacHash(iroha::consensus::Round{1, 1}, "proposal", "commit"); + const std::shared_ptr supermajority_checker = + std::make_shared(); + YacProposalStorage storage = YacProposalStorage( + iroha::consensus::Round{1, 1}, number_of_peers, supermajority_checker); std::vector valid_votes; void SetUp() override { - hash = YacHash(iroha::consensus::Round{1, 1}, "proposal", "commit"); - number_of_peers = 7; - storage = - YacProposalStorage(iroha::consensus::Round{1, 1}, number_of_peers); - valid_votes = [this]() { - std::vector votes; - for (auto i = 0u; i < number_of_peers; ++i) { - votes.push_back(createVote(hash, std::to_string(i))); - } - return votes; - }(); + std::generate_n(std::back_inserter(valid_votes), number_of_peers, [this] { + return createVote(this->hash, std::to_string(this->valid_votes.size())); + }); + + EXPECT_CALL(*supermajority_checker, hasSupermajority(_, _)) + .WillRepeatedly( + Invoke([this](auto c, auto) { return c >= supermajority; })); + EXPECT_CALL(*supermajority_checker, canHaveSupermajority(_, _)) + .WillRepeatedly(Return(true)); } }; @@ -44,17 +52,21 @@ TEST_F(YacProposalStorageTest, YacProposalStorageWhenCommitCase) { "Init storage => insert unique votes => " "expected commit"); - for (auto i = 0u; i < 4; ++i) { - ASSERT_EQ(boost::none, storage.insert(valid_votes.at(i))); - } - - for (auto i = 4u; i < 7; ++i) { - auto commit = storage.insert(valid_votes.at(i)); - log_->info("Commit: {}", logger::opt_to_string(commit, [](auto answer) { - return "value"; - })); - ASSERT_NE(boost::none, commit); - ASSERT_EQ(i + 1, boost::get(*commit).votes.size()); + for (size_t num_inserted = 0; num_inserted < number_of_peers;) { + auto insert_result = storage.insert(valid_votes.at(num_inserted++)); + if (num_inserted < supermajority) { + EXPECT_EQ(boost::none, insert_result) + << "Got an Answer after inserting " << num_inserted + << " votes, before the supermajority reached at " << supermajority; + } else { + // after supermajority reached we have a CommitMessage + ASSERT_NE(boost::none, insert_result) + << "Did not get an Answer after inserting " << num_inserted + << " votes and the supermajority reached at " << supermajority; + ASSERT_EQ(num_inserted, + boost::get(*insert_result).votes.size()) + << " The commit message must have all the previously inserted votes."; + } } } @@ -63,7 +75,7 @@ TEST_F(YacProposalStorageTest, YacProposalStorageWhenInsertNotUnique) { "Init storage => insert not-unique votes => " "expected absence of commit"); - for (auto i = 0; i < 7; ++i) { + for (PeersNumberType i = 0; i < number_of_peers; ++i) { auto fixed_index = 0; ASSERT_EQ(boost::none, storage.insert(valid_votes.at(fixed_index))); } @@ -74,24 +86,31 @@ TEST_F(YacProposalStorageTest, YacProposalStorageWhenRejectCase) { "Init storage => insert votes for reject case => " "expected absence of commit"); - // insert 3 vote for hash 1 - for (auto i = 0; i < 3; ++i) { - ASSERT_EQ(boost::none, storage.insert(valid_votes.at(i))); + size_t num_inserted = 0; + const size_t super_reject = number_of_peers - 2; + + EXPECT_CALL(*supermajority_checker, hasSupermajority(_, _)) + .WillRepeatedly(Return(false)); + EXPECT_CALL(*supermajority_checker, canHaveSupermajority(_, _)) + .WillRepeatedly(Invoke([&num_inserted, &super_reject](auto, auto) { + return num_inserted < super_reject; + })); + + while (num_inserted < number_of_peers) { + auto insert_result = storage.insert(valid_votes.at(num_inserted++)); + if (num_inserted < super_reject) { + EXPECT_EQ(boost::none, insert_result) + << "Got an Answer after inserting " << num_inserted + << " votes, before the supermajority reject reached at " + << super_reject; + } else { + // after supermajority reached we have a CommitMessage + ASSERT_NE(boost::none, insert_result) + << "Did not get an Answer after inserting " << num_inserted + << " votes and the supermajority reject reached at " << super_reject; + ASSERT_EQ(num_inserted, + boost::get(*insert_result).votes.size()) + << " The reject message must have all the previously inserted votes."; + } } - - // insert 2 for other hash - auto other_hash = YacHash(iroha::consensus::Round{1, 1}, - hash.vote_hashes.proposal_hash, - "other_commit"); - for (auto i = 0; i < 2; ++i) { - auto answer = storage.insert( - createVote(other_hash, std::to_string(valid_votes.size() + 1 + i))); - ASSERT_EQ(boost::none, answer); - } - - // insert more one for other hash - auto answer = storage.insert( - createVote(other_hash, std::to_string(2 * valid_votes.size() + 1))); - ASSERT_NE(boost::none, answer); - ASSERT_EQ(6, boost::get(*answer).votes.size()); } diff --git a/test/module/irohad/consensus/yac/yac_rainy_day_test.cpp b/test/module/irohad/consensus/yac/yac_rainy_day_test.cpp index bb1bc6e1f0..9171d214e4 100644 --- a/test/module/irohad/consensus/yac/yac_rainy_day_test.cpp +++ b/test/module/irohad/consensus/yac/yac_rainy_day_test.cpp @@ -3,6 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ +#include "consensus/yac/impl/supermajority_checker_bft.hpp" #include "consensus/yac/storage/yac_proposal_storage.hpp" #include "framework/test_subscriber.hpp" @@ -17,21 +18,22 @@ using namespace iroha::consensus::yac; using namespace framework::test_subscriber; /** - * @given yac consensus with four peers - * @when two peers vote for one hash and two for another + * @given yac consensus with 4 peers + * @when half of peers vote for one hash and the rest for another * @then commit does not happen, instead send_reject is triggered on transport */ TEST_F(YacTest, InvalidCaseWhenNotReceiveSupermajority) { + const size_t N = 4; // number of peers auto my_peers = decltype(default_peers)( - {default_peers.begin(), default_peers.begin() + 4}); - ASSERT_EQ(4, my_peers.size()); + {default_peers.begin(), default_peers.begin() + N}); + ASSERT_EQ(N, my_peers.size()); auto my_order = ClusterOrdering::create(my_peers); ASSERT_TRUE(my_order); initYac(my_order.value()); - EXPECT_CALL(*network, sendState(_, _)).Times(2 * my_peers.size()); + EXPECT_CALL(*network, sendState(_, _)).Times(2 * N); EXPECT_CALL(*timer, deny()).Times(0); @@ -41,10 +43,10 @@ TEST_F(YacTest, InvalidCaseWhenNotReceiveSupermajority) { YacHash hash2(iroha::consensus::Round{1, 1}, "proposal_hash", "block_hash2"); yac->vote(hash1, my_order.value()); - for (auto i = 0; i < 2; ++i) { + for (size_t i = 0; i < N / 2; ++i) { yac->onState({createVote(hash1, std::to_string(i))}); }; - for (auto i = 2; i < 4; ++i) { + for (size_t i = N / 2; i < N; ++i) { yac->onState({createVote(hash2, std::to_string(i))}); }; } @@ -84,8 +86,8 @@ TEST_F(YacTest, InvalidCaseWhenDoesNotVerify) { /** * @given yac consensus with 6 peers - * @when on_reject happens due to 2 peers vote for one hash and 3 peers vote for - * another and then last 6th peer votes for any hash, he directly receives + * @when on_reject happens due to enough peers vote for different hashes + * and then when another peer votes for any hash, he directly receives * reject message, because on_reject already happened * @then reject message will be called in total 7 times (peers size + 1 who * receives reject directly) @@ -110,19 +112,24 @@ TEST_F(YacTest, ValidCaseWhenReceiveOnVoteAfterReject) { EXPECT_CALL(*crypto, verify(_)).WillRepeatedly(Return(true)); - YacHash hash1(iroha::consensus::Round{1, 1}, "proposal_hash", "block_hash"); - YacHash hash2(iroha::consensus::Round{1, 1}, "proposal_hash", "block_hash2"); + const auto makeYacHash = [](size_t i) { + return YacHash(iroha::consensus::Round{1, 1}, + "proposal_hash", + "block_hash" + std::to_string(i)); + }; + SupermajorityCheckerBft super_checker; std::vector votes; - for (size_t i = 0; i < peers_number / 2; ++i) { - auto peer = my_order->getPeers().at(i); - auto pubkey = shared_model::crypto::toBinaryString(peer->pubkey()); - votes.push_back(createVote(hash1, pubkey)); - }; - for (size_t i = peers_number / 2; i < peers_number - 1; ++i) { + std::vector vote_groups; + for (size_t i = 0; + super_checker.canHaveSupermajority(vote_groups, peers_number); + ++i) { + ASSERT_LT(i, peers_number) << "Reject must had already happened when " + "all peers have voted for different hashes."; auto peer = my_order->getPeers().at(i); auto pubkey = shared_model::crypto::toBinaryString(peer->pubkey()); - votes.push_back(createVote(hash2, pubkey)); + votes.push_back(createVote(makeYacHash(i), pubkey)); + vote_groups.push_back({1}); }; for (const auto &vote : votes) { @@ -132,5 +139,5 @@ TEST_F(YacTest, ValidCaseWhenReceiveOnVoteAfterReject) { yac->onState(votes); auto peer = my_order->getPeers().back(); auto pubkey = shared_model::crypto::toBinaryString(peer->pubkey()); - yac->onState({createVote(hash1, pubkey)}); + yac->onState({createVote(makeYacHash(peers_number), pubkey)}); } diff --git a/test/module/irohad/consensus/yac/yac_simple_cold_case_test.cpp b/test/module/irohad/consensus/yac/yac_simple_cold_case_test.cpp index 1faa65e424..e46b935b20 100644 --- a/test/module/irohad/consensus/yac/yac_simple_cold_case_test.cpp +++ b/test/module/irohad/consensus/yac/yac_simple_cold_case_test.cpp @@ -9,6 +9,7 @@ #include #include +#include "consensus/yac/impl/supermajority_checker_bft.hpp" #include "consensus/yac/storage/yac_proposal_storage.hpp" #include "framework/test_subscriber.hpp" @@ -167,14 +168,9 @@ TEST_F(YacTest, PropagateCommitBeforeNotifyingSubscribersApplyReject) { EXPECT_CALL(*crypto, verify(_)).WillRepeatedly(Return(true)); EXPECT_CALL(*timer, deny()).Times(AtLeast(1)); std::vector> messages; - EXPECT_CALL(*network, sendState(_, _)) - .Times(0) - .WillRepeatedly(Invoke( - [&](const auto &, const auto &msg) { messages.push_back(msg); })); + EXPECT_CALL(*network, sendState(_, _)).Times(0); yac->onOutcome().subscribe([&](auto msg) { - // verify that commits are already sent to the network - ASSERT_EQ(0, messages.size()); messages.push_back(boost::get(msg).votes); }); @@ -183,18 +179,19 @@ TEST_F(YacTest, PropagateCommitBeforeNotifyingSubscribersApplyReject) { auto yac_hash = YacHash(iroha::consensus::Round(1, 0), "proposal_hash", "block_hash"); - auto f = (default_peers.size() - 1) / 3; - for (size_t i = 0; i < 2 * f; ++i) { + auto f = (default_peers.size() - 1) + / iroha::consensus::yac::detail::kSupermajorityCheckerKfPlus1Bft; + for (size_t i = 0; i < default_peers.size() - f - 1; ++i) { auto vote = createVote(yac_hash, std::to_string(i)); yac->onState({vote}); commit.push_back(vote); } - auto vote = createVote(yac_hash, std::to_string(2 * f + 1)); + auto vote = createVote(yac_hash, std::to_string(default_peers.size() - f)); RejectMessage reject( {vote, createVote(YacHash(iroha::consensus::Round{1, 1}, "", "my_block"), - std::to_string(2 * f + 2))}); + std::to_string(default_peers.size() - f + 1))}); commit.push_back(vote); yac->onState(reject.votes); diff --git a/test/module/irohad/consensus/yac/yac_sunny_day_test.cpp b/test/module/irohad/consensus/yac/yac_sunny_day_test.cpp index 3b80e47b66..315203c777 100644 --- a/test/module/irohad/consensus/yac/yac_sunny_day_test.cpp +++ b/test/module/irohad/consensus/yac/yac_sunny_day_test.cpp @@ -20,6 +20,12 @@ using namespace iroha::consensus::yac; using namespace framework::test_subscriber; using namespace std; +/** + * @given yac & 6 peers + * @when the 6 peers send the yac votes for the same hash + * @then sendState is called twice per peer + * @and the round keeps open + */ TEST_F(YacTest, ValidCaseWhenReceiveSupermajority) { auto my_peers = decltype(default_peers)( {default_peers.begin(), default_peers.begin() + 4}); @@ -163,6 +169,14 @@ TEST_F(YacTest, ValidCaseWhenSoloConsensus) { ASSERT_TRUE(wrapper.validate()); } +/** + * @given yac & 6 peers + * @when first 5 peers' votes for the same hash are sent to the yac + * @and after that the last peer's vote for the same hash is sent to yac + * @then sendState is not called + * @and round is closed + * @and crypto verification is called once + */ TEST_F(YacTest, ValidCaseWhenVoteAfterCommit) { auto my_peers = decltype(default_peers)( {default_peers.begin(), default_peers.begin() + 4}); diff --git a/test/module/irohad/validation/chain_validation_test.cpp b/test/module/irohad/validation/chain_validation_test.cpp index 0d7560dfa9..5751479a5b 100644 --- a/test/module/irohad/validation/chain_validation_test.cpp +++ b/test/module/irohad/validation/chain_validation_test.cpp @@ -76,9 +76,9 @@ class ChainValidationTest : public ::testing::Test { */ TEST_F(ChainValidationTest, ValidCase) { // Valid previous hash, has supermajority, correct peers subset => valid - shared_model::interface::types::SignatureRangeType block_signatures; + size_t block_signatures_amount; EXPECT_CALL(*supermajority_checker, hasSupermajority(_, _)) - .WillOnce(DoAll(SaveArg<0>(&block_signatures), Return(true))); + .WillOnce(DoAll(SaveArg<0>(&block_signatures_amount), Return(true))); EXPECT_CALL(*query, getLedgerPeers()).WillOnce(Return(peers)); @@ -86,7 +86,7 @@ TEST_F(ChainValidationTest, ValidCase) { .WillOnce(InvokeArgument<1>(ByRef(*block), ByRef(*query), ByRef(hash))); ASSERT_TRUE(validator->validateAndApply(blocks, *storage)); - ASSERT_EQ(block->signatures(), block_signatures); + ASSERT_EQ(boost::size(block->signatures()), block_signatures_amount); } /** @@ -99,9 +99,8 @@ TEST_F(ChainValidationTest, FailWhenDifferentPrevHash) { shared_model::crypto::Hash another_hash = shared_model::crypto::Hash(std::string(32, '1')); - shared_model::interface::types::SignatureRangeType block_signatures; ON_CALL(*supermajority_checker, hasSupermajority(_, _)) - .WillByDefault(DoAll(SaveArg<0>(&block_signatures), Return(true))); + .WillByDefault(Return(true)); EXPECT_CALL(*query, getLedgerPeers()).WillOnce(Return(peers)); @@ -119,9 +118,9 @@ TEST_F(ChainValidationTest, FailWhenDifferentPrevHash) { */ TEST_F(ChainValidationTest, FailWhenNoSupermajority) { // Valid previous hash, no supermajority, correct peers subset => invalid - shared_model::interface::types::SignatureRangeType block_signatures; + size_t block_signatures_amount; EXPECT_CALL(*supermajority_checker, hasSupermajority(_, _)) - .WillOnce(DoAll(SaveArg<0>(&block_signatures), Return(false))); + .WillOnce(DoAll(SaveArg<0>(&block_signatures_amount), Return(false))); EXPECT_CALL(*query, getLedgerPeers()).WillOnce(Return(peers)); @@ -129,5 +128,5 @@ TEST_F(ChainValidationTest, FailWhenNoSupermajority) { .WillOnce(InvokeArgument<1>(ByRef(*block), ByRef(*query), ByRef(hash))); ASSERT_FALSE(validator->validateAndApply(blocks, *storage)); - ASSERT_EQ(block->signatures(), block_signatures); + ASSERT_EQ(boost::size(block->signatures()), block_signatures_amount); } From d75ce7c90dc4c7819444d59c28562649357c23a9 Mon Sep 17 00:00:00 2001 From: Mikhail Boldyrev Date: Tue, 5 Mar 2019 12:27:29 +0300 Subject: [PATCH 05/40] Fuzzing fix (#2136) Signed-off-by: Mikhail Boldyrev --- test/fuzzing/find_fuzz.cpp | 6 ++--- test/fuzzing/mst_fuzz.cpp | 3 +++ test/fuzzing/status_fuzz.cpp | 44 +++++++++++++++++++++--------------- test/fuzzing/torii_fuzz.cpp | 42 +++++++++++++++++++--------------- 4 files changed, 56 insertions(+), 39 deletions(-) diff --git a/test/fuzzing/find_fuzz.cpp b/test/fuzzing/find_fuzz.cpp index 2cdc4a78e1..f911db6b56 100644 --- a/test/fuzzing/find_fuzz.cpp +++ b/test/fuzzing/find_fuzz.cpp @@ -22,7 +22,7 @@ using testing::_; using testing::Return; struct QueryFixture { - std::shared_ptr service_; + std::shared_ptr service_; std::shared_ptr qry_processor_; std::shared_ptr storage_; std::shared_ptr pending_transactions_; @@ -55,8 +55,8 @@ struct QueryFixture { shared_model::interface::Query, shared_model::proto::Query>>(std::move(query_validator), std::move(proto_query_validator)); - service_ = - std::make_shared(qry_processor_, query_factory); + service_ = std::make_shared(qry_processor_, + query_factory); } }; diff --git a/test/fuzzing/mst_fuzz.cpp b/test/fuzzing/mst_fuzz.cpp index 8c3c25b13d..a4cc317e2b 100644 --- a/test/fuzzing/mst_fuzz.cpp +++ b/test/fuzzing/mst_fuzz.cpp @@ -22,6 +22,7 @@ using namespace iroha::network; namespace fuzzing { struct MstFixture { + std::shared_ptr completer_; std::shared_ptr mst_transport_grpc_; MstFixture() { @@ -53,12 +54,14 @@ namespace fuzzing { std::make_shared>(); auto cache = std::make_shared(storage); + completer_ = std::make_shared(); mst_transport_grpc_ = std::make_shared( async_call_, std::move(tx_factory), std::move(parser), std::move(batch_factory), std::move(cache), + completer_, shared_model::crypto::DefaultCryptoAlgorithmType::generateKeypair() .publicKey()); } diff --git a/test/fuzzing/status_fuzz.cpp b/test/fuzzing/status_fuzz.cpp index d9110f6f11..564f1a88cc 100644 --- a/test/fuzzing/status_fuzz.cpp +++ b/test/fuzzing/status_fuzz.cpp @@ -16,6 +16,7 @@ #include "interfaces/iroha_internal/transaction_batch_factory_impl.hpp" #include "interfaces/iroha_internal/transaction_batch_parser_impl.hpp" #include "module/irohad/ametsuchi/ametsuchi_mocks.hpp" +#include "module/irohad/ametsuchi/mock_tx_presence_cache.hpp" #include "module/irohad/multi_sig_transactions/mst_mocks.hpp" #include "module/irohad/network/network_mocks.hpp" #include "synchronizer/synchronizer_common.hpp" @@ -29,14 +30,17 @@ using testing::_; using testing::Return; struct CommandFixture { - std::shared_ptr service_; - std::shared_ptr service_transport_; + std::shared_ptr service_; + std::shared_ptr service_transport_; std::shared_ptr tx_processor_; std::shared_ptr storage_; std::shared_ptr pcs_; std::shared_ptr mst_processor_; std::shared_ptr bq_; - std::shared_ptr consensus_gate_; + std::vector + consensus_gate_objects_{2}; + std::shared_ptr cache_; + std::shared_ptr tx_presence_cache_; rxcpp::subjects::subject prop_notifier_; rxcpp::subjects::subject @@ -98,24 +102,28 @@ struct CommandFixture { transaction_batch_factory = std::make_shared< shared_model::interface::TransactionBatchFactoryImpl>(); - consensus_gate_ = std::make_shared(); - ON_CALL(*consensus_gate_, onOutcome()) - .WillByDefault(Return(consensus_notifier_.get_observable())); - storage_ = std::make_shared(); bq_ = std::make_shared(); EXPECT_CALL(*storage_, getBlockQuery()).WillRepeatedly(Return(bq_)); - service_ = std::make_shared( - tx_processor_, storage_, status_bus, status_factory); - service_transport_ = std::make_shared( - service_, - status_bus, - status_factory, - transaction_factory, - batch_parser, - transaction_batch_factory, - consensus_gate_, - 2); + tx_presence_cache_ = + std::make_shared(); + cache_ = std::make_shared(); + service_ = std::make_shared(tx_processor_, + storage_, + status_bus, + status_factory, + cache_, + tx_presence_cache_); + service_transport_ = + std::make_shared( + service_, + status_bus, + status_factory, + transaction_factory, + batch_parser, + transaction_batch_factory, + rxcpp::observable<>::iterate(consensus_gate_objects_), + 2); } }; diff --git a/test/fuzzing/torii_fuzz.cpp b/test/fuzzing/torii_fuzz.cpp index 3843d6df0f..5de0f84a2b 100644 --- a/test/fuzzing/torii_fuzz.cpp +++ b/test/fuzzing/torii_fuzz.cpp @@ -15,6 +15,7 @@ #include "interfaces/iroha_internal/transaction_batch_factory_impl.hpp" #include "interfaces/iroha_internal/transaction_batch_parser_impl.hpp" #include "module/irohad/ametsuchi/ametsuchi_mocks.hpp" +#include "module/irohad/ametsuchi/mock_tx_presence_cache.hpp" #include "module/irohad/multi_sig_transactions/mst_mocks.hpp" #include "module/irohad/network/network_mocks.hpp" #include "synchronizer/synchronizer_common.hpp" @@ -28,12 +29,15 @@ using testing::Return; struct CommandFixture { - std::shared_ptr service_; - std::shared_ptr service_transport_; + std::shared_ptr service_; + std::shared_ptr service_transport_; std::shared_ptr tx_processor_; std::shared_ptr pcs_; std::shared_ptr mst_processor_; - std::shared_ptr consensus_gate_; + std::vector + consensus_gate_objects_{2}; + std::shared_ptr cache_; + std::shared_ptr tx_presence_cache_; rxcpp::subjects::subject prop_notifier_; rxcpp::subjects::subject @@ -68,8 +72,13 @@ struct CommandFixture { tx_processor_ = std::make_shared( pcs_, mst_processor_, status_bus, status_factory); auto storage = std::make_shared(); - service_ = std::make_shared( - tx_processor_, storage, status_bus, status_factory); + service_ = + std::make_shared(tx_processor_, + storage, + status_bus, + status_factory, + cache_, + tx_presence_cache_); std::unique_ptr> @@ -96,19 +105,16 @@ struct CommandFixture { transaction_batch_factory = std::make_shared< shared_model::interface::TransactionBatchFactoryImpl>(); - consensus_gate_ = std::make_shared(); - ON_CALL(*consensus_gate_, onOutcome()) - .WillByDefault(Return(consensus_notifier_.get_observable())); - - service_transport_ = std::make_shared( - service_, - status_bus, - status_factory, - transaction_factory, - batch_parser, - transaction_batch_factory, - consensus_gate_, - 2); + service_transport_ = + std::make_shared( + service_, + status_bus, + status_factory, + transaction_factory, + batch_parser, + transaction_batch_factory, + rxcpp::observable<>::iterate(consensus_gate_objects_), + 2); } }; From ee9b3bba862b0b35fd5cddeeb7e5d8c7b56ed731 Mon Sep 17 00:00:00 2001 From: Sara Date: Tue, 5 Mar 2019 17:30:22 +0300 Subject: [PATCH 06/40] [ci skip] preparing files for md to rst documentation (#2145) Signed-off-by: Sara --- CONTRIBUTING.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4e65b6be5d..589fdeedb6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,4 @@ -# Contributing guidelines - -:star::tada: First off, thanks for taking the time to contribute! :tada::star: +First off, thanks for taking the time to contribute! The following is a short set of guidelines for contributing to Iroha. From fa29ea550ffd5dc0b0fd2fb185c35e15e2a501db Mon Sep 17 00:00:00 2001 From: Mikhail Boldyrev Date: Wed, 6 Mar 2019 09:53:49 +0300 Subject: [PATCH 07/40] New logger (#2126) Signed-off-by: Mikhail Boldyrev --- cmake/Modules/Findfmt.cmake | 62 ++++ cmake/Modules/Findspdlog.cmake | 52 ++- cmake/dependencies.cmake | 7 +- docs/source/guides/configuration.rst | 38 ++ iroha-cli/CMakeLists.txt | 2 + iroha-cli/client.cpp | 12 +- iroha-cli/client.hpp | 7 +- iroha-cli/grpc_response_handler.hpp | 7 +- iroha-cli/impl/grpc_response_handler.cpp | 8 +- iroha-cli/impl/query_response_handler.cpp | 3 +- .../impl/transaction_response_handler.cpp | 3 +- .../interactive/impl/interactive_cli.cpp | 30 +- .../impl/interactive_query_cli.cpp | 27 +- .../impl/interactive_status_cli.cpp | 12 +- .../impl/interactive_transaction_cli.cpp | 22 +- iroha-cli/interactive/interactive_cli.hpp | 11 +- .../interactive/interactive_query_cli.hpp | 33 +- .../interactive/interactive_status_cli.hpp | 6 +- .../interactive_transaction_cli.hpp | 27 +- iroha-cli/main.cpp | 42 ++- iroha-cli/query_response_handler.hpp | 7 +- iroha-cli/transaction_response_handler.hpp | 8 +- irohad/ametsuchi/CMakeLists.txt | 1 + irohad/ametsuchi/impl/flat_file/flat_file.cpp | 22 +- irohad/ametsuchi/impl/flat_file/flat_file.hpp | 12 +- .../ametsuchi/impl/mutable_storage_impl.cpp | 16 +- .../ametsuchi/impl/mutable_storage_impl.hpp | 7 +- .../ametsuchi/impl/postgres_block_index.cpp | 3 +- .../ametsuchi/impl/postgres_block_index.hpp | 8 +- .../ametsuchi/impl/postgres_block_query.cpp | 5 +- .../ametsuchi/impl/postgres_block_query.hpp | 8 +- .../impl/postgres_query_executor.cpp | 13 +- .../impl/postgres_query_executor.hpp | 11 +- irohad/ametsuchi/impl/postgres_wsv_query.cpp | 5 +- irohad/ametsuchi/impl/postgres_wsv_query.hpp | 8 +- irohad/ametsuchi/impl/storage_impl.cpp | 58 ++- irohad/ametsuchi/impl/storage_impl.hpp | 11 +- irohad/ametsuchi/impl/temporary_wsv_impl.cpp | 17 +- irohad/ametsuchi/impl/temporary_wsv_impl.hpp | 13 +- irohad/consensus/yac/CMakeLists.txt | 1 + irohad/consensus/yac/impl/yac.cpp | 7 +- .../yac/impl/yac_crypto_provider_impl.cpp | 2 - irohad/consensus/yac/impl/yac_gate_impl.cpp | 3 +- irohad/consensus/yac/impl/yac_gate_impl.hpp | 6 +- .../yac/storage/impl/yac_block_storage.cpp | 10 +- .../yac/storage/impl/yac_proposal_storage.cpp | 14 +- .../yac/storage/impl/yac_vote_storage.cpp | 17 +- .../yac/storage/yac_block_storage.hpp | 7 +- .../yac/storage/yac_proposal_storage.hpp | 13 +- .../yac/storage/yac_vote_storage.hpp | 7 +- .../yac/transport/impl/network_impl.cpp | 17 +- .../yac/transport/impl/network_impl.hpp | 8 +- .../yac/transport/yac_pb_converters.hpp | 5 +- irohad/consensus/yac/yac.hpp | 8 +- irohad/main/CMakeLists.txt | 15 + irohad/main/application.cpp | 168 ++++++--- irohad/main/application.hpp | 11 +- irohad/main/assert_config.hpp | 49 --- irohad/main/impl/block_loader_init.cpp | 25 +- irohad/main/impl/block_loader_init.hpp | 14 +- irohad/main/impl/consensus_init.cpp | 32 +- irohad/main/impl/consensus_init.hpp | 6 +- irohad/main/impl/on_demand_ordering_init.cpp | 50 ++- irohad/main/impl/on_demand_ordering_init.hpp | 25 +- irohad/main/impl/raw_block_loader.cpp | 4 +- irohad/main/iroha_conf_literals.cpp | 32 ++ irohad/main/iroha_conf_literals.hpp | 34 ++ irohad/main/iroha_conf_loader.cpp | 345 ++++++++++++++++++ irohad/main/iroha_conf_loader.hpp | 149 +------- irohad/main/irohad.cpp | 92 +++-- irohad/main/raw_block_loader.hpp | 6 +- irohad/main/server_runner.cpp | 5 +- irohad/main/server_runner.hpp | 10 +- .../converters/impl/json_block_factory.cpp | 2 +- .../converters/impl/json_query_factory.cpp | 3 +- .../converters/impl/pb_query_factory.cpp | 3 +- .../model/converters/json_block_factory.hpp | 7 +- .../model/converters/json_query_factory.hpp | 7 +- irohad/model/converters/pb_query_factory.hpp | 7 +- .../generators/impl/transaction_generator.cpp | 11 +- .../generators/transaction_generator.hpp | 5 +- irohad/model/sha3_hash.cpp | 8 +- .../impl/mst_processor.cpp | 3 +- .../impl/mst_processor_impl.cpp | 9 +- .../multi_sig_transactions/mst_processor.hpp | 6 +- .../mst_processor_impl.hpp | 7 +- .../state/impl/mst_state.cpp | 23 +- .../state/mst_state.hpp | 14 +- .../storage/impl/mst_storage.cpp | 2 +- .../storage/impl/mst_storage_impl.cpp | 13 +- .../storage/mst_storage.hpp | 6 +- .../storage/mst_storage_impl.hpp | 8 +- .../transport/impl/mst_transport_grpc.cpp | 41 ++- .../transport/mst_transport_grpc.hpp | 10 +- irohad/network/impl/async_grpc_client.hpp | 7 +- irohad/network/impl/block_loader_impl.cpp | 3 +- irohad/network/impl/block_loader_impl.hpp | 6 +- irohad/network/impl/block_loader_service.cpp | 4 +- irohad/network/impl/block_loader_service.hpp | 6 +- .../impl/peer_communication_service_impl.cpp | 3 +- .../impl/peer_communication_service_impl.hpp | 6 +- .../impl/on_demand_connection_manager.cpp | 5 +- .../impl/on_demand_connection_manager.hpp | 8 +- .../ordering/impl/on_demand_ordering_gate.cpp | 3 +- .../ordering/impl/on_demand_ordering_gate.hpp | 6 +- .../impl/on_demand_ordering_service_impl.cpp | 5 +- .../impl/on_demand_ordering_service_impl.hpp | 10 +- .../impl/on_demand_os_client_grpc.cpp | 11 +- .../impl/on_demand_os_client_grpc.hpp | 9 +- .../impl/on_demand_os_server_grpc.cpp | 3 +- .../impl/on_demand_os_server_grpc.hpp | 6 +- irohad/simulator/impl/simulator.cpp | 3 +- irohad/simulator/impl/simulator.hpp | 6 +- .../synchronizer/impl/synchronizer_impl.cpp | 3 +- .../synchronizer/impl/synchronizer_impl.hpp | 6 +- irohad/torii/command_client.hpp | 6 +- irohad/torii/impl/command_client.cpp | 3 +- irohad/torii/impl/command_service_impl.cpp | 3 +- irohad/torii/impl/command_service_impl.hpp | 6 +- .../impl/command_service_transport_grpc.cpp | 3 +- .../impl/command_service_transport_grpc.hpp | 6 +- irohad/torii/impl/query_service.cpp | 3 +- .../processor/impl/query_processor_impl.cpp | 3 +- .../impl/transaction_processor_impl.cpp | 3 +- .../torii/processor/query_processor_impl.hpp | 6 +- .../processor/transaction_processor_impl.hpp | 6 +- irohad/torii/query_service.hpp | 7 +- .../validation/impl/chain_validator_impl.cpp | 6 +- .../validation/impl/chain_validator_impl.hpp | 11 +- .../impl/stateful_validator_impl.cpp | 6 +- .../impl/stateful_validator_impl.hpp | 6 +- libs/common/files.cpp | 14 +- libs/common/files.hpp | 8 +- libs/crypto/keys_manager_impl.cpp | 4 +- libs/crypto/keys_manager_impl.hpp | 9 +- libs/logger/CMakeLists.txt | 16 +- libs/logger/dummy_logger.hpp | 29 ++ libs/logger/logger.cpp | 54 +-- libs/logger/logger.hpp | 104 ++++-- libs/logger/logger_fwd.hpp | 26 ++ libs/logger/logger_manager.cpp | 83 +++++ libs/logger/logger_manager.hpp | 65 ++++ libs/logger/logger_manager_fwd.hpp | 18 + libs/logger/logger_spdlog.cpp | 92 +++++ libs/logger/logger_spdlog.hpp | 76 ++++ .../protobuf/commands/impl/proto_command.cpp | 10 +- test/framework/CMakeLists.txt | 7 + .../fake_peer/fake_peer.cpp | 27 +- .../fake_peer/fake_peer.hpp | 12 +- .../integration_test_framework.cpp | 76 ++-- .../integration_test_framework.hpp | 12 +- .../integration_framework/iroha_instance.cpp | 9 +- .../integration_framework/iroha_instance.hpp | 16 +- .../integration_framework/test_irohad.hpp | 9 +- test/framework/sql_query.cpp | 2 +- test/framework/sql_query.hpp | 5 +- test/framework/test_logger.cpp | 19 + test/framework/test_logger.hpp | 16 + test/fuzzing/block_loader_fixture.hpp | 6 +- test/fuzzing/consensus_fuzz.cpp | 10 +- test/fuzzing/find_fuzz.cpp | 11 +- test/fuzzing/mst_fuzz.cpp | 10 +- test/fuzzing/ordering_service_fixture.hpp | 4 - test/fuzzing/request_proposal_fuzz.cpp | 7 +- test/fuzzing/send_batches_fuzz.cpp | 17 +- test/fuzzing/status_fuzz.cpp | 29 +- test/fuzzing/torii_fuzz.cpp | 28 +- test/integration/consensus/CMakeLists.txt | 1 + .../consensus/consensus_sunny_day.cpp | 24 +- test/integration/validation/CMakeLists.txt | 1 + .../chain_validator_storage_test.cpp | 3 +- test/module/irohad/ametsuchi/CMakeLists.txt | 7 + .../irohad/ametsuchi/ametsuchi_fixture.hpp | 21 +- .../irohad/ametsuchi/ametsuchi_test.cpp | 3 +- .../irohad/ametsuchi/block_query_test.cpp | 11 +- .../irohad/ametsuchi/flat_file_test.cpp | 33 +- .../ametsuchi/postgres_executor_test.cpp | 4 +- .../postgres_query_executor_test.cpp | 4 +- .../irohad/ametsuchi/storage_init_test.cpp | 24 +- .../ametsuchi/wsv_query_command_test.cpp | 4 +- test/module/irohad/common/CMakeLists.txt | 1 + .../irohad/common/raw_block_loader_test.cpp | 3 +- .../irohad/consensus/yac/CMakeLists.txt | 10 + .../irohad/consensus/yac/network_test.cpp | 7 +- .../yac/supermajority_checker_test.cpp | 3 +- .../consensus/yac/yac_block_storage_test.cpp | 7 +- .../irohad/consensus/yac/yac_common_test.cpp | 3 +- .../irohad/consensus/yac/yac_fixture.hpp | 8 +- .../irohad/consensus/yac/yac_gate_test.cpp | 4 +- .../yac/yac_proposal_storage_test.cpp | 10 +- test/module/irohad/logger/CMakeLists.txt | 1 + test/module/irohad/logger/logger_test.cpp | 22 +- test/module/irohad/main/CMakeLists.txt | 1 + .../module/irohad/main/server_runner_test.cpp | 9 +- test/module/irohad/model/CMakeLists.txt | 3 + .../model/converters/json_block_test.cpp | 7 +- .../converters/json_query_factory_test.cpp | 29 +- .../converters/pb_query_factory_test.cpp | 15 +- .../multi_sig_transactions/CMakeLists.txt | 8 +- .../multi_sig_transactions/mst_mocks.hpp | 2 + .../mst_processor_test.cpp | 21 +- .../mst_test_helpers.hpp | 13 +- .../multi_sig_transactions/state_test.cpp | 46 +-- .../multi_sig_transactions/storage_test.cpp | 8 +- .../multi_sig_transactions/transport_test.cpp | 30 +- test/module/irohad/network/CMakeLists.txt | 1 + .../irohad/network/block_loader_test.cpp | 6 +- test/module/irohad/ordering/CMakeLists.txt | 5 + .../on_demand_connection_manager_test.cpp | 6 +- .../ordering/on_demand_ordering_gate_test.cpp | 4 +- .../on_demand_os_client_grpc_test.cpp | 16 +- .../on_demand_os_server_grpc_test.cpp | 4 +- .../irohad/ordering/on_demand_os_test.cpp | 37 +- .../irohad/pending_txs_storage/CMakeLists.txt | 1 + .../pending_txs_storage_test.cpp | 40 +- test/module/irohad/simulator/CMakeLists.txt | 1 + .../irohad/simulator/simulator_test.cpp | 4 +- .../module/irohad/synchronizer/CMakeLists.txt | 1 + .../irohad/synchronizer/synchronizer_test.cpp | 13 +- test/module/irohad/torii/CMakeLists.txt | 6 + .../torii/command_service_replay_test.cpp | 6 +- .../irohad/torii/command_sync_client_test.cpp | 4 +- .../irohad/torii/processor/CMakeLists.txt | 2 + .../torii/processor/query_processor_test.cpp | 7 +- .../processor/transaction_processor_test.cpp | 6 +- .../irohad/torii/query_service_test.cpp | 5 +- .../irohad/torii/torii_queries_test.cpp | 14 +- .../irohad/torii/torii_service_query_test.cpp | 7 +- .../torii/torii_transport_command_test.cpp | 4 +- test/module/irohad/validation/CMakeLists.txt | 2 + .../validation/chain_validation_test.cpp | 4 +- .../validation/stateful_validator_test.cpp | 7 +- test/module/libs/crypto/CMakeLists.txt | 1 + test/module/libs/crypto/keys_manager_test.cpp | 8 +- test/module/shared_model/CMakeLists.txt | 2 +- test/module/shared_model/interface_test.cpp | 3 +- test/regression/CMakeLists.txt | 1 + test/regression/regression_test.cpp | 5 +- test/system/CMakeLists.txt | 3 + test/system/irohad_test.cpp | 64 +++- 240 files changed, 2742 insertions(+), 1148 deletions(-) create mode 100644 cmake/Modules/Findfmt.cmake delete mode 100644 irohad/main/assert_config.hpp create mode 100644 irohad/main/iroha_conf_literals.cpp create mode 100644 irohad/main/iroha_conf_literals.hpp create mode 100644 irohad/main/iroha_conf_loader.cpp create mode 100644 libs/logger/dummy_logger.hpp create mode 100644 libs/logger/logger_fwd.hpp create mode 100644 libs/logger/logger_manager.cpp create mode 100644 libs/logger/logger_manager.hpp create mode 100644 libs/logger/logger_manager_fwd.hpp create mode 100644 libs/logger/logger_spdlog.cpp create mode 100644 libs/logger/logger_spdlog.hpp create mode 100644 test/framework/test_logger.cpp create mode 100644 test/framework/test_logger.hpp diff --git a/cmake/Modules/Findfmt.cmake b/cmake/Modules/Findfmt.cmake new file mode 100644 index 0000000000..948580593e --- /dev/null +++ b/cmake/Modules/Findfmt.cmake @@ -0,0 +1,62 @@ +add_library(fmt INTERFACE IMPORTED) + +find_path(fmt_INCLUDE_DIR fmt/format.h + PATHS ${EP_PREFIX}/src/fmtlib_fmt/include) +mark_as_advanced(fmt_INCLUDE_DIR) + +# read the fmt version stored in fmt/core.h +unset(fmt_VERSION) +set(fmt_version_file_path ${fmt_INCLUDE_DIR}/fmt/core.h) +if (EXISTS ${fmt_version_file_path}) + file(READ ${fmt_version_file_path} fmt_version_file) + if (fmt_version_file MATCHES "FMT_VERSION ([0-9]+)([0-9][0-9])([0-9][0-9])") + # Use math to skip leading zeros if any. + math(EXPR ver_major ${CMAKE_MATCH_1}) + math(EXPR ver_minor ${CMAKE_MATCH_2}) + math(EXPR ver_patch ${CMAKE_MATCH_3}) + set(fmt_VERSION "${ver_major}.${ver_minor}.${ver_patch}") + endif () +endif () + +if (NOT DEFINED fmt_VERSION OR fmt_VERSION VERSION_LESS fmt_FIND_VERSION) + message(STATUS "Package 'fmt' of version ${fmt_FIND_VERSION} not found. " + "Will download it from git repo.") + + set(GIT_URL https://github.com/fmtlib/fmt.git) + set(fmt_VERSION ${fmt_FIND_VERSION}) + set(GIT_TAG "refs/tags/${fmt_VERSION}") + set_package_properties(fmt + PROPERTIES + URL ${GIT_URL} + DESCRIPTION "String formatting library" + ) + + externalproject_add(fmtlib_fmt + GIT_REPOSITORY ${GIT_URL} + GIT_TAG ${GIT_TAG} + GIT_SHALLOW 1 + CONFIGURE_COMMAND "" # remove configure step + BUILD_COMMAND "" # remove build step + INSTALL_COMMAND "" # remove install step + TEST_COMMAND "" # remove test step + ) + externalproject_get_property(fmtlib_fmt source_dir) + set(fmt_INCLUDE_DIR ${source_dir}/include) + file(MAKE_DIRECTORY ${fmt_INCLUDE_DIR}) + + add_dependencies(fmt fmtlib_fmt) +endif() + +set_target_properties(fmt PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${fmt_INCLUDE_DIR} + ) + +target_compile_definitions(fmt INTERFACE FMT_HEADER_ONLY) + +find_package_handle_standard_args(fmt + REQUIRED_VARS + fmt_INCLUDE_DIR + VERSION_VAR + fmt_VERSION + ) + diff --git a/cmake/Modules/Findspdlog.cmake b/cmake/Modules/Findspdlog.cmake index 0237b97193..846af85689 100644 --- a/cmake/Modules/Findspdlog.cmake +++ b/cmake/Modules/Findspdlog.cmake @@ -1,27 +1,44 @@ add_library(spdlog INTERFACE IMPORTED) -find_path(spdlog_INCLUDE_DIR spdlog/spdlog.h) +find_path(spdlog_INCLUDE_DIR spdlog/spdlog.h + PATHS ${EP_PREFIX}/src/gabime_spdlog/include + ) mark_as_advanced(spdlog_INCLUDE_DIR) -find_package_handle_standard_args(spdlog DEFAULT_MSG - spdlog_INCLUDE_DIR - ) - - -set(URL https://github.com/gabime/spdlog.git) -set(VERSION ccd675a286f457068ee8c823f8207f13c2325b26) -set_target_description(spdlog "Logging library" ${URL} ${VERSION}) - +unset(spdlog_VERSION) +set(spdlog_version_file_path ${spdlog_INCLUDE_DIR}/spdlog/version.h) +if (EXISTS ${spdlog_version_file_path}) + file(READ ${spdlog_version_file_path} spdlog_version_file) + string(REGEX MATCH "SPDLOG_VER_MAJOR ([0-9]*)" _ ${spdlog_version_file}) + set(ver_major ${CMAKE_MATCH_1}) + string(REGEX MATCH "SPDLOG_VER_MINOR ([0-9]*)" _ ${spdlog_version_file}) + set(ver_minor ${CMAKE_MATCH_1}) + string(REGEX MATCH "SPDLOG_VER_PATCH ([0-9]*)" _ ${spdlog_version_file}) + set(ver_patch ${CMAKE_MATCH_1}) + set(spdlog_VERSION "${ver_major}.${ver_minor}.${ver_patch}") +endif() + +if (NOT DEFINED spdlog_VERSION OR spdlog_VERSION VERSION_LESS spdlog_FIND_VERSION) + message(STATUS "Package 'spdlog' of version ${spdlog_FIND_VERSION} not found. " + "Will download it from git repo.") + + set(GIT_URL https://github.com/gabime/spdlog.git) + set(spdlog_VERSION ${spdlog_FIND_VERSION}) + set(GIT_TAG "v${spdlog_VERSION}") + set_package_properties(spdlog + PROPERTIES + URL ${GIT_URL} + DESCRIPTION "Logging library" + ) -if (NOT SPDLOG_FOUND) externalproject_add(gabime_spdlog - GIT_REPOSITORY ${URL} - GIT_TAG ${VERSION} + GIT_REPOSITORY ${GIT_URL} + GIT_TAG ${GIT_TAG} + GIT_SHALLOW 1 CONFIGURE_COMMAND "" # remove configure step BUILD_COMMAND "" # remove build step INSTALL_COMMAND "" # remove install step TEST_COMMAND "" # remove test step - UPDATE_COMMAND "" # remove update step ) externalproject_get_property(gabime_spdlog source_dir) set(spdlog_INCLUDE_DIR ${source_dir}/include) @@ -33,3 +50,10 @@ endif () set_target_properties(spdlog PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${spdlog_INCLUDE_DIR} ) + +find_package_handle_standard_args(spdlog + REQUIRED_VARS + spdlog_INCLUDE_DIR + VERSION_VAR + spdlog_VERSION + ) diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index 2e08b58fd0..c98c646214 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -43,7 +43,7 @@ endif () ############################# # speedlog # ############################# -find_package(spdlog) +find_package(spdlog 1.3.1 REQUIRED) ################################ # protobuf # @@ -202,6 +202,11 @@ endif() ################################### find_package(ed25519) +################################### +# fmt # +################################### +find_package(fmt 5.3.0 REQUIRED) + if (USE_LIBIROHA) find_package(libiroha) endif() diff --git a/docs/source/guides/configuration.rst b/docs/source/guides/configuration.rst index 735f6bf91b..261bd87812 100644 --- a/docs/source/guides/configuration.rst +++ b/docs/source/guides/configuration.rst @@ -31,6 +31,8 @@ Deployment-specific parameters service, consensus and block loader. - ``pg_opt`` is used for setting credentials of PostgreSQL: hostname, port, username and password. +- ``log`` is an optional parameter controlling log output verbosity and format + (see below). Environment-specific parameters ------------------------------- @@ -54,3 +56,39 @@ Environment-specific parameters - ``mst_enable`` enables or disables multisignature transaction support in Iroha. We recommend setting this parameter to ``false`` at the moment until you really need it. + +Logging +------- + +In Iroha logging can be adjusted as granularly as you want. Each component has its own logging configuration with properties inherited from its parent, able to be overriden through config file. This means all the component loggers are organized in a tree with a single root. The relevant section of the configuration file contains the overriden values: + +.. code-block:: json + :linenos: + + "log": { + "level": "info", + "patterns": { + "debug": "don't panic, it's %v.", + "error": "MAMA MIA! %v!!!" + }, + "children": { + "KeysManager": { + "level": "trace" + }, + "Irohad": { + "children": { + "Storage": { + "level": "trace", + "patterns": { + "debug": "thread %t: %v." + } + } + } + } + } + } + +Every part of this config section is optional. +- ``level`` sets the verbosity. Available valies are (in decreasing verbosity order): trace, debug, info, warning, error, critical. +- ``patterns`` controls the formatting of each log string for different verbosity levels. Each value overrides the less verbose levels too. So in the example above, the "don't panic" pattern also applies to info and warning levels, and the trace level pattern is the only one that is not initialized in the config (it will be set to default hardcoded value). +- ``children`` describes the overrides of child nodes. The keys are the names of the components, and the values have the same syntax and semantics as the root log configuration. diff --git a/iroha-cli/CMakeLists.txt b/iroha-cli/CMakeLists.txt index 33df3b0112..3036516012 100644 --- a/iroha-cli/CMakeLists.txt +++ b/iroha-cli/CMakeLists.txt @@ -49,6 +49,8 @@ target_link_libraries(iroha-cli cli-flags_validators keys_manager boost + logger + logger_manager ) add_install_step_for_bin(iroha-cli) diff --git a/iroha-cli/client.cpp b/iroha-cli/client.cpp index 5a603f89b1..cf9d3ca0a5 100644 --- a/iroha-cli/client.cpp +++ b/iroha-cli/client.cpp @@ -15,11 +15,15 @@ namespace iroha_cli { - CliClient::CliClient(std::string target_ip, int port) + CliClient::CliClient(std::string target_ip, + int port, + logger::LoggerPtr pb_qry_factory_log) : command_client_( iroha::network::createClient( - target_ip + ":" + std::to_string(port))), - query_client_(target_ip, port) {} + target_ip + ":" + std::to_string(port)), + pb_qry_factory_log), + query_client_(target_ip, port), + pb_qry_factory_log_(std::move(pb_qry_factory_log)) {} CliClient::Response CliClient::sendTx( const shared_model::interface::Transaction &tx) { @@ -52,7 +56,7 @@ namespace iroha_cli { const shared_model::interface::Query &query) { CliClient::Response response; // Convert to proto and send to Iroha - iroha::model::converters::PbQueryFactory pb_factory; + iroha::model::converters::PbQueryFactory pb_factory(pb_qry_factory_log_); auto proto_query = static_cast(query); iroha::protocol::QueryResponse query_response; response.status = diff --git a/iroha-cli/client.hpp b/iroha-cli/client.hpp index 6ab556d1dc..44e9a758ba 100644 --- a/iroha-cli/client.hpp +++ b/iroha-cli/client.hpp @@ -8,6 +8,7 @@ #include +#include "logger/logger_fwd.hpp" #include "torii/command_client.hpp" #include "torii/query_client.hpp" @@ -31,7 +32,9 @@ namespace iroha_cli { // TODO 13/09/17 luckychess: check if we need more status codes IR-494 enum TxStatus { OK }; - CliClient(std::string target_ip, int port); + CliClient(std::string target_ip, + int port, + logger::LoggerPtr pb_qry_factory_log); /** * Send Transaction to Iroha Peer, i.e. target_ip:port * @param tx @@ -54,6 +57,8 @@ namespace iroha_cli { private: torii::CommandSyncClient command_client_; torii_utils::QuerySyncClient query_client_; + + logger::LoggerPtr pb_qry_factory_log_; }; } // namespace iroha_cli diff --git a/iroha-cli/grpc_response_handler.hpp b/iroha-cli/grpc_response_handler.hpp index d00d16e0a5..3bed1d90d2 100644 --- a/iroha-cli/grpc_response_handler.hpp +++ b/iroha-cli/grpc_response_handler.hpp @@ -6,6 +6,8 @@ #ifndef IROHA_CLI_GRPC_RESPONSE_HANDLER_HPP #define IROHA_CLI_GRPC_RESPONSE_HANDLER_HPP +#include "logger/logger_fwd.hpp" +#include "logger/logger_manager_fwd.hpp" #include "query_response_handler.hpp" #include "transaction_response_handler.hpp" @@ -16,8 +18,7 @@ namespace spdlog { namespace iroha_cli { class GrpcResponseHandler { public: - explicit GrpcResponseHandler( - logger::Logger log = logger::log("GrpcResponseHandler")); + explicit GrpcResponseHandler(logger::LoggerManagerTreePtr log_manager); /** * Handle iroha GRPC TxResponse * @param response @@ -33,7 +34,7 @@ namespace iroha_cli { TransactionResponseHandler tx_handler_; QueryResponseHandler query_handler_; void handleGrpcErrors(grpc::StatusCode code); - std::shared_ptr log_; + logger::LoggerPtr log_; std::unordered_map handler_map_; }; } // namespace iroha_cli diff --git a/iroha-cli/impl/grpc_response_handler.cpp b/iroha-cli/impl/grpc_response_handler.cpp index 1d69f6d6b0..c3341e7bf2 100644 --- a/iroha-cli/impl/grpc_response_handler.cpp +++ b/iroha-cli/impl/grpc_response_handler.cpp @@ -5,12 +5,16 @@ #include "grpc_response_handler.hpp" #include "logger/logger.hpp" +#include "logger/logger_manager.hpp" using namespace grpc; namespace iroha_cli { - GrpcResponseHandler::GrpcResponseHandler(logger::Logger log) - : log_(std::move(log)) { + GrpcResponseHandler::GrpcResponseHandler( + logger::LoggerManagerTreePtr log_manager) + : tx_handler_(log_manager->getChild("Transaction")->getLogger()), + query_handler_(log_manager->getChild("Query")->getLogger()), + log_(log_manager->getChild("Grpc")->getLogger()) { handler_map_[CANCELLED] = "Operation canceled"; handler_map_[UNKNOWN] = "Unknown error"; handler_map_[INVALID_ARGUMENT] = "INVALID_ARGUMENT"; diff --git a/iroha-cli/impl/query_response_handler.cpp b/iroha-cli/impl/query_response_handler.cpp index ba0840ce61..862d6e08ed 100644 --- a/iroha-cli/impl/query_response_handler.cpp +++ b/iroha-cli/impl/query_response_handler.cpp @@ -15,8 +15,7 @@ using namespace iroha::protocol; namespace iroha_cli { - QueryResponseHandler::QueryResponseHandler( - std::shared_ptr log) + QueryResponseHandler::QueryResponseHandler(logger::LoggerPtr log) : log_(std::move(log)) { handler_map_[QueryResponse::ResponseCase::kErrorResponse] = &QueryResponseHandler::handleErrorResponse; diff --git a/iroha-cli/impl/transaction_response_handler.cpp b/iroha-cli/impl/transaction_response_handler.cpp index 4b9bb378f2..51f72900ca 100644 --- a/iroha-cli/impl/transaction_response_handler.cpp +++ b/iroha-cli/impl/transaction_response_handler.cpp @@ -20,8 +20,7 @@ namespace iroha_cli { */ } } - TransactionResponseHandler::TransactionResponseHandler( - std::shared_ptr log) + TransactionResponseHandler::TransactionResponseHandler(logger::LoggerPtr log) : log_(std::move(log)) {} } // namespace iroha_cli diff --git a/iroha-cli/interactive/impl/interactive_cli.cpp b/iroha-cli/interactive/impl/interactive_cli.cpp index 3b4bd5a5b0..8137977921 100644 --- a/iroha-cli/interactive/impl/interactive_cli.cpp +++ b/iroha-cli/interactive/impl/interactive_cli.cpp @@ -8,6 +8,8 @@ #include #include +#include "logger/logger_manager.hpp" + namespace iroha_cli { namespace interactive { @@ -34,12 +36,30 @@ namespace iroha_cli { const std::string &default_peer_ip, int default_port, uint64_t qry_counter, - const std::shared_ptr &provider) + const std::shared_ptr &provider, + logger::LoggerManagerTreePtr response_handler_log_manager, + logger::LoggerPtr pb_qry_factory_log, + logger::LoggerPtr json_qry_factory_log, + logger::LoggerManagerTreePtr log_manager) : creator_(account_name), - tx_cli_(creator_, default_peer_ip, default_port, provider), - query_cli_( - creator_, default_peer_ip, default_port, qry_counter, provider), - statusCli_(default_peer_ip, default_port) { + tx_cli_(creator_, + default_peer_ip, + default_port, + provider, + response_handler_log_manager, + pb_qry_factory_log, + log_manager->getChild("Transaction")->getLogger()), + query_cli_(creator_, + default_peer_ip, + default_port, + qry_counter, + provider, + std::move(response_handler_log_manager), + pb_qry_factory_log, + std::move(json_qry_factory_log), + log_manager->getChild("Query")->getLogger()), + statusCli_( + default_peer_ip, default_port, std::move(pb_qry_factory_log)) { assign_main_handlers(); } diff --git a/iroha-cli/interactive/impl/interactive_query_cli.cpp b/iroha-cli/interactive/impl/interactive_query_cli.cpp index d2f86cd621..893170d3b8 100644 --- a/iroha-cli/interactive/impl/interactive_query_cli.cpp +++ b/iroha-cli/interactive/impl/interactive_query_cli.cpp @@ -15,6 +15,7 @@ #include "datetime/time.hpp" #include "grpc_response_handler.hpp" #include "interactive/interactive_query_cli.hpp" +#include "logger/logger.hpp" #include "model/converters/json_query_factory.hpp" #include "model/converters/pb_query_factory.hpp" #include "model/model_crypto_provider.hpp" // for ModelCryptoProvider @@ -94,14 +95,22 @@ namespace iroha_cli { const std::string &default_peer_ip, int default_port, uint64_t query_counter, - const std::shared_ptr &provider) + const std::shared_ptr &provider, + logger::LoggerManagerTreePtr response_handler_log_manager, + logger::LoggerPtr pb_qry_factory_log, + logger::LoggerPtr json_qry_factory_log, + logger::LoggerPtr log) : current_context_(MAIN), creator_(account_name), default_peer_ip_(default_peer_ip), default_port_(default_port), counter_(query_counter), - provider_(provider) { - log_ = logger::log("InteractiveQueryCli"); + provider_(provider), + response_handler_log_manager_( + std::move(response_handler_log_manager)), + pb_qry_factory_log_(std::move(pb_qry_factory_log)), + json_qry_factory_log_(std::move(json_qry_factory_log)), + log_(std::move(log)) { create_queries_menu(); create_result_menu(); } @@ -261,10 +270,13 @@ namespace iroha_cli { provider_->sign(*query_); - CliClient client(address.value().first, address.value().second); + CliClient client( + address.value().first, address.value().second, pb_qry_factory_log_); auto query = shared_model::proto::Query( - *iroha::model::converters::PbQueryFactory().serialize(query_)); - GrpcResponseHandler{}.handle(client.sendQuery(query)); + *iroha::model::converters::PbQueryFactory(pb_qry_factory_log_) + .serialize(query_)); + GrpcResponseHandler{response_handler_log_manager_}.handle( + client.sendQuery(query)); printEnd(); // Stop parsing return false; @@ -274,7 +286,8 @@ namespace iroha_cli { provider_->sign(*query_); auto path = params[0]; - iroha::model::converters::JsonQueryFactory json_factory; + iroha::model::converters::JsonQueryFactory json_factory( + json_qry_factory_log_); auto json_string = json_factory.serialize(query_); std::ofstream output_file(path); if (not output_file) { diff --git a/iroha-cli/interactive/impl/interactive_status_cli.cpp b/iroha-cli/interactive/impl/interactive_status_cli.cpp index cf76f3b877..07bf9df403 100644 --- a/iroha-cli/interactive/impl/interactive_status_cli.cpp +++ b/iroha-cli/interactive/impl/interactive_status_cli.cpp @@ -36,8 +36,12 @@ namespace iroha_cli { "Transaction has collected all signatures."}}; InteractiveStatusCli::InteractiveStatusCli( - const std::string &default_peer_ip, int default_port) - : default_peer_ip_(default_peer_ip), default_port_(default_port) { + const std::string &default_peer_ip, + int default_port, + logger::LoggerPtr pb_qry_factory_log) + : default_peer_ip_(default_peer_ip), + default_port_(default_port), + pb_qry_factory_log_(std::move(pb_qry_factory_log)) { createActionsMenu(); createResultMenu(); } @@ -138,7 +142,9 @@ namespace iroha_cli { auto status = iroha::protocol::TxStatus::NOT_RECEIVED; iroha::protocol::ToriiResponse answer; if (iroha::hexstringToBytestring(txHash_)) { - answer = CliClient(address.value().first, address.value().second) + answer = CliClient(address.value().first, + address.value().second, + pb_qry_factory_log_) .getTxStatus(txHash_) .answer; status = answer.tx_status(); diff --git a/iroha-cli/interactive/impl/interactive_transaction_cli.cpp b/iroha-cli/interactive/impl/interactive_transaction_cli.cpp index 9a2db5796c..7ab8b8305a 100644 --- a/iroha-cli/interactive/impl/interactive_transaction_cli.cpp +++ b/iroha-cli/interactive/impl/interactive_transaction_cli.cpp @@ -11,6 +11,7 @@ #include "backend/protobuf/transaction.hpp" #include "client.hpp" #include "grpc_response_handler.hpp" +#include "logger/logger.hpp" #include "model/commands/append_role.hpp" #include "model/commands/create_role.hpp" #include "model/commands/detach_role.hpp" @@ -169,13 +170,19 @@ namespace iroha_cli { const std::string &creator_account, const std::string &default_peer_ip, int default_port, - const std::shared_ptr &provider) + const std::shared_ptr &provider, + logger::LoggerManagerTreePtr response_handler_log_manager, + logger::LoggerPtr pb_qry_factory_log, + logger::LoggerPtr log) : current_context_(MAIN), creator_(creator_account), default_peer_ip_(default_peer_ip), default_port_(default_port), - provider_(provider) { - log_ = logger::log("InteractiveTransactionCli"); + provider_(provider), + response_handler_log_manager_( + std::move(response_handler_log_manager)), + pb_qry_factory_log_(std::move(pb_qry_factory_log)), + log_(std::move(log)) { createCommandMenu(); createResultMenu(); } @@ -446,12 +453,13 @@ namespace iroha_cli { provider_->sign(tx); - GrpcResponseHandler response_handler; + GrpcResponseHandler response_handler(response_handler_log_manager_); auto shared_tx = shared_model::proto::Transaction( iroha::model::converters::PbTransactionFactory().serialize(tx)); - response_handler.handle( - CliClient(address.value().first, address.value().second) - .sendTx(shared_tx)); + response_handler.handle(CliClient(address.value().first, + address.value().second, + pb_qry_factory_log_) + .sendTx(shared_tx)); printTxHash(tx); printEnd(); diff --git a/iroha-cli/interactive/interactive_cli.hpp b/iroha-cli/interactive/interactive_cli.hpp index aa3567ae07..c239322bd6 100644 --- a/iroha-cli/interactive/interactive_cli.hpp +++ b/iroha-cli/interactive/interactive_cli.hpp @@ -9,6 +9,7 @@ #include "interactive/interactive_query_cli.hpp" #include "interactive/interactive_status_cli.hpp" #include "interactive/interactive_transaction_cli.hpp" +#include "logger/logger_manager_fwd.hpp" namespace iroha_cli { namespace interactive { @@ -22,13 +23,21 @@ namespace iroha_cli { * @param default_port default port of peer's Iroha Torii * @param qry_counter synchronized nonce for sending queries * @param provider crypto provider to make signatures + * @param response_handler_log_manager for ResponseHandler messages + * @param pb_qry_factory_log for PbQueryFactory mesages + * @param json_qry_factory_log for JsonQueryFactory mesages + * @param log_manager log manager for interactive CLIs */ InteractiveCli( const std::string &account_name, const std::string &default_peer_ip, int default_port, uint64_t qry_counter, - const std::shared_ptr &provider); + const std::shared_ptr &provider, + logger::LoggerManagerTreePtr response_handler_log_manager, + logger::LoggerPtr pb_qry_factory_log, + logger::LoggerPtr json_qry_factory_log, + logger::LoggerManagerTreePtr log_manager); /** * Run interactive cli. Print menu and parse commands */ diff --git a/iroha-cli/interactive/interactive_query_cli.hpp b/iroha-cli/interactive/interactive_query_cli.hpp index 050945ad17..d4e78e2aa0 100644 --- a/iroha-cli/interactive/interactive_query_cli.hpp +++ b/iroha-cli/interactive/interactive_query_cli.hpp @@ -10,21 +10,21 @@ #include #include "interactive/interactive_common_cli.hpp" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" +#include "logger/logger_manager_fwd.hpp" #include "model/generators/query_generator.hpp" namespace iroha { namespace model { class ModelCryptoProvider; struct Query; - } -} + } // namespace model +} // namespace iroha namespace iroha_cli { namespace interactive { class InteractiveQueryCli { public: - /** * Class to form and send Iroha queries in interactive mode * @param creator_account creator's account identification @@ -32,13 +32,21 @@ namespace iroha_cli { * @param default_port of Iroha peer * @param query_counter counter associated with creator's account * @param provider for signing queries + * @param response_handler_log_manager for ResponseHandler mesages + * @param pb_qry_factory_log for PbQueryFactory mesages + * @param json_qry_factory_log for JsonQueryFactory mesages + * @param log for internal messages */ InteractiveQueryCli( const std::string &account_id, const std::string &default_peer_ip, int default_port, uint64_t query_counter, - const std::shared_ptr &provider); + const std::shared_ptr &provider, + logger::LoggerManagerTreePtr response_handler_log_manager, + logger::LoggerPtr pb_qry_factory_log, + logger::LoggerPtr json_qry_factory_log, + logger::LoggerPtr log); /** * Run interactive query command line */ @@ -150,11 +158,20 @@ namespace iroha_cli { // Query generator for new queries iroha::model::generators::QueryGenerator generator_; - // Logger - logger::Logger log_; - // Crypto provider std::shared_ptr provider_; + + /// Logger manager for GrpcResponseHandler + logger::LoggerManagerTreePtr response_handler_log_manager_; + + /// Logger for PbQueryFactory + logger::LoggerPtr pb_qry_factory_log_; + + /// Logger for JsonQueryFactory + logger::LoggerPtr json_qry_factory_log_; + + /// Internal logger + logger::LoggerPtr log_; }; } // namespace interactive } // namespace iroha_cli diff --git a/iroha-cli/interactive/interactive_status_cli.hpp b/iroha-cli/interactive/interactive_status_cli.hpp index 894a7548c5..94ba1201d9 100644 --- a/iroha-cli/interactive/interactive_status_cli.hpp +++ b/iroha-cli/interactive/interactive_status_cli.hpp @@ -10,6 +10,7 @@ #include #include "interactive/interactive_common_cli.hpp" +#include "logger/logger_fwd.hpp" namespace iroha_cli { namespace interactive { @@ -21,7 +22,8 @@ namespace iroha_cli { class InteractiveStatusCli { public: InteractiveStatusCli(const std::string &default_peer_ip, - int default_port); + int default_port, + logger::LoggerPtr pb_qry_factory_log); void run(); private: @@ -57,6 +59,8 @@ namespace iroha_cli { MenuContext currentContext_; std::string txHash_; + + logger::LoggerPtr pb_qry_factory_log_; }; } // namespace interactive } // namespace iroha_cli diff --git a/iroha-cli/interactive/interactive_transaction_cli.hpp b/iroha-cli/interactive/interactive_transaction_cli.hpp index c12c4436bb..9819007b0e 100644 --- a/iroha-cli/interactive/interactive_transaction_cli.hpp +++ b/iroha-cli/interactive/interactive_transaction_cli.hpp @@ -9,15 +9,16 @@ #include #include "interactive/interactive_common_cli.hpp" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" +#include "logger/logger_manager_fwd.hpp" #include "model/generators/transaction_generator.hpp" namespace iroha { namespace model { struct Command; class ModelCryptoProvider; - } -} + } // namespace model +} // namespace iroha namespace iroha_cli { namespace interactive { @@ -29,12 +30,18 @@ namespace iroha_cli { * @param default_peer_ip of Iroha peer * @param default_port of Iroha peer * @param provider for signing transactions + * @param response_handler_log_manager for ResponseHandler messages + * @param pb_qry_factory_log for PbQueryFactory mesages + * @param log for internal messages */ InteractiveTransactionCli( const std::string &creator_account, const std::string &default_peer_ip, int default_port, - const std::shared_ptr &provider); + const std::shared_ptr &provider, + logger::LoggerManagerTreePtr response_handler_log_manager, + logger::LoggerPtr pb_qry_factory_log, + logger::LoggerPtr log); /** * Run interactive query command line */ @@ -172,14 +179,20 @@ namespace iroha_cli { // Commands to be formed std::vector> commands_; - // Logger - logger::Logger log_; - // Crypto provider std::shared_ptr provider_; // Transaction generator iroha::model::generators::TransactionGenerator tx_generator_; + + /// Logger manager for ResponseHandler + logger::LoggerManagerTreePtr response_handler_log_manager_; + + /// Logger for PbQueryFactory + logger::LoggerPtr pb_qry_factory_log_; + + /// Internal logger + logger::LoggerPtr log_; }; } // namespace interactive } // namespace iroha_cli diff --git a/iroha-cli/main.cpp b/iroha-cli/main.cpp index b5ad790085..d75afade1c 100644 --- a/iroha-cli/main.cpp +++ b/iroha-cli/main.cpp @@ -18,6 +18,8 @@ #include "crypto/keys_manager_impl.hpp" #include "grpc_response_handler.hpp" #include "interactive/interactive_cli.hpp" +#include "logger/logger.hpp" +#include "logger/logger_manager.hpp" #include "model/converters/json_block_factory.hpp" #include "model/converters/json_query_factory.hpp" #include "model/converters/pb_block_factory.hpp" @@ -74,7 +76,19 @@ iroha::keypair_t *makeOldModel(const shared_model::crypto::Keypair &keypair) { int main(int argc, char *argv[]) { gflags::ParseCommandLineFlags(&argc, &argv, true); gflags::ShutDownCommandLineFlags(); - auto logger = logger::log("CLI-MAIN"); + auto log_manager = std::make_shared( + logger::LoggerConfig{logger::LogLevel::kInfo, + logger::getDefaultLogPatterns()}) + ->getChild("CLI"); + const auto logger = log_manager->getChild("Main")->getLogger(); + const auto response_handler_log_manager = + log_manager->getChild("ResponseHandler"); + const auto pb_qry_factory_log = + log_manager->getChild("PbQueryFactory")->getLogger(); + const auto json_qry_factory_log = + log_manager->getChild("JsonQueryFactory")->getLogger(); + const auto keys_manager_log = + log_manager->getChild("KeysManager")->getLogger(); // Generate new genesis block now Iroha network if (FLAGS_genesis_block) { BlockGenerator generator; @@ -91,7 +105,7 @@ int main(int argc, char *argv[]) { std::back_inserter(peers_address)); // Generate genesis block transaction = TransactionGenerator().generateGenesisTransaction( - 0, std::move(peers_address)); + 0, std::move(peers_address), keys_manager_log); } else { rapidjson::Document doc; std::ifstream file(FLAGS_genesis_transaction); @@ -132,7 +146,8 @@ int main(int argc, char *argv[]) { logger->error("No account name specified"); return EXIT_FAILURE; } - auto keysManager = iroha::KeysManagerImpl(FLAGS_account_name); + auto keysManager = + iroha::KeysManagerImpl(FLAGS_account_name, keys_manager_log); if (not(FLAGS_pass_phrase.size() == 0 ? keysManager.createKeys() : keysManager.createKeys(FLAGS_pass_phrase))) { @@ -145,8 +160,10 @@ int main(int argc, char *argv[]) { } // Send to Iroha Peer json transaction/query else if (not FLAGS_json_transaction.empty() or not FLAGS_json_query.empty()) { - iroha_cli::CliClient client(FLAGS_peer_ip, FLAGS_torii_port); - iroha_cli::GrpcResponseHandler response_handler; + iroha_cli::CliClient client( + FLAGS_peer_ip, FLAGS_torii_port, pb_qry_factory_log); + iroha_cli::GrpcResponseHandler response_handler( + response_handler_log_manager); if (not FLAGS_json_transaction.empty()) { logger->info( "Send transaction to {}:{} ", FLAGS_peer_ip, FLAGS_torii_port); @@ -176,14 +193,16 @@ int main(int argc, char *argv[]) { std::ifstream file(FLAGS_json_query); std::string str((std::istreambuf_iterator(file)), std::istreambuf_iterator()); - iroha::model::converters::JsonQueryFactory serializer; + iroha::model::converters::JsonQueryFactory serializer( + json_qry_factory_log); auto query_opt = serializer.deserialize(std::move(str)); if (not query_opt) { logger->error("Json has wrong format."); return EXIT_FAILURE; } else { auto query = shared_model::proto::Query( - *iroha::model::converters::PbQueryFactory().serialize(*query_opt)); + *iroha::model::converters::PbQueryFactory(pb_qry_factory_log) + .serialize(*query_opt)); auto response = client.sendQuery(query); response_handler.handle(response); } @@ -200,7 +219,8 @@ int main(int argc, char *argv[]) { logger->error("Path {} not found.", path.string()); return EXIT_FAILURE; } - iroha::KeysManagerImpl manager((path / FLAGS_account_name).string()); + iroha::KeysManagerImpl manager((path / FLAGS_account_name).string(), + keys_manager_log); auto keypair = FLAGS_pass_phrase.size() != 0 ? manager.loadKeys(FLAGS_pass_phrase) : manager.loadKeys(); @@ -222,7 +242,11 @@ int main(int argc, char *argv[]) { FLAGS_torii_port, 0, std::make_shared( - *std::unique_ptr(makeOldModel(*keypair)))); + *std::unique_ptr(makeOldModel(*keypair))), + response_handler_log_manager, + pb_qry_factory_log, + json_qry_factory_log, + log_manager->getChild("InteractiveCli")); interactiveCli.run(); } else { logger->error("Invalid flags"); diff --git a/iroha-cli/query_response_handler.hpp b/iroha-cli/query_response_handler.hpp index 5b317fa67c..53c24db11d 100644 --- a/iroha-cli/query_response_handler.hpp +++ b/iroha-cli/query_response_handler.hpp @@ -11,7 +11,7 @@ #include #include -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" #include "qry_responses.pb.h" namespace iroha_cli { @@ -30,8 +30,7 @@ namespace iroha_cli { class QueryResponseHandler { public: - explicit QueryResponseHandler(std::shared_ptr log = - logger::log("QueryResponseHandler")); + explicit QueryResponseHandler(logger::LoggerPtr log); /** * Handle query response @@ -64,7 +63,7 @@ namespace iroha_cli { std::unordered_map error_handler_map_; - std::shared_ptr log_; + logger::LoggerPtr log_; }; } // namespace iroha_cli diff --git a/iroha-cli/transaction_response_handler.hpp b/iroha-cli/transaction_response_handler.hpp index 5dc413d202..2272e7cc22 100644 --- a/iroha-cli/transaction_response_handler.hpp +++ b/iroha-cli/transaction_response_handler.hpp @@ -8,6 +8,8 @@ #include "client.hpp" // for CliClient::TxStatus (yuck!) +#include "logger/logger_fwd.hpp" + namespace spdlog { class logger; } @@ -16,9 +18,7 @@ namespace iroha_cli { class TransactionResponseHandler { public: - explicit TransactionResponseHandler( - std::shared_ptr log = - logger::log("TransactionResponseHandler")); + explicit TransactionResponseHandler(logger::LoggerPtr log); /** * Handle response from Iroha * @param status of transaction @@ -26,7 +26,7 @@ namespace iroha_cli { void handle(const CliClient::TxStatus status) const; private: - std::shared_ptr log_; + logger::LoggerPtr log_; }; } // namespace iroha_cli diff --git a/irohad/ametsuchi/CMakeLists.txt b/irohad/ametsuchi/CMakeLists.txt index 46ed727805..4695c96840 100644 --- a/irohad/ametsuchi/CMakeLists.txt +++ b/irohad/ametsuchi/CMakeLists.txt @@ -22,6 +22,7 @@ add_library(ametsuchi target_link_libraries(ametsuchi logger + logger_manager rxcpp libs_files common diff --git a/irohad/ametsuchi/impl/flat_file/flat_file.cpp b/irohad/ametsuchi/impl/flat_file/flat_file.cpp index 4bd9aa11ba..b3b73643d2 100644 --- a/irohad/ametsuchi/impl/flat_file/flat_file.cpp +++ b/irohad/ametsuchi/impl/flat_file/flat_file.cpp @@ -14,6 +14,7 @@ #include #include #include "common/files.hpp" +#include "logger/logger.hpp" using namespace iroha::ametsuchi; using Identifier = FlatFile::Identifier; @@ -27,18 +28,17 @@ std::string FlatFile::id_to_name(Identifier id) { } boost::optional> FlatFile::create( - const std::string &path) { - auto log_ = logger::log("FlatFile::create()"); - + const std::string &path, + logger::LoggerPtr log) { boost::system::error_code err; if (not boost::filesystem::is_directory(path, err) and not boost::filesystem::create_directory(path, err)) { - log_->error("Cannot create storage dir: {}\n{}", path, err.message()); + log->error("Cannot create storage dir: {}\n{}", path, err.message()); return boost::none; } - auto res = FlatFile::check_consistency(path); - return std::make_unique(*res, path, private_tag{}); + auto res = FlatFile::check_consistency(path, log); + return std::make_unique(*res, path, private_tag{}, std::move(log)); } bool FlatFile::add(Identifier id, const Bytes &block) { @@ -104,8 +104,8 @@ Identifier FlatFile::last_id() const { } void FlatFile::dropAll() { - iroha::remove_dir_contents(dump_dir_); - auto res = FlatFile::check_consistency(dump_dir_); + iroha::remove_dir_contents(dump_dir_, log_); + auto res = FlatFile::check_consistency(dump_dir_, log_); current_id_.store(*res); } @@ -114,15 +114,13 @@ void FlatFile::dropAll() { FlatFile::FlatFile(Identifier current_id, const std::string &path, FlatFile::private_tag, - logger::Logger log) + logger::LoggerPtr log) : dump_dir_(path), log_{std::move(log)} { current_id_.store(current_id); } boost::optional FlatFile::check_consistency( - const std::string &dump_dir) { - auto log = logger::log("FLAT_FILE"); - + const std::string &dump_dir, logger::LoggerPtr log) { if (dump_dir.empty()) { log->error("check_consistency({}), not directory", dump_dir); return boost::none; diff --git a/irohad/ametsuchi/impl/flat_file/flat_file.hpp b/irohad/ametsuchi/impl/flat_file/flat_file.hpp index 62f1cf51bf..da20283075 100644 --- a/irohad/ametsuchi/impl/flat_file/flat_file.hpp +++ b/irohad/ametsuchi/impl/flat_file/flat_file.hpp @@ -11,7 +11,7 @@ #include #include -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" namespace iroha { namespace ametsuchi { @@ -48,10 +48,11 @@ namespace iroha { /** * Create storage in paths * @param path - target path for creating + * @param log - logger * @return created storage */ static boost::optional> create( - const std::string &path); + const std::string &path, logger::LoggerPtr log); bool add(Identifier id, const Bytes &blob) override; @@ -66,10 +67,11 @@ namespace iroha { * If some block in the middle is missing all blocks following it are * deleted * @param dump_dir - folder of storage + * @param log - log for local messages * @return - last available identifier */ static boost::optional check_consistency( - const std::string &dump_dir); + const std::string &dump_dir, logger::LoggerPtr log); void dropAll() override; @@ -94,7 +96,7 @@ namespace iroha { FlatFile(Identifier last_id, const std::string &path, FlatFile::private_tag, - logger::Logger log = logger::log("FlatFile")); + logger::LoggerPtr log); private: // ----------| private fields |---------- @@ -109,7 +111,7 @@ namespace iroha { */ const std::string dump_dir_; - logger::Logger log_; + logger::LoggerPtr log_; public: ~FlatFile() = default; diff --git a/irohad/ametsuchi/impl/mutable_storage_impl.cpp b/irohad/ametsuchi/impl/mutable_storage_impl.cpp index 3c7d2ac1a3..944ab6aae3 100644 --- a/irohad/ametsuchi/impl/mutable_storage_impl.cpp +++ b/irohad/ametsuchi/impl/mutable_storage_impl.cpp @@ -14,6 +14,8 @@ #include "interfaces/commands/command.hpp" #include "interfaces/common_objects/common_objects_factory.hpp" #include "interfaces/iroha_internal/block.hpp" +#include "logger/logger.hpp" +#include "logger/logger_manager.hpp" namespace iroha { namespace ametsuchi { @@ -22,15 +24,19 @@ namespace iroha { std::shared_ptr cmd_executor, std::unique_ptr sql, std::shared_ptr factory, - logger::Logger log) + logger::LoggerManagerTreePtr log_manager) : top_hash_(top_hash), sql_(std::move(sql)), - peer_query_(std::make_unique( - std::make_shared(*sql_, std::move(factory)))), - block_index_(std::make_unique(*sql_)), + peer_query_( + std::make_unique(std::make_shared( + *sql_, + std::move(factory), + log_manager->getChild("WsvQuery")->getLogger()))), + block_index_(std::make_unique( + *sql_, log_manager->getChild("PostgresBlockIndex")->getLogger())), command_executor_(std::move(cmd_executor)), committed(false), - log_(std::move(log)) { + log_(log_manager->getLogger()) { *sql_ << "BEGIN"; } diff --git a/irohad/ametsuchi/impl/mutable_storage_impl.hpp b/irohad/ametsuchi/impl/mutable_storage_impl.hpp index 736aa81f3e..127bab928f 100644 --- a/irohad/ametsuchi/impl/mutable_storage_impl.hpp +++ b/irohad/ametsuchi/impl/mutable_storage_impl.hpp @@ -13,7 +13,8 @@ #include #include "ametsuchi/command_executor.hpp" #include "interfaces/common_objects/common_objects_factory.hpp" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" +#include "logger/logger_manager_fwd.hpp" namespace iroha { namespace ametsuchi { @@ -30,7 +31,7 @@ namespace iroha { std::unique_ptr sql, std::shared_ptr factory, - logger::Logger log = logger::log("MutableStorage")); + logger::LoggerManagerTreePtr log_manager); bool apply(const shared_model::interface::Block &block) override; @@ -69,7 +70,7 @@ namespace iroha { bool committed; - logger::Logger log_; + logger::LoggerPtr log_; }; } // namespace ametsuchi } // namespace iroha diff --git a/irohad/ametsuchi/impl/postgres_block_index.cpp b/irohad/ametsuchi/impl/postgres_block_index.cpp index ab403f7789..77a37f77d0 100644 --- a/irohad/ametsuchi/impl/postgres_block_index.cpp +++ b/irohad/ametsuchi/impl/postgres_block_index.cpp @@ -12,6 +12,7 @@ #include "interfaces/commands/command_variant.hpp" #include "interfaces/commands/transfer_asset.hpp" #include "interfaces/iroha_internal/block.hpp" +#include "logger/logger.hpp" namespace { // Return transfer asset if command contains it @@ -119,7 +120,7 @@ namespace { namespace iroha { namespace ametsuchi { PostgresBlockIndex::PostgresBlockIndex(soci::session &sql, - logger::Logger log) + logger::LoggerPtr log) : sql_(sql), log_(std::move(log)) {} void PostgresBlockIndex::index( diff --git a/irohad/ametsuchi/impl/postgres_block_index.hpp b/irohad/ametsuchi/impl/postgres_block_index.hpp index 740a5496da..712a2c2a3d 100644 --- a/irohad/ametsuchi/impl/postgres_block_index.hpp +++ b/irohad/ametsuchi/impl/postgres_block_index.hpp @@ -11,15 +11,13 @@ #include "ametsuchi/impl/block_index.hpp" #include "ametsuchi/impl/soci_utils.hpp" #include "interfaces/transaction.hpp" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" namespace iroha { namespace ametsuchi { class PostgresBlockIndex : public BlockIndex { public: - PostgresBlockIndex( - soci::session &sql, - logger::Logger log = logger::log("PostgresBlockIndex")); + PostgresBlockIndex(soci::session &sql, logger::LoggerPtr log); /** * Create several indices for block. Namely: @@ -38,7 +36,7 @@ namespace iroha { private: soci::session &sql_; - logger::Logger log_; + logger::LoggerPtr log_; }; } // namespace ametsuchi } // namespace iroha diff --git a/irohad/ametsuchi/impl/postgres_block_query.cpp b/irohad/ametsuchi/impl/postgres_block_query.cpp index e061355968..13387f0a35 100644 --- a/irohad/ametsuchi/impl/postgres_block_query.cpp +++ b/irohad/ametsuchi/impl/postgres_block_query.cpp @@ -11,6 +11,7 @@ #include "ametsuchi/impl/soci_utils.hpp" #include "common/byteutils.hpp" +#include "logger/logger.hpp" namespace iroha { namespace ametsuchi { @@ -19,7 +20,7 @@ namespace iroha { KeyValueStorage &file_store, std::shared_ptr converter, - logger::Logger log) + logger::LoggerPtr log) : sql_(sql), block_store_(file_store), converter_(std::move(converter)), @@ -30,7 +31,7 @@ namespace iroha { KeyValueStorage &file_store, std::shared_ptr converter, - logger::Logger log) + logger::LoggerPtr log) : psql_(std::move(sql)), sql_(*psql_), block_store_(file_store), diff --git a/irohad/ametsuchi/impl/postgres_block_query.hpp b/irohad/ametsuchi/impl/postgres_block_query.hpp index 26ef79533e..d8c7c29c70 100644 --- a/irohad/ametsuchi/impl/postgres_block_query.hpp +++ b/irohad/ametsuchi/impl/postgres_block_query.hpp @@ -12,7 +12,7 @@ #include #include "ametsuchi/impl/flat_file/flat_file.hpp" #include "interfaces/iroha_internal/block_json_deserializer.hpp" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" namespace iroha { namespace ametsuchi { @@ -29,14 +29,14 @@ namespace iroha { KeyValueStorage &file_store, std::shared_ptr converter, - logger::Logger log = logger::log("PostgresBlockQuery")); + logger::LoggerPtr log); PostgresBlockQuery( std::unique_ptr sql, KeyValueStorage &file_store, std::shared_ptr converter, - logger::Logger log = logger::log("PostgresBlockQuery")); + logger::LoggerPtr log); std::vector getBlocks( shared_model::interface::types::HeightType height, @@ -71,7 +71,7 @@ namespace iroha { std::shared_ptr converter_; - logger::Logger log_; + logger::LoggerPtr log_; }; } // namespace ametsuchi } // namespace iroha diff --git a/irohad/ametsuchi/impl/postgres_query_executor.cpp b/irohad/ametsuchi/impl/postgres_query_executor.cpp index 3f9293f8bb..e7571b63e1 100644 --- a/irohad/ametsuchi/impl/postgres_query_executor.cpp +++ b/irohad/ametsuchi/impl/postgres_query_executor.cpp @@ -34,6 +34,8 @@ #include "interfaces/queries/get_transactions.hpp" #include "interfaces/queries/query.hpp" #include "interfaces/queries/tx_pagination_meta.hpp" +#include "logger/logger.hpp" +#include "logger/logger_manager.hpp" using namespace shared_model::interface::permissions; @@ -275,7 +277,7 @@ namespace iroha { response_factory, std::shared_ptr perm_converter, - logger::Logger log) + logger::LoggerManagerTreePtr log_manager) : sql_(std::move(sql)), block_store_(block_store), pending_txs_storage_(std::move(pending_txs_storage)), @@ -284,9 +286,10 @@ namespace iroha { pending_txs_storage_, std::move(converter), response_factory, - perm_converter), + perm_converter, + log_manager->getChild("Visitor")->getLogger()), query_response_factory_{std::move(response_factory)}, - log_(std::move(log)) {} + log_(log_manager->getLogger()) {} QueryExecutorResult PostgresQueryExecutor::validateAndExecute( const shared_model::interface::Query &query, @@ -348,7 +351,7 @@ namespace iroha { response_factory, std::shared_ptr perm_converter, - logger::Logger log) + logger::LoggerPtr log) : sql_(sql), block_store_(block_store), pending_txs_storage_(std::move(pending_txs_storage)), @@ -398,7 +401,7 @@ namespace iroha { break; } - log_->error(error); + log_->error("{}", error); return query_response_factory_->createErrorQueryResponse( error_type, error, error_code, query_hash_); } diff --git a/irohad/ametsuchi/impl/postgres_query_executor.hpp b/irohad/ametsuchi/impl/postgres_query_executor.hpp index 6115e515b5..4045cb1ee8 100644 --- a/irohad/ametsuchi/impl/postgres_query_executor.hpp +++ b/irohad/ametsuchi/impl/postgres_query_executor.hpp @@ -33,7 +33,8 @@ #include "interfaces/queries/blocks_query.hpp" #include "interfaces/queries/query.hpp" #include "interfaces/query_responses/query_response.hpp" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" +#include "logger/logger_manager_fwd.hpp" namespace iroha { namespace ametsuchi { @@ -58,7 +59,7 @@ namespace iroha { response_factory, std::shared_ptr perm_converter, - logger::Logger log = logger::log("PostgresQueryExecutorVisitor")); + logger::LoggerPtr log); void setCreatorId( const shared_model::interface::types::AccountIdType &creator_id); @@ -234,7 +235,7 @@ namespace iroha { query_response_factory_; std::shared_ptr perm_converter_; - logger::Logger log_; + logger::LoggerPtr log_; }; class PostgresQueryExecutor : public QueryExecutor { @@ -249,7 +250,7 @@ namespace iroha { response_factory, std::shared_ptr perm_converter, - logger::Logger log = logger::log("PostgresQueryExecutor")); + logger::LoggerManagerTreePtr log_manager); QueryExecutorResult validateAndExecute( const shared_model::interface::Query &query, @@ -268,7 +269,7 @@ namespace iroha { PostgresQueryExecutorVisitor visitor_; std::shared_ptr query_response_factory_; - logger::Logger log_; + logger::LoggerPtr log_; }; } // namespace ametsuchi diff --git a/irohad/ametsuchi/impl/postgres_wsv_query.cpp b/irohad/ametsuchi/impl/postgres_wsv_query.cpp index 1e56168888..13c64f5f6a 100644 --- a/irohad/ametsuchi/impl/postgres_wsv_query.cpp +++ b/irohad/ametsuchi/impl/postgres_wsv_query.cpp @@ -9,6 +9,7 @@ #include "ametsuchi/impl/soci_utils.hpp" #include "common/result.hpp" #include "cryptography/public_key.hpp" +#include "logger/logger.hpp" namespace iroha { namespace ametsuchi { @@ -20,13 +21,13 @@ namespace iroha { PostgresWsvQuery::PostgresWsvQuery( soci::session &sql, std::shared_ptr factory, - logger::Logger log) + logger::LoggerPtr log) : sql_(sql), factory_(std::move(factory)), log_(std::move(log)) {} PostgresWsvQuery::PostgresWsvQuery( std::unique_ptr sql, std::shared_ptr factory, - logger::Logger log) + logger::LoggerPtr log) : psql_(std::move(sql)), sql_(*psql_), factory_(std::move(factory)), diff --git a/irohad/ametsuchi/impl/postgres_wsv_query.hpp b/irohad/ametsuchi/impl/postgres_wsv_query.hpp index 2bbf064598..8d4f45579a 100644 --- a/irohad/ametsuchi/impl/postgres_wsv_query.hpp +++ b/irohad/ametsuchi/impl/postgres_wsv_query.hpp @@ -10,7 +10,7 @@ #include #include "interfaces/common_objects/common_objects_factory.hpp" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" namespace iroha { namespace ametsuchi { @@ -20,13 +20,13 @@ namespace iroha { soci::session &sql, std::shared_ptr factory, - logger::Logger log = logger::log("PostgresWsvQuery")); + logger::LoggerPtr log); PostgresWsvQuery( std::unique_ptr sql, std::shared_ptr factory, - logger::Logger log = logger::log("PostgresWsvQuery")); + logger::LoggerPtr log); boost::optional> getSignatories(const shared_model::interface::types::AccountIdType @@ -62,7 +62,7 @@ namespace iroha { std::unique_ptr psql_; soci::session &sql_; std::shared_ptr factory_; - logger::Logger log_; + logger::LoggerPtr log_; }; } // namespace ametsuchi } // namespace iroha diff --git a/irohad/ametsuchi/impl/storage_impl.cpp b/irohad/ametsuchi/impl/storage_impl.cpp index 8fae510ec1..7a6009da2c 100644 --- a/irohad/ametsuchi/impl/storage_impl.cpp +++ b/irohad/ametsuchi/impl/storage_impl.cpp @@ -20,6 +20,8 @@ #include "common/bind.hpp" #include "common/byteutils.hpp" #include "converters/protobuf/json_proto_converter.hpp" +#include "logger/logger.hpp" +#include "logger/logger_manager.hpp" namespace { void prepareStatements(soci::connection_pool &connections, size_t pool_size) { @@ -65,7 +67,7 @@ namespace iroha { perm_converter, size_t pool_size, bool enable_prepared_blocks, - logger::Logger log) + logger::LoggerManagerTreePtr log_manager) : block_store_dir_(std::move(block_store_dir)), postgres_options_(std::move(postgres_options)), block_store_(std::move(block_store)), @@ -73,7 +75,8 @@ namespace iroha { factory_(std::move(factory)), converter_(std::move(converter)), perm_converter_(std::move(perm_converter)), - log_(std::move(log)), + log_manager_(std::move(log_manager)), + log_(log_manager_->getLogger()), pool_size_(pool_size), prepared_blocks_enabled_(enable_prepared_blocks), block_is_prepared(false) { @@ -109,7 +112,10 @@ namespace iroha { return expected::makeValue>( std::make_unique( - std::move(sql), factory_, perm_converter_)); + std::move(sql), + factory_, + perm_converter_, + log_manager_->getChild("TemporaryWorldStateView"))); } expected::Result, std::string> @@ -141,7 +147,8 @@ namespace iroha { }), std::make_shared(*sql, perm_converter_), std::move(sql), - factory_)); + factory_, + log_manager_->getChild("MutableStorageImpl"))); } boost::optional> StorageImpl::createPeerQuery() @@ -180,7 +187,8 @@ namespace iroha { std::move(pending_txs_storage), converter_, std::move(response_factory), - perm_converter_)); + perm_converter_, + log_manager_->getChild("QueryExecutor"))); } bool StorageImpl::insertBlock(const shared_model::interface::Block &block) { @@ -317,17 +325,17 @@ namespace iroha { } expected::Result - StorageImpl::initConnections(std::string block_store_dir) { - auto log_ = logger::log("StorageImpl:initConnection"); - log_->info("Start storage creation"); + StorageImpl::initConnections(std::string block_store_dir, + logger::LoggerPtr log) { + log->info("Start storage creation"); - auto block_store = FlatFile::create(block_store_dir); + auto block_store = FlatFile::create(block_store_dir, log); if (not block_store) { return expected::makeError( (boost::format("Cannot create block store in %s") % block_store_dir) .str()); } - log_->info("block store created"); + log->info("block store created"); return expected::makeValue(ConnectionContext(std::move(*block_store))); } @@ -356,6 +364,7 @@ namespace iroha { std::shared_ptr converter, std::shared_ptr perm_converter, + logger::LoggerManagerTreePtr log_manager, size_t pool_size) { boost::optional string_res = boost::none; @@ -374,7 +383,8 @@ namespace iroha { return expected::makeError(string_res.value()); } - auto ctx_result = initConnections(block_store_dir); + auto ctx_result = + initConnections(block_store_dir, log_manager->getLogger()); auto db_result = initPostgresConnection(postgres_options, pool_size); expected::Result, std::string> storage; ctx_result.match( @@ -394,7 +404,8 @@ namespace iroha { converter, perm_converter, pool_size, - enable_prepared_transactions))); + enable_prepared_transactions, + std::move(log_manager)))); }, [&](expected::Error &error) { storage = error; }); }, @@ -412,7 +423,11 @@ namespace iroha { try { *(storage->sql_) << "COMMIT"; storage->committed = true; - return PostgresWsvQuery(*(storage->sql_), factory_).getPeers() | + return PostgresWsvQuery(*(storage->sql_), + factory_, + log_manager_->getChild("WsvQuery")->getLogger()) + .getPeers() + | [](auto &&peers) { return boost::optional>( std::make_unique( @@ -446,11 +461,15 @@ namespace iroha { } soci::session sql(*connection_); sql << "COMMIT PREPARED '" + prepared_block_name_ + "';"; - PostgresBlockIndex block_index(sql); + PostgresBlockIndex block_index( + sql, log_manager_->getChild("BlockIndex")->getLogger()); block_index.index(block); block_is_prepared = false; - return PostgresWsvQuery(sql, factory_).getPeers() | - [this, &block](auto &&peers) + return PostgresWsvQuery(sql, + factory_, + log_manager_->getChild("WsvQuery")->getLogger()) + .getPeers() + | [this, &block](auto &&peers) -> boost::optional> { if (this->storeBlock(block)) { return boost::optional>( @@ -474,7 +493,9 @@ namespace iroha { return nullptr; } return std::make_shared( - std::make_unique(*connection_), factory_); + std::make_unique(*connection_), + factory_, + log_manager_->getChild("WsvQuery")->getLogger()); } std::shared_ptr StorageImpl::getBlockQuery() const { @@ -486,7 +507,8 @@ namespace iroha { return std::make_shared( std::make_unique(*connection_), *block_store_, - converter_); + converter_, + log_manager_->getChild("PostgresBlockQuery")->getLogger()); } rxcpp::observable> diff --git a/irohad/ametsuchi/impl/storage_impl.hpp b/irohad/ametsuchi/impl/storage_impl.hpp index de7c0f927d..f4e7f1e6d0 100644 --- a/irohad/ametsuchi/impl/storage_impl.hpp +++ b/irohad/ametsuchi/impl/storage_impl.hpp @@ -20,7 +20,8 @@ #include "interfaces/common_objects/common_objects_factory.hpp" #include "interfaces/iroha_internal/block_json_converter.hpp" #include "interfaces/permission_to_string.hpp" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" +#include "logger/logger_manager_fwd.hpp" namespace iroha { namespace ametsuchi { @@ -40,7 +41,7 @@ namespace iroha { const std::string &options_str_without_dbname); static expected::Result initConnections( - std::string block_store_dir); + std::string block_store_dir, logger::LoggerPtr log); static expected::Result, std::string> @@ -56,6 +57,7 @@ namespace iroha { converter, std::shared_ptr perm_converter, + logger::LoggerManagerTreePtr log_manager, size_t pool_size = 10); expected::Result, std::string> @@ -127,7 +129,7 @@ namespace iroha { perm_converter, size_t pool_size, bool enable_prepared_blocks, - logger::Logger log = logger::log("StorageImpl")); + logger::LoggerManagerTreePtr log_manager); /** * Folder with raw blocks @@ -162,7 +164,8 @@ namespace iroha { std::shared_ptr perm_converter_; - logger::Logger log_; + logger::LoggerManagerTreePtr log_manager_; + logger::LoggerPtr log_; mutable std::shared_timed_mutex drop_mutex; diff --git a/irohad/ametsuchi/impl/temporary_wsv_impl.cpp b/irohad/ametsuchi/impl/temporary_wsv_impl.cpp index 1974f53c08..09b23879f4 100644 --- a/irohad/ametsuchi/impl/temporary_wsv_impl.cpp +++ b/irohad/ametsuchi/impl/temporary_wsv_impl.cpp @@ -11,6 +11,8 @@ #include "interfaces/commands/command.hpp" #include "interfaces/permission_to_string.hpp" #include "interfaces/transaction.hpp" +#include "logger/logger.hpp" +#include "logger/logger_manager.hpp" namespace iroha { namespace ametsuchi { @@ -19,11 +21,12 @@ namespace iroha { std::shared_ptr factory, std::shared_ptr perm_converter, - logger::Logger log) + logger::LoggerManagerTreePtr log_manager) : sql_(std::move(sql)), command_executor_(std::make_unique( *sql_, std::move(perm_converter))), - log_(std::move(log)) { + log_manager_(std::move(log_manager)), + log_(log_manager_->getLogger()) { *sql_ << "BEGIN"; } @@ -126,7 +129,10 @@ namespace iroha { std::unique_ptr TemporaryWsvImpl::createSavepoint(const std::string &name) { return std::make_unique( - SavepointWrapperImpl(*this, name)); + SavepointWrapperImpl( + *this, + name, + log_manager_->getChild("SavepointWrapper")->getLogger())); } TemporaryWsvImpl::~TemporaryWsvImpl() { @@ -139,11 +145,12 @@ namespace iroha { TemporaryWsvImpl::SavepointWrapperImpl::SavepointWrapperImpl( const iroha::ametsuchi::TemporaryWsvImpl &wsv, - std::string savepoint_name) + std::string savepoint_name, + logger::LoggerPtr log) : sql_{*wsv.sql_}, savepoint_name_{std::move(savepoint_name)}, is_released_{false}, - log_(logger::log("Temporary wsv's savepoint wrapper")) { + log_(std::move(log)) { sql_ << "SAVEPOINT " + savepoint_name_ + ";"; } diff --git a/irohad/ametsuchi/impl/temporary_wsv_impl.hpp b/irohad/ametsuchi/impl/temporary_wsv_impl.hpp index 48df66cf94..dab5cb6a29 100644 --- a/irohad/ametsuchi/impl/temporary_wsv_impl.hpp +++ b/irohad/ametsuchi/impl/temporary_wsv_impl.hpp @@ -11,7 +11,8 @@ #include #include "ametsuchi/command_executor.hpp" #include "interfaces/common_objects/common_objects_factory.hpp" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" +#include "logger/logger_manager_fwd.hpp" namespace shared_model { namespace interface { @@ -28,7 +29,8 @@ namespace iroha { public: struct SavepointWrapperImpl : public TemporaryWsv::SavepointWrapper { SavepointWrapperImpl(const TemporaryWsvImpl &wsv, - std::string savepoint_name); + std::string savepoint_name, + logger::LoggerPtr log); void release() override; @@ -38,7 +40,7 @@ namespace iroha { soci::session &sql_; std::string savepoint_name_; bool is_released_; - logger::Logger log_; + logger::LoggerPtr log_; }; TemporaryWsvImpl( @@ -47,7 +49,7 @@ namespace iroha { factory, std::shared_ptr perm_converter, - logger::Logger log = logger::log("TemporaryWSV")); + logger::LoggerManagerTreePtr log_manager); expected::Result apply( const shared_model::interface::Transaction &transaction) override; @@ -68,7 +70,8 @@ namespace iroha { std::unique_ptr sql_; std::unique_ptr command_executor_; - logger::Logger log_; + logger::LoggerManagerTreePtr log_manager_; + logger::LoggerPtr log_; }; } // namespace ametsuchi } // namespace iroha diff --git a/irohad/consensus/yac/CMakeLists.txt b/irohad/consensus/yac/CMakeLists.txt index 9c7949cb5e..d835783f54 100644 --- a/irohad/consensus/yac/CMakeLists.txt +++ b/irohad/consensus/yac/CMakeLists.txt @@ -31,6 +31,7 @@ target_link_libraries(yac common rxcpp logger + logger_manager hash consensus_round gate_object diff --git a/irohad/consensus/yac/impl/yac.cpp b/irohad/consensus/yac/impl/yac.cpp index cd7756234c..fb32a08013 100644 --- a/irohad/consensus/yac/impl/yac.cpp +++ b/irohad/consensus/yac/impl/yac.cpp @@ -16,6 +16,7 @@ #include "cryptography/public_key.hpp" #include "cryptography/signed.hpp" #include "interfaces/common_objects/peer.hpp" +#include "logger/logger.hpp" namespace iroha { namespace consensus { @@ -42,7 +43,7 @@ namespace iroha { std::shared_ptr crypto, std::shared_ptr timer, ClusterOrdering order, - logger::Logger log) { + logger::LoggerPtr log) { return std::make_shared( vote_storage, network, crypto, timer, order, std::move(log)); } @@ -52,7 +53,7 @@ namespace iroha { std::shared_ptr crypto, std::shared_ptr timer, ClusterOrdering order, - logger::Logger log) + logger::LoggerPtr log) : vote_storage_(std::move(vote_storage)), network_(std::move(network)), crypto_(std::move(crypto)), @@ -85,7 +86,7 @@ namespace iroha { if (crypto_->verify(state)) { applyState(state); } else { - log_->warn(cryptoError(state)); + log_->warn("{}", cryptoError(state)); } } diff --git a/irohad/consensus/yac/impl/yac_crypto_provider_impl.cpp b/irohad/consensus/yac/impl/yac_crypto_provider_impl.cpp index fa6859b848..1dfae5b74a 100644 --- a/irohad/consensus/yac/impl/yac_crypto_provider_impl.cpp +++ b/irohad/consensus/yac/impl/yac_crypto_provider_impl.cpp @@ -52,8 +52,6 @@ namespace iroha { vote.signature = std::move(sig.value); }, [](iroha::expected::Error &reason) { - logger::log("YacCryptoProvider::getVote") - ->error("Cannot build vote signature: {}", reason.error); }); return vote; diff --git a/irohad/consensus/yac/impl/yac_gate_impl.cpp b/irohad/consensus/yac/impl/yac_gate_impl.cpp index 42020bb044..199ed1cf9c 100644 --- a/irohad/consensus/yac/impl/yac_gate_impl.cpp +++ b/irohad/consensus/yac/impl/yac_gate_impl.cpp @@ -15,6 +15,7 @@ #include "cryptography/public_key.hpp" #include "interfaces/common_objects/signature.hpp" #include "interfaces/iroha_internal/block.hpp" +#include "logger/logger.hpp" #include "simulator/block_creator.hpp" namespace iroha { @@ -28,7 +29,7 @@ namespace iroha { std::shared_ptr block_creator, std::shared_ptr consensus_result_cache, - logger::Logger log) + logger::LoggerPtr log) : hash_gate_(std::move(hash_gate)), orderer_(std::move(orderer)), hash_provider_(std::move(hash_provider)), diff --git a/irohad/consensus/yac/impl/yac_gate_impl.hpp b/irohad/consensus/yac/impl/yac_gate_impl.hpp index 92a147bb4c..72c166ec61 100644 --- a/irohad/consensus/yac/impl/yac_gate_impl.hpp +++ b/irohad/consensus/yac/impl/yac_gate_impl.hpp @@ -12,7 +12,7 @@ #include "consensus/consensus_block_cache.hpp" #include "consensus/yac/yac_hash_provider.hpp" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" namespace iroha { @@ -38,7 +38,7 @@ namespace iroha { std::shared_ptr block_creator, std::shared_ptr consensus_result_cache, - logger::Logger log = logger::log("YacGate")); + logger::LoggerPtr log); void vote(const simulator::BlockCreatorEvent &event) override; rxcpp::observable onOutcome() override; @@ -61,7 +61,7 @@ namespace iroha { std::shared_ptr consensus_result_cache_; - logger::Logger log_; + logger::LoggerPtr log_; boost::optional> current_block_; diff --git a/irohad/consensus/yac/storage/impl/yac_block_storage.cpp b/irohad/consensus/yac/storage/impl/yac_block_storage.cpp index 65583eb78f..28d60f826d 100644 --- a/irohad/consensus/yac/storage/impl/yac_block_storage.cpp +++ b/irohad/consensus/yac/storage/impl/yac_block_storage.cpp @@ -5,7 +5,7 @@ #include "consensus/yac/storage/yac_block_storage.hpp" -using namespace logger; +#include "logger/logger.hpp" namespace iroha { namespace consensus { @@ -16,12 +16,12 @@ namespace iroha { YacBlockStorage::YacBlockStorage( YacHash hash, PeersNumberType peers_in_round, - std::shared_ptr supermajority_checker) + std::shared_ptr supermajority_checker, + logger::LoggerPtr log) : storage_key_(std::move(hash)), peers_in_round_(peers_in_round), - supermajority_checker_(std::move(supermajority_checker)) { - log_ = log("YacBlockStorage"); - } + supermajority_checker_(std::move(supermajority_checker)), + log_(std::move(log)) {} boost::optional YacBlockStorage::insert(VoteMessage msg) { if (validScheme(msg) and uniqueVote(msg)) { diff --git a/irohad/consensus/yac/storage/impl/yac_proposal_storage.cpp b/irohad/consensus/yac/storage/impl/yac_proposal_storage.cpp index e96a9e4005..8882b88aed 100644 --- a/irohad/consensus/yac/storage/impl/yac_proposal_storage.cpp +++ b/irohad/consensus/yac/storage/impl/yac_proposal_storage.cpp @@ -6,6 +6,8 @@ #include "consensus/yac/storage/yac_proposal_storage.hpp" #include +#include "logger/logger.hpp" +#include "logger/logger_manager.hpp" using namespace logger; @@ -34,7 +36,8 @@ namespace iroha { store_hash.vote_hashes.proposal_hash, store_hash.vote_hashes.block_hash), peers_in_round_, - supermajority_checker_); + supermajority_checker_, + log_manager_->getChild("BlockStorage")->getLogger()); } // --------| public api |-------- @@ -42,13 +45,14 @@ namespace iroha { YacProposalStorage::YacProposalStorage( Round store_round, PeersNumberType peers_in_round, - std::shared_ptr supermajority_checker) + std::shared_ptr supermajority_checker, + logger::LoggerManagerTreePtr log_manager) : current_state_(boost::none), storage_key_(store_round), peers_in_round_(peers_in_round), - supermajority_checker_(supermajority_checker) { - log_ = log("ProposalStorage"); - } + supermajority_checker_(supermajority_checker), + log_manager_(std::move(log_manager)), + log_(log_manager_->getLogger()) {} boost::optional YacProposalStorage::insert(VoteMessage msg) { if (shouldInsert(msg)) { diff --git a/irohad/consensus/yac/storage/impl/yac_vote_storage.cpp b/irohad/consensus/yac/storage/impl/yac_vote_storage.cpp index 1bf6f3ec67..47b041e91a 100644 --- a/irohad/consensus/yac/storage/impl/yac_vote_storage.cpp +++ b/irohad/consensus/yac/storage/impl/yac_vote_storage.cpp @@ -11,6 +11,7 @@ #include "common/bind.hpp" #include "consensus/yac/consistency_model.hpp" #include "consensus/yac/storage/yac_proposal_storage.hpp" +#include "logger/logger_manager.hpp" namespace iroha { namespace consensus { @@ -35,10 +36,12 @@ namespace iroha { return val; } if (strategy_->shouldCreateRound(round)) { - return proposal_storages_.emplace(proposal_storages_.end(), - msg.hash.vote_round, - peers_in_round, - supermajority_checker_); + return proposal_storages_.emplace( + proposal_storages_.end(), + msg.hash.vote_round, + peers_in_round, + supermajority_checker_, + log_manager_->getChild("ProposalStorage")); } else { return boost::none; } @@ -59,9 +62,11 @@ namespace iroha { YacVoteStorage::YacVoteStorage( std::shared_ptr cleanup_strategy, - std::unique_ptr supermajority_checker) + std::unique_ptr supermajority_checker, + logger::LoggerManagerTreePtr log_manager) : strategy_(std::move(cleanup_strategy)), - supermajority_checker_(std::move(supermajority_checker)) {} + supermajority_checker_(std::move(supermajority_checker)), + log_manager_(std::move(log_manager)) {} boost::optional YacVoteStorage::store( std::vector state, PeersNumberType peers_in_round) { diff --git a/irohad/consensus/yac/storage/yac_block_storage.hpp b/irohad/consensus/yac/storage/yac_block_storage.hpp index 607065a637..d45a03b97e 100644 --- a/irohad/consensus/yac/storage/yac_block_storage.hpp +++ b/irohad/consensus/yac/storage/yac_block_storage.hpp @@ -14,7 +14,7 @@ #include "consensus/yac/storage/storage_result.hpp" #include "consensus/yac/supermajority_checker.hpp" #include "consensus/yac/yac_types.hpp" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" namespace iroha { namespace consensus { @@ -35,7 +35,8 @@ namespace iroha { YacBlockStorage( YacHash hash, PeersNumberType peers_in_round, - std::shared_ptr supermajority_checker); + std::shared_ptr supermajority_checker, + logger::LoggerPtr log); /** * Try to insert vote to storage @@ -117,7 +118,7 @@ namespace iroha { /** * Storage logger */ - logger::Logger log_; + logger::LoggerPtr log_; }; } // namespace yac diff --git a/irohad/consensus/yac/storage/yac_proposal_storage.hpp b/irohad/consensus/yac/storage/yac_proposal_storage.hpp index e2431a73ae..359954ebee 100644 --- a/irohad/consensus/yac/storage/yac_proposal_storage.hpp +++ b/irohad/consensus/yac/storage/yac_proposal_storage.hpp @@ -15,7 +15,8 @@ #include "consensus/yac/storage/yac_common.hpp" #include "consensus/yac/supermajority_checker.hpp" #include "consensus/yac/yac_types.hpp" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" +#include "logger/logger_manager_fwd.hpp" namespace iroha { namespace consensus { @@ -45,7 +46,8 @@ namespace iroha { YacProposalStorage( Round store_round, PeersNumberType peers_in_round, - std::shared_ptr supermajority_checker); + std::shared_ptr supermajority_checker, + logger::LoggerManagerTreePtr log_manager); /** * Try to insert vote to storage @@ -132,10 +134,15 @@ namespace iroha { */ std::shared_ptr supermajority_checker_; + /** + * Storage logger manager + */ + logger::LoggerManagerTreePtr log_manager_; + /** * Storage logger */ - logger::Logger log_; + logger::LoggerPtr log_; }; } // namespace yac } // namespace consensus diff --git a/irohad/consensus/yac/storage/yac_vote_storage.hpp b/irohad/consensus/yac/storage/yac_vote_storage.hpp index 5a3dbbc535..205ebe663f 100644 --- a/irohad/consensus/yac/storage/yac_vote_storage.hpp +++ b/irohad/consensus/yac/storage/yac_vote_storage.hpp @@ -19,6 +19,7 @@ #include "consensus/yac/storage/yac_proposal_storage.hpp" #include "consensus/yac/supermajority_checker.hpp" #include "consensus/yac/yac_types.hpp" +#include "logger/logger_manager_fwd.hpp" namespace iroha { namespace consensus { @@ -93,10 +94,12 @@ namespace iroha { /** * @param cleanup_strategy - strategy for removing elements from storage * @param consistency_model - consensus consistency model (CFT, BFT). + * @param log_manager - log manager to create component loggers */ YacVoteStorage( std::shared_ptr cleanup_strategy, - std::unique_ptr supermajority_checker); + std::unique_ptr supermajority_checker, + logger::LoggerManagerTreePtr log_manager); /** * Insert votes in storage @@ -159,6 +162,8 @@ namespace iroha { std::shared_ptr strategy_; std::shared_ptr supermajority_checker_; + + logger::LoggerManagerTreePtr log_manager_; }; } // namespace yac diff --git a/irohad/consensus/yac/transport/impl/network_impl.cpp b/irohad/consensus/yac/transport/impl/network_impl.cpp index 2efb1cf349..f49da08140 100644 --- a/irohad/consensus/yac/transport/impl/network_impl.cpp +++ b/irohad/consensus/yac/transport/impl/network_impl.cpp @@ -23,8 +23,9 @@ namespace iroha { NetworkImpl::NetworkImpl( std::shared_ptr> - async_call) - : async_call_(async_call) {} + async_call, + logger::LoggerPtr log) + : async_call_(async_call), log_(std::move(log)) {} void NetworkImpl::subscribe( std::shared_ptr handler) { @@ -45,7 +46,7 @@ namespace iroha { return peers_.at(to.address())->AsyncSendState(context, request, cq); }); - async_call_->log_->info( + log_->info( "Send votes bundle[size={}] to {}", state.size(), to.address()); } @@ -55,23 +56,23 @@ namespace iroha { ::google::protobuf::Empty *response) { std::vector state; for (const auto &pb_vote : request->votes()) { - auto vote = *PbConverters::deserializeVote(pb_vote); + auto vote = *PbConverters::deserializeVote(pb_vote, log_); state.push_back(vote); } if (not sameKeys(state)) { - async_call_->log_->info( + log_->info( "Votes are stateless invalid: proposals are different, or empty " "collection"); return grpc::Status::CANCELLED; } - async_call_->log_->info( - "Receive votes[size={}] from {}", state.size(), context->peer()); + log_->info( + "Received votes[size={}] from {}", state.size(), context->peer()); if (auto notifications = handler_.lock()) { notifications->onState(std::move(state)); } else { - async_call_->log_->error("Unable to lock the subscriber"); + log_->error("Unable to lock the subscriber"); } return grpc::Status::OK; } diff --git a/irohad/consensus/yac/transport/impl/network_impl.hpp b/irohad/consensus/yac/transport/impl/network_impl.hpp index d61f8bd4d7..9030c1210f 100644 --- a/irohad/consensus/yac/transport/impl/network_impl.hpp +++ b/irohad/consensus/yac/transport/impl/network_impl.hpp @@ -16,7 +16,7 @@ #include "consensus/yac/vote_message.hpp" #include "interfaces/common_objects/peer.hpp" #include "interfaces/common_objects/types.hpp" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" #include "network/impl/async_grpc_client.hpp" namespace iroha { @@ -31,7 +31,9 @@ namespace iroha { public: explicit NetworkImpl( std::shared_ptr> - async_call); + async_call, + logger::LoggerPtr log); + void subscribe( std::shared_ptr handler) override; @@ -73,6 +75,8 @@ namespace iroha { */ std::shared_ptr> async_call_; + + logger::LoggerPtr log_; }; } // namespace yac diff --git a/irohad/consensus/yac/transport/yac_pb_converters.hpp b/irohad/consensus/yac/transport/yac_pb_converters.hpp index 4ca8f4babc..3a9e48512b 100644 --- a/irohad/consensus/yac/transport/yac_pb_converters.hpp +++ b/irohad/consensus/yac/transport/yac_pb_converters.hpp @@ -88,7 +88,7 @@ namespace iroha { } static boost::optional deserializeVote( - const proto::Vote &pb_vote) { + const proto::Vote &pb_vote, logger::LoggerPtr log) { static shared_model::proto::ProtoCommonObjectsFactory< shared_model::validation::FieldValidator> factory_; @@ -105,8 +105,7 @@ namespace iroha { std::unique_ptr> &sig) { val = std::move(sig.value); }, [&](iroha::expected::Error &reason) { - logger::log("YacPbConverter::deserializeVote") - ->error(msg, reason.error); + log->error(msg, reason.error); }); }; diff --git a/irohad/consensus/yac/yac.hpp b/irohad/consensus/yac/yac.hpp index ff8d0f57c8..6eda74235a 100644 --- a/irohad/consensus/yac/yac.hpp +++ b/irohad/consensus/yac/yac.hpp @@ -16,7 +16,7 @@ #include "consensus/yac/storage/yac_vote_storage.hpp" // for VoteStorage #include "consensus/yac/transport/yac_network_interface.hpp" // for YacNetworkNotifications #include "consensus/yac/yac_gate.hpp" // for HashGate -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" namespace iroha { namespace consensus { @@ -37,14 +37,14 @@ namespace iroha { std::shared_ptr crypto, std::shared_ptr timer, ClusterOrdering order, - logger::Logger log = logger::log("YAC")); + logger::LoggerPtr log); Yac(YacVoteStorage vote_storage, std::shared_ptr network, std::shared_ptr crypto, std::shared_ptr timer, ClusterOrdering order, - logger::Logger log = logger::log("YAC")); + logger::LoggerPtr log); // ------|Hash gate|------ @@ -98,7 +98,7 @@ namespace iroha { ClusterOrdering cluster_order_; // ------|Logger|------ - logger::Logger log_; + logger::LoggerPtr log_; }; } // namespace yac } // namespace consensus diff --git a/irohad/main/CMakeLists.txt b/irohad/main/CMakeLists.txt index 6b2efc3d00..8a7b0e1c85 100644 --- a/irohad/main/CMakeLists.txt +++ b/irohad/main/CMakeLists.txt @@ -31,6 +31,7 @@ add_library(application ) target_link_libraries(application logger + logger_manager yac yac_transport server_runner @@ -62,6 +63,20 @@ target_link_libraries(irohad rapidjson keys_manager common + iroha_conf_loader + logger + logger_manager ) +add_library(iroha_conf_loader iroha_conf_loader.cpp) +target_link_libraries(iroha_conf_loader + iroha_conf_literals + logger_manager + rapidjson +) + +add_library(iroha_conf_literals iroha_conf_literals.cpp) +add_dependencies(iroha_conf_literals logger) +target_include_directories(iroha_conf_literals PUBLIC ${fmt_INCLUDE_DIR}) + add_install_step_for_bin(irohad) diff --git a/irohad/main/application.cpp b/irohad/main/application.cpp index 7ee51b6192..0d38ccca09 100644 --- a/irohad/main/application.cpp +++ b/irohad/main/application.cpp @@ -20,6 +20,8 @@ #include "cryptography/crypto_provider/crypto_model_signer.hpp" #include "interfaces/iroha_internal/transaction_batch_factory_impl.hpp" #include "interfaces/iroha_internal/transaction_batch_parser_impl.hpp" +#include "logger/logger.hpp" +#include "logger/logger_manager.hpp" #include "main/server_runner.hpp" #include "multi_sig_transactions/gossip_propagation_strategy.hpp" #include "multi_sig_transactions/mst_processor_impl.hpp" @@ -80,6 +82,7 @@ Irohad::Irohad(const std::string &block_store_dir, const shared_model::crypto::Keypair &keypair, std::chrono::milliseconds max_rounds_delay, size_t stale_stream_max_rounds, + logger::LoggerManagerTreePtr logger_manager, const boost::optional &opt_mst_gossip_params) : block_store_dir_(block_store_dir), @@ -95,8 +98,10 @@ Irohad::Irohad(const std::string &block_store_dir, max_rounds_delay_(max_rounds_delay), stale_stream_max_rounds_(stale_stream_max_rounds), opt_mst_gossip_params_(opt_mst_gossip_params), - keypair(keypair) { - log_ = logger::log("IROHAD"); + keypair(keypair), + ordering_init(logger_manager->getLogger()), + log_manager_(std::move(logger_manager)), + log_(log_manager_->getLogger()) { log_->info("created"); // Initializing storage at this point in order to insert genesis block before // initialization of iroha daemon @@ -159,7 +164,8 @@ void Irohad::initStorage() { pg_conn_, common_objects_factory_, std::move(block_converter), - perm_converter); + perm_converter, + log_manager_->getChild("Storage")); storageResult.match( [&](expected::Value> &_storage) { storage = _storage.value; @@ -201,10 +207,14 @@ void Irohad::initBatchParser() { void Irohad::initValidators() { auto factory = std::make_unique>(); - stateful_validator = - std::make_shared(std::move(factory), batch_parser); + auto validators_log_manager = log_manager_->getChild("Validators"); + stateful_validator = std::make_shared( + std::move(factory), + batch_parser, + validators_log_manager->getChild("Stateful")->getLogger()); chain_validator = std::make_shared( - getSupermajorityChecker(kConsensusConsistencyModel)); + getSupermajorityChecker(kConsensusConsistencyModel), + validators_log_manager->getChild("Chain")->getLogger()); log_->info("[Init] => validators"); } @@ -214,7 +224,8 @@ void Irohad::initValidators() { */ void Irohad::initNetworkClient() { async_call_ = - std::make_shared>(); + std::make_shared>( + log_manager_->getChild("AsyncNetworkClient")->getLogger()); } void Irohad::initFactories() { @@ -339,19 +350,21 @@ void Irohad::initOrderingGate() { return reject_delay; }; - ordering_gate = ordering_init.initOrderingGate(max_proposal_size_, - proposal_delay_, - std::move(hashes), - storage, - transaction_factory, - batch_parser, - transaction_batch_factory_, - async_call_, - std::move(factory), - proposal_factory, - persistent_cache, - {blocks.back()->height(), 1}, - delay); + ordering_gate = + ordering_init.initOrderingGate(max_proposal_size_, + proposal_delay_, + std::move(hashes), + storage, + transaction_factory, + batch_parser, + transaction_batch_factory_, + async_call_, + std::move(factory), + proposal_factory, + persistent_cache, + {blocks.back()->height(), 1}, + delay, + log_manager_->getChild("Ordering")); log_->info("[Init] => init ordering gate - [{}]", logger::logBool(ordering_gate)); } @@ -370,12 +383,14 @@ void Irohad::initSimulator() { std::make_unique< shared_model::validation::DefaultUnsignedBlockValidator>(), std::make_unique()); - simulator = std::make_shared(ordering_gate, - stateful_validator, - storage, - storage, - crypto_signer_, - std::move(block_factory)); + simulator = std::make_shared( + ordering_gate, + stateful_validator, + storage, + storage, + crypto_signer_, + std::move(block_factory), + log_manager_->getChild("Simulator")->getLogger()); log_->info("[Init] => init simulator"); } @@ -394,7 +409,10 @@ void Irohad::initConsensusCache() { */ void Irohad::initBlockLoader() { block_loader = - loader_init.initBlockLoader(storage, storage, consensus_result_cache_); + loader_init.initBlockLoader(storage, + storage, + consensus_result_cache_, + log_manager_->getChild("BlockLoader")); log_->info("[Init] => block loader"); } @@ -403,15 +421,17 @@ void Irohad::initBlockLoader() { * Initializing consensus gate */ void Irohad::initConsensusGate() { - consensus_gate = yac_init.initConsensusGate(storage, - simulator, - block_loader, - keypair, - consensus_result_cache_, - vote_delay_, - async_call_, - common_objects_factory_, - kConsensusConsistencyModel); + consensus_gate = + yac_init.initConsensusGate(storage, + simulator, + block_loader, + keypair, + consensus_result_cache_, + vote_delay_, + async_call_, + common_objects_factory_, + kConsensusConsistencyModel, + log_manager_->getChild("Consensus")); consensus_gate->onOutcome().subscribe( consensus_gate_events_subscription, consensus_gate_objects.get_subscriber()); @@ -423,7 +443,12 @@ void Irohad::initConsensusGate() { */ void Irohad::initSynchronizer() { synchronizer = std::make_shared( - consensus_gate, chain_validator, storage, storage, block_loader); + consensus_gate, + chain_validator, + storage, + storage, + block_loader, + log_manager_->getChild("Synchronizer")->getLogger()); log_->info("[Init] => synchronizer"); } @@ -433,7 +458,10 @@ void Irohad::initSynchronizer() { */ void Irohad::initPeerCommunicationService() { pcs = std::make_shared( - ordering_gate, synchronizer, simulator); + ordering_gate, + synchronizer, + simulator, + log_manager_->getChild("PeerCommunicationService")->getLogger()); pcs->onProposal().subscribe([this](const auto &) { log_->info("~~~~~~~~~| PROPOSAL ^_^ |~~~~~~~~~ "); @@ -465,8 +493,14 @@ void Irohad::initStatusBus() { } void Irohad::initMstProcessor() { + auto mst_logger_manager = + log_manager_->getChild("MultiSignatureTransactions"); + auto mst_state_logger = mst_logger_manager->getChild("State")->getLogger(); auto mst_completer = std::make_shared(mst_expiration_time_); - auto mst_storage = std::make_shared(mst_completer); + auto mst_storage = std::make_shared( + mst_completer, + mst_state_logger, + mst_logger_manager->getChild("Storage")->getLogger()); std::shared_ptr mst_propagation; if (is_mst_supported_) { mst_transport = std::make_shared( @@ -476,7 +510,9 @@ void Irohad::initMstProcessor() { transaction_batch_factory_, persistent_cache, mst_completer, - keypair.publicKey()); + keypair.publicKey(), + std::move(mst_state_logger), + mst_logger_manager->getChild("Transport")->getLogger()); mst_propagation = std::make_shared( storage, rxcpp::observe_on_new_thread(), *opt_mst_gossip_params_); } else { @@ -486,7 +522,11 @@ void Irohad::initMstProcessor() { auto mst_time = std::make_shared(); auto fair_mst_processor = std::make_shared( - mst_transport, mst_storage, mst_propagation, mst_time); + mst_transport, + mst_storage, + mst_propagation, + mst_time, + mst_logger_manager->getChild("Processor")->getLogger()); mst_processor = fair_mst_processor; mst_transport->subscribe(fair_mst_processor); log_->info("[Init] => MST processor"); @@ -504,18 +544,24 @@ void Irohad::initPendingTxsStorage() { * Initializing transaction command service */ void Irohad::initTransactionCommandService() { + auto command_service_log_manager = log_manager_->getChild("CommandService"); auto status_factory = std::make_shared(); - auto tx_processor = std::make_shared( - pcs, mst_processor, status_bus_, status_factory); auto cs_cache = std::make_shared<::torii::CommandServiceImpl::CacheType>(); - command_service = - std::make_shared<::torii::CommandServiceImpl>(tx_processor, - storage, - status_bus_, - status_factory, - cs_cache, - persistent_cache); + auto tx_processor = std::make_shared( + pcs, + mst_processor, + status_bus_, + status_factory, + command_service_log_manager->getChild("Processor")->getLogger()); + command_service = std::make_shared<::torii::CommandServiceImpl>( + tx_processor, + storage, + status_bus_, + status_factory, + cs_cache, + persistent_cache, + command_service_log_manager->getLogger()); command_service_transport = std::make_shared<::torii::CommandServiceTransportGrpc>( command_service, @@ -527,7 +573,8 @@ void Irohad::initTransactionCommandService() { consensus_gate_objects.get_observable().map([](const auto &) { return ::torii::CommandServiceTransportGrpc::ConsensusGateEvent{}; }), - stale_stream_max_rounds_); + stale_stream_max_rounds_, + command_service_log_manager->getChild("Transport")->getLogger()); log_->info("[Init] => command service"); } @@ -536,11 +583,16 @@ void Irohad::initTransactionCommandService() { * Initializing query command service */ void Irohad::initQueryService() { + auto query_service_log_manager = log_manager_->getChild("QueryService"); auto query_processor = std::make_shared( - storage, storage, pending_txs_storage_, query_response_factory_); + storage, + storage, + pending_txs_storage_, + query_response_factory_, + query_service_log_manager->getChild("Processor")->getLogger()); - query_service = - std::make_shared<::torii::QueryService>(query_processor, query_factory); + query_service = std::make_shared<::torii::QueryService>( + query_processor, query_factory, query_service_log_manager->getLogger()); log_->info("[Init] => query service"); } @@ -558,13 +610,15 @@ Irohad::RunResult Irohad::run() { // Initializing torii server torii_server = std::make_unique( - listen_ip_ + ":" + std::to_string(torii_port_), false); + listen_ip_ + ":" + std::to_string(torii_port_), + log_manager_->getChild("ToriiServerRunner")->getLogger(), + false); // Initializing internal server internal_server = std::make_unique( listen_ip_ + ":" + std::to_string(internal_port_), - false, - logger::log("InternalServerRunner")); + log_manager_->getChild("InternalServerRunner")->getLogger(), + false); // Run torii server return (torii_server->append(command_service_transport) diff --git a/irohad/main/application.hpp b/irohad/main/application.hpp index 3ab453e297..3bcd1bdd62 100644 --- a/irohad/main/application.hpp +++ b/irohad/main/application.hpp @@ -9,7 +9,8 @@ #include "consensus/consensus_block_cache.hpp" #include "cryptography/crypto_provider/abstract_crypto_model_signer.hpp" #include "interfaces/queries/query.hpp" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" +#include "logger/logger_manager_fwd.hpp" #include "main/impl/block_loader_init.hpp" #include "main/impl/consensus_init.hpp" #include "main/impl/on_demand_ordering_init.hpp" @@ -85,6 +86,7 @@ class Irohad { * transactions * @param stale_stream_max_rounds - maximum number of rounds between * consecutive status emissions + * @param logger_manager - the logger manager to use * @param opt_mst_gossip_params - parameters for Gossip MST propagation * (optional). If not provided, disables mst processing support * TODO mboldyrev 03.11.2018 IR-1844 Refactor the constructor. @@ -101,6 +103,7 @@ class Irohad { const shared_model::crypto::Keypair &keypair, std::chrono::milliseconds max_rounds_delay, size_t stale_stream_max_rounds, + logger::LoggerManagerTreePtr logger_manager, const boost::optional &opt_mst_gossip_params = boost::none); @@ -196,8 +199,6 @@ class Irohad { std::shared_ptr storage; protected: - logger::Logger log_; - // initialization objects iroha::network::OnDemandOrderingInit ordering_init; iroha::consensus::yac::YacInit yac_init; @@ -299,6 +300,10 @@ class Irohad { std::unique_ptr torii_server; std::unique_ptr internal_server; + + logger::LoggerManagerTreePtr log_manager_; ///< application root log manager + + logger::LoggerPtr log_; ///< log for local messages }; #endif // IROHA_APPLICATION_HPP diff --git a/irohad/main/assert_config.hpp b/irohad/main/assert_config.hpp deleted file mode 100644 index a94f979baf..0000000000 --- a/irohad/main/assert_config.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Copyright Soramitsu Co., Ltd. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef IROHA_CLI_ASSERT_CONFIG_HPP -#define IROHA_CLI_ASSERT_CONFIG_HPP - -#include -#include - -namespace assert_config { - /** - * error message helpers that are used in json validation. - */ - inline std::string no_member_error(std::string const &member) { - return "No member '" + member + "'"; - } - - inline std::string type_error(std::string const &value, - std::string const &type) { - return "'" + value + "' is not " + type; - } - - inline std::string parse_error(std::string const &path) { - return "Parse error. JSON file path: " + path + "'"; - } - - /** - * shuts down process when some error occurs. - * @param error - error message - */ - inline void fatal_error(std::string const &error) { - throw std::runtime_error(error); - } - - /** - * shuts down process if a given condition is false. - * @param condition - * @param error - error message - */ - inline void assert_fatal(bool condition, std::string const &error) { - if (!condition) { - fatal_error(error); - } - } -} // namespace assert_config - -#endif // IROHA_CLI_ASSERT_CONFIG_HPP diff --git a/irohad/main/impl/block_loader_init.cpp b/irohad/main/impl/block_loader_init.cpp index 4d39b9df45..9e032b32f1 100644 --- a/irohad/main/impl/block_loader_init.cpp +++ b/irohad/main/impl/block_loader_init.cpp @@ -4,6 +4,8 @@ */ #include "main/impl/block_loader_init.hpp" + +#include "logger/logger_manager.hpp" #include "validators/default_validator.hpp" #include "validators/protobuf/proto_block_validator.hpp" @@ -13,26 +15,33 @@ using namespace iroha::network; auto BlockLoaderInit::createService( std::shared_ptr block_query_factory, - std::shared_ptr consensus_result_cache) { + std::shared_ptr consensus_result_cache, + const logger::LoggerManagerTreePtr &loader_log_manager) { return std::make_shared( - std::move(block_query_factory), std::move(consensus_result_cache)); + std::move(block_query_factory), + std::move(consensus_result_cache), + loader_log_manager->getChild("Network")->getLogger()); } auto BlockLoaderInit::createLoader( - std::shared_ptr peer_query_factory) { + std::shared_ptr peer_query_factory, + logger::LoggerPtr loader_log) { shared_model::proto::ProtoBlockFactory factory( std::make_unique(), std::make_unique()); - return std::make_shared(std::move(peer_query_factory), - std::move(factory)); + return std::make_shared( + std::move(peer_query_factory), std::move(factory), std::move(loader_log)); } std::shared_ptr BlockLoaderInit::initBlockLoader( std::shared_ptr peer_query_factory, std::shared_ptr block_query_factory, - std::shared_ptr consensus_result_cache) { + std::shared_ptr consensus_result_cache, + const logger::LoggerManagerTreePtr &loader_log_manager) { service = createService(std::move(block_query_factory), - std::move(consensus_result_cache)); - loader = createLoader(std::move(peer_query_factory)); + std::move(consensus_result_cache), + loader_log_manager); + loader = createLoader(std::move(peer_query_factory), + loader_log_manager->getLogger()); return loader; } diff --git a/irohad/main/impl/block_loader_init.hpp b/irohad/main/impl/block_loader_init.hpp index a7998a1003..a2da690d6c 100644 --- a/irohad/main/impl/block_loader_init.hpp +++ b/irohad/main/impl/block_loader_init.hpp @@ -8,6 +8,8 @@ #include "ametsuchi/block_query_factory.hpp" #include "consensus/consensus_block_cache.hpp" +#include "logger/logger_fwd.hpp" +#include "logger/logger_manager_fwd.hpp" #include "network/impl/block_loader_impl.hpp" #include "network/impl/block_loader_service.hpp" @@ -22,20 +24,24 @@ namespace iroha { * Create block loader service with given storage * @param block_query_factory - factory to block query component * @param block_cache used to retrieve last block put by consensus + * @param loader_log - the log of the loader subsystem * @return initialized service */ auto createService( std::shared_ptr block_query_factory, - std::shared_ptr block_cache); + std::shared_ptr block_cache, + const logger::LoggerManagerTreePtr &loader_log_manager); /** * Create block loader for loading blocks from given peer factory by top * block * @param peer_query_factory - factory for peer query component creation + * @param loader_log - the log of the loader subsystem * @return initialized loader */ auto createLoader( - std::shared_ptr peer_query_factory); + std::shared_ptr peer_query_factory, + logger::LoggerPtr loader_log); public: /** @@ -43,12 +49,14 @@ namespace iroha { * @param peer_query_factory - factory to peer query component * @param block_query_factory - factory to block query component * @param block_cache used to retrieve last block put by consensus + * @param loader_log - the log of the loader subsystem * @return initialized service */ std::shared_ptr initBlockLoader( std::shared_ptr peer_query_factory, std::shared_ptr block_query_factory, - std::shared_ptr block_cache); + std::shared_ptr block_cache, + const logger::LoggerManagerTreePtr &loader_log_manager); std::shared_ptr loader; std::shared_ptr service; diff --git a/irohad/main/impl/consensus_init.cpp b/irohad/main/impl/consensus_init.cpp index 752591469c..4d70d633dd 100644 --- a/irohad/main/impl/consensus_init.cpp +++ b/irohad/main/impl/consensus_init.cpp @@ -14,6 +14,7 @@ #include "consensus/yac/storage/buffered_cleanup_strategy.hpp" #include "consensus/yac/storage/yac_proposal_storage.hpp" #include "consensus/yac/transport/impl/network_impl.hpp" +#include "logger/logger_manager.hpp" using namespace iroha::consensus::yac; @@ -44,16 +45,19 @@ namespace { std::shared_ptr network, std::shared_ptr common_objects_factory, - ConsistencyModel consistency_model) { + ConsistencyModel consistency_model, + const logger::LoggerManagerTreePtr &consensus_log_manager) { std::shared_ptr cleanup_strategy = std::make_shared(); return Yac::create( YacVoteStorage(cleanup_strategy, - getSupermajorityChecker(consistency_model)), + getSupermajorityChecker(consistency_model), + consensus_log_manager->getChild("VoteStorage")), std::move(network), createCryptoProvider(keypair, std::move(common_objects_factory)), std::move(timer), - initial_order); + initial_order, + consensus_log_manager->getChild("HashGate")->getLogger()); } } // namespace @@ -99,28 +103,34 @@ namespace iroha { async_call, std::shared_ptr common_objects_factory, - ConsistencyModel consistency_model) { + ConsistencyModel consistency_model, + const logger::LoggerManagerTreePtr &consensus_log_manager) { auto peer_orderer = createPeerOrderer(peer_query_factory); - consensus_network_ = std::make_shared(async_call); + consensus_network_ = std::make_shared( + async_call, + consensus_log_manager->getChild("Network")->getLogger()); auto yac = createYac(peer_orderer->getInitialOrdering().value(), keypair, createTimer(vote_delay_milliseconds), consensus_network_, std::move(common_objects_factory), - consistency_model); + consistency_model, + consensus_log_manager); consensus_network_->subscribe(yac); auto hash_provider = createHashProvider(); initialized_ = true; - return std::make_shared(std::move(yac), - std::move(peer_orderer), - hash_provider, - block_creator, - std::move(consensus_result_cache)); + return std::make_shared( + std::move(yac), + std::move(peer_orderer), + hash_provider, + block_creator, + std::move(consensus_result_cache), + consensus_log_manager->getChild("Gate")->getLogger()); } } // namespace yac } // namespace consensus diff --git a/irohad/main/impl/consensus_init.hpp b/irohad/main/impl/consensus_init.hpp index dfe67ce484..9bd85bde36 100644 --- a/irohad/main/impl/consensus_init.hpp +++ b/irohad/main/impl/consensus_init.hpp @@ -7,8 +7,6 @@ #define IROHA_CONSENSUS_INIT_HPP #include -#include -#include #include "ametsuchi/peer_query_factory.hpp" #include "consensus/consensus_block_cache.hpp" @@ -21,6 +19,7 @@ #include "consensus/yac/yac_peer_orderer.hpp" #include "cryptography/keypair.hpp" #include "interfaces/common_objects/common_objects_factory.hpp" +#include "logger/logger_manager_fwd.hpp" #include "network/block_loader.hpp" #include "network/impl/async_grpc_client.hpp" #include "simulator/block_creator.hpp" @@ -43,7 +42,8 @@ namespace iroha { async_call, std::shared_ptr common_objects_factory, - ConsistencyModel consistency_model); + ConsistencyModel consistency_model, + const logger::LoggerManagerTreePtr &consensus_log_manager); std::shared_ptr getConsensusNetwork() const; diff --git a/irohad/main/impl/on_demand_ordering_init.cpp b/irohad/main/impl/on_demand_ordering_init.cpp index 88eec5cd15..9bc559d649 100644 --- a/irohad/main/impl/on_demand_ordering_init.cpp +++ b/irohad/main/impl/on_demand_ordering_init.cpp @@ -11,6 +11,8 @@ #include "datetime/time.hpp" #include "interfaces/common_objects/peer.hpp" #include "interfaces/common_objects/types.hpp" +#include "logger/logger.hpp" +#include "logger/logger_manager.hpp" #include "ordering/impl/on_demand_common.hpp" #include "ordering/impl/on_demand_connection_manager.hpp" #include "ordering/impl/on_demand_ordering_gate.hpp" @@ -41,16 +43,21 @@ namespace { namespace iroha { namespace network { + OnDemandOrderingInit::OnDemandOrderingInit(logger::LoggerPtr log) + : log_(std::move(log)) {} + auto OnDemandOrderingInit::createNotificationFactory( std::shared_ptr> async_call, std::shared_ptr proposal_transport_factory, - std::chrono::milliseconds delay) { + std::chrono::milliseconds delay, + const logger::LoggerManagerTreePtr &ordering_log_manager) { return std::make_shared( std::move(async_call), std::move(proposal_transport_factory), [] { return std::chrono::system_clock::now(); }, - delay); + delay, + ordering_log_manager->getChild("NetworkClient")->getLogger()); } auto OnDemandOrderingInit::createConnectionManager( @@ -59,7 +66,8 @@ namespace iroha { async_call, std::shared_ptr proposal_transport_factory, std::chrono::milliseconds delay, - std::vector initial_hashes) { + std::vector initial_hashes, + const logger::LoggerManagerTreePtr &ordering_log_manager) { // since top block will be the first in notifier observable, hashes of // two previous blocks are prepended const size_t kBeforePreviousTop = 0, kPreviousTop = 1; @@ -179,8 +187,10 @@ namespace iroha { return std::make_shared( createNotificationFactory(std::move(async_call), std::move(proposal_transport_factory), - delay), - peers); + delay, + ordering_log_manager), + peers, + ordering_log_manager->getChild("ConnectionManager")->getLogger()); } auto OnDemandOrderingInit::createGate( @@ -192,7 +202,8 @@ namespace iroha { std::shared_ptr tx_cache, consensus::Round initial_round, std::function delay_func) { + const synchronizer::SynchronizationEvent &)> delay_func, + const logger::LoggerManagerTreePtr &ordering_log_manager) { auto map = [](auto commit) { return matchEvent( commit, @@ -235,16 +246,21 @@ namespace iroha { std::move(cache), std::move(proposal_factory), std::move(tx_cache), - initial_round); + initial_round, + ordering_log_manager->getChild("Gate")->getLogger()); } auto OnDemandOrderingInit::createService( size_t max_size, std::shared_ptr proposal_factory, - std::shared_ptr tx_cache) { + std::shared_ptr tx_cache, + const logger::LoggerManagerTreePtr &ordering_log_manager) { return std::make_shared( - max_size, std::move(proposal_factory), std::move(tx_cache)); + max_size, + std::move(proposal_factory), + std::move(tx_cache), + ordering_log_manager->getChild("Service")->getLogger()); } OnDemandOrderingInit::~OnDemandOrderingInit() { @@ -272,26 +288,30 @@ namespace iroha { std::shared_ptr tx_cache, consensus::Round initial_round, std::function delay_func) { - auto ordering_service = - createService(max_size, proposal_factory, tx_cache); + const synchronizer::SynchronizationEvent &)> delay_func, + logger::LoggerManagerTreePtr ordering_log_manager) { + auto ordering_service = createService( + max_size, proposal_factory, tx_cache, ordering_log_manager); service = std::make_shared( ordering_service, std::move(transaction_factory), std::move(batch_parser), - std::move(transaction_batch_factory)); + std::move(transaction_batch_factory), + ordering_log_manager->getChild("Server")->getLogger()); return createGate( ordering_service, createConnectionManager(std::move(peer_query_factory), std::move(async_call), std::move(proposal_transport_factory), delay, - std::move(initial_hashes)), + std::move(initial_hashes), + ordering_log_manager), std::make_shared(), std::move(proposal_factory), std::move(tx_cache), initial_round, - std::move(delay_func)); + std::move(delay_func), + ordering_log_manager); } } // namespace network diff --git a/irohad/main/impl/on_demand_ordering_init.hpp b/irohad/main/impl/on_demand_ordering_init.hpp index ad8c273a50..9a085d44cc 100644 --- a/irohad/main/impl/on_demand_ordering_init.hpp +++ b/irohad/main/impl/on_demand_ordering_init.hpp @@ -11,7 +11,8 @@ #include "ametsuchi/peer_query_factory.hpp" #include "ametsuchi/tx_presence_cache.hpp" #include "interfaces/iroha_internal/unsafe_proposal_factory.hpp" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" +#include "logger/logger_manager_fwd.hpp" #include "network/impl/async_grpc_client.hpp" #include "network/ordering_gate.hpp" #include "network/peer_communication_service.hpp" @@ -43,7 +44,8 @@ namespace iroha { std::shared_ptr> async_call, std::shared_ptr proposal_transport_factory, - std::chrono::milliseconds delay); + std::chrono::milliseconds delay, + const logger::LoggerManagerTreePtr &ordering_log_manager); /** * Creates connection manager which redirects requests to appropriate @@ -56,7 +58,8 @@ namespace iroha { async_call, std::shared_ptr proposal_transport_factory, std::chrono::milliseconds delay, - std::vector initial_hashes); + std::vector initial_hashes, + const logger::LoggerManagerTreePtr &ordering_log_manager); /** * Creates on-demand ordering gate. \see initOrderingGate for parameters @@ -71,7 +74,8 @@ namespace iroha { std::shared_ptr tx_cache, consensus::Round initial_round, std::function delay_func); + const synchronizer::SynchronizationEvent &)> delay_func, + const logger::LoggerManagerTreePtr &ordering_log_manager); /** * Creates on-demand ordering service. \see initOrderingGate for @@ -81,9 +85,15 @@ namespace iroha { size_t max_size, std::shared_ptr proposal_factory, - std::shared_ptr tx_cache); + std::shared_ptr tx_cache, + const logger::LoggerManagerTreePtr &ordering_log_manager); public: + + /// Constructor. + /// @param log - the logger to use for internal messages. + OnDemandOrderingInit(logger::LoggerPtr log); + ~OnDemandOrderingInit(); /** @@ -129,7 +139,8 @@ namespace iroha { std::shared_ptr tx_cache, consensus::Round initial_round, std::function delay_func); + const synchronizer::SynchronizationEvent &)> delay_func, + logger::LoggerManagerTreePtr ordering_log_manager); /// gRPC service for ordering service std::shared_ptr service; @@ -140,7 +151,7 @@ namespace iroha { notifier; private: - logger::Logger log_ = logger::log("OnDemandOrderingInit"); + logger::LoggerPtr log_; std::vector> current_peers_; diff --git a/irohad/main/impl/raw_block_loader.cpp b/irohad/main/impl/raw_block_loader.cpp index 4d03f23e2d..f28adb84cf 100644 --- a/irohad/main/impl/raw_block_loader.cpp +++ b/irohad/main/impl/raw_block_loader.cpp @@ -10,6 +10,7 @@ #include "backend/protobuf/block.hpp" #include "common/bind.hpp" #include "converters/protobuf/json_proto_converter.hpp" +#include "logger/logger.hpp" namespace iroha { namespace main { @@ -17,7 +18,8 @@ namespace iroha { using shared_model::converters::protobuf::jsonToProto; using shared_model::interface::Block; - BlockLoader::BlockLoader(logger::Logger log) : log_(std::move(log)) {} + BlockLoader::BlockLoader(logger::LoggerPtr log) + : log_(std::move(log)) {} boost::optional> BlockLoader::parseBlock( const std::string &data) { diff --git a/irohad/main/iroha_conf_literals.cpp b/irohad/main/iroha_conf_literals.cpp new file mode 100644 index 0000000000..e209274493 --- /dev/null +++ b/irohad/main/iroha_conf_literals.cpp @@ -0,0 +1,32 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "main/iroha_conf_literals.hpp" + +namespace config_members { + const char *BlockStorePath = "block_store_path"; + const char *ToriiPort = "torii_port"; + const char *InternalPort = "internal_port"; + const char *KeyPairPath = "key_pair_path"; + const char *PgOpt = "pg_opt"; + const char *MaxProposalSize = "max_proposal_size"; + const char *ProposalDelay = "proposal_delay"; + const char *VoteDelay = "vote_delay"; + const char *MstSupport = "mst_enable"; + const char *MstExpirationTime = "mst_expiration_time"; + const char *MaxRoundsDelay = "max_rounds_delay"; + const char *StaleStreamMaxRounds = "stale_stream_max_rounds"; + const char *LogSection = "log"; + const char *LogLevel = "level"; + const char *LogPatternsSection = "patterns"; + const char *LogChildrenSection = "children"; + const std::unordered_map LogLevels{ + {"trace", logger::LogLevel::kTrace}, + {"debug", logger::LogLevel::kDebug}, + {"info", logger::LogLevel::kInfo}, + {"warning", logger::LogLevel::kWarn}, + {"error", logger::LogLevel::kError}, + {"critical", logger::LogLevel::kCritical}}; +} // namespace config_members diff --git a/irohad/main/iroha_conf_literals.hpp b/irohad/main/iroha_conf_literals.hpp new file mode 100644 index 0000000000..e1960b8407 --- /dev/null +++ b/irohad/main/iroha_conf_literals.hpp @@ -0,0 +1,34 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef IROHA_CONF_LITERALS_HPP +#define IROHA_CONF_LITERALS_HPP + +#include +#include + +#include "logger/logger.hpp" + +namespace config_members { + extern const char *BlockStorePath; + extern const char *ToriiPort; + extern const char *InternalPort; + extern const char *KeyPairPath; + extern const char *PgOpt; + extern const char *MaxProposalSize; + extern const char *ProposalDelay; + extern const char *VoteDelay; + extern const char *MstSupport; + extern const char *MstExpirationTime; + extern const char *MaxRoundsDelay; + extern const char *StaleStreamMaxRounds; + extern const char *LogSection; + extern const char *LogLevel; + extern const char *LogPatternsSection; + extern const char *LogChildrenSection; + extern const std::unordered_map LogLevels; +} // namespace config_members + +#endif // IROHA_CONF_LITERALS_HPP diff --git a/irohad/main/iroha_conf_loader.cpp b/irohad/main/iroha_conf_loader.cpp new file mode 100644 index 0000000000..c6d19b9c63 --- /dev/null +++ b/irohad/main/iroha_conf_loader.cpp @@ -0,0 +1,345 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "main/iroha_conf_loader.hpp" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include "main/iroha_conf_literals.hpp" + +/// The length of the string around the error place to print in case of JSON +/// syntax error. +static constexpr size_t kBadJsonPrintLength = 15; + +/// The offset of printed chunk towards file start from the error position. +static constexpr size_t kBadJsonPrintOffsset = 5; + +static_assert(kBadJsonPrintOffsset <= kBadJsonPrintLength, + "The place of error is out of the printed string boundaries!"); + +/** + * Adds the children logger configs from parent logger JSON object to parent + * logger config. The parent logger JSON object is searched for the children + * config section, and the children configs are parsed and created if the + * section is present. + * @param path - current config node path used to denote the possible error + * place. + * @param parent_config - the parent logger config + * @param parent_obj - the parent logger json configuration + */ +void addChildrenLoggerConfigs(const std::string &path, + const logger::LoggerManagerTreePtr &parent_config, + const rapidjson::Value::ConstObject &parent_obj); + +/** + * Overrides the logger configuration with the values from JSON object. + * @param path - current config node path used to denote the possible error + * place. + * @param cfg - the configuration to use as base + * @param obj - the JSON object to take overrides from + */ +void updateLoggerConfig(const std::string &path, + logger::LoggerConfig &cfg, + const rapidjson::Value::ConstObject &obj); + +/** + * Gets a value by a key from a JSON object, if present. + * @param path - current config node path used to denote the possible error + * place. + * @param dest - the variable to store the value + * @param obj - the source JSON object + * @param key - the key for the requested value + * @return true if the value was loaded, otherwise false. + */ +template +bool tryGetValByKey(const std::string &path, + TDest &dest, + const rapidjson::Value::ConstObject &obj, + const TKey &key); + +/** + * Gets an optional value by a key from a JSON object. + * @param path - current config node path used to denote the possible error + * place. + * @param obj - the source JSON object + * @param key - the key for the requested value + * @return the value if present in the JSON object, otherwise boost::none. + */ +template +boost::optional getOptValByKey(const std::string &path, + const rapidjson::Value::ConstObject &obj, + const TKey &key); + +/** + * Adds one sublevel to the path denoting a place in config tree. + * @param parent - the location of the sublevel + * @param child - the name of sublevel + * @return the path to the sublevel + */ +template +inline std::string sublevelPath(std::string parent, TChild child) { + std::stringstream child_sstream; + child_sstream << child; + return std::move(parent) + "/" + child_sstream.str(); +} + +/** + * Gets a value by a key from a JSON object. Throws an exception if the value + * was not loaded. + * @param path - current config node path used to denote the possible error + * place. + * @param dest - the variable to store the value + * @param obj - the source JSON object + * @param key - the key for the requested value + */ +template +void getValByKey(const std::string &path, + TDest &dest, + const rapidjson::Value::ConstObject &obj, + const TKey &key); + +/** + * Throws a runtime exception if the given condition is false. + * @param condition + * @param error - error message + */ +inline void assert_fatal(bool condition, std::string error) { + if (!condition) { + throw std::runtime_error(error); + } +} + +// ------------ getVal(path, dst, src) ------------ +// getVal is a set of functions that load the value from rapidjson::Value to a +// given destination variable. They check the JSON type and throw exception if +// it is wrong. The path argument is used to denote the possible error place. + +template +typename std::enable_if::is_integer>::type +getVal(const std::string &path, TDest &, const rapidjson::Value &) { + BOOST_THROW_EXCEPTION( + std::runtime_error("Wrong type. Should never reach here.")); +} + +template +typename std::enable_if::is_integer>::type getVal( + const std::string &path, TDest &dest, const rapidjson::Value &src) { + assert_fatal(src.IsInt64(), path + " must be an integer"); + const int64_t val = src.GetInt64(); + assert_fatal(val >= std::numeric_limits::min() + && val <= std::numeric_limits::max(), + path + ": integer value out of range"); + dest = val; +} + +template <> +void getVal(const std::string &path, + bool &dest, + const rapidjson::Value &src) { + assert_fatal(src.IsBool(), path + " must be a boolean"); + dest = src.GetBool(); +} + +template <> +void getVal(const std::string &path, + std::string &dest, + const rapidjson::Value &src) { + assert_fatal(src.IsString(), path + " must be a string"); + dest = src.GetString(); +} + +template <> +void getVal(const std::string &path, + logger::LogLevel &dest, + const rapidjson::Value &src) { + std::string level_str; + getVal(path, level_str, src); + const auto it = config_members::LogLevels.find(level_str); + if (it == config_members::LogLevels.end()) { + BOOST_THROW_EXCEPTION(std::runtime_error( + "Wrong log level at " + path + ": must be one of '" + + boost::algorithm::join( + config_members::LogLevels | boost::adaptors::map_keys, "', '") + + "'.")); + } + dest = it->second; +} + +template <> +void getVal(const std::string &path, + logger::LogPatterns &dest, + const rapidjson::Value &src) { + assert_fatal(src.IsObject(), + path + " must be a map from log level to pattern"); + for (const auto &pattern_entry : src.GetObject()) { + logger::LogLevel level; + std::string pattern_str; + getVal(sublevelPath(path, "(level name)"), level, pattern_entry.name); + getVal(sublevelPath(path, "(pattern)"), pattern_str, pattern_entry.value); + dest.setPattern(level, pattern_str); + } +} + +template <> +void getVal(const std::string &path, + logger::LoggerManagerTreePtr &dest, + const rapidjson::Value &src) { + assert_fatal(src.IsObject(), path + " must be a logger tree config"); + logger::LoggerConfig root_config{logger::kDefaultLogLevel, + logger::getDefaultLogPatterns()}; + updateLoggerConfig(path, root_config, src.GetObject()); + dest = std::make_shared( + std::make_shared(std::move(root_config))); + addChildrenLoggerConfigs(path, dest, src.GetObject()); +} + +template <> +void getVal(const std::string &path, + IrohadConfig &dest, + const rapidjson::Value &src) { + assert_fatal(src.IsObject(), + path + " Irohad config top element must be an object."); + const auto obj = src.GetObject(); + getValByKey(path, dest.block_store_path, obj, config_members::BlockStorePath); + getValByKey(path, dest.torii_port, obj, config_members::ToriiPort); + getValByKey(path, dest.internal_port, obj, config_members::InternalPort); + getValByKey(path, dest.pg_opt, obj, config_members::PgOpt); + getValByKey( + path, dest.max_proposal_size, obj, config_members::MaxProposalSize); + getValByKey(path, dest.proposal_delay, obj, config_members::ProposalDelay); + getValByKey(path, dest.vote_delay, obj, config_members::VoteDelay); + getValByKey(path, dest.mst_support, obj, config_members::MstSupport); + getValByKey( + path, dest.mst_expiration_time, obj, config_members::MstExpirationTime); + getValByKey( + path, dest.max_round_delay_ms, obj, config_members::MaxRoundsDelay); + getValByKey(path, + dest.stale_stream_max_rounds, + obj, + config_members::StaleStreamMaxRounds); + getValByKey(path, dest.logger_manager, obj, config_members::LogSection); +} + +// ------------ end of getVal(path, dst, src) ------------ + +void updateLoggerConfig(const std::string &path, + logger::LoggerConfig &cfg, + const rapidjson::Value::ConstObject &obj) { + tryGetValByKey(path, cfg.log_level, obj, config_members::LogLevel); + tryGetValByKey(path, cfg.patterns, obj, config_members::LogPatternsSection); +} + +template +bool tryGetValByKey(const std::string &path, + TDest &dest, + const rapidjson::Value::ConstObject &obj, + const TKey &key) { + const auto it = obj.FindMember(key); + if (it == obj.MemberEnd()) { + return false; + } else { + getVal(sublevelPath(path, key), dest, it->value); + return true; + } +} + +template +bool tryGetValByKey(const std::string &path, + boost::optional &dest, + const rapidjson::Value::ConstObject &obj, + const TKey &key) { + dest = getOptValByKey(path, obj, key); + return true; // value loaded any way, either from file or boost::none +} + +template +boost::optional getOptValByKey(const std::string &path, + const rapidjson::Value::ConstObject &obj, + const TKey &key) { + TDest val; + return boost::make_optional(tryGetValByKey(path, val, obj, key), val); +} + +/** + * Gets a value by a key from a JSON object. Throws an exception if the value + * was not loaded. + * @param path - current config node path used to denote the possible error + * place. + * @param dest - the variable to store the value + * @param obj - the source JSON object + * @param key - the key for the requested value + */ +template +void getValByKey(const std::string &path, + TDest &dest, + const rapidjson::Value::ConstObject &obj, + const TKey &key) { + assert_fatal(tryGetValByKey(path, dest, obj, key), + path + " has no key '" + key + "'."); +} + +void addChildrenLoggerConfigs(const std::string &path, + const logger::LoggerManagerTreePtr &parent_config, + const rapidjson::Value::ConstObject &parent_obj) { + const auto it = parent_obj.FindMember(config_members::LogChildrenSection); + if (it != parent_obj.MemberEnd()) { + auto children_section_path = + sublevelPath(path, config_members::LogChildrenSection); + for (const auto &child_json : it->value.GetObject()) { + assert_fatal(child_json.name.IsString(), + "Child logger key must be a string holding its tag."); + assert_fatal(child_json.value.IsObject(), + "Child logger value must be a JSON object."); + auto child_tag = child_json.name.GetString(); + const auto child_obj = child_json.value.GetObject(); + auto child_path = sublevelPath(children_section_path, child_tag); + auto child_conf = parent_config->registerChild( + std::move(child_tag), + getOptValByKey( + child_path, child_obj, config_members::LogLevel), + getOptValByKey( + child_path, child_obj, config_members::LogPatternsSection)); + addChildrenLoggerConfigs(std::move(child_path), child_conf, child_obj); + } + } +} + +std::string reportJsonParsingError(const rapidjson::Document &doc, + const std::string &conf_path, + std::istream &input) { + const size_t error_offset = doc.GetErrorOffset(); + // This ensures the unsigned string beginning position does not cross zero: + const size_t print_offset = + std::max(error_offset, kBadJsonPrintOffsset) - kBadJsonPrintOffsset; + input.seekg(print_offset); + std::string json_error_buf(kBadJsonPrintLength, 0); + input.readsome(&json_error_buf[0], kBadJsonPrintLength); + return "JSON parse error [" + conf_path + "] " + "(near `" + json_error_buf + + "'): " + std::string(rapidjson::GetParseError_En(doc.GetParseError())); +} + +IrohadConfig parse_iroha_config(const std::string &conf_path) { + const rapidjson::Document doc{[&conf_path] { + rapidjson::Document doc; + std::ifstream ifs_iroha(conf_path); + rapidjson::IStreamWrapper isw(ifs_iroha); + doc.ParseStream(isw); + assert_fatal(not doc.HasParseError(), + reportJsonParsingError(doc, conf_path, ifs_iroha)); + return doc; + }()}; + + IrohadConfig config; + getVal("", config, doc); + return config; +} diff --git a/irohad/main/iroha_conf_loader.hpp b/irohad/main/iroha_conf_loader.hpp index e4cc218d3b..12ed324f15 100644 --- a/irohad/main/iroha_conf_loader.hpp +++ b/irohad/main/iroha_conf_loader.hpp @@ -6,140 +6,31 @@ #ifndef IROHA_CONF_LOADER_HPP #define IROHA_CONF_LOADER_HPP -#include #include - -#include -#include -#include -#include - -#include "main/assert_config.hpp" - -namespace config_members { - const char *BlockStorePath = "block_store_path"; - const char *ToriiPort = "torii_port"; - const char *InternalPort = "internal_port"; - const char *KeyPairPath = "key_pair_path"; - const char *PgOpt = "pg_opt"; - const char *MaxProposalSize = "max_proposal_size"; - const char *ProposalDelay = "proposal_delay"; - const char *VoteDelay = "vote_delay"; - const char *MstSupport = "mst_enable"; - const char *MstExpirationTime = "mst_expiration_time"; - const char *MaxRoundsDelay = "max_rounds_delay"; - const char *StaleStreamMaxRounds = "stale_stream_max_rounds"; -} // namespace config_members - -static constexpr size_t kBadJsonPrintLength = 15; -static constexpr size_t kBadJsonPrintOffsset = 5; -static_assert(kBadJsonPrintOffsset <= kBadJsonPrintLength, - "The place of error is out of the printed string boundaries!"); - -std::string reportJsonParsingError(const rapidjson::Document &doc, - const std::string &conf_path, - std::istream &input) { - const size_t error_offset = doc.GetErrorOffset(); - // This ensures the unsigned string beginning position does not cross zero: - const size_t print_offset = - std::max(error_offset, kBadJsonPrintOffsset) - kBadJsonPrintOffsset; - input.seekg(print_offset); - std::string json_error_buf(kBadJsonPrintLength, 0); - input.readsome(&json_error_buf[0], kBadJsonPrintLength); - return "JSON parse error [" + conf_path + "] " + "(near `" + json_error_buf - + "'): " + std::string(rapidjson::GetParseError_En(doc.GetParseError())); -} +#include + +#include "logger/logger_manager.hpp" + +struct IrohadConfig { + std::string block_store_path; + uint16_t torii_port; + uint16_t internal_port; + std::string pg_opt; + uint32_t max_proposal_size; + uint32_t proposal_delay; + uint32_t vote_delay; + bool mst_support; + boost::optional mst_expiration_time; + boost::optional max_round_delay_ms; + boost::optional stale_stream_max_rounds; + boost::optional logger_manager; +}; /** * parse and assert trusted peers json in `iroha.conf` * @param conf_path is a path to iroha's config - * @return rapidjson::Document is a parsed equivalent of that file + * @return a parsed equivalent of that file */ -inline rapidjson::Document parse_iroha_config(const std::string &conf_path) { - namespace ac = assert_config; - namespace mbr = config_members; - rapidjson::Document doc; - std::ifstream ifs_iroha(conf_path); - rapidjson::IStreamWrapper isw(ifs_iroha); - const std::string kStrType = "string"; - const std::string kUintType = "uint"; - const std::string kBoolType = "bool"; - doc.ParseStream(isw); - auto &allocator = doc.GetAllocator(); - ac::assert_fatal(not doc.HasParseError(), - reportJsonParsingError(doc, conf_path, ifs_iroha)); - - ac::assert_fatal(doc.HasMember(mbr::BlockStorePath), - ac::no_member_error(mbr::BlockStorePath)); - ac::assert_fatal(doc[mbr::BlockStorePath].IsString(), - ac::type_error(mbr::BlockStorePath, kStrType)); - - ac::assert_fatal(doc.HasMember(mbr::ToriiPort), - ac::no_member_error(mbr::ToriiPort)); - ac::assert_fatal(doc[mbr::ToriiPort].IsUint(), - ac::type_error(mbr::ToriiPort, kUintType)); - - ac::assert_fatal(doc.HasMember(mbr::InternalPort), - ac::no_member_error(mbr::InternalPort)); - ac::assert_fatal(doc[mbr::InternalPort].IsUint(), - ac::type_error(mbr::InternalPort, kUintType)); - - ac::assert_fatal(doc.HasMember(mbr::PgOpt), ac::no_member_error(mbr::PgOpt)); - ac::assert_fatal(doc[mbr::PgOpt].IsString(), - ac::type_error(mbr::PgOpt, kStrType)); - - ac::assert_fatal(doc.HasMember(mbr::MaxProposalSize), - ac::no_member_error(mbr::MaxProposalSize)); - ac::assert_fatal(doc[mbr::MaxProposalSize].IsUint(), - ac::type_error(mbr::MaxProposalSize, kUintType)); - - ac::assert_fatal(doc.HasMember(mbr::ProposalDelay), - ac::no_member_error(mbr::ProposalDelay)); - ac::assert_fatal(doc[mbr::ProposalDelay].IsUint(), - ac::type_error(mbr::ProposalDelay, kUintType)); - - ac::assert_fatal(doc.HasMember(mbr::VoteDelay), - ac::no_member_error(mbr::VoteDelay)); - ac::assert_fatal(doc[mbr::VoteDelay].IsUint(), - ac::type_error(mbr::VoteDelay, kUintType)); - - ac::assert_fatal(doc.HasMember(mbr::MstSupport), - ac::no_member_error(mbr::MstSupport)); - ac::assert_fatal(doc[mbr::MstSupport].IsBool(), - ac::type_error(mbr::MstSupport, kBoolType)); - - // This way of initialization of unspecified config parameters is going to - // be substituted very soon with completely different approach introduced by - // pr https://github.com/hyperledger/iroha/pull/2126 - const auto kMaxRoundsDelayDefault = 3000u; - const auto kStaleStreamMaxRoundsDefault = 2u; - const auto kMstExpirationTimeDefault = 1440u; - - if (not doc.HasMember(mbr::MstExpirationTime)) { - rapidjson::Value key(mbr::MstExpirationTime, allocator); - doc.AddMember(key, kMstExpirationTimeDefault, allocator); - } else { - ac::assert_fatal(doc[mbr::MstExpirationTime].IsUint(), - ac::type_error(mbr::MstExpirationTime, kUintType)); - } - - if (not doc.HasMember(mbr::MaxRoundsDelay)) { - rapidjson::Value key(mbr::MaxRoundsDelay, allocator); - doc.AddMember(key, kMaxRoundsDelayDefault, allocator); - } else { - ac::assert_fatal(doc[mbr::MaxRoundsDelay].IsUint(), - ac::type_error(mbr::MaxRoundsDelay, kUintType)); - } - - if (not doc.HasMember(mbr::StaleStreamMaxRounds)) { - rapidjson::Value key(mbr::StaleStreamMaxRounds, allocator); - doc.AddMember(key, kStaleStreamMaxRoundsDefault, allocator); - } else { - ac::assert_fatal(doc[mbr::StaleStreamMaxRounds].IsUint(), - ac::type_error(mbr::StaleStreamMaxRounds, kUintType)); - } - - return doc; -} +IrohadConfig parse_iroha_config(const std::string &conf_path); #endif // IROHA_CONF_LOADER_HPP diff --git a/irohad/main/irohad.cpp b/irohad/main/irohad.cpp index 592b0b144b..f1f5f962de 100644 --- a/irohad/main/irohad.cpp +++ b/irohad/main/irohad.cpp @@ -12,11 +12,18 @@ #include "ametsuchi/storage.hpp" #include "common/result.hpp" #include "crypto/keys_manager_impl.hpp" +#include "logger/logger.hpp" +#include "logger/logger_manager.hpp" #include "main/application.hpp" +#include "main/iroha_conf_literals.hpp" #include "main/iroha_conf_loader.hpp" #include "main/raw_block_loader.hpp" static const std::string kListenIp = "0.0.0.0"; +static const std::string kLogSettingsFromConfigFile = "config_file"; +static const uint32_t kMstExpirationTimeDefault = 1440; +static const uint32_t kMaxRoundsDelayDefault = 3000; +static const uint32_t kStaleStreamMaxRoundsDefault = 2; /** * Gflag validator. @@ -70,46 +77,71 @@ DEFINE_validator(keypair_name, &validate_keypair_name); */ DEFINE_bool(overwrite_ledger, false, "Overwrite ledger data if existing"); -static bool validateVerbosity(const char *flagname, int32_t val) { - if (val >= 0 && val <= 6) +static bool validateVerbosity(const char *flagname, const std::string &val) { + if (val == kLogSettingsFromConfigFile) { return true; - - std::cout << "Invalid value for " << flagname << ": should be in range [0, 6]" - << std::endl; - return false; + } + const auto it = config_members::LogLevels.find(val); + if (it == config_members::LogLevels.end()) { + std::cerr << "Invalid value for " << flagname << ": should be one of '" + << kLogSettingsFromConfigFile; + for (const auto &level : config_members::LogLevels) { + std::cerr << "', '" << level.first; + } + std::cerr << "'." << std::endl; + return false; + } + return true; } /// Verbosity flag for spdlog configuration -DEFINE_int32(verbosity, spdlog::level::info, "Log verbosity"); -DEFINE_validator(verbosity, validateVerbosity); +DEFINE_string(verbosity, kLogSettingsFromConfigFile, "Log verbosity"); +DEFINE_validator(verbosity, &validateVerbosity); std::promise exit_requested; +logger::LoggerManagerTreePtr getDefaultLogManager() { + return std::make_shared(logger::LoggerConfig{ + logger::LogLevel::kInfo, logger::getDefaultLogPatterns()}); +} + int main(int argc, char *argv[]) { // Parsing command line arguments gflags::ParseCommandLineFlags(&argc, &argv, true); - spdlog::set_level(spdlog::level::level_enum(FLAGS_verbosity)); + logger::LoggerManagerTreePtr log_manager; + logger::LoggerPtr log; - auto log = logger::log("MAIN"); - log->info("start"); + // If the global log level override was set in the command line arguments, + // create a logger manager with the given log level for all subsystems: + if (FLAGS_verbosity != kLogSettingsFromConfigFile) { + logger::LoggerConfig cfg; + cfg.log_level = config_members::LogLevels.at(FLAGS_verbosity); + log_manager = std::make_shared(std::move(cfg)); + log = log_manager->getChild("Init")->getLogger(); + } // Check if validators are registered. if (not config_validator_registered or not keypair_name_validator_registered) { // Abort execution if not - log->error("Flag validator is not registered"); + if (log) { + log->error("Flag validator is not registered"); + } return EXIT_FAILURE; } - namespace mbr = config_members; - // Reading iroha configuration file - auto config = parse_iroha_config(FLAGS_config); + const auto config = parse_iroha_config(FLAGS_config); + if (not log_manager) { + log_manager = config.logger_manager.value_or(getDefaultLogManager()); + log = log_manager->getChild("Init")->getLogger(); + } log->info("config initialized"); // Reading public and private key files - iroha::KeysManagerImpl keysManager(FLAGS_keypair_name); + iroha::KeysManagerImpl keysManager( + FLAGS_keypair_name, log_manager->getChild("KeysManager")->getLogger()); auto keypair = keysManager.loadKeys(); // Check if both keys are read properly if (not keypair) { @@ -120,20 +152,23 @@ int main(int argc, char *argv[]) { // Configuring iroha daemon Irohad irohad( - config[mbr::BlockStorePath].GetString(), - config[mbr::PgOpt].GetString(), + config.block_store_path, + config.pg_opt, kListenIp, // TODO(mboldyrev) 17/10/2018: add a parameter in // config file and/or command-line arguments? - config[mbr::ToriiPort].GetUint(), - config[mbr::InternalPort].GetUint(), - config[mbr::MaxProposalSize].GetUint(), - std::chrono::milliseconds(config[mbr::ProposalDelay].GetUint()), - std::chrono::milliseconds(config[mbr::VoteDelay].GetUint()), - std::chrono::minutes(config[mbr::MstExpirationTime].GetUint()), + config.torii_port, + config.internal_port, + config.max_proposal_size, + std::chrono::milliseconds(config.proposal_delay), + std::chrono::milliseconds(config.vote_delay), + std::chrono::minutes( + config.mst_expiration_time.value_or(kMstExpirationTimeDefault)), *keypair, - std::chrono::milliseconds(config[mbr::MaxRoundsDelay].GetUint()), - config[mbr::StaleStreamMaxRounds].GetUint(), - boost::make_optional(config[mbr::MstSupport].GetBool(), + std::chrono::milliseconds( + config.max_round_delay_ms.value_or(kMaxRoundsDelayDefault)), + config.stale_stream_max_rounds.value_or(kStaleStreamMaxRoundsDefault), + log_manager->getChild("Irohad"), + boost::make_optional(config.mst_support, iroha::GossipPropagationStrategyParams{})); // Check if iroha daemon storage was successfully initialized @@ -176,7 +211,8 @@ int main(int argc, char *argv[]) { "Passed genesis block will be ignored without --overwrite_ledger " "flag. Restoring existing state."); } else { - iroha::main::BlockLoader loader; + iroha::main::BlockLoader loader( + log_manager->getChild("GenesisBlockLoader")->getLogger()); auto file = loader.loadFile(FLAGS_genesis_block); auto block = loader.parseBlock(file.value()); diff --git a/irohad/main/raw_block_loader.hpp b/irohad/main/raw_block_loader.hpp index 92d555999c..d6cb996bca 100644 --- a/irohad/main/raw_block_loader.hpp +++ b/irohad/main/raw_block_loader.hpp @@ -11,7 +11,7 @@ #include -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" namespace shared_model { namespace interface { @@ -29,7 +29,7 @@ namespace iroha { */ class BlockLoader { public: - explicit BlockLoader(logger::Logger log = logger::log("BlockLoader")); + explicit BlockLoader(logger::LoggerPtr log); /** * Parse block from file @@ -47,7 +47,7 @@ namespace iroha { boost::optional loadFile(const std::string &path); private: - logger::Logger log_; + logger::LoggerPtr log_; }; } // namespace main diff --git a/irohad/main/server_runner.cpp b/irohad/main/server_runner.cpp index 75297af8e3..30472899d7 100644 --- a/irohad/main/server_runner.cpp +++ b/irohad/main/server_runner.cpp @@ -6,12 +6,13 @@ #include "main/server_runner.hpp" #include +#include "logger/logger.hpp" const auto kPortBindError = "Cannot bind server to address %s"; ServerRunner::ServerRunner(const std::string &address, - bool reuse, - logger::Logger log) + logger::LoggerPtr log, + bool reuse) : log_(std::move(log)), serverAddress_(address), reuse_(reuse) {} ServerRunner &ServerRunner::append(std::shared_ptr service) { diff --git a/irohad/main/server_runner.hpp b/irohad/main/server_runner.hpp index 495a8a2393..d8c08e303e 100644 --- a/irohad/main/server_runner.hpp +++ b/irohad/main/server_runner.hpp @@ -9,7 +9,7 @@ #include #include #include "common/result.hpp" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" /** * Class runs Torii server for handling queries and commands. @@ -19,12 +19,12 @@ class ServerRunner { /** * Constructor. Initialize a new instance of ServerRunner class. * @param address - the address the server will be bind to in URI form - * @param reuse - allow multiple sockets to bind to the same port * @param log to print progress to + * @param reuse - allow multiple sockets to bind to the same port */ explicit ServerRunner(const std::string &address, - bool reuse = true, - logger::Logger log = logger::log("ServerRunner")); + logger::LoggerPtr log, + bool reuse = true); /** * Adds a new grpc service to be run. @@ -55,7 +55,7 @@ class ServerRunner { void shutdown(const std::chrono::system_clock::time_point &deadline); private: - logger::Logger log_; + logger::LoggerPtr log_; std::unique_ptr serverInstance_; std::mutex waitForServer_; diff --git a/irohad/model/converters/impl/json_block_factory.cpp b/irohad/model/converters/impl/json_block_factory.cpp index 36e6cb46e7..6d712e2a46 100644 --- a/irohad/model/converters/impl/json_block_factory.cpp +++ b/irohad/model/converters/impl/json_block_factory.cpp @@ -11,7 +11,7 @@ namespace iroha { namespace model { namespace converters { - JsonBlockFactory::JsonBlockFactory(logger::Logger log) + JsonBlockFactory::JsonBlockFactory(logger::LoggerPtr log) : log_{std::move(log)} {} Document JsonBlockFactory::serialize(const Block &block) { diff --git a/irohad/model/converters/impl/json_query_factory.cpp b/irohad/model/converters/impl/json_query_factory.cpp index b853129647..1ae1bc24d8 100644 --- a/irohad/model/converters/impl/json_query_factory.cpp +++ b/irohad/model/converters/impl/json_query_factory.cpp @@ -5,6 +5,7 @@ #include "model/converters/json_query_factory.hpp" +#include "logger/logger.hpp" #include "model/queries/get_account.hpp" #include "model/queries/get_account_assets.hpp" #include "model/queries/get_account_detail.hpp" @@ -18,7 +19,7 @@ using namespace rapidjson; namespace iroha { namespace model { namespace converters { - JsonQueryFactory::JsonQueryFactory(logger::Logger log) + JsonQueryFactory::JsonQueryFactory(logger::LoggerPtr log) : log_{std::move(log)} { deserializers_ = { {"GetAccount", &JsonQueryFactory::deserializeGetAccount}, diff --git a/irohad/model/converters/impl/pb_query_factory.cpp b/irohad/model/converters/impl/pb_query_factory.cpp index cebfbc3056..13e23960b8 100644 --- a/irohad/model/converters/impl/pb_query_factory.cpp +++ b/irohad/model/converters/impl/pb_query_factory.cpp @@ -5,6 +5,7 @@ #include "model/converters/pb_query_factory.hpp" +#include "logger/logger.hpp" #include "model/common.hpp" #include "model/queries/get_account.hpp" #include "model/queries/get_account_assets.hpp" @@ -19,7 +20,7 @@ namespace iroha { namespace model { namespace converters { - PbQueryFactory::PbQueryFactory(logger::Logger log) + PbQueryFactory::PbQueryFactory(logger::LoggerPtr log) : log_{std::move(log)} { serializers_[typeid(GetAccount)] = &PbQueryFactory::serializeGetAccount; serializers_[typeid(GetAccountAssets)] = diff --git a/irohad/model/converters/json_block_factory.hpp b/irohad/model/converters/json_block_factory.hpp index eb3acbe5b5..a88cebe8a9 100644 --- a/irohad/model/converters/json_block_factory.hpp +++ b/irohad/model/converters/json_block_factory.hpp @@ -6,7 +6,7 @@ #ifndef IROHA_JSON_BLOCK_FACTORY_HPP #define IROHA_JSON_BLOCK_FACTORY_HPP -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" #include "model/block.hpp" #include "model/converters/json_transaction_factory.hpp" @@ -16,15 +16,14 @@ namespace iroha { class JsonBlockFactory { public: - explicit JsonBlockFactory( - logger::Logger log = logger::log("JsonBlockFactory")); + explicit JsonBlockFactory(logger::LoggerPtr log); rapidjson::Document serialize(const Block &block); boost::optional deserialize(const rapidjson::Document &document); private: JsonTransactionFactory factory_; - logger::Logger log_; + logger::LoggerPtr log_; }; } // namespace converters diff --git a/irohad/model/converters/json_query_factory.hpp b/irohad/model/converters/json_query_factory.hpp index 290ab40b2e..7ad98acbd2 100644 --- a/irohad/model/converters/json_query_factory.hpp +++ b/irohad/model/converters/json_query_factory.hpp @@ -8,7 +8,7 @@ #include #include -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" #include "model/common.hpp" #include "model/converters/json_common.hpp" #include "model/query.hpp" @@ -19,8 +19,7 @@ namespace iroha { namespace converters { class JsonQueryFactory { public: - explicit JsonQueryFactory( - logger::Logger log = logger::log("JsonQueryFactory")); + explicit JsonQueryFactory(logger::LoggerPtr log); /** * get query from string json @@ -99,7 +98,7 @@ namespace iroha { std::shared_ptr query); // Logger - std::shared_ptr log_; + logger::LoggerPtr log_; }; } // namespace converters } // namespace model diff --git a/irohad/model/converters/pb_query_factory.hpp b/irohad/model/converters/pb_query_factory.hpp index 5faa372798..2eea9a642c 100644 --- a/irohad/model/converters/pb_query_factory.hpp +++ b/irohad/model/converters/pb_query_factory.hpp @@ -7,7 +7,7 @@ #include #include -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" #include "model/common.hpp" #include "model/query.hpp" #include "queries.pb.h" @@ -37,8 +37,7 @@ namespace iroha { boost::optional serialize( std::shared_ptr query) const; - explicit PbQueryFactory( - logger::Logger log = logger::log("PbQueryFactory")); + explicit PbQueryFactory(logger::LoggerPtr log); private: // Query serializer: @@ -75,7 +74,7 @@ namespace iroha { std::shared_ptr) const; std::unordered_map serializers_; - logger::Logger log_; + logger::LoggerPtr log_; }; } // namespace converters diff --git a/irohad/model/generators/impl/transaction_generator.cpp b/irohad/model/generators/impl/transaction_generator.cpp index d013d0b400..92d956b99c 100644 --- a/irohad/model/generators/impl/transaction_generator.cpp +++ b/irohad/model/generators/impl/transaction_generator.cpp @@ -23,14 +23,17 @@ namespace iroha { } Transaction TransactionGenerator::generateGenesisTransaction( - ts64_t timestamp, std::vector peers_address) { + ts64_t timestamp, + std::vector peers_address, + logger::LoggerPtr keys_manager_logger) { Transaction tx; tx.created_ts = timestamp; tx.creator_account_id = ""; CommandGenerator command_generator; // Add peers for (size_t i = 0; i < peers_address.size(); ++i) { - KeysManagerImpl manager("node" + std::to_string(i)); + KeysManagerImpl manager("node" + std::to_string(i), + keys_manager_logger); manager.createKeys(); auto keypair = *std::unique_ptr( makeOldModel(*manager.loadKeys())); @@ -52,13 +55,13 @@ namespace iroha { tx.commands.push_back( command_generator.generateCreateAsset("coin", "test", precision)); // Create accounts - KeysManagerImpl manager("admin@test"); + KeysManagerImpl manager("admin@test", keys_manager_logger); manager.createKeys(); auto keypair = *std::unique_ptr( makeOldModel(*manager.loadKeys())); tx.commands.push_back(command_generator.generateCreateAccount( "admin", "test", keypair.pubkey)); - manager = KeysManagerImpl("test@test"); + manager = KeysManagerImpl("test@test", std::move(keys_manager_logger)); manager.createKeys(); keypair = *std::unique_ptr( makeOldModel(*manager.loadKeys())); diff --git a/irohad/model/generators/transaction_generator.hpp b/irohad/model/generators/transaction_generator.hpp index 8d48b00736..aa429858ba 100644 --- a/irohad/model/generators/transaction_generator.hpp +++ b/irohad/model/generators/transaction_generator.hpp @@ -6,6 +6,7 @@ #ifndef IROHA_TRANSACTION_GENERATOR_HPP #define IROHA_TRANSACTION_GENERATOR_HPP +#include "logger/logger_fwd.hpp" #include "model/generators/command_generator.hpp" #include "model/generators/signature_generator.hpp" #include "model/transaction.hpp" @@ -23,7 +24,9 @@ namespace iroha { * @return */ Transaction generateGenesisTransaction( - ts64_t timestamp, std::vector peers_address); + ts64_t timestamp, + std::vector peers_address, + logger::LoggerPtr keys_manager_logger); /** * Generate transaction from give meta data and commands list diff --git a/irohad/model/sha3_hash.cpp b/irohad/model/sha3_hash.cpp index 7965a3ed6e..aee67549b7 100644 --- a/irohad/model/sha3_hash.cpp +++ b/irohad/model/sha3_hash.cpp @@ -7,12 +7,18 @@ #include "converters/pb_common.hpp" #include "converters/pb_query_factory.hpp" #include "converters/pb_transaction_factory.hpp" +#include "logger/logger_spdlog.hpp" namespace iroha { // TODO: 24.01.2018 @victordrobny: remove factories IR-850 const static model::converters::PbTransactionFactory tx_factory; const static model::converters::PbBlockFactory block_factory; - const static model::converters::PbQueryFactory query_factory; + // TODO 10.01.2019 mboldyrev: initialize query_factory logger using config + const static model::converters::PbQueryFactory query_factory( + std::make_shared( + "QueryFactory", + std::make_shared(logger::LoggerConfig{ + logger::kDefaultLogLevel, logger::getDefaultLogPatterns()}))); hash256_t hash(const model::Transaction &tx) { auto &&pb_dat = tx_factory.serialize(tx); diff --git a/irohad/multi_sig_transactions/impl/mst_processor.cpp b/irohad/multi_sig_transactions/impl/mst_processor.cpp index 4edd2a61ce..1e3e9ae647 100644 --- a/irohad/multi_sig_transactions/impl/mst_processor.cpp +++ b/irohad/multi_sig_transactions/impl/mst_processor.cpp @@ -7,7 +7,8 @@ namespace iroha { - MstProcessor::MstProcessor(logger::Logger log) : log_(std::move(log)) {} + MstProcessor::MstProcessor(logger::LoggerPtr log) + : log_(std::move(log)) {} void MstProcessor::propagateBatch(const DataType &batch) { this->propagateBatchImpl(batch); diff --git a/irohad/multi_sig_transactions/impl/mst_processor_impl.cpp b/irohad/multi_sig_transactions/impl/mst_processor_impl.cpp index ad76bcfa1c..b433019a5c 100644 --- a/irohad/multi_sig_transactions/impl/mst_processor_impl.cpp +++ b/irohad/multi_sig_transactions/impl/mst_processor_impl.cpp @@ -5,6 +5,7 @@ #include +#include "logger/logger.hpp" #include "multi_sig_transactions/mst_processor_impl.hpp" namespace iroha { @@ -13,14 +14,16 @@ namespace iroha { std::shared_ptr transport, std::shared_ptr storage, std::shared_ptr strategy, - std::shared_ptr time_provider) - : MstProcessor(logger::log("FairMstProcessor")), + std::shared_ptr time_provider, + logger::LoggerPtr log) + : MstProcessor(log), // use the same logger in base class transport_(std::move(transport)), storage_(std::move(storage)), strategy_(std::move(strategy)), time_provider_(std::move(time_provider)), propagation_subscriber_(strategy_->emitter().subscribe( - [this](auto data) { this->onPropagate(data); })) {} + [this](auto data) { this->onPropagate(data); })), + log_(std::move(log)) {} FairMstProcessor::~FairMstProcessor() { propagation_subscriber_.unsubscribe(); diff --git a/irohad/multi_sig_transactions/mst_processor.hpp b/irohad/multi_sig_transactions/mst_processor.hpp index 0e7a85a5b4..f6602071e0 100644 --- a/irohad/multi_sig_transactions/mst_processor.hpp +++ b/irohad/multi_sig_transactions/mst_processor.hpp @@ -9,7 +9,7 @@ #include #include #include -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" #include "multi_sig_transactions/mst_types.hpp" #include "multi_sig_transactions/state/mst_state.hpp" @@ -56,9 +56,9 @@ namespace iroha { virtual ~MstProcessor() = default; protected: - explicit MstProcessor(logger::Logger log = logger::log("MstProcessor")); + explicit MstProcessor(logger::LoggerPtr log); - logger::Logger log_; + logger::LoggerPtr log_; private: // ------------------------| inheritance interface |------------------------ diff --git a/irohad/multi_sig_transactions/mst_processor_impl.hpp b/irohad/multi_sig_transactions/mst_processor_impl.hpp index aa08889ba6..6cf3aca34f 100644 --- a/irohad/multi_sig_transactions/mst_processor_impl.hpp +++ b/irohad/multi_sig_transactions/mst_processor_impl.hpp @@ -8,7 +8,7 @@ #include #include "cryptography/public_key.hpp" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" #include "multi_sig_transactions/mst_processor.hpp" #include "multi_sig_transactions/mst_propagation_strategy.hpp" #include "multi_sig_transactions/mst_time_provider.hpp" @@ -33,7 +33,8 @@ namespace iroha { FairMstProcessor(std::shared_ptr transport, std::shared_ptr storage, std::shared_ptr strategy, - std::shared_ptr time_provider); + std::shared_ptr time_provider, + logger::LoggerPtr log); ~FairMstProcessor(); @@ -107,6 +108,8 @@ namespace iroha { /// use for tracking the propagation subscription rxcpp::composite_subscription propagation_subscriber_; + + logger::LoggerPtr log_; }; } // namespace iroha diff --git a/irohad/multi_sig_transactions/state/impl/mst_state.cpp b/irohad/multi_sig_transactions/state/impl/mst_state.cpp index e2f6de8eff..c3b79caa6f 100644 --- a/irohad/multi_sig_transactions/state/impl/mst_state.cpp +++ b/irohad/multi_sig_transactions/state/impl/mst_state.cpp @@ -12,6 +12,7 @@ #include "common/set.hpp" #include "interfaces/iroha_internal/transaction_batch.hpp" #include "interfaces/transaction.hpp" +#include "logger/logger.hpp" namespace iroha { @@ -44,22 +45,23 @@ namespace iroha { // ------------------------------| public api |------------------------------- - MstState MstState::empty(const CompleterType &completer) { - return MstState(completer); + MstState MstState::empty(logger::LoggerPtr log, + const CompleterType &completer) { + return MstState(completer, std::move(log)); } StateUpdateResult MstState::operator+=(const DataType &rhs) { auto state_update = StateUpdateResult{ - std::make_shared(MstState::empty(completer_)), - std::make_shared(MstState::empty(completer_))}; + std::make_shared(MstState::empty(log_, completer_)), + std::make_shared(MstState::empty(log_, completer_))}; insertOne(state_update, rhs); return state_update; } StateUpdateResult MstState::operator+=(const MstState &rhs) { auto state_update = StateUpdateResult{ - std::make_shared(MstState::empty(completer_)), - std::make_shared(MstState::empty(completer_))}; + std::make_shared(MstState::empty(log_, completer_)), + std::make_shared(MstState::empty(log_, completer_))}; for (auto &&rhs_tx : rhs.internal_state_) { insertOne(state_update, rhs_tx); } @@ -68,7 +70,8 @@ namespace iroha { MstState MstState::operator-(const MstState &rhs) const { return MstState(this->completer_, - set_difference(this->internal_state_, rhs.internal_state_)); + set_difference(this->internal_state_, rhs.internal_state_), + log_); } bool MstState::operator==(const MstState &rhs) const { @@ -90,7 +93,7 @@ namespace iroha { } MstState MstState::eraseByTime(const TimeType &time) { - MstState out = MstState::empty(completer_); + MstState out = MstState::empty(log_, completer_); while (not index_.empty() and (*completer_)(index_.top(), time)) { auto iter = internal_state_.find(index_.top()); @@ -134,12 +137,12 @@ namespace iroha { return inserted_new_signatures; } - MstState::MstState(const CompleterType &completer, logger::Logger log) + MstState::MstState(const CompleterType &completer, logger::LoggerPtr log) : MstState(completer, InternalStateType{}, std::move(log)) {} MstState::MstState(const CompleterType &completer, const InternalStateType &transactions, - logger::Logger log) + logger::LoggerPtr log) : completer_(completer), internal_state_(transactions.begin(), transactions.end()), index_(transactions.begin(), transactions.end()), diff --git a/irohad/multi_sig_transactions/state/mst_state.hpp b/irohad/multi_sig_transactions/state/mst_state.hpp index 2403e9f0a0..bb7b87d356 100644 --- a/irohad/multi_sig_transactions/state/mst_state.hpp +++ b/irohad/multi_sig_transactions/state/mst_state.hpp @@ -6,11 +6,12 @@ #ifndef IROHA_MST_STATE_HPP #define IROHA_MST_STATE_HPP +#include #include #include #include -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" #include "multi_sig_transactions/hash.hpp" #include "multi_sig_transactions/mst_types.hpp" @@ -82,10 +83,12 @@ namespace iroha { /** * Create empty state + * @param log - the logger to use in the new object * @param completer - strategy for determine completed and expired batches * @return empty mst state */ - static MstState empty(const CompleterType &completer); + static MstState empty(logger::LoggerPtr log, + const CompleterType &completer); /** * Add batch to current state @@ -164,12 +167,11 @@ namespace iroha { using IndexType = std::priority_queue, Less>; - explicit MstState(const CompleterType &completer, - logger::Logger log = logger::log("MstState")); + MstState(const CompleterType &completer, logger::LoggerPtr log); MstState(const CompleterType &completer, const InternalStateType &transactions, - logger::Logger log = logger::log("MstState")); + logger::LoggerPtr log); /** * Insert batch in own state and push it in out_completed_state or @@ -193,7 +195,7 @@ namespace iroha { IndexType index_; - logger::Logger log_; + logger::LoggerPtr log_; }; } // namespace iroha diff --git a/irohad/multi_sig_transactions/storage/impl/mst_storage.cpp b/irohad/multi_sig_transactions/storage/impl/mst_storage.cpp index 5b6b15c86b..b0959e14da 100644 --- a/irohad/multi_sig_transactions/storage/impl/mst_storage.cpp +++ b/irohad/multi_sig_transactions/storage/impl/mst_storage.cpp @@ -8,7 +8,7 @@ #include "multi_sig_transactions/storage/mst_storage.hpp" namespace iroha { - MstStorage::MstStorage(logger::Logger log) : log_{std::move(log)} {} + MstStorage::MstStorage(logger::LoggerPtr log) : log_{std::move(log)} {} StateUpdateResult MstStorage::apply( const shared_model::crypto::PublicKey &target_peer_key, diff --git a/irohad/multi_sig_transactions/storage/impl/mst_storage_impl.cpp b/irohad/multi_sig_transactions/storage/impl/mst_storage_impl.cpp index d577622c28..69a7814a9e 100644 --- a/irohad/multi_sig_transactions/storage/impl/mst_storage_impl.cpp +++ b/irohad/multi_sig_transactions/storage/impl/mst_storage_impl.cpp @@ -12,17 +12,22 @@ namespace iroha { const shared_model::crypto::PublicKey &target_peer_key) { auto target_state_iter = peer_states_.find(target_peer_key); if (target_state_iter == peer_states_.end()) { - return peer_states_.insert({target_peer_key, MstState::empty(completer_)}) + return peer_states_ + .insert( + {target_peer_key, MstState::empty(mst_state_logger_, completer_)}) .first; } return target_state_iter; } // -----------------------------| interface API |----------------------------- - MstStorageStateImpl::MstStorageStateImpl(const CompleterType &completer) - : MstStorage(), + MstStorageStateImpl::MstStorageStateImpl(const CompleterType &completer, + logger::LoggerPtr mst_state_logger, + logger::LoggerPtr log) + : MstStorage(log), completer_(completer), - own_state_(MstState::empty(completer_)) {} + own_state_(MstState::empty(mst_state_logger, completer_)), + mst_state_logger_(std::move(mst_state_logger)) {} auto MstStorageStateImpl::applyImpl( const shared_model::crypto::PublicKey &target_peer_key, diff --git a/irohad/multi_sig_transactions/storage/mst_storage.hpp b/irohad/multi_sig_transactions/storage/mst_storage.hpp index d07a07a15b..5173b65320 100644 --- a/irohad/multi_sig_transactions/storage/mst_storage.hpp +++ b/irohad/multi_sig_transactions/storage/mst_storage.hpp @@ -9,7 +9,7 @@ #include #include "cryptography/public_key.hpp" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" #include "multi_sig_transactions/mst_types.hpp" #include "multi_sig_transactions/state/mst_state.hpp" @@ -83,7 +83,7 @@ namespace iroha { /** * Constructor provide initialization of protected fields, such as logger. */ - explicit MstStorage(logger::Logger log = logger::log("MstStorage")); + explicit MstStorage(logger::LoggerPtr log); private: virtual auto applyImpl( @@ -112,7 +112,7 @@ namespace iroha { mutable std::mutex mutex_; protected: - logger::Logger log_; + logger::LoggerPtr log_; }; } // namespace iroha #endif // IROHA_MST_STORAGE_HPP diff --git a/irohad/multi_sig_transactions/storage/mst_storage_impl.hpp b/irohad/multi_sig_transactions/storage/mst_storage_impl.hpp index 45c59f0df5..3f3d616952 100644 --- a/irohad/multi_sig_transactions/storage/mst_storage_impl.hpp +++ b/irohad/multi_sig_transactions/storage/mst_storage_impl.hpp @@ -7,6 +7,7 @@ #define IROHA_MST_STORAGE_IMPL_HPP #include +#include "logger/logger_fwd.hpp" #include "multi_sig_transactions/hash.hpp" #include "multi_sig_transactions/storage/mst_storage.hpp" @@ -25,7 +26,9 @@ namespace iroha { public: // ----------------------------| interface API |---------------------------- - explicit MstStorageStateImpl(const CompleterType &completer); + MstStorageStateImpl(const CompleterType &completer, + logger::LoggerPtr mst_state_logger, + logger::LoggerPtr log); auto applyImpl(const shared_model::crypto::PublicKey &target_peer_key, const MstState &new_state) @@ -56,6 +59,9 @@ namespace iroha { iroha::model::BlobHasher> peer_states_; MstState own_state_; + + logger::LoggerPtr mst_state_logger_; ///< Logger for created MstState + ///< objects. }; } // namespace iroha diff --git a/irohad/multi_sig_transactions/transport/impl/mst_transport_grpc.cpp b/irohad/multi_sig_transactions/transport/impl/mst_transport_grpc.cpp index 1a0c7c5b87..2459d0c0a0 100644 --- a/irohad/multi_sig_transactions/transport/impl/mst_transport_grpc.cpp +++ b/irohad/multi_sig_transactions/transport/impl/mst_transport_grpc.cpp @@ -13,6 +13,7 @@ #include "backend/protobuf/transaction.hpp" #include "interfaces/iroha_internal/transaction_batch.hpp" #include "interfaces/transaction.hpp" +#include "logger/logger.hpp" #include "validators/field_validator.hpp" using namespace iroha; @@ -34,14 +35,18 @@ MstTransportGrpc::MstTransportGrpc( transaction_batch_factory, std::shared_ptr tx_presence_cache, std::shared_ptr mst_completer, - shared_model::crypto::PublicKey my_key) + shared_model::crypto::PublicKey my_key, + logger::LoggerPtr mst_state_logger, + logger::LoggerPtr log) : async_call_(std::move(async_call)), transaction_factory_(std::move(transaction_factory)), batch_parser_(std::move(batch_parser)), batch_factory_(std::move(transaction_batch_factory)), tx_presence_cache_(std::move(tx_presence_cache)), mst_completer_(std::move(mst_completer)), - my_key_(shared_model::crypto::toBinaryString(my_key)) {} + my_key_(shared_model::crypto::toBinaryString(my_key)), + mst_state_logger_(std::move(mst_state_logger)), + log_(std::move(log)) {} shared_model::interface::types::SharedTxsCollectionType MstTransportGrpc::deserializeTransactions(const transport::MstState *request) { @@ -58,10 +63,9 @@ MstTransportGrpc::deserializeTransactions(const transport::MstState *request) { }, [&](const iroha::expected::Error &error) { - async_call_->log_->info( - "Transaction deserialization failed: hash {}, {}", - error.error.hash, - error.error.error); + log_->info("Transaction deserialization failed: hash {}, {}", + error.error.hash, + error.error.error); return false; }); }) @@ -77,13 +81,13 @@ grpc::Status MstTransportGrpc::SendState( ::grpc::ServerContext *context, const ::iroha::network::transport::MstState *request, ::google::protobuf::Empty *response) { - async_call_->log_->info("MstState Received"); + log_->info("MstState Received"); auto transactions = deserializeTransactions(request); auto batches = batch_parser_->parseBatches(transactions); - MstState new_state = MstState::empty(mst_completer_); + MstState new_state = MstState::empty(mst_state_logger_, mst_completer_); for (auto &batch : batches) { batch_factory_->createTransactionBatch(batch).match( @@ -92,8 +96,8 @@ grpc::Status MstTransportGrpc::SendState( auto cache_presence = tx_presence_cache_->check(*value.value); if (not cache_presence) { // TODO andrei 30.11.18 IR-51 Handle database error - async_call_->log_->warn( - "Check tx presence database error. Batch: {}", *value.value); + log_->warn("Check tx presence database error. Batch: {}", + *value.value); return; } auto is_replay = std::any_of( @@ -112,26 +116,23 @@ grpc::Status MstTransportGrpc::SendState( } }, [&](iroha::expected::Error &error) { - async_call_->log_->warn("Batch deserialization failed: {}", - error.error); + log_->warn("Batch deserialization failed: {}", error.error); }); } - async_call_->log_->info("batches in MstState: {}", - new_state.getBatches().size()); + log_->info("batches in MstState: {}", new_state.getBatches().size()); shared_model::crypto::PublicKey source_key(request->source_peer_key()); auto key_invalid_reason = shared_model::validation::validatePubkey(source_key); if (key_invalid_reason) { - async_call_->log_->info( - "Dropping received MST State due to invalid public key: {}", - *key_invalid_reason); + log_->info("Dropping received MST State due to invalid public key: {}", + *key_invalid_reason); return grpc::Status::OK; } if (new_state.isEmpty()) { - async_call_->log_->info( + log_->info( "All transactions from received MST state have been processed already, " "nothing to propagate to MST processor"); return grpc::Status::OK; @@ -140,7 +141,7 @@ grpc::Status MstTransportGrpc::SendState( if (auto subscriber = subscriber_.lock()) { subscriber->onNewState(source_key, std::move(new_state)); } else { - async_call_->log_->warn("No subscriber for MST SendState event is set"); + log_->warn("No subscriber for MST SendState event is set"); } return grpc::Status::OK; @@ -153,7 +154,7 @@ void MstTransportGrpc::subscribe( void MstTransportGrpc::sendState(const shared_model::interface::Peer &to, ConstRefState providing_state) { - async_call_->log_->info("Propagate MstState to peer {}", to.address()); + log_->info("Propagate MstState to peer {}", to.address()); sendStateAsyncImpl(to, providing_state, my_key_, *async_call_); } diff --git a/irohad/multi_sig_transactions/transport/mst_transport_grpc.hpp b/irohad/multi_sig_transactions/transport/mst_transport_grpc.hpp index d75837ad24..16ec9c531e 100644 --- a/irohad/multi_sig_transactions/transport/mst_transport_grpc.hpp +++ b/irohad/multi_sig_transactions/transport/mst_transport_grpc.hpp @@ -14,7 +14,7 @@ #include "interfaces/iroha_internal/abstract_transport_factory.hpp" #include "interfaces/iroha_internal/transaction_batch_factory.hpp" #include "interfaces/iroha_internal/transaction_batch_parser.hpp" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" #include "multi_sig_transactions/state/mst_state.hpp" #include "network/impl/async_grpc_client.hpp" @@ -43,7 +43,9 @@ namespace iroha { transaction_batch_factory, std::shared_ptr tx_presence_cache, std::shared_ptr mst_completer, - shared_model::crypto::PublicKey my_key); + shared_model::crypto::PublicKey my_key, + logger::LoggerPtr mst_state_logger, + logger::LoggerPtr log); /** * Server part of grpc SendState method call @@ -82,6 +84,10 @@ namespace iroha { /// source peer key for MST propogation messages std::shared_ptr mst_completer_; const std::string my_key_; + + logger::LoggerPtr mst_state_logger_; ///< Logger for created MstState + ///< objects. + logger::LoggerPtr log_; ///< Logger for local use. }; void sendStateAsync(const shared_model::interface::Peer &to, diff --git a/irohad/network/impl/async_grpc_client.hpp b/irohad/network/impl/async_grpc_client.hpp index 5e8ea7535c..02eadb4578 100644 --- a/irohad/network/impl/async_grpc_client.hpp +++ b/irohad/network/impl/async_grpc_client.hpp @@ -24,8 +24,7 @@ namespace iroha { template class AsyncGrpcClient { public: - explicit AsyncGrpcClient( - logger::Logger log = logger::log("AsyncGrpcClient")) + explicit AsyncGrpcClient(logger::LoggerPtr log) : thread_(&AsyncGrpcClient::asyncCompleteRpc, this), log_(std::move(log)) {} @@ -53,7 +52,6 @@ namespace iroha { grpc::CompletionQueue cq_; std::thread thread_; - logger::Logger log_; /** * State and data information of gRPC call @@ -80,6 +78,9 @@ namespace iroha { call->response_reader = lambda(&call->context, &cq_); call->response_reader->Finish(&call->reply, &call->status, call); } + + private: + logger::LoggerPtr log_; }; } // namespace network } // namespace iroha diff --git a/irohad/network/impl/block_loader_impl.cpp b/irohad/network/impl/block_loader_impl.cpp index 085c3c9e73..99e15f3fb1 100644 --- a/irohad/network/impl/block_loader_impl.cpp +++ b/irohad/network/impl/block_loader_impl.cpp @@ -11,6 +11,7 @@ #include "builders/protobuf/transport_builder.hpp" #include "common/bind.hpp" #include "interfaces/common_objects/peer.hpp" +#include "logger/logger.hpp" #include "network/impl/grpc_channel_builder.hpp" using namespace iroha::ametsuchi; @@ -27,7 +28,7 @@ namespace { BlockLoaderImpl::BlockLoaderImpl( std::shared_ptr peer_query_factory, shared_model::proto::ProtoBlockFactory factory, - logger::Logger log) + logger::LoggerPtr log) : peer_query_factory_(std::move(peer_query_factory)), block_factory_(std::move(factory)), log_(std::move(log)) {} diff --git a/irohad/network/impl/block_loader_impl.hpp b/irohad/network/impl/block_loader_impl.hpp index 0ee8a41aed..40f6ac71bc 100644 --- a/irohad/network/impl/block_loader_impl.hpp +++ b/irohad/network/impl/block_loader_impl.hpp @@ -13,7 +13,7 @@ #include "ametsuchi/peer_query_factory.hpp" #include "backend/protobuf/proto_block_factory.hpp" #include "loader.grpc.pb.h" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" namespace iroha { namespace network { @@ -22,7 +22,7 @@ namespace iroha { BlockLoaderImpl( std::shared_ptr peer_query_factory, shared_model::proto::ProtoBlockFactory factory, - logger::Logger log = logger::log("BlockLoaderImpl")); + logger::LoggerPtr log); rxcpp::observable> retrieveBlocks( @@ -57,7 +57,7 @@ namespace iroha { std::shared_ptr peer_query_factory_; shared_model::proto::ProtoBlockFactory block_factory_; - logger::Logger log_; + logger::LoggerPtr log_; }; } // namespace network } // namespace iroha diff --git a/irohad/network/impl/block_loader_service.cpp b/irohad/network/impl/block_loader_service.cpp index 90a1745a3f..0b0045a003 100644 --- a/irohad/network/impl/block_loader_service.cpp +++ b/irohad/network/impl/block_loader_service.cpp @@ -4,8 +4,10 @@ */ #include "network/impl/block_loader_service.hpp" + #include "backend/protobuf/block.hpp" #include "common/bind.hpp" +#include "logger/logger.hpp" using namespace iroha; using namespace iroha::ametsuchi; @@ -15,7 +17,7 @@ BlockLoaderService::BlockLoaderService( std::shared_ptr block_query_factory, std::shared_ptr consensus_result_cache, - logger::Logger log) + logger::LoggerPtr log) : block_query_factory_(std::move(block_query_factory)), consensus_result_cache_(std::move(consensus_result_cache)), log_(std::move(log)) {} diff --git a/irohad/network/impl/block_loader_service.hpp b/irohad/network/impl/block_loader_service.hpp index 71b0c57dbb..8f11ff549d 100644 --- a/irohad/network/impl/block_loader_service.hpp +++ b/irohad/network/impl/block_loader_service.hpp @@ -9,7 +9,7 @@ #include "ametsuchi/block_query_factory.hpp" #include "consensus/consensus_block_cache.hpp" #include "loader.grpc.pb.h" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" namespace iroha { namespace network { @@ -19,7 +19,7 @@ namespace iroha { std::shared_ptr block_query_factory, std::shared_ptr consensus_result_cache, - logger::Logger log = logger::log("BlockLoaderService")); + logger::LoggerPtr log); grpc::Status retrieveBlocks( ::grpc::ServerContext *context, @@ -34,7 +34,7 @@ namespace iroha { std::shared_ptr block_query_factory_; std::shared_ptr consensus_result_cache_; - logger::Logger log_; + logger::LoggerPtr log_; }; } // namespace network } // namespace iroha diff --git a/irohad/network/impl/peer_communication_service_impl.cpp b/irohad/network/impl/peer_communication_service_impl.cpp index b90053f01e..e8e400e8ff 100644 --- a/irohad/network/impl/peer_communication_service_impl.cpp +++ b/irohad/network/impl/peer_communication_service_impl.cpp @@ -6,6 +6,7 @@ #include "network/impl/peer_communication_service_impl.hpp" #include "interfaces/iroha_internal/transaction_batch.hpp" +#include "logger/logger.hpp" #include "network/ordering_gate.hpp" #include "simulator/verified_proposal_creator.hpp" #include "synchronizer/synchronizer.hpp" @@ -17,7 +18,7 @@ namespace iroha { std::shared_ptr synchronizer, std::shared_ptr proposal_creator, - logger::Logger log) + logger::LoggerPtr log) : ordering_gate_(std::move(ordering_gate)), synchronizer_(std::move(synchronizer)), proposal_creator_(std::move(proposal_creator)), diff --git a/irohad/network/impl/peer_communication_service_impl.hpp b/irohad/network/impl/peer_communication_service_impl.hpp index bbe4bac05c..007e54cb0f 100644 --- a/irohad/network/impl/peer_communication_service_impl.hpp +++ b/irohad/network/impl/peer_communication_service_impl.hpp @@ -8,7 +8,7 @@ #include "network/peer_communication_service.hpp" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" namespace iroha { namespace simulator { @@ -28,7 +28,7 @@ namespace iroha { std::shared_ptr ordering_gate, std::shared_ptr synchronizer, std::shared_ptr proposal_creator, - logger::Logger log = logger::log("PCS")); + logger::LoggerPtr log); void propagate_batch( std::shared_ptr batch) @@ -46,7 +46,7 @@ namespace iroha { std::shared_ptr ordering_gate_; std::shared_ptr synchronizer_; std::shared_ptr proposal_creator_; - logger::Logger log_; + logger::LoggerPtr log_; }; } // namespace network } // namespace iroha diff --git a/irohad/ordering/impl/on_demand_connection_manager.cpp b/irohad/ordering/impl/on_demand_connection_manager.cpp index f918b692db..5f3dc5eff3 100644 --- a/irohad/ordering/impl/on_demand_connection_manager.cpp +++ b/irohad/ordering/impl/on_demand_connection_manager.cpp @@ -7,6 +7,7 @@ #include #include "interfaces/iroha_internal/proposal.hpp" +#include "logger/logger.hpp" #include "ordering/impl/on_demand_common.hpp" using namespace iroha; @@ -15,7 +16,7 @@ using namespace iroha::ordering; OnDemandConnectionManager::OnDemandConnectionManager( std::shared_ptr factory, rxcpp::observable peers, - logger::Logger log) + logger::LoggerPtr log) : log_(std::move(log)), factory_(std::move(factory)), subscription_(peers.subscribe([this](const auto &peers) { @@ -29,7 +30,7 @@ OnDemandConnectionManager::OnDemandConnectionManager( std::shared_ptr factory, rxcpp::observable peers, CurrentPeers initial_peers, - logger::Logger log) + logger::LoggerPtr log) : OnDemandConnectionManager(std::move(factory), peers, std::move(log)) { // using start_with(initial_peers) results in deadlock initializeConnections(initial_peers); diff --git a/irohad/ordering/impl/on_demand_connection_manager.hpp b/irohad/ordering/impl/on_demand_connection_manager.hpp index 894de4e610..de4bfb9e12 100644 --- a/irohad/ordering/impl/on_demand_connection_manager.hpp +++ b/irohad/ordering/impl/on_demand_connection_manager.hpp @@ -11,7 +11,7 @@ #include #include -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" namespace iroha { namespace ordering { @@ -52,13 +52,13 @@ namespace iroha { OnDemandConnectionManager( std::shared_ptr factory, rxcpp::observable peers, - logger::Logger log = logger::log("OnDemandConnectionManager")); + logger::LoggerPtr log); OnDemandConnectionManager( std::shared_ptr factory, rxcpp::observable peers, CurrentPeers initial_peers, - logger::Logger log = logger::log("OnDemandConnectionManager")); + logger::LoggerPtr log); ~OnDemandConnectionManager() override; @@ -82,7 +82,7 @@ namespace iroha { */ void initializeConnections(const CurrentPeers &peers); - logger::Logger log_; + logger::LoggerPtr log_; std::shared_ptr factory_; rxcpp::composite_subscription subscription_; diff --git a/irohad/ordering/impl/on_demand_ordering_gate.cpp b/irohad/ordering/impl/on_demand_ordering_gate.cpp index 9e602ab071..a10664c40f 100644 --- a/irohad/ordering/impl/on_demand_ordering_gate.cpp +++ b/irohad/ordering/impl/on_demand_ordering_gate.cpp @@ -12,6 +12,7 @@ #include "ametsuchi/tx_presence_cache.hpp" #include "common/visitor.hpp" #include "interfaces/iroha_internal/transaction_batch_parser_impl.hpp" +#include "logger/logger.hpp" #include "ordering/impl/on_demand_common.hpp" using namespace iroha; @@ -25,7 +26,7 @@ OnDemandOrderingGate::OnDemandOrderingGate( std::shared_ptr factory, std::shared_ptr tx_cache, consensus::Round initial_round, - logger::Logger log) + logger::LoggerPtr log) : log_(std::move(log)), ordering_service_(std::move(ordering_service)), network_client_(std::move(network_client)), diff --git a/irohad/ordering/impl/on_demand_ordering_gate.hpp b/irohad/ordering/impl/on_demand_ordering_gate.hpp index 068c54f2a6..b23b643635 100644 --- a/irohad/ordering/impl/on_demand_ordering_gate.hpp +++ b/irohad/ordering/impl/on_demand_ordering_gate.hpp @@ -15,7 +15,7 @@ #include "interfaces/common_objects/types.hpp" #include "interfaces/iroha_internal/proposal.hpp" #include "interfaces/iroha_internal/unsafe_proposal_factory.hpp" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" #include "ordering/impl/ordering_gate_cache/ordering_gate_cache.hpp" #include "ordering/on_demand_ordering_service.hpp" @@ -63,7 +63,7 @@ namespace iroha { factory, std::shared_ptr tx_cache, consensus::Round initial_round, - logger::Logger log = logger::log("OnDemandOrderingGate")); + logger::LoggerPtr log); ~OnDemandOrderingGate() override; @@ -93,7 +93,7 @@ namespace iroha { std::shared_ptr proposal) const; - logger::Logger log_; + logger::LoggerPtr log_; std::shared_ptr ordering_service_; std::shared_ptr network_client_; rxcpp::composite_subscription events_subscription_; diff --git a/irohad/ordering/impl/on_demand_ordering_service_impl.cpp b/irohad/ordering/impl/on_demand_ordering_service_impl.cpp index a3fbfc9432..14575f4175 100644 --- a/irohad/ordering/impl/on_demand_ordering_service_impl.cpp +++ b/irohad/ordering/impl/on_demand_ordering_service_impl.cpp @@ -20,6 +20,7 @@ #include "interfaces/iroha_internal/proposal.hpp" #include "interfaces/iroha_internal/transaction_batch.hpp" #include "interfaces/transaction.hpp" +#include "logger/logger.hpp" using namespace iroha; using namespace iroha::ordering; @@ -30,9 +31,9 @@ OnDemandOrderingServiceImpl::OnDemandOrderingServiceImpl( std::shared_ptr proposal_factory, std::shared_ptr tx_cache, + logger::LoggerPtr log, size_t number_of_proposals, - const consensus::Round &initial_round, - logger::Logger log) + const consensus::Round &initial_round) : transaction_limit_(transaction_limit), number_of_proposals_(number_of_proposals), proposal_factory_(std::move(proposal_factory)), diff --git a/irohad/ordering/impl/on_demand_ordering_service_impl.hpp b/irohad/ordering/impl/on_demand_ordering_service_impl.hpp index 2881deb03a..744886f0d1 100644 --- a/irohad/ordering/impl/on_demand_ordering_service_impl.hpp +++ b/irohad/ordering/impl/on_demand_ordering_service_impl.hpp @@ -14,7 +14,7 @@ #include #include "interfaces/iroha_internal/unsafe_proposal_factory.hpp" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" #include "ordering/impl/on_demand_common.hpp" namespace iroha { @@ -30,20 +30,20 @@ namespace iroha { * proposal * @param proposal_factory - used to generate proposals * @param tx_cache - cache of transactions + * @param log to print progress * @param number_of_proposals - number of stored proposals, older will be * removed. Default value is 3 * @param initial_round - first round of agreement. * Default value is {2, kFirstRejectRound} since genesis block height is 1 - * @param log to print progress */ OnDemandOrderingServiceImpl( size_t transaction_limit, std::shared_ptr proposal_factory, std::shared_ptr tx_cache, + logger::LoggerPtr log, size_t number_of_proposals = 3, - const consensus::Round &initial_round = {2, kFirstRejectRound}, - logger::Logger log = logger::log("OnDemandOrderingServiceImpl")); + const consensus::Round &initial_round = {2, kFirstRejectRound}); // --------------------- | OnDemandOrderingService |_--------------------- @@ -123,7 +123,7 @@ namespace iroha { /** * Logger instance */ - logger::Logger log_; + logger::LoggerPtr log_; }; } // namespace ordering } // namespace iroha diff --git a/irohad/ordering/impl/on_demand_os_client_grpc.cpp b/irohad/ordering/impl/on_demand_os_client_grpc.cpp index e55700e716..273b2cc31a 100644 --- a/irohad/ordering/impl/on_demand_os_client_grpc.cpp +++ b/irohad/ordering/impl/on_demand_os_client_grpc.cpp @@ -9,6 +9,7 @@ #include "backend/protobuf/transaction.hpp" #include "interfaces/common_objects/peer.hpp" #include "interfaces/iroha_internal/transaction_batch.hpp" +#include "logger/logger.hpp" #include "network/impl/grpc_channel_builder.hpp" using namespace iroha; @@ -22,7 +23,7 @@ OnDemandOsClientGrpc::OnDemandOsClientGrpc( std::shared_ptr proposal_factory, std::function time_provider, std::chrono::milliseconds proposal_request_timeout, - logger::Logger log) + logger::LoggerPtr log) : log_(std::move(log)), stub_(std::move(stub)), async_call_(std::move(async_call)), @@ -86,11 +87,13 @@ OnDemandOsClientGrpcFactory::OnDemandOsClientGrpcFactory( async_call, std::shared_ptr proposal_factory, std::function time_provider, - OnDemandOsClientGrpc::TimeoutType proposal_request_timeout) + OnDemandOsClientGrpc::TimeoutType proposal_request_timeout, + logger::LoggerPtr client_log) : async_call_(std::move(async_call)), proposal_factory_(std::move(proposal_factory)), time_provider_(time_provider), - proposal_request_timeout_(proposal_request_timeout) {} + proposal_request_timeout_(proposal_request_timeout), + client_log_(std::move(client_log)) {} std::unique_ptr OnDemandOsClientGrpcFactory::create( const shared_model::interface::Peer &to) { @@ -100,5 +103,5 @@ std::unique_ptr OnDemandOsClientGrpcFactory::create( proposal_factory_, time_provider_, proposal_request_timeout_, - logger::log("OnDemandOsClientGrpc")); + client_log_); } diff --git a/irohad/ordering/impl/on_demand_os_client_grpc.hpp b/irohad/ordering/impl/on_demand_os_client_grpc.hpp index a41e69193a..6543d9e42f 100644 --- a/irohad/ordering/impl/on_demand_os_client_grpc.hpp +++ b/irohad/ordering/impl/on_demand_os_client_grpc.hpp @@ -9,6 +9,7 @@ #include "ordering/on_demand_os_transport.hpp" #include "interfaces/iroha_internal/abstract_transport_factory.hpp" +#include "logger/logger_fwd.hpp" #include "network/impl/async_grpc_client.hpp" #include "ordering.grpc.pb.h" @@ -39,7 +40,7 @@ namespace iroha { std::shared_ptr proposal_factory, std::function time_provider, std::chrono::milliseconds proposal_request_timeout, - logger::Logger log = logger::log("OnDemandOsClientGrpc")); + logger::LoggerPtr log); void onBatches(consensus::Round round, CollectionType batches) override; @@ -47,7 +48,7 @@ namespace iroha { consensus::Round round) override; private: - logger::Logger log_; + logger::LoggerPtr log_; std::unique_ptr stub_; std::shared_ptr> async_call_; @@ -64,7 +65,8 @@ namespace iroha { async_call, std::shared_ptr proposal_factory, std::function time_provider, - OnDemandOsClientGrpc::TimeoutType proposal_request_timeout); + OnDemandOsClientGrpc::TimeoutType proposal_request_timeout, + logger::LoggerPtr client_log); /** * Create connection with insecure gRPC channel defined by @@ -81,6 +83,7 @@ namespace iroha { std::shared_ptr proposal_factory_; std::function time_provider_; std::chrono::milliseconds proposal_request_timeout_; + logger::LoggerPtr client_log_; }; } // namespace transport diff --git a/irohad/ordering/impl/on_demand_os_server_grpc.cpp b/irohad/ordering/impl/on_demand_os_server_grpc.cpp index 444f04f134..dc668b7433 100644 --- a/irohad/ordering/impl/on_demand_os_server_grpc.cpp +++ b/irohad/ordering/impl/on_demand_os_server_grpc.cpp @@ -12,6 +12,7 @@ #include "backend/protobuf/proposal.hpp" #include "common/bind.hpp" #include "interfaces/iroha_internal/transaction_batch.hpp" +#include "logger/logger.hpp" using namespace iroha::ordering; using namespace iroha::ordering::transport; @@ -23,7 +24,7 @@ OnDemandOsServerGrpc::OnDemandOsServerGrpc( batch_parser, std::shared_ptr transaction_batch_factory, - logger::Logger log) + logger::LoggerPtr log) : ordering_service_(ordering_service), transaction_factory_(std::move(transaction_factory)), batch_parser_(std::move(batch_parser)), diff --git a/irohad/ordering/impl/on_demand_os_server_grpc.hpp b/irohad/ordering/impl/on_demand_os_server_grpc.hpp index b9cc974a1c..144fd9ab5d 100644 --- a/irohad/ordering/impl/on_demand_os_server_grpc.hpp +++ b/irohad/ordering/impl/on_demand_os_server_grpc.hpp @@ -11,7 +11,7 @@ #include "interfaces/iroha_internal/abstract_transport_factory.hpp" #include "interfaces/iroha_internal/transaction_batch_factory.hpp" #include "interfaces/iroha_internal/transaction_batch_parser.hpp" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" #include "ordering.grpc.pb.h" namespace iroha { @@ -35,7 +35,7 @@ namespace iroha { batch_parser, std::shared_ptr transaction_batch_factory, - logger::Logger log = logger::log("OnDemandOsServerGrpc")); + logger::LoggerPtr log); grpc::Status SendBatches(::grpc::ServerContext *context, const proto::BatchesRequest *request, @@ -61,7 +61,7 @@ namespace iroha { std::shared_ptr batch_factory_; - logger::Logger log_; + logger::LoggerPtr log_; }; } // namespace transport diff --git a/irohad/simulator/impl/simulator.cpp b/irohad/simulator/impl/simulator.cpp index 61614c0e39..7e5c8ee362 100644 --- a/irohad/simulator/impl/simulator.cpp +++ b/irohad/simulator/impl/simulator.cpp @@ -9,6 +9,7 @@ #include "common/bind.hpp" #include "interfaces/iroha_internal/block.hpp" #include "interfaces/iroha_internal/proposal.hpp" +#include "logger/logger.hpp" namespace iroha { namespace simulator { @@ -21,7 +22,7 @@ namespace iroha { std::shared_ptr crypto_signer, std::unique_ptr block_factory, - logger::Logger log) + logger::LoggerPtr log) : validator_(std::move(statefulValidator)), ametsuchi_factory_(std::move(factory)), block_query_factory_(block_query_factory), diff --git a/irohad/simulator/impl/simulator.hpp b/irohad/simulator/impl/simulator.hpp index 5d8ee91665..9e392f0dba 100644 --- a/irohad/simulator/impl/simulator.hpp +++ b/irohad/simulator/impl/simulator.hpp @@ -14,7 +14,7 @@ #include "ametsuchi/temporary_factory.hpp" #include "cryptography/crypto_provider/abstract_crypto_model_signer.hpp" #include "interfaces/iroha_internal/unsafe_block_factory.hpp" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" #include "network/ordering_gate.hpp" #include "validation/stateful_validator.hpp" @@ -34,7 +34,7 @@ namespace iroha { std::shared_ptr crypto_signer, std::unique_ptr block_factory, - logger::Logger log = logger::log("Simulator")); + logger::LoggerPtr log); ~Simulator() override; @@ -67,7 +67,7 @@ namespace iroha { std::unique_ptr block_factory_; - logger::Logger log_; + logger::LoggerPtr log_; // last block std::shared_ptr last_block; diff --git a/irohad/synchronizer/impl/synchronizer_impl.cpp b/irohad/synchronizer/impl/synchronizer_impl.cpp index 6af915e2dc..e5b2dd2e23 100644 --- a/irohad/synchronizer/impl/synchronizer_impl.cpp +++ b/irohad/synchronizer/impl/synchronizer_impl.cpp @@ -11,6 +11,7 @@ #include "ametsuchi/mutable_storage.hpp" #include "common/visitor.hpp" #include "interfaces/iroha_internal/block.hpp" +#include "logger/logger.hpp" namespace iroha { namespace synchronizer { @@ -21,7 +22,7 @@ namespace iroha { std::shared_ptr mutable_factory, std::shared_ptr block_query_factory, std::shared_ptr block_loader, - logger::Logger log) + logger::LoggerPtr log) : validator_(std::move(validator)), mutable_factory_(std::move(mutable_factory)), block_query_factory_(std::move(block_query_factory)), diff --git a/irohad/synchronizer/impl/synchronizer_impl.hpp b/irohad/synchronizer/impl/synchronizer_impl.hpp index aaa816cd1f..abe2eaaef5 100644 --- a/irohad/synchronizer/impl/synchronizer_impl.hpp +++ b/irohad/synchronizer/impl/synchronizer_impl.hpp @@ -10,7 +10,7 @@ #include "ametsuchi/mutable_factory.hpp" #include "ametsuchi/peer_query_factory.hpp" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" #include "network/block_loader.hpp" #include "network/consensus_gate.hpp" #include "validation/chain_validator.hpp" @@ -31,7 +31,7 @@ namespace iroha { std::shared_ptr mutable_factory, std::shared_ptr block_query_factory, std::shared_ptr block_loader, - logger::Logger log = logger::log("Synchronizer")); + logger::LoggerPtr log); ~SynchronizerImpl() override; @@ -67,7 +67,7 @@ namespace iroha { rxcpp::subjects::subject notifier_; rxcpp::composite_subscription subscription_; - logger::Logger log_; + logger::LoggerPtr log_; }; } // namespace synchronizer diff --git a/irohad/torii/command_client.hpp b/irohad/torii/command_client.hpp index 23d71f4608..02ced79deb 100644 --- a/irohad/torii/command_client.hpp +++ b/irohad/torii/command_client.hpp @@ -11,7 +11,7 @@ #include #include -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" namespace torii { @@ -22,7 +22,7 @@ namespace torii { public: CommandSyncClient( std::unique_ptr stub, - logger::Logger log = logger::log("CommandSyncClient")); + logger::LoggerPtr log); /** * requests tx to a torii server and returns response (blocking, sync) @@ -58,7 +58,7 @@ namespace torii { private: std::unique_ptr stub_; - logger::Logger log_; + logger::LoggerPtr log_; }; } // namespace torii diff --git a/irohad/torii/impl/command_client.cpp b/irohad/torii/impl/command_client.cpp index 847efc7baa..b528212b59 100644 --- a/irohad/torii/impl/command_client.cpp +++ b/irohad/torii/impl/command_client.cpp @@ -8,6 +8,7 @@ #include #include "common/byteutils.hpp" +#include "logger/logger.hpp" #include "torii/command_client.hpp" #include "transaction.pb.h" @@ -18,7 +19,7 @@ namespace torii { CommandSyncClient::CommandSyncClient( std::unique_ptr stub, - logger::Logger log) + logger::LoggerPtr log) : stub_(std::move(stub)), log_(std::move(log)) {} grpc::Status CommandSyncClient::Torii(const Transaction &tx) const { diff --git a/irohad/torii/impl/command_service_impl.cpp b/irohad/torii/impl/command_service_impl.cpp index ddb56693cb..863c1d240b 100644 --- a/irohad/torii/impl/command_service_impl.cpp +++ b/irohad/torii/impl/command_service_impl.cpp @@ -13,6 +13,7 @@ #include "common/visitor.hpp" #include "interfaces/iroha_internal/transaction_batch.hpp" #include "interfaces/transaction_responses/not_received_tx_response.hpp" +#include "logger/logger.hpp" namespace iroha { namespace torii { @@ -25,7 +26,7 @@ namespace iroha { status_factory, std::shared_ptr cache, std::shared_ptr tx_presence_cache, - logger::Logger log) + logger::LoggerPtr log) : tx_processor_(std::move(tx_processor)), storage_(std::move(storage)), status_bus_(std::move(status_bus)), diff --git a/irohad/torii/impl/command_service_impl.hpp b/irohad/torii/impl/command_service_impl.hpp index 7306ea2092..52ffa9b1cd 100644 --- a/irohad/torii/impl/command_service_impl.hpp +++ b/irohad/torii/impl/command_service_impl.hpp @@ -13,7 +13,7 @@ #include "cache/cache.hpp" #include "cryptography/hash.hpp" #include "interfaces/iroha_internal/tx_status_factory.hpp" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" #include "torii/processor/transaction_processor.hpp" #include "torii/status_bus.hpp" @@ -47,7 +47,7 @@ namespace iroha { status_factory, std::shared_ptr cache, std::shared_ptr tx_presence_cache, - logger::Logger log = logger::log("CommandServiceImpl")); + logger::LoggerPtr log); ~CommandServiceImpl() override; @@ -105,7 +105,7 @@ namespace iroha { rxcpp::composite_subscription status_subscription_; - logger::Logger log_; + logger::LoggerPtr log_; }; } // namespace torii diff --git a/irohad/torii/impl/command_service_transport_grpc.cpp b/irohad/torii/impl/command_service_transport_grpc.cpp index 0374a53ffe..91636ae534 100644 --- a/irohad/torii/impl/command_service_transport_grpc.cpp +++ b/irohad/torii/impl/command_service_transport_grpc.cpp @@ -21,6 +21,7 @@ #include "interfaces/iroha_internal/transaction_batch_parser.hpp" #include "interfaces/iroha_internal/tx_status_factory.hpp" #include "interfaces/transaction.hpp" +#include "logger/logger.hpp" #include "torii/status_bus.hpp" namespace iroha { @@ -38,7 +39,7 @@ namespace iroha { transaction_batch_factory, rxcpp::observable consensus_gate_objects, int maximum_rounds_without_update, - logger::Logger log) + logger::LoggerPtr log) : command_service_(std::move(command_service)), status_bus_(std::move(status_bus)), status_factory_(std::move(status_factory)), diff --git a/irohad/torii/impl/command_service_transport_grpc.hpp b/irohad/torii/impl/command_service_transport_grpc.hpp index 45f03864e0..9c64e9d2f8 100644 --- a/irohad/torii/impl/command_service_transport_grpc.hpp +++ b/irohad/torii/impl/command_service_transport_grpc.hpp @@ -12,7 +12,7 @@ #include "endpoint.pb.h" #include "interfaces/common_objects/transaction_sequence_common.hpp" #include "interfaces/iroha_internal/abstract_transport_factory.hpp" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" namespace iroha { namespace torii { @@ -65,7 +65,7 @@ namespace iroha { transaction_batch_factory, rxcpp::observable consensus_gate_objects, int maximum_rounds_without_update, - logger::Logger log = logger::log("CommandServiceTransportGrpc")); + logger::LoggerPtr log); /** * Torii call via grpc @@ -132,7 +132,7 @@ namespace iroha { batch_parser_; std::shared_ptr batch_factory_; - logger::Logger log_; + logger::LoggerPtr log_; rxcpp::observable consensus_gate_objects_; const int maximum_rounds_without_update_; diff --git a/irohad/torii/impl/query_service.cpp b/irohad/torii/impl/query_service.cpp index 44f3373f8d..8e6abf3d3b 100644 --- a/irohad/torii/impl/query_service.cpp +++ b/irohad/torii/impl/query_service.cpp @@ -9,6 +9,7 @@ #include "backend/protobuf/query_responses/proto_query_response.hpp" #include "cryptography/default_hash_provider.hpp" #include "interfaces/iroha_internal/abstract_transport_factory.hpp" +#include "logger/logger.hpp" #include "validators/default_validator.hpp" namespace iroha { @@ -17,7 +18,7 @@ namespace iroha { QueryService::QueryService( std::shared_ptr query_processor, std::shared_ptr query_factory, - logger::Logger log) + logger::LoggerPtr log) : query_processor_{std::move(query_processor)}, query_factory_{std::move(query_factory)}, log_{std::move(log)} {} diff --git a/irohad/torii/processor/impl/query_processor_impl.cpp b/irohad/torii/processor/impl/query_processor_impl.cpp index 90641a8cb6..b52c23cc1b 100644 --- a/irohad/torii/processor/impl/query_processor_impl.cpp +++ b/irohad/torii/processor/impl/query_processor_impl.cpp @@ -12,6 +12,7 @@ #include "interfaces/query_responses/block_query_response.hpp" #include "interfaces/query_responses/block_response.hpp" #include "interfaces/query_responses/query_response.hpp" +#include "logger/logger.hpp" #include "validation/utils.hpp" namespace iroha { @@ -23,7 +24,7 @@ namespace iroha { std::shared_ptr pending_transactions, std::shared_ptr response_factory, - logger::Logger log) + logger::LoggerPtr log) : storage_{std::move(storage)}, qry_exec_{std::move(qry_exec)}, pending_transactions_{std::move(pending_transactions)}, diff --git a/irohad/torii/processor/impl/transaction_processor_impl.cpp b/irohad/torii/processor/impl/transaction_processor_impl.cpp index cf009a0e86..d00f7f86c4 100644 --- a/irohad/torii/processor/impl/transaction_processor_impl.cpp +++ b/irohad/torii/processor/impl/transaction_processor_impl.cpp @@ -11,6 +11,7 @@ #include "interfaces/iroha_internal/proposal.hpp" #include "interfaces/iroha_internal/transaction_batch.hpp" #include "interfaces/iroha_internal/transaction_sequence.hpp" +#include "logger/logger.hpp" #include "validation/stateful_validator_common.hpp" namespace iroha { @@ -48,7 +49,7 @@ namespace iroha { std::shared_ptr status_bus, std::shared_ptr status_factory, - logger::Logger log) + logger::LoggerPtr log) : pcs_(std::move(pcs)), mst_processor_(std::move(mst_processor)), status_bus_(std::move(status_bus)), diff --git a/irohad/torii/processor/query_processor_impl.hpp b/irohad/torii/processor/query_processor_impl.hpp index e7e40f1bef..d4e8e2985a 100644 --- a/irohad/torii/processor/query_processor_impl.hpp +++ b/irohad/torii/processor/query_processor_impl.hpp @@ -8,7 +8,7 @@ #include "ametsuchi/storage.hpp" #include "interfaces/iroha_internal/query_response_factory.hpp" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" #include "torii/processor/query_processor.hpp" namespace iroha { @@ -26,7 +26,7 @@ namespace iroha { pending_transactions, std::shared_ptr response_factory, - logger::Logger log = logger::log("QueryProcessorImpl")); + logger::LoggerPtr log); std::unique_ptr queryHandle( const shared_model::interface::Query &qry) override; @@ -46,7 +46,7 @@ namespace iroha { std::shared_ptr response_factory_; - logger::Logger log_; + logger::LoggerPtr log_; }; } // namespace torii diff --git a/irohad/torii/processor/transaction_processor_impl.hpp b/irohad/torii/processor/transaction_processor_impl.hpp index 107071ef6a..9258e01605 100644 --- a/irohad/torii/processor/transaction_processor_impl.hpp +++ b/irohad/torii/processor/transaction_processor_impl.hpp @@ -14,7 +14,7 @@ #include "interfaces/common_objects/transaction_sequence_common.hpp" #include "interfaces/iroha_internal/tx_status_factory.hpp" #include "interfaces/transaction_responses/tx_response.hpp" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" #include "multi_sig_transactions/mst_processor.hpp" #include "network/peer_communication_service.hpp" #include "torii/status_bus.hpp" @@ -36,7 +36,7 @@ namespace iroha { std::shared_ptr status_bus, std::shared_ptr status_factory, - logger::Logger log = logger::log("TxProcessor")); + logger::LoggerPtr log); void batchHandle( std::shared_ptr @@ -62,7 +62,7 @@ namespace iroha { // creates transaction status std::shared_ptr status_factory_; - logger::Logger log_; + logger::LoggerPtr log_; // TODO: [IR-1665] Akvinikym 29.08.18: Refactor method publishStatus(..) /** diff --git a/irohad/torii/query_service.hpp b/irohad/torii/query_service.hpp index 11f979034e..e6d1382781 100644 --- a/irohad/torii/query_service.hpp +++ b/irohad/torii/query_service.hpp @@ -15,10 +15,9 @@ #include "backend/protobuf/queries/proto_query.hpp" #include "builders/protobuf/transport_builder.hpp" #include "cache/cache.hpp" +#include "logger/logger_fwd.hpp" #include "torii/processor/query_processor.hpp" -#include "logger/logger.hpp" - namespace shared_model { namespace interface { template @@ -43,7 +42,7 @@ namespace iroha { QueryService( std::shared_ptr query_processor, std::shared_ptr query_factory, - logger::Logger log = logger::log("Query Service")); + logger::LoggerPtr log); QueryService(const QueryService &) = delete; QueryService &operator=(const QueryService &) = delete; @@ -76,7 +75,7 @@ namespace iroha { shared_model::crypto::Hash::Hasher> cache_; - logger::Logger log_; + logger::LoggerPtr log_; }; } // namespace torii } // namespace iroha diff --git a/irohad/validation/impl/chain_validator_impl.cpp b/irohad/validation/impl/chain_validator_impl.cpp index dc07ad5b80..e74ffbec0a 100644 --- a/irohad/validation/impl/chain_validator_impl.cpp +++ b/irohad/validation/impl/chain_validator_impl.cpp @@ -11,6 +11,7 @@ #include "cryptography/public_key.hpp" #include "interfaces/common_objects/peer.hpp" #include "interfaces/iroha_internal/block.hpp" +#include "logger/logger.hpp" #include "validation/utils.hpp" namespace iroha { @@ -18,8 +19,9 @@ namespace iroha { ChainValidatorImpl::ChainValidatorImpl( std::shared_ptr supermajority_checker, - logger::Logger log) - : supermajority_checker_(supermajority_checker), log_(std::move(log)) {} + logger::LoggerPtr log) + : supermajority_checker_(supermajority_checker), + log_(std::move(log)) {} bool ChainValidatorImpl::validateAndApply( rxcpp::observable> diff --git a/irohad/validation/impl/chain_validator_impl.hpp b/irohad/validation/impl/chain_validator_impl.hpp index 7287f78a40..af42fc8e3c 100644 --- a/irohad/validation/impl/chain_validator_impl.hpp +++ b/irohad/validation/impl/chain_validator_impl.hpp @@ -11,7 +11,7 @@ #include #include "interfaces/common_objects/types.hpp" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" namespace shared_model { namespace interface { @@ -34,10 +34,9 @@ namespace iroha { namespace validation { class ChainValidatorImpl : public ChainValidator { public: - explicit ChainValidatorImpl( - std::shared_ptr - supermajority_checker, - logger::Logger log = logger::log("ChainValidator")); + ChainValidatorImpl(std::shared_ptr + supermajority_checker, + logger::LoggerPtr log); bool validateAndApply( rxcpp::observable> @@ -71,7 +70,7 @@ namespace iroha { std::shared_ptr supermajority_checker_; - logger::Logger log_; + logger::LoggerPtr log_; }; } // namespace validation } // namespace iroha diff --git a/irohad/validation/impl/stateful_validator_impl.cpp b/irohad/validation/impl/stateful_validator_impl.cpp index 860c096cb4..a80ed18f24 100644 --- a/irohad/validation/impl/stateful_validator_impl.cpp +++ b/irohad/validation/impl/stateful_validator_impl.cpp @@ -14,6 +14,7 @@ #include #include "common/result.hpp" #include "interfaces/iroha_internal/batch_meta.hpp" +#include "logger/logger.hpp" #include "validation/utils.hpp" namespace iroha { @@ -46,7 +47,6 @@ namespace iroha { * @param txs to be validated * @param temporary_wsv to apply transactions on * @param transactions_errors_log to write errors to - * @param log to write errors to console * @param batch_parser to parse batches from transaction range * @return range of transactions, which passed stateful validation */ @@ -54,7 +54,6 @@ namespace iroha { const shared_model::interface::types::TransactionsCollectionType &txs, ametsuchi::TemporaryWsv &temporary_wsv, validation::TransactionsErrors &transactions_errors_log, - const logger::Logger &log, const shared_model::interface::TransactionBatchParser &batch_parser) { std::vector validation_results; validation_results.reserve(boost::size(txs)); @@ -117,7 +116,7 @@ namespace iroha { std::unique_ptr factory, std::shared_ptr batch_parser, - logger::Logger log) + logger::LoggerPtr log) : factory_(std::move(factory)), batch_parser_(std::move(batch_parser)), log_(std::move(log)) {} @@ -134,7 +133,6 @@ namespace iroha { validateTransactions(proposal.transactions(), temporaryWsv, validation_result->rejected_transactions, - log_, *batch_parser_); // Since proposal came from ordering gate it was already validated. diff --git a/irohad/validation/impl/stateful_validator_impl.hpp b/irohad/validation/impl/stateful_validator_impl.hpp index 54f5e332e9..53c750df0f 100644 --- a/irohad/validation/impl/stateful_validator_impl.hpp +++ b/irohad/validation/impl/stateful_validator_impl.hpp @@ -10,7 +10,7 @@ #include "interfaces/iroha_internal/transaction_batch_parser.hpp" #include "interfaces/iroha_internal/unsafe_proposal_factory.hpp" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" namespace iroha { namespace validation { @@ -25,7 +25,7 @@ namespace iroha { factory, std::shared_ptr batch_parser, - logger::Logger log = logger::log("SFV")); + logger::LoggerPtr log); std::unique_ptr validate( const shared_model::interface::Proposal &proposal, @@ -35,7 +35,7 @@ namespace iroha { std::unique_ptr factory_; std::shared_ptr batch_parser_; - logger::Logger log_; + logger::LoggerPtr log_; }; } // namespace validation diff --git a/libs/common/files.cpp b/libs/common/files.cpp index 075254a7f4..e62aac6d37 100644 --- a/libs/common/files.cpp +++ b/libs/common/files.cpp @@ -10,31 +10,31 @@ #include #include "logger/logger.hpp" -void iroha::remove_dir_contents(const std::string &dump_dir) { - auto log = logger::log("common::remove_all"); +void iroha::remove_dir_contents(const std::string &dir, + const logger::LoggerPtr &log) { boost::system::error_code error_code; - bool exists = boost::filesystem::exists(dump_dir, error_code); + bool exists = boost::filesystem::exists(dir, error_code); if (error_code != boost::system::errc::success) { log->error(error_code.message()); return; } if (not exists) { - log->error("Directory does not exist {}", dump_dir); + log->error("Directory does not exist {}", dir); return; } - bool is_dir = boost::filesystem::is_directory(dump_dir, error_code); + bool is_dir = boost::filesystem::is_directory(dir, error_code); if (error_code != boost::system::errc::success) { log->error(error_code.message()); return; } if (not is_dir) { - log->error("{} is not a directory", dump_dir); + log->error("{} is not a directory", dir); return; } - for (auto entry : boost::filesystem::directory_iterator(dump_dir)) { + for (auto entry : boost::filesystem::directory_iterator(dir)) { boost::filesystem::remove_all(entry.path(), error_code); if (error_code != boost::system::errc::success) log->error(error_code.message()); diff --git a/libs/common/files.hpp b/libs/common/files.hpp index 9254ed78c4..a2a26ae7bf 100644 --- a/libs/common/files.hpp +++ b/libs/common/files.hpp @@ -8,6 +8,8 @@ #include +#include "logger/logger_fwd.hpp" + /** * This source file contains common methods related to files */ @@ -16,8 +18,10 @@ namespace iroha { /** * Remove all files and directories inside a folder. * Keeps the target folder. - * @param dump_dir - target folder + * @param dir - target folder + * @param log - a log for local messages */ - void remove_dir_contents(const std::string &dump_dir); + void remove_dir_contents(const std::string &dir, + const logger::LoggerPtr &log); } // namespace iroha #endif // IROHA_FILES_HPP diff --git a/libs/crypto/keys_manager_impl.cpp b/libs/crypto/keys_manager_impl.cpp index ccdaf027db..24ca93d58a 100644 --- a/libs/crypto/keys_manager_impl.cpp +++ b/libs/crypto/keys_manager_impl.cpp @@ -45,7 +45,7 @@ namespace iroha { KeysManagerImpl::KeysManagerImpl( const std::string &account_id, const boost::filesystem::path &path_to_keypair, - logger::Logger log) + logger::LoggerPtr log) : path_to_keypair_(path_to_keypair), account_id_(account_id), log_(std::move(log)) {} @@ -56,7 +56,7 @@ namespace iroha { * account_id. */ KeysManagerImpl::KeysManagerImpl(const std::string account_id, - logger::Logger log) + logger::LoggerPtr log) : KeysManagerImpl(account_id, "", std::move(log)) {} bool KeysManagerImpl::validate(const Keypair &keypair) const { diff --git a/libs/crypto/keys_manager_impl.hpp b/libs/crypto/keys_manager_impl.hpp index d376485653..3e9b7ea89e 100644 --- a/libs/crypto/keys_manager_impl.hpp +++ b/libs/crypto/keys_manager_impl.hpp @@ -11,7 +11,7 @@ #include #include #include "cryptography/keypair.hpp" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" namespace iroha { @@ -26,15 +26,14 @@ namespace iroha { */ KeysManagerImpl(const std::string &account_id, const boost::filesystem::path &path_to_keypair, - logger::Logger log = logger::log("KeysManagerImpl")); + logger::LoggerPtr log); /** * Initialize key manager for a specific account * @param account_id - fully qualified account id, e.g. admin@test * @param log to print progress */ - KeysManagerImpl(const std::string account_id, - logger::Logger log = logger::log("KeysManagerImpl")); + KeysManagerImpl(const std::string account_id, logger::LoggerPtr log); bool createKeys() override; @@ -74,7 +73,7 @@ namespace iroha { boost::filesystem::path path_to_keypair_; std::string account_id_; - logger::Logger log_; + logger::LoggerPtr log_; }; } // namespace iroha #endif // IROHA_KEYS_MANAGER_IMPL_HPP diff --git a/libs/logger/CMakeLists.txt b/libs/logger/CMakeLists.txt index f9fadec2cb..2f4df236fc 100644 --- a/libs/logger/CMakeLists.txt +++ b/libs/logger/CMakeLists.txt @@ -3,7 +3,21 @@ # SPDX-License-Identifier: Apache-2.0 # -add_library(logger STATIC logger.cpp) +add_library(logger + logger.cpp + logger_spdlog.cpp +) target_link_libraries(logger + fmt + spdlog +) +# ensure that externals are downloaded before any dependants +add_dependencies(logger + fmt spdlog ) + +add_library(logger_manager logger_manager.cpp) +target_link_libraries(logger_manager + logger +) diff --git a/libs/logger/dummy_logger.hpp b/libs/logger/dummy_logger.hpp new file mode 100644 index 0000000000..152c4cba27 --- /dev/null +++ b/libs/logger/dummy_logger.hpp @@ -0,0 +1,29 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef LOGGER_DUMMY_LOGGER_HPP +#define LOGGER_DUMMY_LOGGER_HPP + +#include "logger/logger.hpp" + +namespace logger { + + class DummyLogger : public Logger { + protected: + void logInternal(Level level, const std::string &s) const override {} + + bool shouldLog(Level level) const override { + return false; + } + }; + + LoggerPtr getDummyLoggerPtr() { + static std::shared_ptr log = std::make_shared(); + return log; + } + +} // namespace logger + +#endif // LOGGER_DUMMY_LOGGER_HPP diff --git a/libs/logger/logger.cpp b/libs/logger/logger.cpp index 2fd87dcc05..950d0368ec 100644 --- a/libs/logger/logger.cpp +++ b/libs/logger/logger.cpp @@ -6,61 +6,11 @@ #include "logger/logger.hpp" namespace logger { - const std::string end = "\033[0m"; - std::string red(const std::string &string) { - const std::string red_start = "\033[31m"; - return red_start + string + end; - } - - std::string yellow(const std::string &string) { - const std::string yellow_start = "\033[33m"; - return yellow_start + string + end; - } - - std::string output(const std::string &string) { - return yellow("---> " + string); - } - - std::string input(const std::string &string) { - return red("<--- " + string); - } - - static void setGlobalPattern(spdlog::logger &logger) { - logger.set_pattern("[%Y-%m-%d %H:%M:%S.%F] %n %v"); - } - - static void setDebugPattern(spdlog::logger &logger) { - logger.set_pattern("[%Y-%m-%d %H:%M:%S.%F][th:%t][%l] %n %v"); - } - - static std::shared_ptr createLogger(const std::string &tag, - bool debug_mode = true) { - auto logger = spdlog::stdout_color_mt(tag); - if (debug_mode) { - setDebugPattern(*logger); - } else { - setGlobalPattern(*logger); - } - return logger; - } - - Logger log(const std::string &tag) { - static std::mutex mutex; - std::lock_guard lock(mutex); - auto logger = spdlog::get(tag); - if (logger == nullptr) { - logger = createLogger(tag); - } - return logger; - } - - Logger testLog(const std::string &tag) { - return log(tag); - } + const LogLevel kDefaultLogLevel = LogLevel::kInfo; std::string boolRepr(bool value) { return value ? "true" : "false"; } -} // namespace logger +} diff --git a/libs/logger/logger.hpp b/libs/logger/logger.hpp index 03e0404a76..d9cf8a2a33 100644 --- a/libs/logger/logger.hpp +++ b/libs/logger/logger.hpp @@ -3,8 +3,10 @@ * SPDX-License-Identifier: Apache-2.0 */ -#ifndef IROHA_SPDLOG_LOGGER_LOGGER_HPP -#define IROHA_SPDLOG_LOGGER_LOGGER_HPP +#ifndef IROHA_LOGGER_LOGGER_HPP +#define IROHA_LOGGER_LOGGER_HPP + +#include "logger/logger_fwd.hpp" #include #include // for std::accumulate @@ -18,35 +20,79 @@ auto operator<<(StreamType &os, const T &object) return os << object.toString(); } -#include -#include +#include +#include namespace logger { - using Logger = std::shared_ptr; - - std::string red(const std::string &string); - - std::string yellow(const std::string &string); - - std::string output(const std::string &string); - - std::string input(const std::string &string); - - /** - * Provide logger object - * @param tag - tagging name for identifiing logger - * @return logger object - */ - Logger log(const std::string &tag); - - /** - * Provide logger for using in test purposes; - * This logger write data only for console - * @param tag - tagging name for identifiing logger - * @return logger object - */ - Logger testLog(const std::string &tag); + enum class LogLevel; + + extern const LogLevel kDefaultLogLevel; + + /// Log levels + enum class LogLevel { + kTrace, + kDebug, + kInfo, + kWarn, + kError, + kCritical, + }; + + class Logger { + public: + using Level = LogLevel; + + virtual ~Logger() = default; + + // --- Logging functions --- + + template + void trace(const std::string &format, const Args &... args) const { + log(LogLevel::kTrace, format, args...); + } + + template + void debug(const std::string &format, const Args &... args) const { + log(LogLevel::kDebug, format, args...); + } + + template + void info(const std::string &format, const Args &... args) const { + log(LogLevel::kInfo, format, args...); + } + + template + void warn(const std::string &format, const Args &... args) const { + log(LogLevel::kWarn, format, args...); + } + + template + void error(const std::string &format, const Args &... args) const { + log(LogLevel::kError, format, args...); + } + + template + void critical(const std::string &format, const Args &... args) const { + log(LogLevel::kCritical, format, args...); + } + + template + void log(Level level, + const std::string &format, + const Args &... args) const { + if (shouldLog(level)) { + logInternal(level, fmt::format(format, args...)); + } + } + + protected: + virtual void logInternal(Level level, const std::string &s) const = 0; + + /// Whether the configured logging level is at least as verbose as the + /// one given in parameter. + virtual bool shouldLog(Level level) const = 0; + }; /** * Convert bool value to human readable string repr @@ -111,4 +157,4 @@ namespace logger { } // namespace logger -#endif +#endif // IROHA_LOGGER_LOGGER_HPP diff --git a/libs/logger/logger_fwd.hpp b/libs/logger/logger_fwd.hpp new file mode 100644 index 0000000000..a267b0caf0 --- /dev/null +++ b/libs/logger/logger_fwd.hpp @@ -0,0 +1,26 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef IROHA_LOGGER_LOGGER_FWD_HPP +#define IROHA_LOGGER_LOGGER_FWD_HPP + +#include + +/* It is preferable to include this header in files that do not contain + * dereferencing of LoggerPtr and do not use the Logger class functions, because + * the actual Logger class definition contains template member functions that + * use template library functions, thus making the preprocessed source file much + * bigger. + */ + +namespace logger { + + class Logger; + + using LoggerPtr = std::shared_ptr; + +} // namespace logger + +#endif // IROHA_LOGGER_LOGGER_FWD_HPP diff --git a/libs/logger/logger_manager.cpp b/libs/logger/logger_manager.cpp new file mode 100644 index 0000000000..04297f7de1 --- /dev/null +++ b/libs/logger/logger_manager.cpp @@ -0,0 +1,83 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "logger/logger_manager.hpp" + +#include + +static const std::string kTagHierarchySeparator = "/"; + +static inline std::string joinTags(const std::string &parent, + const std::string &child) { + return parent.empty() ? child : parent + kTagHierarchySeparator + child; +} + +namespace logger { + + LoggerManagerTree::LoggerManagerTree(ConstLoggerConfigPtr config) + : config_(std::move(config)){}; + + LoggerManagerTree::LoggerManagerTree(LoggerConfig config) + : config_(std::make_shared(std::move(config))){}; + + LoggerManagerTree::LoggerManagerTree(std::string full_tag, + std::string node_tag, + ConstLoggerConfigPtr config) + : node_tag_(std::move(node_tag)), + full_tag_(std::move(full_tag)), + config_(std::move(config)){}; + + LoggerManagerTreePtr LoggerManagerTree::registerChild( + std::string tag, + boost::optional log_level, + boost::optional patterns) { + LoggerConfig child_config{ + log_level.value_or(config_->log_level), + patterns ? LogPatterns{*std::move(patterns)} : config_->patterns}; + // Operator new is employed due to private visibility of used constructor. + LoggerManagerTreePtr child(new LoggerManagerTree( + joinTags(full_tag_, tag), + tag, + std::make_shared(std::move(child_config)))); + auto map_elem = std::make_pair( + std::move(tag), std::move(child)); + std::lock_guard lock(children_mutex_); + return children_.emplace(std::move(map_elem)).first->second; + } + + LoggerPtr LoggerManagerTree::getLogger() { + LoggerPtr logger = + std::atomic_load_explicit(&logger_, std::memory_order_acquire); + if (not logger) { + LoggerPtr new_logger = std::make_shared(full_tag_, config_); + while (not logger) { + if (std::atomic_compare_exchange_weak_explicit( + &logger_, + &logger, + new_logger, + std::memory_order_release, + std::memory_order_acquire)) { + return new_logger; + } + } + } + return logger; + } + + LoggerManagerTreePtr LoggerManagerTree::getChild(const std::string &tag) { + std::lock_guard lock(children_mutex_); + const auto child_it = children_.find(tag); + if (child_it != children_.end()) { + return child_it->second; + } + // If a node for this child is not found in the tree config, create a + // new standalone logger using this logger's settings. + LoggerManagerTreePtr new_child( + new LoggerManagerTree(joinTags(full_tag_, tag), tag, config_)); + return children_.emplace(std::make_pair(tag, std::move(new_child))) + .first->second; + } + +} // namespace logger diff --git a/libs/logger/logger_manager.hpp b/libs/logger/logger_manager.hpp new file mode 100644 index 0000000000..2cb2723248 --- /dev/null +++ b/libs/logger/logger_manager.hpp @@ -0,0 +1,65 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef IROHA_SPDLOG_LOGGER_MANAGER_HPP +#define IROHA_SPDLOG_LOGGER_MANAGER_HPP + +#include "logger/logger_manager_fwd.hpp" + +#include +#include +#include +#include + +#include +#include "logger/logger_spdlog.hpp" + +namespace logger { + + /** + * A node of logger managers tree. It stores the configuration needed to + * create a logger corresponding to this node and its children. Thread safe. + */ + class LoggerManagerTree { + public: + explicit LoggerManagerTree(ConstLoggerConfigPtr config); + + explicit LoggerManagerTree(LoggerConfig config); + + /** + * Register a child configuration. The new child's configuration parameters + * are taken from the parent optionally overrided by the arguments. Thread + * safe. + * + * @param tag - the child's tag, without any parents' prefixes + * @param log_level - override the log level for the new child + * @param patterns - override the patterns + */ + LoggerManagerTreePtr registerChild(std::string tag, + boost::optional log_level, + boost::optional patterns); + + /// Get this node's logger. Thread safe. + LoggerPtr getLogger(); + + /// Get non-const child node by tag, if present. Thread safe. + LoggerManagerTreePtr getChild(const std::string &tag); + + private: + LoggerManagerTree(std::string full_tag, + std::string node_tag, + ConstLoggerConfigPtr config); + + const std::string node_tag_; + const std::string full_tag_; + const ConstLoggerConfigPtr config_; + std::shared_ptr logger_; + std::unordered_map children_; + std::mutex children_mutex_; + }; + +} // namespace logger + +#endif // IROHA_SPDLOG_LOGGER_MANAGER_HPP diff --git a/libs/logger/logger_manager_fwd.hpp b/libs/logger/logger_manager_fwd.hpp new file mode 100644 index 0000000000..30967f8bb7 --- /dev/null +++ b/libs/logger/logger_manager_fwd.hpp @@ -0,0 +1,18 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef IROHA_SPDLOG_LOGGER_MANAGER_FWD_HPP +#define IROHA_SPDLOG_LOGGER_MANAGER_FWD_HPP + +#include + +namespace logger { + + class LoggerManagerTree; + using LoggerManagerTreePtr = std::shared_ptr; + +} // namespace logger + +#endif // IROHA_SPDLOG_LOGGER_MANAGER_FWD_HPP diff --git a/libs/logger/logger_spdlog.cpp b/libs/logger/logger_spdlog.cpp new file mode 100644 index 0000000000..fe5a19af46 --- /dev/null +++ b/libs/logger/logger_spdlog.cpp @@ -0,0 +1,92 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "logger/logger_spdlog.hpp" + +#include +#include + +#define SPDLOG_FMT_EXTERNAL + +#include +#include +#include + +namespace { + + spdlog::level::level_enum getSpdlogLogLevel(logger::LogLevel level) { + static const std::map + kSpdLogLevels = { + {logger::LogLevel::kTrace, spdlog::level::trace}, + {logger::LogLevel::kDebug, spdlog::level::debug}, + {logger::LogLevel::kInfo, spdlog::level::info}, + {logger::LogLevel::kWarn, spdlog::level::warn}, + {logger::LogLevel::kError, spdlog::level::err}, + {logger::LogLevel::kCritical, spdlog::level::critical}}; + const auto it = kSpdLogLevels.find(level); + BOOST_ASSERT_MSG(it != kSpdLogLevels.end(), "Unknown log level!"); + return it == kSpdLogLevels.end() + ? kSpdLogLevels.at(logger::kDefaultLogLevel) + : it->second; + } + + std::shared_ptr getOrCreateLogger(const std::string tag) { + std::shared_ptr logger; + try { + logger = spdlog::stdout_color_mt(tag); + } catch (const spdlog::spdlog_ex &) { + logger = spdlog::get(tag); + } + assert(logger); + return logger; + } + +} // namespace + +namespace logger { + + LogPatterns getDefaultLogPatterns() { + static std::atomic_flag is_initialized = ATOMIC_FLAG_INIT; + static LogPatterns default_patterns; + if (not is_initialized.test_and_set()) { + default_patterns.setPattern( + LogLevel::kTrace, R"([%Y-%m-%d %H:%M:%S.%F] [th:%t] [%5l] [%n]: %v)"); + default_patterns.setPattern(LogLevel::kInfo, + R"([%Y-%m-%d %H:%M:%S.%F] [%L] [%n]: %v)"); + } + return default_patterns; + } + + void LogPatterns::setPattern(LogLevel level, std::string pattern) { + patterns_[level] = pattern; + } + + std::string LogPatterns::getPattern(LogLevel level) const { + for (auto it = patterns_.rbegin(); it != patterns_.rend(); ++it) { + if (it->first <= level) { + return it->second; + } + } + return getDefaultLogPatterns().getPattern(level); + } + + LoggerSpdlog::LoggerSpdlog(std::string tag, ConstLoggerConfigPtr config) + : tag_(tag), config_(std::move(config)), logger_(getOrCreateLogger(tag)) { + setupLogger(); + } + + void LoggerSpdlog::setupLogger() { + logger_->set_level(getSpdlogLogLevel(config_->log_level)); + logger_->set_pattern(config_->patterns.getPattern(config_->log_level)); + } + + void LoggerSpdlog::logInternal(Level level, const std::string &s) const { + logger_->log(getSpdlogLogLevel(config_->log_level), s); + } + + bool LoggerSpdlog::shouldLog(Level level) const { + return config_->log_level <= level; + } +} // namespace logger diff --git a/libs/logger/logger_spdlog.hpp b/libs/logger/logger_spdlog.hpp new file mode 100644 index 0000000000..9b38187ed4 --- /dev/null +++ b/libs/logger/logger_spdlog.hpp @@ -0,0 +1,76 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef IROHA_LOGGER_SPDLOG_HPP +#define IROHA_LOGGER_SPDLOG_HPP + +#include "logger/logger.hpp" + +#include +#include +#include + +namespace spdlog { + class logger; +} + +namespace logger { + + class LogPatterns; + struct LoggerConfig; + + using ConstLoggerConfigPtr = std::shared_ptr; + + LogPatterns getDefaultLogPatterns(); + + /// Patterns for logging depending on the log level. + class LogPatterns { + public: + /// Set a logging pattern for the given level. + void setPattern(LogLevel level, std::string pattern); + + /** + * Get the logging pattern for the given level. If not set, get the + * next present more verbose level pattern, if any, or the default + * pattern. + */ + std::string getPattern(LogLevel level) const; + + private: + std::map patterns_; + }; + + // TODO mboldyrev 29.12.2018 IR-188 Add sink options (console, file, syslog) + struct LoggerConfig { + LogLevel log_level; + LogPatterns patterns; + }; + + class LoggerSpdlog : public Logger { + public: + /** + * @param tag - the tag for logging (aka logger name) + * @param config - logger configuration + */ + LoggerSpdlog(std::string tag, ConstLoggerConfigPtr config); + + private: + void logInternal(Level level, const std::string &s) const override; + + /// Whether the configured logging level is at least as verbose as the + /// one given in parameter. + bool shouldLog(Level level) const override; + + /// Set Spdlog logger level and pattern. + void setupLogger(); + + const std::string tag_; + const ConstLoggerConfigPtr config_; + const std::shared_ptr logger_; + }; + +} // namespace logger + +#endif // IROHA_LOGGER_SPDLOG_HPP diff --git a/shared_model/backend/protobuf/commands/impl/proto_command.cpp b/shared_model/backend/protobuf/commands/impl/proto_command.cpp index d249a3aff9..c37b17f5aa 100644 --- a/shared_model/backend/protobuf/commands/impl/proto_command.cpp +++ b/shared_model/backend/protobuf/commands/impl/proto_command.cpp @@ -65,8 +65,6 @@ namespace shared_model { }()}; CommandVariantType ivariant_{variant_}; - - logger::Logger log_{logger::log("ProtoCommand")}; }; Command::Command(Command &&o) noexcept = default; @@ -82,12 +80,8 @@ namespace shared_model { } Command *Command::clone() const { - logError("tried to clone a proto command, which is uncloneable"); - std::terminate(); - } - - void Command::logError(const std::string &message) const { - impl_->log_->error(message); + throw std::runtime_error( + "tried to clone a proto command, which is uncloneable"); } } // namespace proto diff --git a/test/framework/CMakeLists.txt b/test/framework/CMakeLists.txt index 7eb29b4266..4a1e72fc03 100644 --- a/test/framework/CMakeLists.txt +++ b/test/framework/CMakeLists.txt @@ -8,6 +8,12 @@ target_link_libraries(test_subscriber_testing rxcpp ) +add_library(test_logger test_logger.cpp) +target_link_libraries(test_logger + logger + logger_manager + ) + add_library(integration_framework integration_framework/integration_test_framework.cpp integration_framework/iroha_instance.cpp @@ -26,6 +32,7 @@ target_link_libraries(integration_framework shared_model_cryptography_model server_runner mst_transport + test_logger ) add_library(common_test_constants common_constants.cpp) diff --git a/test/framework/integration_framework/fake_peer/fake_peer.cpp b/test/framework/integration_framework/fake_peer/fake_peer.cpp index eccc9d0d3e..e30e341491 100644 --- a/test/framework/integration_framework/fake_peer/fake_peer.cpp +++ b/test/framework/integration_framework/fake_peer/fake_peer.cpp @@ -14,6 +14,8 @@ #include "cryptography/keypair.hpp" #include "framework/result_fixture.hpp" #include "interfaces/common_objects/common_objects_factory.hpp" +#include "logger/logger.hpp" +#include "logger/logger_manager.hpp" #include "main/server_runner.hpp" #include "multi_sig_transactions/transport/mst_transport_grpc.hpp" #include "network/impl/async_grpc_client.hpp" @@ -62,8 +64,13 @@ namespace integration_framework { std::shared_ptr transaction_batch_factory, std::shared_ptr tx_presence_cache, + logger::LoggerManagerTreePtr log_manager, bool agree_all_proposals) - : common_objects_factory_(common_objects_factory), + : log_(log_manager->getLogger()), + log_manager_(std::move(log_manager)), + consensus_log_manager_(log_manager_->getChild("Consensus")), + mst_log_manager_(log_manager_->getChild("MultiSignatureTransactions")), + common_objects_factory_(common_objects_factory), listen_ip_(listen_ip), internal_port_(internal_port), keypair_(std::make_unique( @@ -71,7 +78,8 @@ namespace integration_framework { this_peer_(createPeer( common_objects_factory, getAddress(), keypair_->publicKey())), real_peer_(real_peer), - async_call_(std::make_shared()), + async_call_(std::make_shared( + log_manager_->getChild("AsyncNetworkClient")->getLogger())), mst_transport_(std::make_shared( async_call_, transaction_factory, @@ -79,16 +87,16 @@ namespace integration_framework { transaction_batch_factory, tx_presence_cache, std::make_shared(std::chrono::minutes(0)), - keypair_->publicKey())), - yac_transport_(std::make_shared(async_call_)), + keypair_->publicKey(), + mst_log_manager_->getChild("State")->getLogger(), + mst_log_manager_->getChild("Transport")->getLogger())), + yac_transport_(std::make_shared( + async_call_, + consensus_log_manager_->getChild("Transport")->getLogger())), yac_network_notifier_(std::make_shared()), yac_crypto_(std::make_shared( *keypair_, common_objects_factory)) { yac_transport_->subscribe(yac_network_notifier_); - log_ = logger::log( - "IntegrationTestFramework " - "(fake peer at " - + getAddress() + ")"); if (agree_all_proposals) { enableAgreeAllProposals(); } @@ -97,7 +105,8 @@ namespace integration_framework { void FakePeer::run() { // start instance log_->info("starting listening server"); - internal_server_ = std::make_unique(getAddress()); + internal_server_ = std::make_unique( + getAddress(), log_manager_->getChild("InternalServer")->getLogger()); internal_server_->append(yac_transport_) .append(mst_transport_) .run() diff --git a/test/framework/integration_framework/fake_peer/fake_peer.hpp b/test/framework/integration_framework/fake_peer/fake_peer.hpp index b198d63f24..9972164d31 100644 --- a/test/framework/integration_framework/fake_peer/fake_peer.hpp +++ b/test/framework/integration_framework/fake_peer/fake_peer.hpp @@ -12,7 +12,8 @@ #include #include "framework/integration_framework/fake_peer/yac_network_notifier.hpp" #include "interfaces/iroha_internal/abstract_transport_factory.hpp" -#include "logger/logger.hpp" +#include "logger/logger_fwd.hpp" +#include "logger/logger_manager_fwd.hpp" #include "network/impl/async_grpc_client.hpp" namespace shared_model { @@ -72,6 +73,7 @@ namespace integration_framework { * @param transaction_factory - transaction_factory * @param batch_parser - batch_parser * @param transaction_batch_factory - transaction_batch_factory + * @param log_manager - log manager * @param gree_all_proposals - whether this peer should agree all proposals */ FakePeer( @@ -87,6 +89,7 @@ namespace integration_framework { std::shared_ptr transaction_batch_factory, std::shared_ptr tx_presence_cache, + logger::LoggerManagerTreePtr log_manager, bool agree_all_proposals = true); /// Start the fake peer. @@ -146,6 +149,11 @@ namespace integration_framework { using YacTransport = iroha::consensus::yac::NetworkImpl; using AsyncCall = iroha::network::AsyncGrpcClient; + logger::LoggerPtr log_; + logger::LoggerManagerTreePtr log_manager_; + logger::LoggerManagerTreePtr consensus_log_manager_; + logger::LoggerManagerTreePtr mst_log_manager_; + std::shared_ptr common_objects_factory_; @@ -169,8 +177,6 @@ namespace integration_framework { std::shared_ptr yac_crypto_; rxcpp::subscription proposal_agreer_subscription_; - - logger::Logger log_; }; } // namespace integration_framework diff --git a/test/framework/integration_framework/integration_test_framework.cpp b/test/framework/integration_framework/integration_test_framework.cpp index 016c5ce98b..c9cf41bb25 100644 --- a/test/framework/integration_framework/integration_test_framework.cpp +++ b/test/framework/integration_framework/integration_test_framework.cpp @@ -31,9 +31,12 @@ #include "framework/integration_framework/port_guard.hpp" #include "framework/integration_framework/test_irohad.hpp" #include "framework/result_fixture.hpp" +#include "framework/test_logger.hpp" #include "interfaces/iroha_internal/transaction_batch_factory_impl.hpp" #include "interfaces/iroha_internal/transaction_batch_parser_impl.hpp" #include "interfaces/permissions.hpp" +#include "logger/logger.hpp" +#include "logger/logger_manager.hpp" #include "module/irohad/ametsuchi/tx_presence_cache_stub.hpp" #include "module/shared_model/builders/protobuf/block.hpp" #include "module/shared_model/builders/protobuf/proposal.hpp" @@ -71,6 +74,14 @@ namespace { std::string kLocalHost = "127.0.0.1"; constexpr size_t kDefaultToriiPort = 11501; constexpr size_t kDefaultInternalPort = 50541; + + std::string format_address(std::string ip, + integration_framework::PortGuard::PortType port) { + ip.append(":"); + ip.append(std::to_string(port)); + return ip; + } + } // namespace namespace integration_framework { @@ -83,21 +94,29 @@ namespace integration_framework { const std::string &block_store_path, milliseconds proposal_waiting, milliseconds block_waiting, - milliseconds tx_response_waiting) - : port_guard_(std::make_unique()), + milliseconds tx_response_waiting, + logger::LoggerManagerTreePtr log_manager) + : log_(log_manager->getLogger()), + log_manager_(std::move(log_manager)), + port_guard_(std::make_unique()), torii_port_(port_guard_->getPort(kDefaultToriiPort)), internal_port_(port_guard_->getPort(kDefaultInternalPort)), - iroha_instance_(std::make_shared(mst_support, - block_store_path, - kLocalHost, - torii_port_, - internal_port_, - dbname)), + iroha_instance_( + std::make_shared(mst_support, + block_store_path, + kLocalHost, + torii_port_, + internal_port_, + log_manager_->getChild("Irohad"), + log_, + dbname)), command_client_( iroha::network::createClient( - kLocalHost + ":" + std::to_string(torii_port_))), + format_address(kLocalHost, torii_port_)), + log_manager_->getChild("CommandClient")->getLogger()), query_client_(kLocalHost, torii_port_), - async_call_(std::make_shared()), + async_call_(std::make_shared( + log_manager_->getChild("AsyncCall")->getLogger())), proposal_waiting(proposal_waiting), block_waiting(block_waiting), tx_response_waiting(tx_response_waiting), @@ -113,8 +132,9 @@ namespace integration_framework { std::make_shared< shared_model::interface::TransactionBatchFactoryImpl>()), tx_presence_cache_(std::make_shared()), - yac_transport_( - std::make_shared(async_call_)), + yac_transport_(std::make_shared( + async_call_, + log_manager_->getChild("ConsensusTransport")->getLogger())), cleanup_on_exit_(cleanup_on_exit) {} IntegrationTestFramework::~IntegrationTestFramework() { @@ -144,16 +164,19 @@ namespace integration_framework { assert(this_peer_ && "this_peer_ is needed for fake peers initialization, " "but not set"); for (auto &promise_and_key : fake_peers_promises_) { - auto fake_peer = - std::make_shared(kLocalHost, - port_guard_->getPort(kDefaultInternalPort), - promise_and_key.second, - this_peer_, - common_objects_factory_, - transaction_factory_, - batch_parser_, - transaction_batch_factory_, - tx_presence_cache_); + const auto port = port_guard_->getPort(kDefaultInternalPort); + auto fake_peer = std::make_shared( + kLocalHost, + port, + promise_and_key.second, + this_peer_, + common_objects_factory_, + transaction_factory_, + batch_parser_, + transaction_batch_factory_, + tx_presence_cache_, + log_manager_->getChild("FakePeer") + ->getChild("at " + format_address(kLocalHost, port))); fake_peers_.emplace_back(fake_peer); promise_and_key.first.set_value(fake_peer); } @@ -170,7 +193,7 @@ namespace integration_framework { shared_model::proto::TransactionBuilder() .creatorAccountId(kAdminId) .createdTime(iroha::time::now()) - .addPeer(kLocalHost + ":" + std::to_string(internal_port_), + .addPeer(format_address(kLocalHost, internal_port_), key.publicKey()) .createRole(kAdminRole, all_perms) .createRole(kDefaultRole, {}) @@ -240,7 +263,7 @@ namespace integration_framework { log_->info("init state"); // peer initialization common_objects_factory_ - ->createPeer(kLocalHost + ":" + std::to_string(internal_port_), + ->createPeer(format_address(kLocalHost, internal_port_), keypair.publicKey()) .match( [this](iroha::expected::Result< @@ -635,4 +658,9 @@ namespace integration_framework { boost::filesystem::remove_all(iroha_instance_->block_store_dir_); } } + + logger::LoggerManagerTreePtr getDefaultItfLogManager() { + return getTestLoggerManager()->getChild("IntegrationFramework"); + } + } // namespace integration_framework diff --git a/test/framework/integration_framework/integration_test_framework.hpp b/test/framework/integration_framework/integration_test_framework.hpp index 94efc430df..33637d6ce2 100644 --- a/test/framework/integration_framework/integration_test_framework.hpp +++ b/test/framework/integration_framework/integration_test_framework.hpp @@ -25,6 +25,7 @@ #include "interfaces/common_objects/peer.hpp" #include "interfaces/iroha_internal/transaction_sequence.hpp" #include "logger/logger.hpp" +#include "logger/logger_manager_fwd.hpp" #include "multi_sig_transactions/state/mst_state.hpp" #include "network/impl/async_grpc_client.hpp" #include "network/mst_transport.hpp" @@ -65,6 +66,9 @@ namespace integration_framework { class FakePeer; class PortGuard; + /// Get the default logger of ITF. + logger::LoggerManagerTreePtr getDefaultItfLogManager(); + class IntegrationTestFramework { private: using ProposalType = @@ -86,6 +90,7 @@ namespace integration_framework { * @param block_store_path - specifies path where blocks will be stored * @param proposal_waiting - timeout for next proposal appearing * @param block_waiting - timeout for next committed block appearing + * @param log_manager - log manager */ explicit IntegrationTestFramework( size_t maximum_proposal_size, @@ -98,7 +103,8 @@ namespace integration_framework { .string(), milliseconds proposal_waiting = milliseconds(20000), milliseconds block_waiting = milliseconds(20000), - milliseconds tx_response_waiting = milliseconds(10000)); + milliseconds tx_response_waiting = milliseconds(10000), + logger::LoggerManagerTreePtr log_manager = getDefaultItfLogManager()); ~IntegrationTestFramework(); @@ -383,6 +389,9 @@ namespace integration_framework { std::map> responses_queues_; + logger::LoggerPtr log_; + logger::LoggerManagerTreePtr log_manager_; + std::unique_ptr port_guard_; size_t torii_port_; size_t internal_port_; @@ -428,7 +437,6 @@ namespace integration_framework { private: void makeFakePeers(); - logger::Logger log_ = logger::log("IntegrationTestFramework"); std::mutex queue_mu; std::condition_variable queue_cond; bool cleanup_on_exit_; diff --git a/test/framework/integration_framework/iroha_instance.cpp b/test/framework/integration_framework/iroha_instance.cpp index 234ec6c422..754216e042 100644 --- a/test/framework/integration_framework/iroha_instance.cpp +++ b/test/framework/integration_framework/iroha_instance.cpp @@ -12,6 +12,7 @@ #include "cryptography/keypair.hpp" #include "framework/config_helper.hpp" #include "framework/integration_framework/test_irohad.hpp" +#include "logger/logger.hpp" using namespace std::chrono_literals; @@ -22,6 +23,8 @@ namespace integration_framework { const std::string &listen_ip, size_t torii_port, size_t internal_port, + logger::LoggerManagerTreePtr irohad_log_manager, + logger::LoggerPtr log, const boost::optional &dbname) : block_store_dir_(block_store_path), pg_conn_(getPostgreCredsOrDefault(dbname)), @@ -38,7 +41,9 @@ namespace integration_framework { max_rounds_delay_(0ms), stale_stream_max_rounds_(2), opt_mst_gossip_params_(boost::make_optional( - mst_support, iroha::GossipPropagationStrategyParams{})) {} + mst_support, iroha::GossipPropagationStrategyParams{})), + irohad_log_manager_(std::move(irohad_log_manager)), + log_(std::move(log)) {} void IrohaInstance::makeGenesis(const shared_model::interface::Block &block) { instance_->storage->reset(); @@ -77,6 +82,8 @@ namespace integration_framework { key_pair, max_rounds_delay_, stale_stream_max_rounds_, + irohad_log_manager_, + log_, opt_mst_gossip_params_); } diff --git a/test/framework/integration_framework/iroha_instance.hpp b/test/framework/integration_framework/iroha_instance.hpp index 7c85b58e68..18be06d1c6 100644 --- a/test/framework/integration_framework/iroha_instance.hpp +++ b/test/framework/integration_framework/iroha_instance.hpp @@ -6,13 +6,16 @@ #ifndef IROHA_IROHA_INSTANCE_HPP #define IROHA_IROHA_INSTANCE_HPP -#include -#include -#include #include #include #include + +#include +#include +#include #include "ametsuchi/impl/postgres_options.hpp" +#include "logger/logger_fwd.hpp" +#include "logger/logger_manager_fwd.hpp" #include "multi_sig_transactions/gossip_propagation_strategy_params.hpp" namespace shared_model { @@ -35,6 +38,8 @@ namespace integration_framework { * @param listen_ip - ip address for opening ports (internal & torii) * @param torii_port - port to bind Torii service to * @param internal_port - port for internal irohad communication + * @param irohad_log_manager - the log manager for irohad + * @param log - the log for internal messages * @param dbname is a name of postgres database */ IrohaInstance(bool mst_support, @@ -42,6 +47,8 @@ namespace integration_framework { const std::string &listen_ip, size_t torii_port, size_t internal_port, + logger::LoggerManagerTreePtr irohad_log_manager, + logger::LoggerPtr log, const boost::optional &dbname = boost::none); void makeGenesis(const shared_model::interface::Block &block); @@ -78,6 +85,9 @@ namespace integration_framework { private: std::shared_ptr instance_; + logger::LoggerManagerTreePtr irohad_log_manager_; + + logger::LoggerPtr log_; boost::optional mst_gossip_emitting_period_; boost::optional mst_gossip_amount_per_once_; diff --git a/test/framework/integration_framework/test_irohad.hpp b/test/framework/integration_framework/test_irohad.hpp index 9d017fdcea..6f224df9c1 100644 --- a/test/framework/integration_framework/test_irohad.hpp +++ b/test/framework/integration_framework/test_irohad.hpp @@ -28,6 +28,8 @@ namespace integration_framework { const shared_model::crypto::Keypair &keypair, std::chrono::milliseconds max_rounds_delay, size_t stale_stream_max_rounds, + logger::LoggerManagerTreePtr irohad_log_manager, + logger::LoggerPtr log, const boost::optional &opt_mst_gossip_params = boost::none) : Irohad(block_store_dir, @@ -42,7 +44,9 @@ namespace integration_framework { keypair, max_rounds_delay, stale_stream_max_rounds, - opt_mst_gossip_params) {} + std::move(irohad_log_manager), + opt_mst_gossip_params), + log_(std::move(log)) {} auto &getCommandService() { return command_service; @@ -91,6 +95,9 @@ namespace integration_framework { log_->warn("Tried to terminate without internal server"); } } + + private: + logger::LoggerPtr log_; }; } // namespace integration_framework diff --git a/test/framework/sql_query.cpp b/test/framework/sql_query.cpp index 8e5f1a66f9..4e3b37f498 100644 --- a/test/framework/sql_query.cpp +++ b/test/framework/sql_query.cpp @@ -55,7 +55,7 @@ namespace framework { SqlQuery::SqlQuery( soci::session &sql, std::shared_ptr factory, - logger::Logger log) + logger::LoggerPtr log) : sql_{sql}, factory_{std::move(factory)}, log_{std::move(log)} {} bool SqlQuery::hasAccountGrantablePermission( diff --git a/test/framework/sql_query.hpp b/test/framework/sql_query.hpp index ec6c660afd..33374e0338 100644 --- a/test/framework/sql_query.hpp +++ b/test/framework/sql_query.hpp @@ -10,6 +10,7 @@ #include #include +#include "framework/test_logger.hpp" #include "interfaces/common_objects/account.hpp" #include "interfaces/common_objects/account_asset.hpp" #include "interfaces/common_objects/asset.hpp" @@ -35,7 +36,7 @@ namespace framework { SqlQuery(soci::session &sql, std::shared_ptr factory, - logger::Logger log = logger::log("SqlQuery")); + logger::LoggerPtr log = getTestLogger("SqlQuery")); /** * Check if permitee has permission on account @@ -135,7 +136,7 @@ namespace framework { private: soci::session &sql_; std::shared_ptr factory_; - logger::Logger log_; + logger::LoggerPtr log_; /** * Executes given lambda of type F, catches exceptions if any, logs the diff --git a/test/framework/test_logger.cpp b/test/framework/test_logger.cpp new file mode 100644 index 0000000000..86de19c388 --- /dev/null +++ b/test/framework/test_logger.cpp @@ -0,0 +1,19 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "framework/test_logger.hpp" + +#include "logger/logger_manager.hpp" + +logger::LoggerManagerTreePtr getTestLoggerManager() { + static logger::LoggerManagerTreePtr log_manager( + std::make_shared(logger::LoggerConfig{ + logger::LogLevel::kInfo, logger::getDefaultLogPatterns()})); + return log_manager->getChild("Test"); +} + +logger::LoggerPtr getTestLogger(const std::string &tag) { + return getTestLoggerManager()->getChild(tag)->getLogger(); +} diff --git a/test/framework/test_logger.hpp b/test/framework/test_logger.hpp new file mode 100644 index 0000000000..00f96a6767 --- /dev/null +++ b/test/framework/test_logger.hpp @@ -0,0 +1,16 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef TEST_FRAMEWORK_TEST_LOGGER_HPP +#define TEST_FRAMEWORK_TEST_LOGGER_HPP + +#include "logger/logger_fwd.hpp" +#include "logger/logger_manager_fwd.hpp" + +logger::LoggerManagerTreePtr getTestLoggerManager(); + +logger::LoggerPtr getTestLogger(const std::string &tag); + +#endif // TEST_FRAMEWORK_TEST_LOGGER_HPP diff --git a/test/fuzzing/block_loader_fixture.hpp b/test/fuzzing/block_loader_fixture.hpp index 25c5475e81..97e9df7da7 100644 --- a/test/fuzzing/block_loader_fixture.hpp +++ b/test/fuzzing/block_loader_fixture.hpp @@ -13,6 +13,7 @@ #include "consensus/consensus_block_cache.hpp" #include "cryptography/crypto_provider/crypto_defaults.hpp" +#include "logger/dummy_logger.hpp" #include "module/irohad/ametsuchi/ametsuchi_mocks.hpp" #include "network/impl/block_loader_service.hpp" @@ -28,16 +29,13 @@ namespace fuzzing { std::shared_ptr block_loader_service_; BlockLoaderFixture() { - // BlockLoaderService produces lots of error logs about missing blocks - spdlog::set_level(spdlog::level::critical); - storage_ = std::make_shared>(); block_query_factory_ = std::make_shared>(); block_cache_ = std::make_shared(); block_loader_service_ = std::make_shared( - block_query_factory_, block_cache_); + block_query_factory_, block_cache_, logger::getDummyLoggerPtr()); EXPECT_CALL(*block_query_factory_, createBlockQuery()) .WillRepeatedly(Return(boost::make_optional( std::shared_ptr(storage_)))); diff --git a/test/fuzzing/consensus_fuzz.cpp b/test/fuzzing/consensus_fuzz.cpp index 3ec36a19b8..80384671ac 100644 --- a/test/fuzzing/consensus_fuzz.cpp +++ b/test/fuzzing/consensus_fuzz.cpp @@ -9,6 +9,7 @@ #include "consensus/yac/transport/impl/network_impl.hpp" +#include "logger/dummy_logger.hpp" #include "module/irohad/consensus/yac/mock_yac_network.hpp" using namespace testing; @@ -23,14 +24,13 @@ namespace fuzzing { std::shared_ptr network_; ConsensusFixture() { - spdlog::set_level(spdlog::level::critical); - notifications_ = std::make_shared< NiceMock>(); async_call_ = std::make_shared< - iroha::network::AsyncGrpcClient>(); - network_ = - std::make_shared(async_call_); + iroha::network::AsyncGrpcClient>( + logger::getDummyLoggerPtr()); + network_ = std::make_shared( + async_call_, logger::getDummyLoggerPtr()); network_->subscribe(notifications_); } }; diff --git a/test/fuzzing/find_fuzz.cpp b/test/fuzzing/find_fuzz.cpp index f911db6b56..a2ada53858 100644 --- a/test/fuzzing/find_fuzz.cpp +++ b/test/fuzzing/find_fuzz.cpp @@ -10,6 +10,7 @@ #include "backend/protobuf/proto_query_response_factory.hpp" #include "backend/protobuf/proto_transport_factory.hpp" #include "libfuzzer/libfuzzer_macro.h" +#include "logger/dummy_logger.hpp" #include "module/irohad/ametsuchi/ametsuchi_mocks.hpp" #include "module/irohad/network/network_mocks.hpp" #include "module/irohad/pending_txs_storage/pending_txs_storage_mock.hpp" @@ -40,7 +41,11 @@ struct QueryFixture { auto query_response_factory_ = std::make_shared(); qry_processor_ = std::make_shared( - storage_, storage_, pending_transactions_, query_response_factory_); + storage_, + storage_, + pending_transactions_, + query_response_factory_, + logger::getDummyLoggerPtr()); std::unique_ptr> @@ -55,8 +60,8 @@ struct QueryFixture { shared_model::interface::Query, shared_model::proto::Query>>(std::move(query_validator), std::move(proto_query_validator)); - service_ = std::make_shared(qry_processor_, - query_factory); + service_ = std::make_shared( + qry_processor_, query_factory, logger::getDummyLoggerPtr()); } }; diff --git a/test/fuzzing/mst_fuzz.cpp b/test/fuzzing/mst_fuzz.cpp index a4cc317e2b..1ed42ded08 100644 --- a/test/fuzzing/mst_fuzz.cpp +++ b/test/fuzzing/mst_fuzz.cpp @@ -12,6 +12,7 @@ #include "backend/protobuf/proto_transport_factory.hpp" #include "interfaces/iroha_internal/transaction_batch_factory_impl.hpp" #include "interfaces/iroha_internal/transaction_batch_parser_impl.hpp" +#include "logger/dummy_logger.hpp" #include "module/irohad/ametsuchi/ametsuchi_mocks.hpp" #include "module/irohad/multi_sig_transactions/mst_test_helpers.hpp" #include "multi_sig_transactions/transport/mst_transport_grpc.hpp" @@ -26,10 +27,9 @@ namespace fuzzing { std::shared_ptr mst_transport_grpc_; MstFixture() { - spdlog::set_level(spdlog::level::err); - auto async_call_ = std::make_shared< - iroha::network::AsyncGrpcClient>(); + iroha::network::AsyncGrpcClient>( + logger::getDummyLoggerPtr()); // TODO luckychess 25.12.2018 Component initialisation reuse // IR-1886, IR-142 std::unique_ptr> interface_transaction_validator = diff --git a/test/fuzzing/request_proposal_fuzz.cpp b/test/fuzzing/request_proposal_fuzz.cpp index 88de77896e..6308354333 100644 --- a/test/fuzzing/request_proposal_fuzz.cpp +++ b/test/fuzzing/request_proposal_fuzz.cpp @@ -6,6 +6,7 @@ #include "fuzzing/ordering_service_fixture.hpp" #include "ametsuchi/impl/tx_presence_cache_impl.hpp" +#include "logger/dummy_logger.hpp" #include "module/irohad/ametsuchi/ametsuchi_mocks.hpp" struct RequestProposalFixture : public fuzzing::OrderingServiceFixture { @@ -26,12 +27,14 @@ struct RequestProposalFixture : public fuzzing::OrderingServiceFixture { ordering_service_ = std::make_shared( transaction_limit, std::move(proposal_factory_), - std::move(persistent_cache_)); + std::move(persistent_cache_), + logger::getDummyLoggerPtr()); server_ = std::make_shared(ordering_service_, transaction_factory_, batch_parser_, - transaction_batch_factory_); + transaction_batch_factory_, + logger::getDummyLoggerPtr()); } }; diff --git a/test/fuzzing/send_batches_fuzz.cpp b/test/fuzzing/send_batches_fuzz.cpp index c36d08d292..c1be7e5b68 100644 --- a/test/fuzzing/send_batches_fuzz.cpp +++ b/test/fuzzing/send_batches_fuzz.cpp @@ -4,6 +4,7 @@ */ #include "ametsuchi/impl/tx_presence_cache_impl.hpp" +#include "logger/dummy_logger.hpp" #include "module/irohad/ametsuchi/ametsuchi_mocks.hpp" #include "ordering_service_fixture.hpp" @@ -21,12 +22,16 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, std::size_t size) { auto storage = std::make_shared>(); auto cache = std::make_shared(storage); ordering_service_ = std::make_shared( - data[0], std::move(proposal_factory), std::move(cache)); - server_ = std::make_shared( - ordering_service_, - fixture.transaction_factory_, - fixture.batch_parser_, - fixture.transaction_batch_factory_); + data[0], + std::move(proposal_factory), + std::move(cache), + logger::getDummyLoggerPtr()); + server_ = + std::make_shared(ordering_service_, + fixture.transaction_factory_, + fixture.batch_parser_, + fixture.transaction_batch_factory_, + logger::getDummyLoggerPtr()); proto::BatchesRequest request; if (protobuf_mutator::libfuzzer::LoadProtoInput( diff --git a/test/fuzzing/status_fuzz.cpp b/test/fuzzing/status_fuzz.cpp index 564f1a88cc..6363be66f6 100644 --- a/test/fuzzing/status_fuzz.cpp +++ b/test/fuzzing/status_fuzz.cpp @@ -15,6 +15,7 @@ #include "backend/protobuf/transaction.hpp" #include "interfaces/iroha_internal/transaction_batch_factory_impl.hpp" #include "interfaces/iroha_internal/transaction_batch_parser_impl.hpp" +#include "logger/dummy_logger.hpp" #include "module/irohad/ametsuchi/ametsuchi_mocks.hpp" #include "module/irohad/ametsuchi/mock_tx_presence_cache.hpp" #include "module/irohad/multi_sig_transactions/mst_mocks.hpp" @@ -53,8 +54,6 @@ struct CommandFixture { rxcpp::subjects::subject consensus_notifier_; CommandFixture() { - spdlog::set_level(spdlog::level::err); - pcs_ = std::make_shared(); EXPECT_CALL(*pcs_, onProposal()) .WillRepeatedly(Return(prop_notifier_.get_observable())); @@ -63,7 +62,8 @@ struct CommandFixture { EXPECT_CALL(*pcs_, onVerifiedProposal()) .WillRepeatedly(Return(vprop_notifier_.get_observable())); - mst_processor_ = std::make_shared(); + mst_processor_ = + std::make_shared(logger::getDummyLoggerPtr()); EXPECT_CALL(*mst_processor_, onStateUpdateImpl()) .WillRepeatedly(Return(mst_state_notifier_.get_observable())); EXPECT_CALL(*mst_processor_, onPreparedBatchesImpl()) @@ -75,7 +75,11 @@ struct CommandFixture { auto status_factory = std::make_shared(); tx_processor_ = std::make_shared( - pcs_, mst_processor_, status_bus, status_factory); + pcs_, + mst_processor_, + status_bus, + status_factory, + logger::getDummyLoggerPtr()); std::unique_ptr> @@ -108,12 +112,14 @@ struct CommandFixture { tx_presence_cache_ = std::make_shared(); cache_ = std::make_shared(); - service_ = std::make_shared(tx_processor_, - storage_, - status_bus, - status_factory, - cache_, - tx_presence_cache_); + service_ = std::make_shared( + tx_processor_, + storage_, + status_bus, + status_factory, + cache_, + tx_presence_cache_, + logger::getDummyLoggerPtr()); service_transport_ = std::make_shared( service_, @@ -123,7 +129,8 @@ struct CommandFixture { batch_parser, transaction_batch_factory, rxcpp::observable<>::iterate(consensus_gate_objects_), - 2); + 2, + logger::getDummyLoggerPtr()); } }; diff --git a/test/fuzzing/torii_fuzz.cpp b/test/fuzzing/torii_fuzz.cpp index 5de0f84a2b..a7b635e780 100644 --- a/test/fuzzing/torii_fuzz.cpp +++ b/test/fuzzing/torii_fuzz.cpp @@ -14,6 +14,7 @@ #include "backend/protobuf/transaction.hpp" #include "interfaces/iroha_internal/transaction_batch_factory_impl.hpp" #include "interfaces/iroha_internal/transaction_batch_parser_impl.hpp" +#include "logger/dummy_logger.hpp" #include "module/irohad/ametsuchi/ametsuchi_mocks.hpp" #include "module/irohad/ametsuchi/mock_tx_presence_cache.hpp" #include "module/irohad/multi_sig_transactions/mst_mocks.hpp" @@ -58,7 +59,8 @@ struct CommandFixture { EXPECT_CALL(*pcs_, onVerifiedProposal()) .WillRepeatedly(Return(vprop_notifier_.get_observable())); - mst_processor_ = std::make_shared(); + mst_processor_ = + std::make_shared(logger::getDummyLoggerPtr()); EXPECT_CALL(*mst_processor_, onStateUpdateImpl()) .WillRepeatedly(Return(mst_state_notifier_.get_observable())); EXPECT_CALL(*mst_processor_, onPreparedBatchesImpl()) @@ -70,15 +72,20 @@ struct CommandFixture { auto status_factory = std::make_shared(); tx_processor_ = std::make_shared( - pcs_, mst_processor_, status_bus, status_factory); + pcs_, + mst_processor_, + status_bus, + status_factory, + logger::getDummyLoggerPtr()); auto storage = std::make_shared(); - service_ = - std::make_shared(tx_processor_, - storage, - status_bus, - status_factory, - cache_, - tx_presence_cache_); + service_ = std::make_shared( + tx_processor_, + storage, + status_bus, + status_factory, + cache_, + tx_presence_cache_, + logger::getDummyLoggerPtr()); std::unique_ptr> @@ -114,7 +121,8 @@ struct CommandFixture { batch_parser, transaction_batch_factory, rxcpp::observable<>::iterate(consensus_gate_objects_), - 2); + 2, + logger::getDummyLoggerPtr()); } }; diff --git a/test/integration/consensus/CMakeLists.txt b/test/integration/consensus/CMakeLists.txt index 5c51151add..cf9a9d7ece 100644 --- a/test/integration/consensus/CMakeLists.txt +++ b/test/integration/consensus/CMakeLists.txt @@ -8,4 +8,5 @@ target_link_libraries(consensus_sunny_day yac yac_transport shared_model_stateless_validation + test_logger ) diff --git a/test/integration/consensus/consensus_sunny_day.cpp b/test/integration/consensus/consensus_sunny_day.cpp index a92458e03b..6bba5df866 100644 --- a/test/integration/consensus/consensus_sunny_day.cpp +++ b/test/integration/consensus/consensus_sunny_day.cpp @@ -15,7 +15,9 @@ #include "consensus/yac/yac.hpp" #include "cryptography/crypto_provider/crypto_defaults.hpp" +#include "framework/test_logger.hpp" #include "framework/test_subscriber.hpp" +#include "logger/logger_manager.hpp" #include "module/irohad/consensus/yac/mock_yac_crypto_provider.hpp" #include "module/irohad/consensus/yac/yac_test_util.hpp" #include "module/shared_model/interface_mocks.hpp" @@ -93,8 +95,10 @@ class ConsensusSunnyDayTest : public ::testing::Test { cleanup_strategy = std::make_shared(); auto async_call = std::make_shared< - iroha::network::AsyncGrpcClient>(); - network = std::make_shared(async_call); + iroha::network::AsyncGrpcClient>( + getTestLogger("AsyncCall")); + network = + std::make_shared(async_call, getTestLogger("YacNetwork")); crypto = std::make_shared(std::to_string(my_num)); timer = std::make_shared([this] { // static factory with a single thread @@ -107,13 +111,15 @@ class ConsensusSunnyDayTest : public ::testing::Test { auto order = ClusterOrdering::create(default_peers); ASSERT_TRUE(order); - yac = - Yac::create(YacVoteStorage(cleanup_strategy, - getSupermajorityChecker(kConsistencyModel)), - network, - crypto, - timer, - order.value()); + yac = Yac::create( + YacVoteStorage(cleanup_strategy, + getSupermajorityChecker(kConsistencyModel), + getTestLoggerManager()->getChild("YacVoteStorage")), + network, + crypto, + timer, + order.value(), + getTestLogger("Yac")); network->subscribe(yac); grpc::ServerBuilder builder; diff --git a/test/integration/validation/CMakeLists.txt b/test/integration/validation/CMakeLists.txt index a7fe93eed8..e8f544ba76 100644 --- a/test/integration/validation/CMakeLists.txt +++ b/test/integration/validation/CMakeLists.txt @@ -7,4 +7,5 @@ target_link_libraries(chain_validator_storage_test ametsuchi_fixture yac chain_validator + test_logger ) diff --git a/test/integration/validation/chain_validator_storage_test.cpp b/test/integration/validation/chain_validator_storage_test.cpp index c2232c86b1..229ea0d829 100644 --- a/test/integration/validation/chain_validator_storage_test.cpp +++ b/test/integration/validation/chain_validator_storage_test.cpp @@ -12,6 +12,7 @@ #include "cryptography/crypto_provider/crypto_defaults.hpp" #include "cryptography/default_hash_provider.hpp" #include "cryptography/keypair.hpp" +#include "framework/test_logger.hpp" #include "module/shared_model/builders/protobuf/block.hpp" // TODO mboldyrev 14.02.2019 IR-324 Use supermajority checker mock @@ -25,7 +26,7 @@ namespace iroha { void SetUp() override { ametsuchi::AmetsuchiTest::SetUp(); validator = std::make_shared( - supermajority_checker); + supermajority_checker, getTestLogger("ChainValidator")); for (size_t i = 0; i < 5; ++i) { keys.push_back(shared_model::crypto::DefaultCryptoAlgorithmType:: diff --git a/test/module/irohad/ametsuchi/CMakeLists.txt b/test/module/irohad/ametsuchi/CMakeLists.txt index 4e37cb0321..911eae73bf 100644 --- a/test/module/irohad/ametsuchi/CMakeLists.txt +++ b/test/module/irohad/ametsuchi/CMakeLists.txt @@ -8,17 +8,20 @@ target_link_libraries(ametsuchi_test ametsuchi ametsuchi_fixture shared_model_stateless_validation + test_logger ) addtest(wsv_query_command_test wsv_query_command_test.cpp) target_link_libraries(wsv_query_command_test ametsuchi ametsuchi_fixture + test_logger ) addtest(flat_file_test flat_file_test.cpp) target_link_libraries(flat_file_test ametsuchi + test_logger ) addtest(block_query_test block_query_test.cpp) @@ -33,6 +36,7 @@ target_link_libraries(storage_init_test ametsuchi integration_framework_config_helper shared_model_proto_backend + test_logger ) addtest(postgres_options_test postgres_options_test.cpp) @@ -47,6 +51,7 @@ target_link_libraries(postgres_executor_test ametsuchi commands_mocks_factory framework_sql_query + test_logger ) addtest(postgres_query_executor_test postgres_query_executor_test.cpp) @@ -55,6 +60,7 @@ target_link_libraries(postgres_query_executor_test ametsuchi_fixture ametsuchi commands_mocks_factory + test_logger ) addtest(tx_presence_cache_test tx_presence_cache_test.cpp) @@ -70,4 +76,5 @@ target_link_libraries(ametsuchi_fixture INTERFACE framework_sql_query SOCI::core SOCI::postgresql + test_logger ) diff --git a/test/module/irohad/ametsuchi/ametsuchi_fixture.hpp b/test/module/irohad/ametsuchi/ametsuchi_fixture.hpp index c77a501714..3b945d0b4b 100644 --- a/test/module/irohad/ametsuchi/ametsuchi_fixture.hpp +++ b/test/module/irohad/ametsuchi/ametsuchi_fixture.hpp @@ -19,7 +19,9 @@ #include "common/files.hpp" #include "framework/config_helper.hpp" #include "framework/sql_query.hpp" +#include "framework/test_logger.hpp" #include "logger/logger.hpp" +#include "logger/logger_manager.hpp" #include "validators/field_validator.hpp" namespace iroha { @@ -40,8 +42,13 @@ namespace iroha { std::make_shared(); auto converter = std::make_shared(); - StorageImpl::create( - block_store_path, pgopt_, factory, converter, perm_converter_) + + StorageImpl::create(block_store_path, + pgopt_, + factory, + converter, + perm_converter_, + getTestLoggerManager()->getChild("Storage")) .match([&](iroha::expected::Value> &_storage) { storage = _storage.value; }, [](iroha::expected::Error &error) { @@ -70,6 +77,13 @@ namespace iroha { shared_model::validation::FieldValidator>> factory; + /* Since + * - both the storage and the logger config it uses are static + * - storage uses the logger at destruction + * we need to ensure the static logger config is destroyed after the + * static storage + */ + static logger::LoggerPtr storage_logger_; static std::shared_ptr storage; static std::unique_ptr sql_query; @@ -193,6 +207,9 @@ CREATE TABLE IF NOT EXISTS index_by_id_height_asset ( AmetsuchiTest::perm_converter_ = nullptr; std::shared_ptr AmetsuchiTest::sql = nullptr; + // hold the storage static logger while the static storage is alive + logger::LoggerPtr AmetsuchiTest::storage_logger_ = + getTestLoggerManager()->getChild("Storage")->getLogger(); std::shared_ptr AmetsuchiTest::storage = nullptr; std::unique_ptr AmetsuchiTest::sql_query = nullptr; diff --git a/test/module/irohad/ametsuchi/ametsuchi_test.cpp b/test/module/irohad/ametsuchi/ametsuchi_test.cpp index 2268c14ab3..812199265f 100644 --- a/test/module/irohad/ametsuchi/ametsuchi_test.cpp +++ b/test/module/irohad/ametsuchi/ametsuchi_test.cpp @@ -13,6 +13,7 @@ #include "builders/default_builders.hpp" #include "builders/protobuf/transaction.hpp" #include "framework/result_fixture.hpp" +#include "framework/test_logger.hpp" #include "framework/test_subscriber.hpp" #include "module/irohad/ametsuchi/ametsuchi_fixture.hpp" #include "module/shared_model/builders/protobuf/test_block_builder.hpp" @@ -390,7 +391,7 @@ shared_model::proto::Block getBlock() { } TEST_F(AmetsuchiTest, TestingStorageWhenInsertBlock) { - auto log = logger::testLog("TestStorage"); + auto log = getTestLogger("TestStorage"); log->info( "Test case: create storage " "=> insert block " diff --git a/test/module/irohad/ametsuchi/block_query_test.cpp b/test/module/irohad/ametsuchi/block_query_test.cpp index 27fb8951d1..10a423e69a 100644 --- a/test/module/irohad/ametsuchi/block_query_test.cpp +++ b/test/module/irohad/ametsuchi/block_query_test.cpp @@ -12,6 +12,7 @@ #include "common/byteutils.hpp" #include "converters/protobuf/json_proto_converter.hpp" #include "framework/result_fixture.hpp" +#include "framework/test_logger.hpp" #include "module/irohad/ametsuchi/ametsuchi_fixture.hpp" #include "module/irohad/ametsuchi/mock_key_value_storage.hpp" #include "module/shared_model/builders/protobuf/test_block_builder.hpp" @@ -26,18 +27,20 @@ class BlockQueryTest : public AmetsuchiTest { void SetUp() override { AmetsuchiTest::SetUp(); - auto tmp = FlatFile::create(block_store_path); + auto tmp = FlatFile::create(block_store_path, getTestLogger("FlatFile")); ASSERT_TRUE(tmp); file = std::move(*tmp); mock_file = std::make_shared(); sql = std::make_unique(*soci::factory_postgresql(), pgopt_); - index = std::make_shared(*sql); + index = + std::make_shared(*sql, getTestLogger("BlockIndex")); auto converter = std::make_shared(); - blocks = std::make_shared(*sql, *file, converter); + blocks = std::make_shared( + *sql, *file, converter, getTestLogger("BlockQuery")); empty_blocks = std::make_shared( - *sql, *mock_file, converter, logger::log("PostgresBlockQueryEmpty")); + *sql, *mock_file, converter, getTestLogger("PostgresBlockQueryEmpty")); *sql << init_; diff --git a/test/module/irohad/ametsuchi/flat_file_test.cpp b/test/module/irohad/ametsuchi/flat_file_test.cpp index 02377a43ef..e4d5a1647e 100644 --- a/test/module/irohad/ametsuchi/flat_file_test.cpp +++ b/test/module/irohad/ametsuchi/flat_file_test.cpp @@ -4,20 +4,19 @@ */ #include "ametsuchi/impl/flat_file/flat_file.hpp" + #include #include #include #include "common/byteutils.hpp" #include "common/files.hpp" - +#include "framework/test_logger.hpp" #include "logger/logger.hpp" using namespace iroha::ametsuchi; namespace fs = boost::filesystem; using Identifier = FlatFile::Identifier; -static logger::Logger log_ = logger::testLog("BlockStore"); - class BlStore_Test : public ::testing::Test { protected: void SetUp() override { @@ -31,10 +30,12 @@ class BlStore_Test : public ::testing::Test { (fs::temp_directory_path() / fs::unique_path()).string(); std::vector block; + logger::LoggerPtr flat_file_log_ = getTestLogger("FlatFile"); + logger::LoggerPtr log_ = getTestLogger("BlockStoreTest"); }; TEST_F(BlStore_Test, Read_Write_Test) { - auto store = FlatFile::create(block_store_path); + auto store = FlatFile::create(block_store_path, flat_file_log_); ASSERT_TRUE(store); auto bl_store = std::move(*store); auto id = 1u; @@ -59,7 +60,7 @@ TEST_F(BlStore_Test, BlockStoreWhenRemoveBlock) { "----------| create blockstore and insert 3 elements " "|----------"); - auto store = FlatFile::create(block_store_path); + auto store = FlatFile::create(block_store_path, flat_file_log_); ASSERT_TRUE(store); auto bl_store = std::move(*store); @@ -74,7 +75,7 @@ TEST_F(BlStore_Test, BlockStoreWhenRemoveBlock) { log_->info("----------| remove second and init new storage |----------"); fs::remove(fs::path(block_store_path) / "0000000000000002"); - auto store = FlatFile::create(block_store_path); + auto store = FlatFile::create(block_store_path, flat_file_log_); ASSERT_TRUE(store); auto bl_store = std::move(*store); auto res = bl_store->last_id(); @@ -86,7 +87,7 @@ TEST_F(BlStore_Test, BlockStoreWhenAbsentFolder) { "----------| Check that folder is absent => create => " "make storage => remove storage |----------"); fs::remove_all(block_store_path); - auto store = FlatFile::create(block_store_path); + auto store = FlatFile::create(block_store_path, flat_file_log_); ASSERT_TRUE(store); auto bl_store = std::move(*store); auto id = 1u; @@ -102,7 +103,7 @@ TEST_F(BlStore_Test, BlockStoreWhenAbsentFolder) { * @then new block storage has all blocks from the folder */ TEST_F(BlStore_Test, BlockStoreInitializationFromNonemptyFolder) { - auto store = FlatFile::create(block_store_path); + auto store = FlatFile::create(block_store_path, flat_file_log_); ASSERT_TRUE(store); auto bl_store1 = std::move(*store); @@ -111,7 +112,7 @@ TEST_F(BlStore_Test, BlockStoreInitializationFromNonemptyFolder) { bl_store1->add(2u, std::vector(1000, 5)); // create second block storage from the same folder - auto store2 = FlatFile::create(block_store_path); + auto store2 = FlatFile::create(block_store_path, flat_file_log_); ASSERT_TRUE(store2); auto bl_store2 = std::move(*store2); @@ -124,7 +125,7 @@ TEST_F(BlStore_Test, BlockStoreInitializationFromNonemptyFolder) { * @then check consistency fails */ TEST_F(BlStore_Test, EmptyDumpDir) { - auto res = FlatFile::check_consistency(""); + auto res = FlatFile::check_consistency("", flat_file_log_); ASSERT_FALSE(res); } @@ -134,7 +135,7 @@ TEST_F(BlStore_Test, EmptyDumpDir) { * @then get() fails */ TEST_F(BlStore_Test, GetNonExistingFile) { - auto store = FlatFile::create(block_store_path); + auto store = FlatFile::create(block_store_path, flat_file_log_); ASSERT_TRUE(store); auto bl_store = std::move(*store); Identifier id = 98759385; // random number that does not exist @@ -148,7 +149,7 @@ TEST_F(BlStore_Test, GetNonExistingFile) { * @then directory() returns bock store path */ TEST_F(BlStore_Test, GetDirectory) { - auto store = FlatFile::create(block_store_path); + auto store = FlatFile::create(block_store_path, flat_file_log_); ASSERT_TRUE(store); auto bl_store = std::move(*store); ASSERT_EQ(bl_store->directory(), block_store_path); @@ -160,7 +161,7 @@ TEST_F(BlStore_Test, GetDirectory) { * @then get() fails */ TEST_F(BlStore_Test, GetDeniedBlock) { - auto store = FlatFile::create(block_store_path); + auto store = FlatFile::create(block_store_path, flat_file_log_); ASSERT_TRUE(store); auto bl_store = std::move(*store); auto id = 1u; @@ -179,7 +180,7 @@ TEST_F(BlStore_Test, GetDeniedBlock) { * @then add() fails */ TEST_F(BlStore_Test, AddExistingId) { - auto store = FlatFile::create(block_store_path); + auto store = FlatFile::create(block_store_path, flat_file_log_); ASSERT_TRUE(store); auto bl_store = std::move(*store); auto id = 1u; @@ -197,7 +198,7 @@ TEST_F(BlStore_Test, AddExistingId) { * @then FlatFile creation fails */ TEST_F(BlStore_Test, WriteEmptyFolder) { - auto bl_store = FlatFile::create(""); + auto bl_store = FlatFile::create("", flat_file_log_); ASSERT_FALSE(bl_store); } @@ -207,7 +208,7 @@ TEST_F(BlStore_Test, WriteEmptyFolder) { * @then add() fails */ TEST_F(BlStore_Test, WriteDeniedFolder) { - auto store = FlatFile::create(block_store_path); + auto store = FlatFile::create(block_store_path, flat_file_log_); ASSERT_TRUE(store); auto bl_store = std::move(*store); auto id = 1u; diff --git a/test/module/irohad/ametsuchi/postgres_executor_test.cpp b/test/module/irohad/ametsuchi/postgres_executor_test.cpp index 2dcd4c3428..4550754ebf 100644 --- a/test/module/irohad/ametsuchi/postgres_executor_test.cpp +++ b/test/module/irohad/ametsuchi/postgres_executor_test.cpp @@ -8,6 +8,7 @@ #include "ametsuchi/impl/postgres_wsv_query.hpp" #include "backend/protobuf/proto_permission_to_string.hpp" #include "framework/result_fixture.hpp" +#include "framework/test_logger.hpp" #include "module/irohad/ametsuchi/ametsuchi_fixture.hpp" #include "module/irohad/ametsuchi/ametsuchi_mocks.hpp" #include "module/shared_model/interface_mocks.hpp" @@ -44,7 +45,8 @@ namespace iroha { auto factory = std::make_shared>(); - wsv_query = std::make_unique(*sql, factory); + wsv_query = std::make_unique( + *sql, factory, getTestLogger("WcvQuery")); PostgresCommandExecutor::prepareStatements(*sql); executor = std::make_unique(*sql, perm_converter); diff --git a/test/module/irohad/ametsuchi/postgres_query_executor_test.cpp b/test/module/irohad/ametsuchi/postgres_query_executor_test.cpp index 5696ad62ca..cddb73788e 100644 --- a/test/module/irohad/ametsuchi/postgres_query_executor_test.cpp +++ b/test/module/irohad/ametsuchi/postgres_query_executor_test.cpp @@ -20,6 +20,7 @@ #include "backend/protobuf/proto_query_response_factory.hpp" #include "datetime/time.hpp" #include "framework/result_fixture.hpp" +#include "framework/test_logger.hpp" #include "interfaces/common_objects/types.hpp" #include "interfaces/permissions.hpp" #include "interfaces/query_responses/account_asset_response.hpp" @@ -1050,7 +1051,8 @@ namespace iroha { auto factory = std::make_shared>(); - auto block_store = FlatFile::create(block_store_dir); + auto block_store = + FlatFile::create(block_store_dir, getTestLogger("FlatFile")); ASSERT_TRUE(block_store); this->block_store = std::move(block_store.get()); createDefaultAccount(); diff --git a/test/module/irohad/ametsuchi/storage_init_test.cpp b/test/module/irohad/ametsuchi/storage_init_test.cpp index b2d669f5b1..5d75a77c4e 100644 --- a/test/module/irohad/ametsuchi/storage_init_test.cpp +++ b/test/module/irohad/ametsuchi/storage_init_test.cpp @@ -3,17 +3,20 @@ * SPDX-License-Identifier: Apache-2.0 */ +#include "ametsuchi/impl/storage_impl.hpp" + #include #include #include #include #include #include -#include "ametsuchi/impl/storage_impl.hpp" #include "backend/protobuf/common_objects/proto_common_objects_factory.hpp" #include "backend/protobuf/proto_block_json_converter.hpp" #include "backend/protobuf/proto_permission_to_string.hpp" #include "framework/config_helper.hpp" +#include "framework/test_logger.hpp" +#include "logger/logger_manager.hpp" #include "validators/field_validator.hpp" using namespace iroha::ametsuchi; @@ -61,6 +64,9 @@ class StorageInitTest : public ::testing::Test { sql << query; boost::filesystem::remove_all(block_store_path); } + + logger::LoggerManagerTreePtr storage_log_manager_{ + getTestLoggerManager()->getChild("Storage")}; }; /** @@ -70,8 +76,12 @@ class StorageInitTest : public ::testing::Test { */ TEST_F(StorageInitTest, CreateStorageWithDatabase) { std::shared_ptr storage; - StorageImpl::create( - block_store_path, pgopt_, factory, converter, perm_converter_) + StorageImpl::create(block_store_path, + pgopt_, + factory, + converter, + perm_converter_, + storage_log_manager_) .match( [&storage](const Value> &value) { storage = value.value; @@ -95,8 +105,12 @@ TEST_F(StorageInitTest, CreateStorageWithDatabase) { TEST_F(StorageInitTest, CreateStorageWithInvalidPgOpt) { std::string pg_opt = "host=localhost port=5432 users=nonexistinguser dbname=test"; - StorageImpl::create( - block_store_path, pg_opt, factory, converter, perm_converter_) + StorageImpl::create(block_store_path, + pg_opt, + factory, + converter, + perm_converter_, + storage_log_manager_) .match( [](const Value> &) { FAIL() << "storage created, but should not"; diff --git a/test/module/irohad/ametsuchi/wsv_query_command_test.cpp b/test/module/irohad/ametsuchi/wsv_query_command_test.cpp index 936c95ae53..54cd01b631 100644 --- a/test/module/irohad/ametsuchi/wsv_query_command_test.cpp +++ b/test/module/irohad/ametsuchi/wsv_query_command_test.cpp @@ -8,6 +8,7 @@ #include "ametsuchi/impl/postgres_wsv_command.hpp" #include "ametsuchi/impl/postgres_wsv_query.hpp" #include "framework/result_fixture.hpp" +#include "framework/test_logger.hpp" #include "module/irohad/ametsuchi/ametsuchi_fixture.hpp" #include "module/shared_model/interface_mocks.hpp" @@ -24,7 +25,8 @@ namespace iroha { pgopt_); command = std::make_unique(*sql); - query = std::make_unique(*sql, factory); + query = std::make_unique( + *sql, factory, getTestLogger("WsvQuery")); *sql << init_; } diff --git a/test/module/irohad/common/CMakeLists.txt b/test/module/irohad/common/CMakeLists.txt index d1c0eb25e6..73b9c63590 100644 --- a/test/module/irohad/common/CMakeLists.txt +++ b/test/module/irohad/common/CMakeLists.txt @@ -11,4 +11,5 @@ target_link_libraries(blob_converter_test AddTest(raw_block_loader_test raw_block_loader_test.cpp) target_link_libraries(raw_block_loader_test raw_block_loader + test_logger ) diff --git a/test/module/irohad/common/raw_block_loader_test.cpp b/test/module/irohad/common/raw_block_loader_test.cpp index f704d8c39d..312e8319f4 100644 --- a/test/module/irohad/common/raw_block_loader_test.cpp +++ b/test/module/irohad/common/raw_block_loader_test.cpp @@ -6,6 +6,7 @@ #include "main/raw_block_loader.hpp" #include +#include "framework/test_logger.hpp" #include "interfaces/iroha_internal/block.hpp" #include "interfaces/transaction.hpp" @@ -17,7 +18,7 @@ using iroha::main::BlockLoader; * @then check that the block is correct */ TEST(BlockLoaderTest, BlockLoaderJsonParsing) { - BlockLoader loader; + BlockLoader loader(getTestLogger("BlockLoader")); auto str = R"({ "block_v1": { diff --git a/test/module/irohad/consensus/yac/CMakeLists.txt b/test/module/irohad/consensus/yac/CMakeLists.txt index 53f7b439c6..93309885bf 100644 --- a/test/module/irohad/consensus/yac/CMakeLists.txt +++ b/test/module/irohad/consensus/yac/CMakeLists.txt @@ -10,31 +10,37 @@ target_link_libraries(cluster_order_test addtest(yac_cold_case_test yac_simple_cold_case_test.cpp) target_link_libraries(yac_cold_case_test yac + test_logger ) addtest(yac_sunny_day_test yac_sunny_day_test.cpp) target_link_libraries(yac_sunny_day_test yac + test_logger ) addtest(yac_rainy_day_test yac_rainy_day_test.cpp) target_link_libraries(yac_rainy_day_test yac + test_logger ) addtest(yac_unknown_peer_test yac_unknown_peer_test.cpp) target_link_libraries(yac_unknown_peer_test yac + test_logger ) addtest(yac_block_storage_test yac_block_storage_test.cpp) target_link_libraries(yac_block_storage_test yac + test_logger ) addtest(yac_proposal_storage_test yac_proposal_storage_test.cpp) target_link_libraries(yac_proposal_storage_test yac + test_logger ) addtest(yac_timer_test timer_test.cpp) @@ -46,6 +52,7 @@ addtest(yac_network_test network_test.cpp) target_link_libraries(yac_network_test yac yac_transport + test_logger ) addtest(yac_peer_orderer_test peer_orderer_test.cpp) @@ -56,6 +63,7 @@ target_link_libraries(yac_peer_orderer_test addtest(yac_gate_test yac_gate_test.cpp) target_link_libraries(yac_gate_test yac + test_logger ) addtest(yac_hash_provider_test yac_hash_provider_test.cpp) @@ -66,6 +74,7 @@ target_link_libraries(yac_hash_provider_test addtest(yac_common_test yac_common_test.cpp) target_link_libraries(yac_common_test yac + test_logger ) addtest(yac_crypto_provider_test yac_crypto_provider_test.cpp) @@ -77,6 +86,7 @@ target_link_libraries(yac_crypto_provider_test addtest(supermajority_checker_test supermajority_checker_test.cpp) target_link_libraries(supermajority_checker_test yac + test_logger ) addtest(buffered_cleanup_strategy_test buffered_cleanup_strategy_test.cpp) diff --git a/test/module/irohad/consensus/yac/network_test.cpp b/test/module/irohad/consensus/yac/network_test.cpp index e6200546c3..7744a9c6ff 100644 --- a/test/module/irohad/consensus/yac/network_test.cpp +++ b/test/module/irohad/consensus/yac/network_test.cpp @@ -7,6 +7,7 @@ #include +#include "framework/test_logger.hpp" #include "module/irohad/consensus/yac/mock_yac_crypto_provider.hpp" #include "module/irohad/consensus/yac/mock_yac_network.hpp" #include "module/irohad/consensus/yac/yac_test_util.hpp" @@ -26,8 +27,10 @@ namespace iroha { void SetUp() override { notifications = std::make_shared(); async_call = std::make_shared< - network::AsyncGrpcClient>(); - network = std::make_shared(async_call); + network::AsyncGrpcClient>( + getTestLogger("AsyncCall")); + network = std::make_shared(async_call, + getTestLogger("YacNetwork")); message.hash.vote_hashes.proposal_hash = "proposal"; message.hash.vote_hashes.block_hash = "block"; diff --git a/test/module/irohad/consensus/yac/supermajority_checker_test.cpp b/test/module/irohad/consensus/yac/supermajority_checker_test.cpp index f5a4aa66c0..d26ad8d963 100644 --- a/test/module/irohad/consensus/yac/supermajority_checker_test.cpp +++ b/test/module/irohad/consensus/yac/supermajority_checker_test.cpp @@ -14,13 +14,14 @@ #include "consensus/yac/supermajority_checker.hpp" #include "logger/logger.hpp" +#include "framework/test_logger.hpp" #include "module/shared_model/interface_mocks.hpp" using namespace iroha::consensus::yac; using ::testing::ReturnRefOfCopy; -static logger::Logger log_ = logger::testLog("YacCommon"); +static logger::LoggerPtr log_ = getTestLogger("YacCommon"); static const std::map kf1_param{ {ConsistencyModel::kCft, detail::kSupermajorityCheckerKfPlus1Cft}, diff --git a/test/module/irohad/consensus/yac/yac_block_storage_test.cpp b/test/module/irohad/consensus/yac/yac_block_storage_test.cpp index 4ef97c5de0..cae425411a 100644 --- a/test/module/irohad/consensus/yac/yac_block_storage_test.cpp +++ b/test/module/irohad/consensus/yac/yac_block_storage_test.cpp @@ -11,6 +11,7 @@ #include "consensus/yac/storage/yac_proposal_storage.hpp" #include "logger/logger.hpp" +#include "framework/test_logger.hpp" #include "module/irohad/consensus/yac/yac_test_util.hpp" using namespace iroha::consensus::yac; @@ -19,7 +20,7 @@ using namespace iroha::consensus::yac; static const iroha::consensus::yac::ConsistencyModel kConsistencyModel = iroha::consensus::yac::ConsistencyModel::kBft; -static logger::Logger log_ = logger::testLog("YacBlockStorage"); +static logger::LoggerPtr log_ = getTestLogger("YacBlockStorage"); class YacBlockStorageTest : public ::testing::Test { public: @@ -34,7 +35,8 @@ class YacBlockStorageTest : public ::testing::Test { YacBlockStorage(hash, number_of_peers, // todo mboldyrev 13.12.2018 IR-324 use mock super checker - getSupermajorityChecker(kConsistencyModel)); + getSupermajorityChecker(kConsistencyModel), + getTestLogger("YacBlockStorage")); std::vector valid_votes; void SetUp() override { @@ -75,6 +77,7 @@ TEST_F(YacBlockStorageTest, YacBlockStorageWhenNotCommittedAndCommitAcheive) { decltype(YacBlockStorageTest::valid_votes) for_insert(valid_votes.begin() + 1, valid_votes.end()); auto insert_commit = storage.insert(for_insert); + ASSERT_TRUE(insert_commit) << "Must be a commit!"; ASSERT_EQ(number_of_peers, boost::get(*insert_commit).votes.size()); } diff --git a/test/module/irohad/consensus/yac/yac_common_test.cpp b/test/module/irohad/consensus/yac/yac_common_test.cpp index 006e30237e..fbb14eb038 100644 --- a/test/module/irohad/consensus/yac/yac_common_test.cpp +++ b/test/module/irohad/consensus/yac/yac_common_test.cpp @@ -10,12 +10,13 @@ #include "consensus/yac/storage/yac_proposal_storage.hpp" #include "logger/logger.hpp" +#include "framework/test_logger.hpp" #include "module/irohad/consensus/yac/yac_test_util.hpp" using namespace iroha::consensus; using namespace iroha::consensus::yac; -static logger::Logger log_ = logger::testLog("YacCommon"); +static logger::LoggerPtr log_ = getTestLogger("YacCommon"); TEST(YacCommonTest, SameProposalTest) { log_->info("-----------| Verify ok and fail cases |-----------"); diff --git a/test/module/irohad/consensus/yac/yac_fixture.hpp b/test/module/irohad/consensus/yac/yac_fixture.hpp index 0a5be760f1..3c49a19ff8 100644 --- a/test/module/irohad/consensus/yac/yac_fixture.hpp +++ b/test/module/irohad/consensus/yac/yac_fixture.hpp @@ -12,6 +12,8 @@ #include "consensus/yac/storage/buffered_cleanup_strategy.hpp" #include "consensus/yac/yac.hpp" +#include "framework/test_logger.hpp" +#include "logger/logger_manager.hpp" #include "module/irohad/consensus/yac/mock_yac_crypto_provider.hpp" #include "module/irohad/consensus/yac/mock_yac_network.hpp" #include "module/irohad/consensus/yac/mock_yac_timer.hpp" @@ -62,11 +64,13 @@ namespace iroha { YacVoteStorage( std::make_shared< iroha::consensus::yac::BufferedCleanupStrategy>(), - getSupermajorityChecker(kConsistencyModel)), + getSupermajorityChecker(kConsistencyModel), + getTestLoggerManager()->getChild("YacVoteStorage")), network, crypto, timer, - ordering); + ordering, + getTestLogger("Yac")); network->subscribe(yac); } }; diff --git a/test/module/irohad/consensus/yac/yac_gate_test.cpp b/test/module/irohad/consensus/yac/yac_gate_test.cpp index 3d6b40a8b3..372a4c7e3f 100644 --- a/test/module/irohad/consensus/yac/yac_gate_test.cpp +++ b/test/module/irohad/consensus/yac/yac_gate_test.cpp @@ -13,6 +13,7 @@ #include "cryptography/crypto_provider/crypto_defaults.hpp" #include "framework/test_subscriber.hpp" +#include "framework/test_logger.hpp" #include "module/irohad/consensus/yac/mock_yac_hash_gate.hpp" #include "module/irohad/consensus/yac/mock_yac_hash_provider.hpp" #include "module/irohad/consensus/yac/mock_yac_peer_orderer.hpp" @@ -90,7 +91,8 @@ class YacGateTest : public ::testing::Test { std::move(peer_orderer_ptr), hash_provider, block_creator, - block_cache); + block_cache, + getTestLogger("YacGateImpl")); } iroha::consensus::Round round{1, 1}; diff --git a/test/module/irohad/consensus/yac/yac_proposal_storage_test.cpp b/test/module/irohad/consensus/yac/yac_proposal_storage_test.cpp index 894dba0b3f..2eef474d53 100644 --- a/test/module/irohad/consensus/yac/yac_proposal_storage_test.cpp +++ b/test/module/irohad/consensus/yac/yac_proposal_storage_test.cpp @@ -11,13 +11,15 @@ #include "consensus/yac/storage/yac_common.hpp" #include "logger/logger.hpp" +#include "framework/test_logger.hpp" +#include "logger/logger_manager.hpp" #include "module/irohad/consensus/yac/mock_yac_supermajority_checker.hpp" #include "module/irohad/consensus/yac/yac_test_util.hpp" using namespace iroha::consensus::yac; using namespace ::testing; -static logger::Logger log_ = logger::testLog("YacProposalStorage"); +static logger::LoggerPtr log_ = getTestLogger("YacProposalStorage"); class YacProposalStorageTest : public ::testing::Test { public: @@ -31,10 +33,14 @@ class YacProposalStorageTest : public ::testing::Test { const std::shared_ptr supermajority_checker = std::make_shared(); YacProposalStorage storage = YacProposalStorage( - iroha::consensus::Round{1, 1}, number_of_peers, supermajority_checker); + iroha::consensus::Round{1, 1}, + number_of_peers, + supermajority_checker, + getTestLoggerManager()->getChild("YacProposalStorage")); std::vector valid_votes; void SetUp() override { + valid_votes.reserve(number_of_peers); std::generate_n(std::back_inserter(valid_votes), number_of_peers, [this] { return createVote(this->hash, std::to_string(this->valid_votes.size())); }); diff --git a/test/module/irohad/logger/CMakeLists.txt b/test/module/irohad/logger/CMakeLists.txt index 7c99757cc5..ef490a1f38 100644 --- a/test/module/irohad/logger/CMakeLists.txt +++ b/test/module/irohad/logger/CMakeLists.txt @@ -8,5 +8,6 @@ AddTest(logger_test logger_test.cpp) target_link_libraries(logger_test logger + logger_manager ) diff --git a/test/module/irohad/logger/logger_test.cpp b/test/module/irohad/logger/logger_test.cpp index 358e1dda57..9b17dbd2ac 100644 --- a/test/module/irohad/logger/logger_test.cpp +++ b/test/module/irohad/logger/logger_test.cpp @@ -3,22 +3,20 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include "logger/logger.hpp" +#include "logger/logger_manager.hpp" #include #include -TEST(LoggerTest, getLoggerTest) { - auto one_logger = logger::log("one_logger"); - one_logger->info("one logger"); - - auto another_logger = logger::log("another_logger"); - another_logger->warn("another logger"); - another_logger->info("temporal output {}, {}", 123, "string param"); - another_logger->info(logger::red("color output")); - another_logger->info( - logger::yellow("color args output {} // note: require char *").c_str(), - "=^._.^="); +TEST(LoggerTest, basicStandaloneLoggerTest) { + logger::LoggerConfig config; + config.log_level = logger::LogLevel::kInfo; + logger::LoggerManagerTree manager( + std::make_unique(std::move(config))); + auto a_logger = manager.getChild("test info logger")->getLogger(); + a_logger->trace("testing a standalone logger: trace"); + a_logger->info("testing a standalone logger: info"); + a_logger->error("testing a standalone logger: error"); } TEST(LoggerTest, boolReprTest) { diff --git a/test/module/irohad/main/CMakeLists.txt b/test/module/irohad/main/CMakeLists.txt index d368e27148..7ca451e6be 100644 --- a/test/module/irohad/main/CMakeLists.txt +++ b/test/module/irohad/main/CMakeLists.txt @@ -7,4 +7,5 @@ addtest(server_runner_test server_runner_test.cpp) target_link_libraries(server_runner_test server_runner endpoint + test_logger ) diff --git a/test/module/irohad/main/server_runner_test.cpp b/test/module/irohad/main/server_runner_test.cpp index bed577243d..0f69da9bca 100644 --- a/test/module/irohad/main/server_runner_test.cpp +++ b/test/module/irohad/main/server_runner_test.cpp @@ -7,6 +7,7 @@ #include #include "endpoint.grpc.pb.h" // any gRPC service is required for test +#include "framework/test_logger.hpp" #include "main/server_runner.hpp" boost::format address{"0.0.0.0:%d"}; @@ -21,7 +22,7 @@ auto port_visitor = iroha::make_visitor( */ TEST(ServerRunnerTest, SamePortNoReuse) { ServerRunner first_runner( - (address % 0).str(), true, logger::log("ServerRunner1")); + (address % 0).str(), getTestLogger("ServerRunner1"), true); auto first_query_service = std::make_shared(); auto result = first_runner.append(first_query_service).run(); @@ -29,7 +30,7 @@ TEST(ServerRunnerTest, SamePortNoReuse) { ASSERT_NE(0, port); ServerRunner second_runner( - (address % port).str(), false, logger::log("ServerRunner2")); + (address % port).str(), getTestLogger("ServerRunner2"), false); auto second_query_service = std::make_shared(); result = second_runner.append(second_query_service).run(); @@ -44,7 +45,7 @@ TEST(ServerRunnerTest, SamePortNoReuse) { */ TEST(ServerRunnerTest, SamePortWithReuse) { ServerRunner first_runner( - (address % 0).str(), true, logger::log("ServerRunner1")); + (address % 0).str(), getTestLogger("ServerRunner1"), true); auto first_query_service = std::make_shared(); auto result = first_runner.append(first_query_service).run(); @@ -52,7 +53,7 @@ TEST(ServerRunnerTest, SamePortWithReuse) { ASSERT_NE(0, port); ServerRunner second_runner( - (address % port).str(), true, logger::log("ServerRunner2")); + (address % port).str(), getTestLogger("ServerRunner2"), true); auto second_query_service = std::make_shared(); result = second_runner.append(second_query_service).run(); diff --git a/test/module/irohad/model/CMakeLists.txt b/test/module/irohad/model/CMakeLists.txt index 537fdf2083..a49d701855 100644 --- a/test/module/irohad/model/CMakeLists.txt +++ b/test/module/irohad/model/CMakeLists.txt @@ -49,16 +49,19 @@ target_link_libraries(json_transaction_converter_test addtest(json_block_converter_test converters/json_block_test.cpp) target_link_libraries(json_block_converter_test json_model_converters + test_logger ) addtest(json_query_factory_test converters/json_query_factory_test.cpp) target_link_libraries(json_query_factory_test model_generators json_model_converters + test_logger ) addtest(pb_query_factory_test converters/pb_query_factory_test.cpp) target_link_libraries(pb_query_factory_test model_generators pb_model_converters + test_logger ) diff --git a/test/module/irohad/model/converters/json_block_test.cpp b/test/module/irohad/model/converters/json_block_test.cpp index 9e1026ac2d..05af1997d4 100644 --- a/test/module/irohad/model/converters/json_block_test.cpp +++ b/test/module/irohad/model/converters/json_block_test.cpp @@ -3,16 +3,19 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include #include "model/converters/json_block_factory.hpp" +#include + +#include "framework/test_logger.hpp" + using namespace iroha; using namespace iroha::model; using namespace iroha::model::converters; class JsonBlockTest : public ::testing::Test { public: - JsonBlockFactory factory; + JsonBlockFactory factory{getTestLogger("JsonBlockFactory")}; }; TEST_F(JsonBlockTest, ValidWhenWellFormed) { diff --git a/test/module/irohad/model/converters/json_query_factory_test.cpp b/test/module/irohad/model/converters/json_query_factory_test.cpp index 62d36c0e8f..e94ef2a774 100644 --- a/test/module/irohad/model/converters/json_query_factory_test.cpp +++ b/test/module/irohad/model/converters/json_query_factory_test.cpp @@ -4,8 +4,11 @@ */ #include "model/converters/json_query_factory.hpp" + #include #include + +#include "framework/test_logger.hpp" #include "model/converters/json_common.hpp" #include "model/generators/query_generator.hpp" #include "model/generators/signature_generator.hpp" @@ -18,8 +21,10 @@ using namespace iroha::model; using namespace iroha::model::converters; using namespace iroha::model::generators; +const auto json_query_factory_logger = getTestLogger("JsonQueryFactory"); + void runQueryTest(std::shared_ptr val) { - JsonQueryFactory queryFactory; + JsonQueryFactory queryFactory(json_query_factory_logger); auto json = queryFactory.serialize(val); auto ser_val = queryFactory.deserialize(json); ASSERT_TRUE(ser_val); @@ -28,7 +33,7 @@ void runQueryTest(std::shared_ptr val) { } TEST(QuerySerializerTest, ClassHandlerTest) { - JsonQueryFactory factory; + JsonQueryFactory factory(json_query_factory_logger); std::vector> commands = { std::make_shared(), std::make_shared(), @@ -43,7 +48,7 @@ TEST(QuerySerializerTest, ClassHandlerTest) { } TEST(QuerySerializerTest, DeserializeGetAccountWhenValid) { - JsonQueryFactory querySerializer; + JsonQueryFactory querySerializer(json_query_factory_logger); auto json_query = R"({ "signature":{ @@ -62,7 +67,7 @@ TEST(QuerySerializerTest, DeserializeGetAccountWhenValid) { } TEST(QuerySerializerTest, DeserializeGetAccountWhenInvalid) { - JsonQueryFactory querySerializer; + JsonQueryFactory querySerializer(json_query_factory_logger); auto json_query = R"({ "created_ts":0, "creator_account_id":"123", @@ -74,7 +79,7 @@ TEST(QuerySerializerTest, DeserializeGetAccountWhenInvalid) { } TEST(QuerySerializerTest, DeserializeGetAccountAssetsWhenValid) { - JsonQueryFactory querySerializer; + JsonQueryFactory querySerializer(json_query_factory_logger); auto json_query = R"({ "signature":{ "pubkey":"2323232323232323232323232323232323232323232323232323232323232323", @@ -98,7 +103,7 @@ TEST(QuerySerializerTest, DeserializeGetAccountAssetsWhenValid) { * @then Validate the invalid hash is skipped and the only valid deserialized. */ TEST(QuerySerializerTest, DeserializeGetAccountDetailWhenValid) { - JsonQueryFactory querySerializer; + JsonQueryFactory querySerializer(json_query_factory_logger); auto json_query = R"({ "signature":{ "pubkey":"2323232323232323232323232323232323232323232323232323232323232323", @@ -117,7 +122,7 @@ TEST(QuerySerializerTest, DeserializeGetAccountDetailWhenValid) { } TEST(QuerySerializerTest, DeserializeWhenUnknownType) { - JsonQueryFactory querySerializer; + JsonQueryFactory querySerializer(json_query_factory_logger); auto json_query = R"( "signature":{ "pubkey":"2323232323232323232323232323232323232323232323232323232323232323", @@ -140,7 +145,7 @@ TEST(QuerySerializerTest, DeserializeWhenUnknownType) { * @then Validate the invalid hash is skipped and the only valid deserialized. */ TEST(QuerySerialzierTest, DeserializeGetTransactionsWithInvalidHash) { - JsonQueryFactory queryFactory; + JsonQueryFactory queryFactory(json_query_factory_logger); iroha::hash256_t valid_size_hash{}; valid_size_hash[0] = 1; QueryGenerator queryGenerator; @@ -174,7 +179,7 @@ TEST(QuerySerialzierTest, DeserializeGetTransactionsWithInvalidHash) { } TEST(QuerySerializerTest, SerializeGetAccount) { - JsonQueryFactory queryFactory; + JsonQueryFactory queryFactory(json_query_factory_logger); QueryGenerator queryGenerator; auto val = queryGenerator.generateGetAccount(0, "123", 0, "test"); auto json = queryFactory.serialize(val); @@ -185,7 +190,7 @@ TEST(QuerySerializerTest, SerializeGetAccount) { } TEST(QuerySerializerTest, SerializeGetAccountAssets) { - JsonQueryFactory queryFactory; + JsonQueryFactory queryFactory(json_query_factory_logger); QueryGenerator queryGenerator; auto val = queryGenerator.generateGetAccountAssets(0, "123", 0, "test", "coin"); @@ -197,7 +202,7 @@ TEST(QuerySerializerTest, SerializeGetAccountAssets) { } TEST(QuerySerializerTest, SerializeGetAccountTransactions) { - JsonQueryFactory queryFactory; + JsonQueryFactory queryFactory(json_query_factory_logger); QueryGenerator queryGenerator; auto val = queryGenerator.generateGetAccountTransactions(0, "123", 0, "test"); auto json = queryFactory.serialize(val); @@ -217,7 +222,7 @@ TEST(QuerySerializerTest, SerialiizeGetTransactions) { } TEST(QuerySerializerTest, SerializeGetSignatories) { - JsonQueryFactory queryFactory; + JsonQueryFactory queryFactory(json_query_factory_logger); QueryGenerator queryGenerator; auto val = queryGenerator.generateGetSignatories(0, "123", 0, "test"); auto json = queryFactory.serialize(val); diff --git a/test/module/irohad/model/converters/pb_query_factory_test.cpp b/test/module/irohad/model/converters/pb_query_factory_test.cpp index 0bef13151e..1b341a1ea8 100644 --- a/test/module/irohad/model/converters/pb_query_factory_test.cpp +++ b/test/module/irohad/model/converters/pb_query_factory_test.cpp @@ -5,6 +5,7 @@ #include +#include "framework/test_logger.hpp" #include "model/converters/pb_query_factory.hpp" #include "model/converters/pb_transaction_factory.hpp" #include "model/generators/query_generator.hpp" @@ -16,8 +17,10 @@ using namespace iroha::model::converters; using namespace iroha::model::generators; using namespace iroha::model; +const auto pb_query_factory_logger = getTestLogger("PbQueryFactory"); + void runQueryTest(std::shared_ptr query) { - PbQueryFactory query_factory; + PbQueryFactory query_factory(pb_query_factory_logger); auto pb_query = query_factory.serialize(query); ASSERT_TRUE(pb_query); auto res_query = query_factory.deserialize(pb_query.value()); @@ -32,7 +35,7 @@ TEST(PbQueryFactoryTest, SerializeGetAccount) { auto creator_account_id = "creator"; auto query_counter = 222u; auto account_id = "test"; - PbQueryFactory query_factory; + PbQueryFactory query_factory(pb_query_factory_logger); QueryGenerator query_generator; auto query = query_generator.generateGetAccount( created_time, creator_account_id, query_counter, account_id); @@ -58,7 +61,7 @@ TEST(PbQueryFactoryTest, SerializeGetAccount) { } TEST(PbQueryFactoryTest, SerializeGetAccountAssets) { - PbQueryFactory query_factory; + PbQueryFactory query_factory(pb_query_factory_logger); QueryGenerator query_generator; auto query = query_generator.generateGetAccountAssets(0, "123", 0, "test", "coin"); @@ -77,7 +80,7 @@ TEST(PbQueryFactoryTest, SerializeGetAccountAssets) { * @then Return Protobuf Data */ TEST(PbQueryFactoryTest, SerializeGetAccountDetail) { - PbQueryFactory query_factory; + PbQueryFactory query_factory(pb_query_factory_logger); QueryGenerator query_generator; auto query = query_generator.generateGetAccountDetail(0, "123", 0, "test", "test2"); @@ -89,7 +92,7 @@ TEST(PbQueryFactoryTest, SerializeGetAccountDetail) { } TEST(PbQueryFactoryTest, SerializeGetAccountTransactions) { - PbQueryFactory query_factory; + PbQueryFactory query_factory(pb_query_factory_logger); QueryGenerator query_generator; auto query = query_generator.generateGetAccountTransactions(0, "123", 0, "test"); @@ -112,7 +115,7 @@ TEST(PbQueryFactoryTest, SerializeGetTransactions) { } TEST(PbQueryFactoryTest, SerializeGetSignatories) { - PbQueryFactory query_factory; + PbQueryFactory query_factory(pb_query_factory_logger); QueryGenerator query_generator; auto query = query_generator.generateGetSignatories(0, "123", 0, "test"); auto pb_query = query_factory.serialize(query); diff --git a/test/module/irohad/multi_sig_transactions/CMakeLists.txt b/test/module/irohad/multi_sig_transactions/CMakeLists.txt index 69d7c56187..cc00644825 100644 --- a/test/module/irohad/multi_sig_transactions/CMakeLists.txt +++ b/test/module/irohad/multi_sig_transactions/CMakeLists.txt @@ -4,7 +4,7 @@ AddTest(state_test state_test.cpp) target_link_libraries(state_test mst_state - logger + test_logger shared_model_proto_builders shared_model_stateless_validation shared_model_interfaces_factories @@ -13,7 +13,7 @@ target_link_libraries(state_test AddTest(storage_test storage_test.cpp) target_link_libraries(storage_test mst_storage - logger + test_logger shared_model_proto_builders shared_model_stateless_validation shared_model_interfaces_factories @@ -28,7 +28,7 @@ AddTest(transport_test transport_test.cpp) target_link_libraries(transport_test mst_transport mst_processor - logger + test_logger shared_model_cryptography shared_model_stateless_validation shared_model_proto_backend @@ -37,7 +37,7 @@ target_link_libraries(transport_test AddTest(mst_processor_test mst_processor_test.cpp) target_link_libraries(mst_processor_test mst_processor - logger + test_logger shared_model_proto_builders shared_model_stateless_validation ) diff --git a/test/module/irohad/multi_sig_transactions/mst_mocks.hpp b/test/module/irohad/multi_sig_transactions/mst_mocks.hpp index 4860f90bb3..fbdbfee445 100644 --- a/test/module/irohad/multi_sig_transactions/mst_mocks.hpp +++ b/test/module/irohad/multi_sig_transactions/mst_mocks.hpp @@ -7,6 +7,7 @@ #define IROHA_MST_MOCKS_HPP #include +#include "logger/logger_fwd.hpp" #include "multi_sig_transactions/mst_processor.hpp" #include "multi_sig_transactions/mst_propagation_strategy.hpp" #include "multi_sig_transactions/mst_time_provider.hpp" @@ -52,6 +53,7 @@ namespace iroha { }; struct MockMstProcessor : public MstProcessor { + MockMstProcessor(logger::LoggerPtr log) : MstProcessor(std::move(log)) {} MOCK_METHOD1(propagateBatchImpl, void(const DataType &)); MOCK_CONST_METHOD0(onStateUpdateImpl, rxcpp::observable>()); diff --git a/test/module/irohad/multi_sig_transactions/mst_processor_test.cpp b/test/module/irohad/multi_sig_transactions/mst_processor_test.cpp index 54e78d5445..4e4aacd004 100644 --- a/test/module/irohad/multi_sig_transactions/mst_processor_test.cpp +++ b/test/module/irohad/multi_sig_transactions/mst_processor_test.cpp @@ -7,6 +7,7 @@ #include #include "cryptography/keypair.hpp" #include "datetime/time.hpp" +#include "framework/test_logger.hpp" #include "framework/test_subscriber.hpp" #include "logger/logger.hpp" #include "module/irohad/multi_sig_transactions/mst_mocks.hpp" @@ -15,7 +16,7 @@ #include "multi_sig_transactions/mst_processor_impl.hpp" #include "multi_sig_transactions/storage/mst_storage_impl.hpp" -auto log_ = logger::log("MstProcessorTest"); +auto log_ = getTestLogger("MstProcessorTest"); using namespace iroha; using namespace framework::test_subscriber; @@ -48,8 +49,10 @@ class MstProcessorTest : public testing::Test { protected: void SetUp() override { transport = std::make_shared(); - storage = std::make_shared( - std::make_shared()); + storage = + std::make_shared(std::make_shared(), + getTestLogger("MstState"), + getTestLogger("MstStorage")); propagation_strategy = std::make_shared(); EXPECT_CALL(*propagation_strategy, emitter()) @@ -59,8 +62,12 @@ class MstProcessorTest : public testing::Test { EXPECT_CALL(*time_provider, getCurrentTime()) .WillRepeatedly(Return(time_now)); - mst_processor = std::make_shared( - transport, storage, propagation_strategy, time_provider); + mst_processor = + std::make_shared(transport, + storage, + propagation_strategy, + time_provider, + getTestLogger("FairMstProcessor")); } }; @@ -247,7 +254,8 @@ TEST_F(MstProcessorTest, onUpdateFromTransportUsecase) { // ---------------------------------| when |---------------------------------- shared_model::crypto::PublicKey another_peer_key("another_pubkey"); - auto transported_state = MstState::empty(std::make_shared()); + auto transported_state = MstState::empty(getTestLogger("MstState"), + std::make_shared()); transported_state += addSignaturesFromKeyPairs( makeTestBatch(txBuilder(1, time_now, quorum)), 0, makeKey()); mst_processor->onNewState(another_peer_key, transported_state); @@ -298,6 +306,7 @@ TEST_F(MstProcessorTest, emptyStatePropagation) { "another", shared_model::interface::types::PubkeyType("sign_one")); auto another_peer_state = MstState::empty( + getTestLogger("MstState"), std::make_shared(std::chrono::minutes(0))); another_peer_state += makeTestBatch(txBuilder(1)); diff --git a/test/module/irohad/multi_sig_transactions/mst_test_helpers.hpp b/test/module/irohad/multi_sig_transactions/mst_test_helpers.hpp index 0bcf48802a..0713a44a04 100644 --- a/test/module/irohad/multi_sig_transactions/mst_test_helpers.hpp +++ b/test/module/irohad/multi_sig_transactions/mst_test_helpers.hpp @@ -11,15 +11,13 @@ #include "cryptography/crypto_provider/crypto_defaults.hpp" #include "datetime/time.hpp" #include "framework/batch_helper.hpp" +#include "framework/test_logger.hpp" #include "interfaces/common_objects/types.hpp" +#include "logger/logger.hpp" #include "module/shared_model/builders/protobuf/test_transaction_builder.hpp" #include "multi_sig_transactions/mst_types.hpp" #include "multi_sig_transactions/state/mst_state.hpp" -#include "logger/logger.hpp" - -static logger::Logger mst_helpers_log_ = logger::log("MST_HELPERS"); - inline auto makeKey() { return shared_model::crypto::DefaultCryptoAlgorithmType::generateKeypair(); } @@ -43,6 +41,8 @@ auto makeTestBatch(TxBuilders... builders) { template auto addSignatures(Batch &&batch, int tx_number, Signatures... signatures) { + static logger::LoggerPtr log_ = getTestLogger("addSignatures"); + auto insert_signatures = [&](auto &&sig_pair) { batch->addSignature(tx_number, sig_pair.first, sig_pair.second); }; @@ -55,9 +55,8 @@ auto addSignatures(Batch &&batch, int tx_number, Signatures... signatures) { // use unused variable (void)temp; - mst_helpers_log_->info( - "Number of signatures was inserted {}", - boost::size(batch->transactions().at(tx_number)->signatures())); + log_->info("Number of signatures was inserted {}", + boost::size(batch->transactions().at(tx_number)->signatures())); return std::forward(batch); } diff --git a/test/module/irohad/multi_sig_transactions/state_test.cpp b/test/module/irohad/multi_sig_transactions/state_test.cpp index 5bd4998b1e..b17ba1e53b 100644 --- a/test/module/irohad/multi_sig_transactions/state_test.cpp +++ b/test/module/irohad/multi_sig_transactions/state_test.cpp @@ -4,6 +4,7 @@ */ #include +#include "framework/test_logger.hpp" #include "logger/logger.hpp" #include "module/irohad/multi_sig_transactions/mst_test_helpers.hpp" #include "multi_sig_transactions/state/mst_state.hpp" @@ -12,7 +13,8 @@ using namespace std; using namespace iroha; using namespace iroha::model; -auto log_ = logger::log("MstStateTest"); +auto mst_state_log_ = getTestLogger("MstState"); +auto log_ = getTestLogger("MstStateTest"); auto completer_ = std::make_shared(); /** @@ -21,7 +23,7 @@ auto completer_ = std::make_shared(); * @then checks that state contains the inserted batch */ TEST(StateTest, CreateState) { - auto state = MstState::empty(completer_); + auto state = MstState::empty(mst_state_log_, completer_); ASSERT_EQ(0, state.getBatches().size()); auto tx = addSignatures( makeTestBatch(txBuilder(1)), 0, makeSignature("1", "pub_key_1")); @@ -36,7 +38,7 @@ TEST(StateTest, CreateState) { * @then checks that signatures are merged into the state */ TEST(StateTest, UpdateExistingState) { - auto state = MstState::empty(completer_); + auto state = MstState::empty(mst_state_log_, completer_); auto time = iroha::time::now(); auto first_signature = makeSignature("1", "pub_key_1"); @@ -60,7 +62,7 @@ TEST(StateTest, UpdateExistingState) { * @then "contains" method shows presence of the batch */ TEST(StateTest, ContainsMethodFindsInsertedBatch) { - auto state = MstState::empty(completer_); + auto state = MstState::empty(mst_state_log_, completer_); auto first_signature = makeSignature("1", "pub_key_1"); auto batch = makeTestBatch(txBuilder(1, iroha::time::now())); @@ -76,7 +78,7 @@ TEST(StateTest, ContainsMethodFindsInsertedBatch) { * @then "contains" method shows absence of the batch */ TEST(StateTest, ContainsMethodDoesNotFindNonInsertedBatch) { - auto state = MstState::empty(completer_); + auto state = MstState::empty(mst_state_log_, completer_); auto batch = makeTestBatch(txBuilder(1, iroha::time::now())); EXPECT_FALSE(state.contains(batch)); @@ -90,7 +92,7 @@ TEST(StateTest, ContainsMethodDoesNotFindNonInsertedBatch) { TEST(StateTest, UpdateStateWhenTransactionsSame) { log_->info("Create empty state => insert two equal transaction"); - auto state = MstState::empty(completer_); + auto state = MstState::empty(mst_state_log_, completer_); auto time = iroha::time::now(); state += addSignatures( @@ -117,7 +119,7 @@ TEST(StateTest, UpdateStateWhenTransactionsSame) { TEST(StateTest, DifferentSignaturesUnionTest) { log_->info("Create two states => merge them"); - auto state1 = MstState::empty(completer_); + auto state1 = MstState::empty(mst_state_log_, completer_); state1 += addSignatures(makeTestBatch(txBuilder(1)), 0, makeSignature("1", "1")); @@ -129,7 +131,7 @@ TEST(StateTest, DifferentSignaturesUnionTest) { ASSERT_EQ(3, state1.getBatches().size()); - auto state2 = MstState::empty(completer_); + auto state2 = MstState::empty(mst_state_log_, completer_); state2 += addSignatures(makeTestBatch(txBuilder(4)), 0, makeSignature("4", "4")); state2 += @@ -154,8 +156,8 @@ TEST(StateTest, UnionStateWhenSameTransactionHaveDifferentSignatures) { auto time = iroha::time::now(); - auto state1 = MstState::empty(completer_); - auto state2 = MstState::empty(completer_); + auto state1 = MstState::empty(mst_state_log_, completer_); + auto state2 = MstState::empty(mst_state_log_, completer_); state1 += addSignatures( makeTestBatch(txBuilder(1, time)), 0, makeSignature("1", "1")); @@ -183,7 +185,7 @@ TEST(StateTest, UnionStateWhenSameTransactionHaveDifferentSignatures) { TEST(StateTest, UnionStateWhenTransactionsSame) { auto time = iroha::time::now(); - auto state1 = MstState::empty(completer_); + auto state1 = MstState::empty(mst_state_log_, completer_); state1 += addSignatures( makeTestBatch(txBuilder(1, time)), 0, makeSignature("1", "1")); state1 += addSignatures( @@ -191,7 +193,7 @@ TEST(StateTest, UnionStateWhenTransactionsSame) { ASSERT_EQ(2, state1.getBatches().size()); - auto state2 = MstState::empty(completer_); + auto state2 = MstState::empty(mst_state_log_, completer_); state2 += addSignatures( makeTestBatch(txBuilder(1, time)), 0, makeSignature("1", "1")); state2 += addSignatures( @@ -219,11 +221,11 @@ TEST(StateTest, DifferenceTest) { auto common_batch = makeTestBatch(txBuilder(1, time)); auto another_batch = makeTestBatch(txBuilder(3)); - auto state1 = MstState::empty(completer_); + auto state1 = MstState::empty(mst_state_log_, completer_); state1 += addSignatures(common_batch, 0, first_signature); state1 += addSignatures(common_batch, 0, second_signature); - auto state2 = MstState::empty(completer_); + auto state2 = MstState::empty(mst_state_log_, completer_); state2 += addSignatures(common_batch, 0, second_signature); state2 += addSignatures(common_batch, 0, third_signature); state2 += addSignatures(another_batch, 0, another_signature); @@ -245,7 +247,7 @@ TEST(StateTest, UpdateTxUntillQuorum) { auto quorum = 3u; auto time = iroha::time::now(); - auto state = MstState::empty(completer_); + auto state = MstState::empty(mst_state_log_, completer_); auto state_after_one_tx = state += addSignatures( makeTestBatch(txBuilder(1, time, quorum)), 0, makeSignature("1", "1")); @@ -276,7 +278,7 @@ TEST(StateTest, UpdateStateWithNewStateUntilQuorum) { auto keypair = makeKey(); auto time = iroha::time::now(); - auto state1 = MstState::empty(completer_); + auto state1 = MstState::empty(mst_state_log_, completer_); state1 += addSignatures(makeTestBatch(txBuilder(1, time, quorum)), 0, makeSignature("1_1", "1_1")); @@ -286,7 +288,7 @@ TEST(StateTest, UpdateStateWithNewStateUntilQuorum) { makeTestBatch(txBuilder(2, time)), 0, makeSignature("3", "3")); ASSERT_EQ(2, state1.getBatches().size()); - auto state2 = MstState::empty(completer_); + auto state2 = MstState::empty(mst_state_log_, completer_); state2 += addSignatures(makeTestBatch(txBuilder(1, time, quorum)), 0, makeSignature("1_2", "1_2")); @@ -314,7 +316,7 @@ TEST(StateTest, TimeIndexInsertionByTx) { 0, makeSignature("1_1", "1_1")); - auto state = MstState::empty(completer_); + auto state = MstState::empty(mst_state_log_, completer_); state += prepared_batch; @@ -334,7 +336,7 @@ TEST(StateTest, TimeIndexInsertionByAddState) { auto quorum = 3u; auto time = iroha::time::now(); - auto state1 = MstState::empty(completer_); + auto state1 = MstState::empty(mst_state_log_, completer_); state1 += addSignatures(makeTestBatch(txBuilder(1, time, quorum)), 0, makeSignature("1_1", "1_1")); @@ -342,7 +344,7 @@ TEST(StateTest, TimeIndexInsertionByAddState) { 0, makeSignature("1_2", "1_2")); - auto state2 = MstState::empty(completer_); + auto state2 = MstState::empty(mst_state_log_, completer_); state2 += addSignatures( makeTestBatch(txBuilder(2, time)), 0, makeSignature("2", "2")); state2 += addSignatures( @@ -363,13 +365,13 @@ TEST(StateTest, TimeIndexInsertionByAddState) { TEST(StateTest, RemovingTestWhenByTimeHasExpired) { auto time = iroha::time::now(); - auto state1 = MstState::empty(completer_); + auto state1 = MstState::empty(mst_state_log_, completer_); state1 += addSignatures( makeTestBatch(txBuilder(1, time)), 0, makeSignature("2", "2")); state1 += addSignatures( makeTestBatch(txBuilder(2, time)), 0, makeSignature("2", "2")); - auto state2 = MstState::empty(completer_); + auto state2 = MstState::empty(mst_state_log_, completer_); auto diff_state = state1 - state2; diff --git a/test/module/irohad/multi_sig_transactions/storage_test.cpp b/test/module/irohad/multi_sig_transactions/storage_test.cpp index 4cb5af5ee9..398dbb9893 100644 --- a/test/module/irohad/multi_sig_transactions/storage_test.cpp +++ b/test/module/irohad/multi_sig_transactions/storage_test.cpp @@ -5,13 +5,14 @@ #include #include +#include "framework/test_logger.hpp" #include "logger/logger.hpp" #include "module/irohad/multi_sig_transactions/mst_test_helpers.hpp" #include "multi_sig_transactions/storage/mst_storage_impl.hpp" using namespace iroha; -auto log_ = logger::log("MstStorageTest"); +auto log_ = getTestLogger("MstStorageTest"); class StorageTest : public testing::Test { public: @@ -19,7 +20,8 @@ class StorageTest : public testing::Test { void SetUp() override { completer_ = std::make_shared(); - storage = std::make_shared(completer_); + storage = std::make_shared( + completer_, getTestLogger("MstState"), getTestLogger("MstStorage")); fillOwnState(); } @@ -43,7 +45,7 @@ TEST_F(StorageTest, StorageWhenApplyOtherState) { "create state with default peers and other state => " "apply state"); - auto new_state = MstState::empty(completer_); + auto new_state = MstState::empty(getTestLogger("MstState"), completer_); new_state += makeTestBatch(txBuilder(5, creation_time)); new_state += makeTestBatch(txBuilder(6, creation_time)); new_state += makeTestBatch(txBuilder(7, creation_time)); diff --git a/test/module/irohad/multi_sig_transactions/transport_test.cpp b/test/module/irohad/multi_sig_transactions/transport_test.cpp index 6f4b4b332a..5687c0e219 100644 --- a/test/module/irohad/multi_sig_transactions/transport_test.cpp +++ b/test/module/irohad/multi_sig_transactions/transport_test.cpp @@ -8,6 +8,7 @@ #include #include "backend/protobuf/common_objects/proto_common_objects_factory.hpp" #include "backend/protobuf/proto_transport_factory.hpp" +#include "framework/test_logger.hpp" #include "interfaces/iroha_internal/transaction_batch_factory_impl.hpp" #include "interfaces/iroha_internal/transaction_batch_parser_impl.hpp" #include "module/irohad/ametsuchi/ametsuchi_mocks.hpp" @@ -30,8 +31,8 @@ using ::testing::Invoke; class TransportTest : public ::testing::Test { public: TransportTest() - : async_call_( - std::make_shared>()), + : async_call_(std::make_shared>( + getTestLogger("AsyncClient"))), parser_(std::make_shared()), batch_factory_(std::make_shared()), tx_presence_cache_( @@ -95,14 +96,16 @@ TEST_F(TransportTest, SendAndReceive) { std::move(batch_factory_), std::move(tx_presence_cache_), completer_, - my_key_.publicKey()); + my_key_.publicKey(), + getTestLogger("MstState"), + getTestLogger("MstTransportGrpc")); transport->subscribe(mst_notification_transport_); std::mutex mtx; std::condition_variable cv; auto time = iroha::time::now(); - auto state = iroha::MstState::empty(completer_); + auto state = iroha::MstState::empty(getTestLogger("MstState"), completer_); state += addSignaturesFromKeyPairs( makeTestBatch(txBuilder(1, time)), 0, makeKey()); state += addSignaturesFromKeyPairs( @@ -174,18 +177,21 @@ TEST_F(TransportTest, ReplayAttack) { shared_model::proto::Transaction>>(std::move(interface_tx_validator), std::move(proto_tx_validator)); - auto transport = std::make_shared(std::move(async_call_), - std::move(tx_factory), - std::move(parser_), - std::move(batch_factory_), - tx_presence_cache_, - completer_, - my_key_.publicKey()); + auto transport = + std::make_shared(std::move(async_call_), + std::move(tx_factory), + std::move(parser_), + std::move(batch_factory_), + tx_presence_cache_, + completer_, + my_key_.publicKey(), + getTestLogger("MstState"), + getTestLogger("MstTransportGrpc")); transport->subscribe(mst_notification_transport_); auto batch = makeTestBatch(txBuilder(1), txBuilder(2)); - auto state = iroha::MstState::empty(completer_); + auto state = iroha::MstState::empty(getTestLogger("MstState"), completer_); state += addSignaturesFromKeyPairs( addSignaturesFromKeyPairs(batch, 0, makeKey()), 1, makeKey()); diff --git a/test/module/irohad/network/CMakeLists.txt b/test/module/irohad/network/CMakeLists.txt index 085fcedf37..e41adabf3d 100644 --- a/test/module/irohad/network/CMakeLists.txt +++ b/test/module/irohad/network/CMakeLists.txt @@ -9,4 +9,5 @@ target_link_libraries(block_loader_test block_loader_service shared_model_cryptography shared_model_default_builders + test_logger ) diff --git a/test/module/irohad/network/block_loader_test.cpp b/test/module/irohad/network/block_loader_test.cpp index 2c667c4dc8..f8ad63f0ca 100644 --- a/test/module/irohad/network/block_loader_test.cpp +++ b/test/module/irohad/network/block_loader_test.cpp @@ -13,6 +13,7 @@ #include "cryptography/crypto_provider/crypto_defaults.hpp" #include "cryptography/hash.hpp" #include "datetime/time.hpp" +#include "framework/test_logger.hpp" #include "framework/test_subscriber.hpp" #include "module/irohad/ametsuchi/mock_block_query.hpp" #include "module/irohad/ametsuchi/mock_block_query_factory.hpp" @@ -59,9 +60,10 @@ class BlockLoaderTest : public testing::Test { peer_query_factory, shared_model::proto::ProtoBlockFactory( std::move(validator_ptr), - std::make_unique>())); + std::make_unique>()), + getTestLogger("BlockLoader")); service = std::make_shared( - block_query_factory, block_cache, logger::log("BlockLoaderService")); + block_query_factory, block_cache, getTestLogger("BlockLoaderService")); grpc::ServerBuilder builder; int port = 0; diff --git a/test/module/irohad/ordering/CMakeLists.txt b/test/module/irohad/ordering/CMakeLists.txt index f29e06dac8..9ab4475f02 100644 --- a/test/module/irohad/ordering/CMakeLists.txt +++ b/test/module/irohad/ordering/CMakeLists.txt @@ -6,21 +6,25 @@ target_link_libraries(on_demand_os_test on_demand_ordering_service shared_model_default_builders ametsuchi + test_logger ) addtest(on_demand_os_client_grpc_test on_demand_os_client_grpc_test.cpp) target_link_libraries(on_demand_os_client_grpc_test on_demand_ordering_service_transport_grpc + test_logger ) addtest(on_demand_os_server_grpc_test on_demand_os_server_grpc_test.cpp) target_link_libraries(on_demand_os_server_grpc_test on_demand_ordering_service_transport_grpc + test_logger ) addtest(on_demand_connection_manager_test on_demand_connection_manager_test.cpp) target_link_libraries(on_demand_connection_manager_test on_demand_connection_manager + test_logger ) addtest(on_demand_ordering_gate_test on_demand_ordering_gate_test.cpp) @@ -28,6 +32,7 @@ target_link_libraries(on_demand_ordering_gate_test on_demand_ordering_gate ordering_gate_common shared_model_interfaces_factories + test_logger ) addtest(on_demand_cache_test on_demand_cache_test.cpp) diff --git a/test/module/irohad/ordering/on_demand_connection_manager_test.cpp b/test/module/irohad/ordering/on_demand_connection_manager_test.cpp index 8fbaf931a1..0525ae8fb1 100644 --- a/test/module/irohad/ordering/on_demand_connection_manager_test.cpp +++ b/test/module/irohad/ordering/on_demand_connection_manager_test.cpp @@ -7,6 +7,7 @@ #include #include +#include "framework/test_logger.hpp" #include "interfaces/iroha_internal/proposal.hpp" #include "module/irohad/ordering/ordering_mocks.hpp" #include "module/shared_model/interface_mocks.hpp" @@ -45,7 +46,10 @@ struct OnDemandConnectionManagerTest : public ::testing::Test { } manager = std::make_shared( - factory, peers.get_observable(), cpeers); + factory, + peers.get_observable(), + cpeers, + getTestLogger("OsConnectionManager")); } OnDemandConnectionManager::CurrentPeers cpeers; diff --git a/test/module/irohad/ordering/on_demand_ordering_gate_test.cpp b/test/module/irohad/ordering/on_demand_ordering_gate_test.cpp index 2d53cf228f..6b4eb84343 100644 --- a/test/module/irohad/ordering/on_demand_ordering_gate_test.cpp +++ b/test/module/irohad/ordering/on_demand_ordering_gate_test.cpp @@ -7,6 +7,7 @@ #include #include +#include "framework/test_logger.hpp" #include "framework/test_subscriber.hpp" #include "interfaces/iroha_internal/transaction_batch_impl.hpp" #include "module/irohad/ametsuchi/mock_tx_presence_cache.hpp" @@ -52,7 +53,8 @@ class OnDemandOrderingGateTest : public ::testing::Test { cache, std::move(ufactory), tx_cache, - initial_round); + initial_round, + getTestLogger("OrderingGate")); } rxcpp::subjects::subject rounds; diff --git a/test/module/irohad/ordering/on_demand_os_client_grpc_test.cpp b/test/module/irohad/ordering/on_demand_os_client_grpc_test.cpp index abd5c4e374..8af2a79c4d 100644 --- a/test/module/irohad/ordering/on_demand_os_client_grpc_test.cpp +++ b/test/module/irohad/ordering/on_demand_os_client_grpc_test.cpp @@ -10,6 +10,7 @@ #include "backend/protobuf/proto_transport_factory.hpp" #include "backend/protobuf/transaction.hpp" #include "framework/mock_stream.h" +#include "framework/test_logger.hpp" #include "interfaces/iroha_internal/proposal.hpp" #include "interfaces/iroha_internal/transaction_batch_impl.hpp" #include "module/shared_model/validators/validators.hpp" @@ -45,18 +46,21 @@ class OnDemandOsClientGrpcTest : public ::testing::Test { auto ustub = std::make_unique(); stub = ustub.get(); async_call = - std::make_shared>(); + std::make_shared>( + getTestLogger("AsyncCall")); auto validator = std::make_unique(); proposal_validator = validator.get(); auto proto_validator = std::make_unique(); proto_proposal_validator = proto_validator.get(); proposal_factory = std::make_shared( std::move(validator), std::move(proto_validator)); - client = std::make_shared(std::move(ustub), - async_call, - proposal_factory, - [&] { return timepoint; }, - timeout); + client = + std::make_shared(std::move(ustub), + async_call, + proposal_factory, + [&] { return timepoint; }, + timeout, + getTestLogger("OdOsClientGrpc")); } proto::MockOnDemandOrderingStub *stub; diff --git a/test/module/irohad/ordering/on_demand_os_server_grpc_test.cpp b/test/module/irohad/ordering/on_demand_os_server_grpc_test.cpp index ab83983377..d3109e5712 100644 --- a/test/module/irohad/ordering/on_demand_os_server_grpc_test.cpp +++ b/test/module/irohad/ordering/on_demand_os_server_grpc_test.cpp @@ -9,6 +9,7 @@ #include "backend/protobuf/proposal.hpp" #include "backend/protobuf/proto_transport_factory.hpp" #include "backend/protobuf/transaction.hpp" +#include "framework/test_logger.hpp" #include "interfaces/iroha_internal/transaction_batch_impl.hpp" #include "interfaces/iroha_internal/transaction_batch_parser_impl.hpp" #include "module/irohad/ordering/mock_on_demand_os_notification.hpp" @@ -50,7 +51,8 @@ struct OnDemandOsServerGrpcTest : public ::testing::Test { std::make_shared(notification, std::move(transaction_factory), std::move(batch_parser), - batch_factory); + batch_factory, + getTestLogger("OdOsServerGrpc")); } std::shared_ptr notification; diff --git a/test/module/irohad/ordering/on_demand_os_test.cpp b/test/module/irohad/ordering/on_demand_os_test.cpp index 4fb4be9293..552c6ec0f8 100644 --- a/test/module/irohad/ordering/on_demand_os_test.cpp +++ b/test/module/irohad/ordering/on_demand_os_test.cpp @@ -12,6 +12,7 @@ #include "backend/protobuf/proto_proposal_factory.hpp" #include "builders/protobuf/transaction.hpp" #include "datetime/time.hpp" +#include "framework/test_logger.hpp" #include "interfaces/iroha_internal/transaction_batch_impl.hpp" #include "module/irohad/ametsuchi/ametsuchi_mocks.hpp" #include "module/shared_model/interface_mocks.hpp" @@ -61,11 +62,13 @@ class OnDemandOsTest : public ::testing::Test { _))) .WillByDefault(Return(std::vector{ iroha::ametsuchi::tx_cache_status_responses::Missing()})); - os = std::make_shared(transaction_limit, - std::move(factory), - std::move(tx_cache), - proposal_limit, - initial_round); + os = std::make_shared( + transaction_limit, + std::move(factory), + std::move(tx_cache), + getTestLogger("OdOrderingService"), + proposal_limit, + initial_round); } /** @@ -168,11 +171,13 @@ TEST_F(OnDemandOsTest, DISABLED_ConcurrentInsert) { shared_model::proto::ProtoProposalFactory>(); auto tx_cache = std::make_unique>(); - os = std::make_shared(large_tx_limit, - std::move(factory), - std::move(tx_cache), - proposal_limit, - initial_round); + os = std::make_shared( + large_tx_limit, + std::move(factory), + std::move(tx_cache), + getTestLogger("OdOrderingService"), + proposal_limit, + initial_round); auto call = [this](auto bounds) { for (auto i = bounds.first; i < bounds.second; ++i) { @@ -263,11 +268,13 @@ TEST_F(OnDemandOsTest, UseFactoryForProposal) { }); return result; })); - os = std::make_shared(transaction_limit, - std::move(factory), - std::move(tx_cache), - proposal_limit, - initial_round); + os = std::make_shared( + transaction_limit, + std::move(factory), + std::move(tx_cache), + getTestLogger("OdOrderingService"), + proposal_limit, + initial_round); EXPECT_CALL(*mock_factory, unsafeCreateProposal(_, _, _)) .WillOnce(Return(ByMove(makeMockProposal()))); diff --git a/test/module/irohad/pending_txs_storage/CMakeLists.txt b/test/module/irohad/pending_txs_storage/CMakeLists.txt index 969528f59f..47b812ac6b 100644 --- a/test/module/irohad/pending_txs_storage/CMakeLists.txt +++ b/test/module/irohad/pending_txs_storage/CMakeLists.txt @@ -14,4 +14,5 @@ target_link_libraries(pending_txs_storage_test shared_model_stateless_validation shared_model_proto_backend shared_model_interfaces_factories + test_logger ) diff --git a/test/module/irohad/pending_txs_storage/pending_txs_storage_test.cpp b/test/module/irohad/pending_txs_storage/pending_txs_storage_test.cpp index dae338c09d..bb2746502c 100644 --- a/test/module/irohad/pending_txs_storage/pending_txs_storage_test.cpp +++ b/test/module/irohad/pending_txs_storage/pending_txs_storage_test.cpp @@ -6,6 +6,7 @@ #include #include #include "datetime/time.hpp" +#include "framework/test_logger.hpp" #include "module/irohad/multi_sig_transactions/mst_test_helpers.hpp" #include "multi_sig_transactions/state/mst_state.hpp" #include "pending_txs_storage/impl/pending_txs_storage_impl.hpp" @@ -32,6 +33,9 @@ class PendingTxsStorageFixture : public ::testing::Test { std::shared_ptr completer_ = std::make_shared(std::chrono::minutes(0)); + + logger::LoggerPtr mst_state_log_{getTestLogger("MstState")}; + logger::LoggerPtr log_{getTestLogger("PendingTxsStorageFixture")}; }; /** @@ -42,8 +46,8 @@ class PendingTxsStorageFixture : public ::testing::Test { * @then the transactions can be added to MST state successfully */ TEST_F(PendingTxsStorageFixture, FixutureSelfCheck) { - auto state = - std::make_shared(iroha::MstState::empty(completer_)); + auto state = std::make_shared( + iroha::MstState::empty(mst_state_log_, completer_)); auto transactions = addSignatures(makeTestBatch(txBuilder(1, getUniqueTime()), @@ -64,8 +68,8 @@ TEST_F(PendingTxsStorageFixture, FixutureSelfCheck) { * @then list of pending transactions can be received for all batch creators */ TEST_F(PendingTxsStorageFixture, InsertionTest) { - auto state = - std::make_shared(iroha::MstState::empty(completer_)); + auto state = std::make_shared( + iroha::MstState::empty(mst_state_log_, completer_)); auto transactions = addSignatures( makeTestBatch(txBuilder(2, getUniqueTime(), 2, "alice@iroha"), txBuilder(2, getUniqueTime(), 2, "bob@iroha")), @@ -103,10 +107,10 @@ TEST_F(PendingTxsStorageFixture, InsertionTest) { * @then pending transactions response is also updated */ TEST_F(PendingTxsStorageFixture, SignaturesUpdate) { - auto state1 = - std::make_shared(iroha::MstState::empty(completer_)); - auto state2 = - std::make_shared(iroha::MstState::empty(completer_)); + auto state1 = std::make_shared( + iroha::MstState::empty(mst_state_log_, completer_)); + auto state2 = std::make_shared( + iroha::MstState::empty(mst_state_log_, completer_)); auto transactions = addSignatures( makeTestBatch(txBuilder(3, getUniqueTime(), 3, "alice@iroha")), 0, @@ -138,8 +142,8 @@ TEST_F(PendingTxsStorageFixture, SignaturesUpdate) { * @then users receives correct responses */ TEST_F(PendingTxsStorageFixture, SeveralBatches) { - auto state = - std::make_shared(iroha::MstState::empty(completer_)); + auto state = std::make_shared( + iroha::MstState::empty(mst_state_log_, completer_)); auto batch1 = addSignatures( makeTestBatch(txBuilder(2, getUniqueTime(), 2, "alice@iroha"), txBuilder(2, getUniqueTime(), 2, "bob@iroha")), @@ -180,16 +184,16 @@ TEST_F(PendingTxsStorageFixture, SeveralBatches) { * @then updates don't overwrite the whole storage state */ TEST_F(PendingTxsStorageFixture, SeparateBatchesDoNotOverwriteStorage) { - auto state1 = - std::make_shared(iroha::MstState::empty(completer_)); + auto state1 = std::make_shared( + iroha::MstState::empty(mst_state_log_, completer_)); auto batch1 = addSignatures( makeTestBatch(txBuilder(2, getUniqueTime(), 2, "alice@iroha"), txBuilder(2, getUniqueTime(), 2, "bob@iroha")), 0, makeSignature("1", "pub_key_1")); *state1 += batch1; - auto state2 = - std::make_shared(iroha::MstState::empty(completer_)); + auto state2 = std::make_shared( + iroha::MstState::empty(mst_state_log_, completer_)); auto batch2 = addSignatures( makeTestBatch(txBuilder(2, getUniqueTime(), 2, "alice@iroha"), txBuilder(3, getUniqueTime(), 3, "alice@iroha")), @@ -222,8 +226,8 @@ TEST_F(PendingTxsStorageFixture, SeparateBatchesDoNotOverwriteStorage) { * @then storage removes the batch */ TEST_F(PendingTxsStorageFixture, PreparedBatch) { - auto state = - std::make_shared(iroha::MstState::empty(completer_)); + auto state = std::make_shared( + iroha::MstState::empty(mst_state_log_, completer_)); std::shared_ptr batch = addSignatures( makeTestBatch(txBuilder(3, getUniqueTime(), 3, "alice@iroha")), @@ -258,8 +262,8 @@ TEST_F(PendingTxsStorageFixture, PreparedBatch) { * @then storage removes the batch */ TEST_F(PendingTxsStorageFixture, ExpiredBatch) { - auto state = - std::make_shared(iroha::MstState::empty(completer_)); + auto state = std::make_shared( + iroha::MstState::empty(mst_state_log_, completer_)); std::shared_ptr batch = addSignatures( makeTestBatch(txBuilder(3, getUniqueTime(), 3, "alice@iroha")), diff --git a/test/module/irohad/simulator/CMakeLists.txt b/test/module/irohad/simulator/CMakeLists.txt index c1340617d9..49315e8b75 100644 --- a/test/module/irohad/simulator/CMakeLists.txt +++ b/test/module/irohad/simulator/CMakeLists.txt @@ -9,4 +9,5 @@ target_link_libraries(simulator_test shared_model_stateless_validation shared_model_cryptography_model shared_model_proto_backend + test_logger ) diff --git a/test/module/irohad/simulator/simulator_test.cpp b/test/module/irohad/simulator/simulator_test.cpp index 84eafa5d1b..8c49e97324 100644 --- a/test/module/irohad/simulator/simulator_test.cpp +++ b/test/module/irohad/simulator/simulator_test.cpp @@ -13,6 +13,7 @@ #include "backend/protobuf/transaction.hpp" #include "builders/protobuf/transaction.hpp" #include "datetime/time.hpp" +#include "framework/test_logger.hpp" #include "framework/test_subscriber.hpp" #include "module/irohad/ametsuchi/mock_block_query.hpp" #include "module/irohad/ametsuchi/mock_block_query_factory.hpp" @@ -70,7 +71,8 @@ class SimulatorTest : public ::testing::Test { factory, block_query_factory, crypto_signer, - std::move(block_factory)); + std::move(block_factory), + getTestLogger("Simulator")); } consensus::Round round; diff --git a/test/module/irohad/synchronizer/CMakeLists.txt b/test/module/irohad/synchronizer/CMakeLists.txt index a4a5d54fd4..eb1a760413 100644 --- a/test/module/irohad/synchronizer/CMakeLists.txt +++ b/test/module/irohad/synchronizer/CMakeLists.txt @@ -12,4 +12,5 @@ target_link_libraries(synchronizer_test shared_model_interfaces_factories shared_model_default_builders consensus_round + test_logger ) diff --git a/test/module/irohad/synchronizer/synchronizer_test.cpp b/test/module/irohad/synchronizer/synchronizer_test.cpp index 7cd76567ee..a9bdb797b2 100644 --- a/test/module/irohad/synchronizer/synchronizer_test.cpp +++ b/test/module/irohad/synchronizer/synchronizer_test.cpp @@ -8,6 +8,7 @@ #include #include #include "backend/protobuf/block.hpp" +#include "framework/test_logger.hpp" #include "framework/test_subscriber.hpp" #include "module/irohad/ametsuchi/mock_block_query.hpp" #include "module/irohad/ametsuchi/mock_block_query_factory.hpp" @@ -71,11 +72,13 @@ class SynchronizerTest : public ::testing::Test { ON_CALL(*block_query, getTopBlockHeight()) .WillByDefault(Return(kHeight - 1)); - synchronizer = std::make_shared(consensus_gate, - chain_validator, - mutable_factory, - block_query_factory, - block_loader); + synchronizer = + std::make_shared(consensus_gate, + chain_validator, + mutable_factory, + block_query_factory, + block_loader, + getTestLogger("Synchronizer")); peer = makePeer("127.0.0.1", shared_model::crypto::PublicKey("111")); ledger_peers = std::make_shared(PeerList{peer}); diff --git a/test/module/irohad/torii/CMakeLists.txt b/test/module/irohad/torii/CMakeLists.txt index 6b50cee8a7..37a6ec3a16 100644 --- a/test/module/irohad/torii/CMakeLists.txt +++ b/test/module/irohad/torii/CMakeLists.txt @@ -11,6 +11,7 @@ target_link_libraries(torii_transport_command_test torii_service command_client gate_object + test_logger ) addtest(torii_queries_test torii_queries_test.cpp) @@ -20,12 +21,14 @@ target_link_libraries(torii_queries_test query_client server_runner processors + test_logger ) addtest(query_service_test query_service_test.cpp) target_link_libraries(query_service_test torii_service query_client + test_logger ) addtest(torii_service_query_test torii_service_query_test.cpp) @@ -33,6 +36,7 @@ target_link_libraries(torii_service_query_test torii_service server_runner query_client + test_logger ) addtest(command_sync_client_test @@ -42,6 +46,7 @@ target_link_libraries(command_sync_client_test command_client server_runner endpoint + test_logger ) addtest(command_service_replay_test @@ -49,4 +54,5 @@ addtest(command_service_replay_test ) target_link_libraries(command_service_replay_test torii_service + test_logger ) diff --git a/test/module/irohad/torii/command_service_replay_test.cpp b/test/module/irohad/torii/command_service_replay_test.cpp index 98ce419592..778fc80003 100644 --- a/test/module/irohad/torii/command_service_replay_test.cpp +++ b/test/module/irohad/torii/command_service_replay_test.cpp @@ -9,8 +9,8 @@ #include "ametsuchi/tx_cache_response.hpp" #include "backend/protobuf/proto_tx_status_factory.hpp" #include "framework/batch_helper.hpp" +#include "framework/test_logger.hpp" #include "interfaces/iroha_internal/transaction_batch_impl.hpp" -#include "logger/logger.hpp" #include "module/irohad/ametsuchi/mock_storage.hpp" #include "module/irohad/ametsuchi/mock_tx_presence_cache.hpp" #include "module/irohad/torii/torii_mocks.hpp" @@ -41,7 +41,7 @@ class CommandServiceReplayTest : public ::testing::Test { std::make_shared(); tx_presence_cache = std::make_shared(); - log = logger::log("CommandServiceReplayTest"); + log = getTestLogger("CommandServiceReplayTest"); cache = std::make_shared(); command_service = std::make_shared( @@ -72,7 +72,7 @@ class CommandServiceReplayTest : public ::testing::Test { std::shared_ptr status_bus; std::shared_ptr tx_status_factory; std::shared_ptr tx_presence_cache; - logger::Logger log; + logger::LoggerPtr log; std::shared_ptr cache; std::shared_ptr command_service; }; diff --git a/test/module/irohad/torii/command_sync_client_test.cpp b/test/module/irohad/torii/command_sync_client_test.cpp index 5b3094d463..7cd1c1f8e6 100644 --- a/test/module/irohad/torii/command_sync_client_test.cpp +++ b/test/module/irohad/torii/command_sync_client_test.cpp @@ -9,6 +9,7 @@ #include #include "endpoint_mock.grpc.pb.h" #include "framework/mock_stream.h" +#include "framework/test_logger.hpp" using testing::_; using testing::Invoke; @@ -19,7 +20,8 @@ class CommandSyncClientTest : public testing::Test { void SetUp() override { auto ustub = std::make_unique(); stub = ustub.get(); - client = std::make_shared(std::move(ustub)); + client = std::make_shared( + std::move(ustub), getTestLogger("CommandSyncClient")); } iroha::protocol::MockCommandService_v1Stub *stub; diff --git a/test/module/irohad/torii/processor/CMakeLists.txt b/test/module/irohad/torii/processor/CMakeLists.txt index c88f7d76b6..1d47287bb0 100644 --- a/test/module/irohad/torii/processor/CMakeLists.txt +++ b/test/module/irohad/torii/processor/CMakeLists.txt @@ -8,6 +8,7 @@ addtest(transaction_processor_test transaction_processor_test.cpp) target_link_libraries(transaction_processor_test processors shared_model_stateless_validation + test_logger ) # Testing of query processor @@ -15,4 +16,5 @@ addtest(query_processor_test query_processor_test.cpp) target_link_libraries(query_processor_test processors shared_model_cryptography + test_logger ) diff --git a/test/module/irohad/torii/processor/query_processor_test.cpp b/test/module/irohad/torii/processor/query_processor_test.cpp index 19e88d6a19..a902d0a79e 100644 --- a/test/module/irohad/torii/processor/query_processor_test.cpp +++ b/test/module/irohad/torii/processor/query_processor_test.cpp @@ -9,6 +9,7 @@ #include "backend/protobuf/query_responses/proto_error_query_response.hpp" #include "cryptography/crypto_provider/crypto_defaults.hpp" #include "cryptography/keypair.hpp" +#include "framework/test_logger.hpp" #include "framework/test_subscriber.hpp" #include "interfaces/query_responses/block_query_response.hpp" #include "module/irohad/ametsuchi/mock_block_query.hpp" @@ -42,7 +43,11 @@ class QueryProcessorTest : public ::testing::Test { query_response_factory = std::make_shared(); qpi = std::make_shared( - storage, storage, nullptr, query_response_factory); + storage, + storage, + nullptr, + query_response_factory, + getTestLogger("QueryProcessor")); EXPECT_CALL(*storage, getBlockQuery()) .WillRepeatedly(Return(block_queries)); EXPECT_CALL(*storage, createQueryExecutor(_, _)) diff --git a/test/module/irohad/torii/processor/transaction_processor_test.cpp b/test/module/irohad/torii/processor/transaction_processor_test.cpp index efbf9a4a70..550cf7ded7 100644 --- a/test/module/irohad/torii/processor/transaction_processor_test.cpp +++ b/test/module/irohad/torii/processor/transaction_processor_test.cpp @@ -12,6 +12,7 @@ #include "builders/default_builders.hpp" #include "builders/protobuf/transaction.hpp" #include "framework/batch_helper.hpp" +#include "framework/test_logger.hpp" #include "framework/test_subscriber.hpp" #include "interfaces/iroha_internal/transaction_batch.hpp" #include "interfaces/iroha_internal/transaction_sequence_factory.hpp" @@ -40,7 +41,7 @@ class TransactionProcessorTest : public ::testing::Test { public: void SetUp() override { pcs = std::make_shared(); - mst = std::make_shared(); + mst = std::make_shared(getTestLogger("MstProcessor")); EXPECT_CALL(*pcs, on_commit()) .WillRepeatedly(Return(commit_notifier.get_observable())); @@ -59,7 +60,8 @@ class TransactionProcessorTest : public ::testing::Test { pcs, mst, status_bus, - std::make_shared()); + std::make_shared(), + getTestLogger("TransactionProcessor")); } auto base_tx() { diff --git a/test/module/irohad/torii/query_service_test.cpp b/test/module/irohad/torii/query_service_test.cpp index cf9a90b0a3..b61af32145 100644 --- a/test/module/irohad/torii/query_service_test.cpp +++ b/test/module/irohad/torii/query_service_test.cpp @@ -8,6 +8,7 @@ #include "backend/protobuf/proto_transport_factory.hpp" #include "backend/protobuf/query_responses/proto_query_response.hpp" #include "builders/protobuf/queries.hpp" +#include "framework/test_logger.hpp" #include "module/irohad/torii/processor/mock_query_processor.hpp" #include "utils/query_error_response_visitor.hpp" #include "validators/protobuf/proto_query_validator.hpp" @@ -54,8 +55,8 @@ class QueryServiceTest : public ::testing::Test { } void init() { - query_service = - std::make_shared(query_processor, query_factory); + query_service = std::make_shared( + query_processor, query_factory, getTestLogger("QueryService")); } std::unique_ptr getResponse() { diff --git a/test/module/irohad/torii/torii_queries_test.cpp b/test/module/irohad/torii/torii_queries_test.cpp index 83135fefa8..0c4b2cd260 100644 --- a/test/module/irohad/torii/torii_queries_test.cpp +++ b/test/module/irohad/torii/torii_queries_test.cpp @@ -5,6 +5,7 @@ #include #include "crypto/keypair.hpp" +#include "framework/test_logger.hpp" #include "module/irohad/ametsuchi/mock_block_query.hpp" #include "module/irohad/ametsuchi/mock_query_executor.hpp" #include "module/irohad/ametsuchi/mock_storage.hpp" @@ -54,7 +55,8 @@ using ErrorQueryType = class ToriiQueriesTest : public testing::Test { public: virtual void SetUp() { - runner = std::make_unique(ip + ":0"); + runner = std::make_unique(ip + ":0", + getTestLogger("ServerRunner")); wsv_query = std::make_shared(); block_query = std::make_shared(); query_executor = std::make_shared(); @@ -73,11 +75,17 @@ class ToriiQueriesTest : public testing::Test { std::shared_ptr(query_executor)))); auto qpi = std::make_shared( - storage, storage, pending_txs_storage, query_response_factory); + storage, + storage, + pending_txs_storage, + query_response_factory, + getTestLogger("QueryProcessor")); //----------- Server run ---------------- initQueryFactory(); - runner->append(std::make_unique(qpi, query_factory)) + runner + ->append(std::make_unique( + qpi, query_factory, getTestLogger("QueryService"))) .run() .match( [this](iroha::expected::Value port) { diff --git a/test/module/irohad/torii/torii_service_query_test.cpp b/test/module/irohad/torii/torii_service_query_test.cpp index 893f00d9ac..9d4f1dc7cf 100644 --- a/test/module/irohad/torii/torii_service_query_test.cpp +++ b/test/module/irohad/torii/torii_service_query_test.cpp @@ -11,6 +11,7 @@ #include "backend/protobuf/query_responses/proto_query_response.hpp" #include "builders/default_builders.hpp" #include "builders/protobuf/queries.hpp" +#include "framework/test_logger.hpp" #include "main/server_runner.hpp" #include "module/irohad/torii/processor/mock_query_processor.hpp" #include "module/shared_model/builders/protobuf/test_query_builder.hpp" @@ -31,7 +32,7 @@ using ::testing::Truly; class ToriiQueryServiceTest : public ::testing::Test { public: virtual void SetUp() { - runner = std::make_unique(ip + ":0"); + runner = std::make_unique(ip + ":0", getTestLogger("ServerRunner")); // ----------- Command Service -------------- query_processor = std::make_shared(); @@ -39,8 +40,8 @@ class ToriiQueryServiceTest : public ::testing::Test { //----------- Server run ---------------- initQueryFactory(); runner - ->append(std::make_unique(query_processor, - query_factory)) + ->append(std::make_unique( + query_processor, query_factory, getTestLogger("QueryService"))) .run() .match( [this](iroha::expected::Value port) { diff --git a/test/module/irohad/torii/torii_transport_command_test.cpp b/test/module/irohad/torii/torii_transport_command_test.cpp index c69c697a76..705f58babb 100644 --- a/test/module/irohad/torii/torii_transport_command_test.cpp +++ b/test/module/irohad/torii/torii_transport_command_test.cpp @@ -16,6 +16,7 @@ #include "cryptography/public_key.hpp" #include "endpoint.pb.h" #include "endpoint_mock.grpc.pb.h" +#include "framework/test_logger.hpp" #include "interfaces/iroha_internal/transaction_batch.hpp" #include "interfaces/iroha_internal/transaction_batch_factory_impl.hpp" #include "interfaces/iroha_internal/transaction_batch_parser_impl.hpp" @@ -84,7 +85,8 @@ class CommandServiceTransportGrpcTest : public testing::Test { batch_parser, batch_factory, rxcpp::observable<>::iterate(gate_objects), - gate_objects.size()); + gate_objects.size(), + getTestLogger("CommandServiceTransportGrpc")); } std::shared_ptr status_bus; diff --git a/test/module/irohad/validation/CMakeLists.txt b/test/module/irohad/validation/CMakeLists.txt index 4c927941a9..a5c590f8fd 100644 --- a/test/module/irohad/validation/CMakeLists.txt +++ b/test/module/irohad/validation/CMakeLists.txt @@ -5,6 +5,7 @@ addtest(chain_validation_test chain_validation_test.cpp) target_link_libraries(chain_validation_test chain_validator shared_model_default_builders + test_logger ) addtest(stateful_validator_test stateful_validator_test.cpp) @@ -12,4 +13,5 @@ target_link_libraries(stateful_validator_test stateful_validator shared_model_default_builders shared_model_proto_backend + test_logger ) diff --git a/test/module/irohad/validation/chain_validation_test.cpp b/test/module/irohad/validation/chain_validation_test.cpp index 5751479a5b..8126b931e7 100644 --- a/test/module/irohad/validation/chain_validation_test.cpp +++ b/test/module/irohad/validation/chain_validation_test.cpp @@ -6,6 +6,7 @@ #include "validation/impl/chain_validator_impl.hpp" #include +#include "framework/test_logger.hpp" #include "module/irohad/ametsuchi/mock_mutable_storage.hpp" #include "module/irohad/ametsuchi/mock_peer_query.hpp" #include "module/irohad/consensus/yac/mock_yac_supermajority_checker.hpp" @@ -28,7 +29,8 @@ using ::testing::SaveArg; class ChainValidationTest : public ::testing::Test { public: void SetUp() override { - validator = std::make_shared(supermajority_checker); + validator = std::make_shared( + supermajority_checker, getTestLogger("ChainValidato")); storage = std::make_shared(); query = std::make_shared(); peers = std::vector>(); diff --git a/test/module/irohad/validation/stateful_validator_test.cpp b/test/module/irohad/validation/stateful_validator_test.cpp index 493f3b8c19..101828496c 100644 --- a/test/module/irohad/validation/stateful_validator_test.cpp +++ b/test/module/irohad/validation/stateful_validator_test.cpp @@ -11,6 +11,7 @@ #include "backend/protobuf/proto_proposal_factory.hpp" #include "common/result.hpp" #include "cryptography/crypto_provider/crypto_defaults.hpp" +#include "framework/test_logger.hpp" #include "interfaces/iroha_internal/batch_meta.hpp" #include "interfaces/iroha_internal/transaction_batch_parser_impl.hpp" #include "interfaces/transaction.hpp" @@ -102,8 +103,10 @@ class Validator : public testing::Test { shared_model::validation::DefaultProposalValidator>>(); parser = std::make_shared(); - sfv = std::make_shared(std::move(factory), - std::move(parser)); + sfv = std::make_shared( + std::move(factory), + std::move(parser), + getTestLogger("StatefulValidator")); temp_wsv_mock = std::make_shared(); } diff --git a/test/module/libs/crypto/CMakeLists.txt b/test/module/libs/crypto/CMakeLists.txt index 77e9c749b2..d7b18f87d2 100644 --- a/test/module/libs/crypto/CMakeLists.txt +++ b/test/module/libs/crypto/CMakeLists.txt @@ -19,4 +19,5 @@ target_link_libraries(signature_test AddTest(keys_manager_test keys_manager_test.cpp) target_link_libraries(keys_manager_test keys_manager + test_logger ) diff --git a/test/module/libs/crypto/keys_manager_test.cpp b/test/module/libs/crypto/keys_manager_test.cpp index c8ef392e48..5037c25250 100644 --- a/test/module/libs/crypto/keys_manager_test.cpp +++ b/test/module/libs/crypto/keys_manager_test.cpp @@ -9,6 +9,7 @@ #include #include #include "crypto/keys_manager_impl.hpp" +#include "framework/test_logger.hpp" #define PUBKEY_HEX_SIZE 64 #define PRIVKEY_HEX_SIZE 64 @@ -48,7 +49,8 @@ class KeyManager : public ::testing::Test { "00576e02f23c8c694c322796cb3ef494829fdf484f4b42312fb7d776fbd5123b"s; const std::string prikey = "36f028580bb02cc8272a9a020f4200e346e276ae664e45ee80745574e2f5ab80"s; - KeysManagerImpl manager = KeysManagerImpl(filepath); + const logger::LoggerPtr kKeysManagerLogger = getTestLogger("KeysManager"); + KeysManagerImpl manager = KeysManagerImpl(filepath, kKeysManagerLogger); const std::string passphrase = "test"; const std::string nonexistent = (boost::filesystem::temp_directory_path() / boost::filesystem::unique_path()) @@ -125,6 +127,8 @@ TEST_F(KeyManager, LoadInaccessiblePrikey) { TEST_F(KeyManager, CreateKeypairInNonexistentDir) { KeysManagerImpl manager = - KeysManagerImpl(boost::filesystem::unique_path().string(), nonexistent); + KeysManagerImpl(boost::filesystem::unique_path().string(), + nonexistent, + kKeysManagerLogger); ASSERT_FALSE(manager.createKeys(passphrase)); } diff --git a/test/module/shared_model/CMakeLists.txt b/test/module/shared_model/CMakeLists.txt index 51188050c7..741006f3f8 100644 --- a/test/module/shared_model/CMakeLists.txt +++ b/test/module/shared_model/CMakeLists.txt @@ -19,7 +19,7 @@ AddTest(interface_test ) target_link_libraries(interface_test shared_model_default_builders - logger + test_logger ) add_library(commands_mocks_factory diff --git a/test/module/shared_model/interface_test.cpp b/test/module/shared_model/interface_test.cpp index d5ca85c119..db5f51fe54 100644 --- a/test/module/shared_model/interface_test.cpp +++ b/test/module/shared_model/interface_test.cpp @@ -7,6 +7,7 @@ #include "logger/logger.hpp" #include "builders/protobuf/transaction.hpp" +#include "framework/test_logger.hpp" class TransactionFixture : public ::testing::Test { public: @@ -18,7 +19,7 @@ class TransactionFixture : public ::testing::Test { shared_model::crypto::Keypair keypair; shared_model::interface::types::TimestampType time; - logger::Logger log = logger::log("TransactionFixture"); + logger::LoggerPtr log = getTestLogger("TransactionFixture"); auto makeTx() { log->info("keypair = {}, timestemp = {}", keypair, time); diff --git a/test/regression/CMakeLists.txt b/test/regression/CMakeLists.txt index 03b7016e13..5c5a428fd8 100644 --- a/test/regression/CMakeLists.txt +++ b/test/regression/CMakeLists.txt @@ -8,6 +8,7 @@ addtest(regression_test regression_test.cpp) target_link_libraries(regression_test application integration_framework + test_logger ) addtest(query_regression_test query_test.cpp) diff --git a/test/regression/regression_test.cpp b/test/regression/regression_test.cpp index 6fc74d5915..27eb3f73a6 100644 --- a/test/regression/regression_test.cpp +++ b/test/regression/regression_test.cpp @@ -11,11 +11,14 @@ #include "cryptography/crypto_provider/crypto_defaults.hpp" #include "framework/common_constants.hpp" #include "framework/integration_framework/integration_test_framework.hpp" +#include "framework/test_logger.hpp" #include "interfaces/query_responses/transactions_response.hpp" using namespace common_constants; using shared_model::interface::permissions::Role; +static logger::LoggerPtr log_ = getTestLogger("RegressionTest"); + /** * @given ITF instance with Iroha * @when existing ITF instance was not gracefully shutdown @@ -126,7 +129,7 @@ TEST(RegressionTest, StateRecovery) { // test launch if ITF was failed for some reason. If there are some blocks, // then checkProposal will fail with "missed proposal" error, because of // incorrect calculation of chain height. - iroha::remove_dir_contents(path); + iroha::remove_dir_contents(path, log_); { integration_framework::IntegrationTestFramework( diff --git a/test/system/CMakeLists.txt b/test/system/CMakeLists.txt index a316911189..8de588413c 100644 --- a/test/system/CMakeLists.txt +++ b/test/system/CMakeLists.txt @@ -16,6 +16,9 @@ target_link_libraries(irohad_test command_client query_client keys_manager + iroha_conf_literals + logger + logger_manager ) add_dependencies(irohad_test irohad) add_definitions(-DPATHIROHAD="${PROJECT_BINARY_DIR}/bin") diff --git a/test/system/irohad_test.cpp b/test/system/irohad_test.cpp index d7dee9c091..6066f7d688 100644 --- a/test/system/irohad_test.cpp +++ b/test/system/irohad_test.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -20,6 +21,9 @@ #include "crypto/keys_manager_impl.hpp" #include "integration/acceptance/acceptance_fixture.hpp" #include "interfaces/query_responses/roles_response.hpp" +#include "logger/logger.hpp" +#include "logger/logger_manager.hpp" +#include "main/iroha_conf_literals.hpp" #include "main/iroha_conf_loader.hpp" #include "network/impl/grpc_channel_builder.hpp" #include "torii/command_client.hpp" @@ -36,29 +40,51 @@ using namespace std::chrono_literals; using namespace common_constants; using iroha::operator|; +static logger::LoggerManagerTreePtr getIrohadTestLoggerManager() { + static logger::LoggerManagerTreePtr irohad_test_logger_manager; + if (!irohad_test_logger_manager) { + irohad_test_logger_manager = + std::make_shared(logger::LoggerConfig{ + logger::LogLevel::kInfo, logger::getDefaultLogPatterns()}); + } + return irohad_test_logger_manager->getChild("IrohadTest"); +} + class IrohadTest : public AcceptanceFixture { public: - IrohadTest() : kAddress("127.0.0.1"), kPort(50051) {} + IrohadTest() + : kAddress("127.0.0.1"), + kPort(50051), + test_data_path_(boost::filesystem::path(PATHTESTDATA)), + keys_manager_( + kAdminId, + test_data_path_, + getIrohadTestLoggerManager()->getChild("KeysManager")->getLogger()), + log_(getIrohadTestLoggerManager()->getLogger()) {} void SetUp() override { setPaths(); - auto config = parse_iroha_config(path_config_.string()); + + rapidjson::Document doc; + std::ifstream ifs_iroha(path_config_.string()); + rapidjson::IStreamWrapper isw(ifs_iroha); + doc.ParseStream(isw); + ASSERT_FALSE(doc.HasParseError()) + << "Failed to parse irohad config at " << path_config_.string(); blockstore_path_ = (boost::filesystem::temp_directory_path() / boost::filesystem::unique_path()) .string(); pgopts_ = integration_framework::getPostgresCredsOrDefault( - config[config_members::PgOpt].GetString()); + doc[config_members::PgOpt].GetString()); // we need a separate file here in case if target environment // has custom database connection options set // via environment variables - auto config_copy_json = parse_iroha_config(path_config_.string()); - config_copy_json[config_members::BlockStorePath].SetString( - blockstore_path_.data(), blockstore_path_.size()); - config_copy_json[config_members::PgOpt].SetString(pgopts_.data(), - pgopts_.size()); + doc[config_members::BlockStorePath].SetString(blockstore_path_.data(), + blockstore_path_.size()); + doc[config_members::PgOpt].SetString(pgopts_.data(), pgopts_.size()); rapidjson::StringBuffer sb; rapidjson::PrettyWriter writer(sb); - config_copy_json.Accept(writer); + doc.Accept(writer); std::string config_copy_string = sb.GetString(); std::ofstream copy_file(config_copy_); copy_file.write(config_copy_string.data(), config_copy_string.size()); @@ -149,7 +175,8 @@ class IrohadTest : public AcceptanceFixture { torii::CommandSyncClient client( iroha::network::createClient( - kAddress + ":" + std::to_string(kPort))); + kAddress + ":" + std::to_string(kPort)), + getIrohadTestLoggerManager()->getChild("CommandClient")->getLogger()); client.Torii(tx.getTransport()); auto resub_counter(resubscribe_attempts); @@ -180,7 +207,6 @@ class IrohadTest : public AcceptanceFixture { void setPaths() { path_irohad_ = boost::filesystem::path(PATHIROHAD); irohad_executable = path_irohad_ / "irohad"; - test_data_path_ = boost::filesystem::path(PATHTESTDATA); path_config_ = test_data_path_ / "config.sample"; path_genesis_ = test_data_path_ / "genesis.block"; path_keypair_ = test_data_path_ / "node0"; @@ -240,6 +266,9 @@ DROP TABLE IF EXISTS position_by_account_asset; std::string pgopts_; std::string blockstore_path_; std::string config_copy_; + iroha::KeysManagerImpl keys_manager_; + + logger::LoggerPtr log_; }; /** @@ -261,8 +290,7 @@ TEST_F(IrohadTest, RunIrohad) { TEST_F(IrohadTest, SendTx) { launchIroha(); - auto key_manager = iroha::KeysManagerImpl(kAdminId, test_data_path_); - auto key_pair = key_manager.loadKeys(); + auto key_pair = keys_manager_.loadKeys(); ASSERT_TRUE(key_pair); SCOPED_TRACE("From send transaction test"); @@ -277,8 +305,8 @@ TEST_F(IrohadTest, SendTx) { */ TEST_F(IrohadTest, SendQuery) { launchIroha(); - auto key_manager = iroha::KeysManagerImpl(kAdminId, test_data_path_); - auto key_pair = key_manager.loadKeys(); + + auto key_pair = keys_manager_.loadKeys(); ASSERT_TRUE(key_pair); iroha::protocol::QueryResponse response; @@ -304,8 +332,7 @@ TEST_F(IrohadTest, SendQuery) { TEST_F(IrohadTest, RestartWithOverwriteLedger) { launchIroha(); - auto key_manager = iroha::KeysManagerImpl(kAdminId, test_data_path_); - auto key_pair = key_manager.loadKeys(); + auto key_pair = keys_manager_.loadKeys(); ASSERT_TRUE(key_pair); SCOPED_TRACE("From restart with --overwrite-ledger flag test"); @@ -336,8 +363,7 @@ TEST_F(IrohadTest, RestartWithOverwriteLedger) { TEST_F(IrohadTest, RestartWithoutResetting) { launchIroha(); - auto key_manager = iroha::KeysManagerImpl(kAdminId, test_data_path_); - auto key_pair = key_manager.loadKeys(); + auto key_pair = keys_manager_.loadKeys(); ASSERT_TRUE(key_pair); SCOPED_TRACE("From restart without resetting test"); From 2d52df54f1c45b5e5bff89cee107f41133d7e440 Mon Sep 17 00:00:00 2001 From: Konstantin Munichev Date: Wed, 6 Mar 2019 13:07:09 +0300 Subject: [PATCH 08/40] Add deb packages debug build support (#2144) Signed-off-by: Konstantin Munichev --- cmake/functions.cmake | 4 ++-- cmake/release.cmake | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cmake/functions.cmake b/cmake/functions.cmake index 3f319f96fc..f9161fcdbb 100644 --- a/cmake/functions.cmake +++ b/cmake/functions.cmake @@ -125,7 +125,7 @@ endmacro() macro(add_install_step_for_bin target) install(TARGETS ${target} RUNTIME DESTINATION bin - CONFIGURATIONS Release + CONFIGURATIONS ${CMAKE_BUILD_TYPE} COMPONENT binaries) endmacro() @@ -137,7 +137,7 @@ macro(add_install_step_for_lib libpath) install(FILES ${lib_major_minor_patch} DESTINATION lib - CONFIGURATIONS Release + CONFIGURATIONS ${CMAKE_BUILD_TYPE} COMPONENT libraries) endmacro() diff --git a/cmake/release.cmake b/cmake/release.cmake index e9dbf6783b..73628ae880 100644 --- a/cmake/release.cmake +++ b/cmake/release.cmake @@ -24,8 +24,11 @@ endif() SET(CPACK_PACKAGE_VERSION ${IROHA_VERSION}) message(STATUS "[IROHA_VERSION] '${IROHA_VERSION}'") - -SET(CPACK_STRIP_FILES TRUE) +if (CMAKE_BUILD_TYPE MATCHES Release) + SET(CPACK_STRIP_FILES TRUE) +else() + SET(CPACK_STRIP_FILES FALSE) +endif() set(CPACK_COMPONENTS_ALL binaries libraries) From 15c88f289824330b7e6ae55e35aa86ae3e7581d7 Mon Sep 17 00:00:00 2001 From: Igor Egorov Date: Wed, 6 Mar 2019 14:43:53 +0300 Subject: [PATCH 09/40] Decrease quantity of allocated proto query objects (#2148) Avoid creation of redundant proto query object. Signed-off-by: Igor Egorov --- .../backend/protobuf/queries/impl/proto_query.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/shared_model/backend/protobuf/queries/impl/proto_query.cpp b/shared_model/backend/protobuf/queries/impl/proto_query.cpp index bf039ed0d1..9aff554f01 100644 --- a/shared_model/backend/protobuf/queries/impl/proto_query.cpp +++ b/shared_model/backend/protobuf/queries/impl/proto_query.cpp @@ -51,14 +51,14 @@ namespace shared_model { TransportType proto_; ProtoQueryVariantType variant_{[this] { - const auto &ar = proto_; + auto &ar = proto_; int which = ar.payload() .GetDescriptor() ->FindFieldByNumber(ar.payload().query_case()) ->index_in_oneof(); - return shared_model::detail::variant_impl:: - template load(std::forward(ar), - which); + return shared_model::detail::variant_impl< + ProtoQueryListType>::template load(ar, + which); }()}; QueryVariantType ivariant_{variant_}; From 229a8d3cf8a6f323ef51eebb5348dc70d632d40e Mon Sep 17 00:00:00 2001 From: Mikhail Boldyrev Date: Thu, 7 Mar 2019 16:17:18 +0300 Subject: [PATCH 10/40] Torii: TransactionProcessorImpl listens for storage commits (#2143) * TransactionProcessorImpl uses Storage::on_commit() Signed-off-by: Mikhail Boldyrev --- irohad/main/application.cpp | 1 + .../impl/transaction_processor_impl.cpp | 42 +++++++------------ .../processor/transaction_processor_impl.hpp | 3 ++ test/fuzzing/status_fuzz.cpp | 5 +-- test/fuzzing/torii_fuzz.cpp | 7 +++- .../processor/transaction_processor_test.cpp | 29 ++++--------- 6 files changed, 35 insertions(+), 52 deletions(-) diff --git a/irohad/main/application.cpp b/irohad/main/application.cpp index 0d38ccca09..2dd8ee8e05 100644 --- a/irohad/main/application.cpp +++ b/irohad/main/application.cpp @@ -553,6 +553,7 @@ void Irohad::initTransactionCommandService() { mst_processor, status_bus_, status_factory, + storage->on_commit(), command_service_log_manager->getChild("Processor")->getLogger()); command_service = std::make_shared<::torii::CommandServiceImpl>( tx_processor, diff --git a/irohad/torii/processor/impl/transaction_processor_impl.cpp b/irohad/torii/processor/impl/transaction_processor_impl.cpp index d00f7f86c4..3c714e8748 100644 --- a/irohad/torii/processor/impl/transaction_processor_impl.cpp +++ b/irohad/torii/processor/impl/transaction_processor_impl.cpp @@ -49,6 +49,8 @@ namespace iroha { std::shared_ptr status_bus, std::shared_ptr status_factory, + rxcpp::observable> + commits, logger::LoggerPtr log) : pcs_(std::move(pcs)), mst_processor_(std::move(mst_processor)), @@ -83,33 +85,19 @@ namespace iroha { }); // commit transactions - pcs_->on_commit().subscribe( - [this](synchronizer::SynchronizationEvent sync_event) { - bool has_at_least_one_committed = false; - sync_event.synced_blocks.subscribe( - // on next - [this, &has_at_least_one_committed](auto model_block) { - for (const auto &tx : model_block->transactions()) { - const auto &hash = tx.hash(); - log_->info("SynchronizationEvent Committed: {}", - hash.hex()); - this->publishStatus(TxStatusType::kCommitted, hash); - has_at_least_one_committed = true; - } - for (const auto &rejected_tx_hash : - model_block->rejected_transactions_hashes()) { - log_->info("SynchronizationEvent Rejected: {}", - rejected_tx_hash.hex()); - this->publishStatus(TxStatusType::kRejected, - rejected_tx_hash); - } - }, - // on complete - [this, &has_at_least_one_committed] { - if (not has_at_least_one_committed) { - log_->info("there are no transactions to be committed"); - } - }); + commits.subscribe( + // on next + [this](auto block) { + for (const auto &tx : block->transactions()) { + const auto &hash = tx.hash(); + log_->debug("Committed transaction: {}", hash.hex()); + this->publishStatus(TxStatusType::kCommitted, hash); + } + for (const auto &rejected_tx_hash : + block->rejected_transactions_hashes()) { + log_->debug("Rejected transaction: {}", rejected_tx_hash.hex()); + this->publishStatus(TxStatusType::kRejected, rejected_tx_hash); + } }); mst_processor_->onStateUpdate().subscribe([this](auto &&state) { diff --git a/irohad/torii/processor/transaction_processor_impl.hpp b/irohad/torii/processor/transaction_processor_impl.hpp index 9258e01605..c92ba1acf3 100644 --- a/irohad/torii/processor/transaction_processor_impl.hpp +++ b/irohad/torii/processor/transaction_processor_impl.hpp @@ -28,6 +28,7 @@ namespace iroha { * @param mst_processor is a handler for multisignature transactions * @param status_bus is a common notifier for tx statuses * @param status_factory creates transaction statuses + * @param commits - an observable on committed blocks * @param log to print the progress */ TransactionProcessorImpl( @@ -36,6 +37,8 @@ namespace iroha { std::shared_ptr status_bus, std::shared_ptr status_factory, + rxcpp::observable> + commits, logger::LoggerPtr log); void batchHandle( diff --git a/test/fuzzing/status_fuzz.cpp b/test/fuzzing/status_fuzz.cpp index 6363be66f6..12a0e2a88f 100644 --- a/test/fuzzing/status_fuzz.cpp +++ b/test/fuzzing/status_fuzz.cpp @@ -46,7 +46,7 @@ struct CommandFixture { rxcpp::subjects::subject prop_notifier_; rxcpp::subjects::subject vprop_notifier_; - rxcpp::subjects::subject + rxcpp::subjects::subject> commit_notifier_; rxcpp::subjects::subject mst_notifier_; rxcpp::subjects::subject> @@ -57,8 +57,6 @@ struct CommandFixture { pcs_ = std::make_shared(); EXPECT_CALL(*pcs_, onProposal()) .WillRepeatedly(Return(prop_notifier_.get_observable())); - EXPECT_CALL(*pcs_, on_commit()) - .WillRepeatedly(Return(commit_notifier_.get_observable())); EXPECT_CALL(*pcs_, onVerifiedProposal()) .WillRepeatedly(Return(vprop_notifier_.get_observable())); @@ -79,6 +77,7 @@ struct CommandFixture { mst_processor_, status_bus, status_factory, + commit_notifier_.get_observable(), logger::getDummyLoggerPtr()); std::unique_ptr vprop_notifier_; rxcpp::subjects::subject - commit_notifier_; + sync_event_notifier_; rxcpp::subjects::subject mst_notifier_; rxcpp::subjects::subject> mst_state_notifier_; rxcpp::subjects::subject consensus_notifier_; + rxcpp::subjects::subject> + commit_notifier_; CommandFixture() { pcs_ = std::make_shared(); EXPECT_CALL(*pcs_, onProposal()) .WillRepeatedly(Return(prop_notifier_.get_observable())); EXPECT_CALL(*pcs_, on_commit()) - .WillRepeatedly(Return(commit_notifier_.get_observable())); + .WillRepeatedly(Return(sync_event_notifier_.get_observable())); EXPECT_CALL(*pcs_, onVerifiedProposal()) .WillRepeatedly(Return(vprop_notifier_.get_observable())); @@ -76,6 +78,7 @@ struct CommandFixture { mst_processor_, status_bus, status_factory, + commit_notifier_.get_observable(), logger::getDummyLoggerPtr()); auto storage = std::make_shared(); service_ = std::make_shared( diff --git a/test/module/irohad/torii/processor/transaction_processor_test.cpp b/test/module/irohad/torii/processor/transaction_processor_test.cpp index 550cf7ded7..b49bd7a7ee 100644 --- a/test/module/irohad/torii/processor/transaction_processor_test.cpp +++ b/test/module/irohad/torii/processor/transaction_processor_test.cpp @@ -43,8 +43,6 @@ class TransactionProcessorTest : public ::testing::Test { pcs = std::make_shared(); mst = std::make_shared(getTestLogger("MstProcessor")); - EXPECT_CALL(*pcs, on_commit()) - .WillRepeatedly(Return(commit_notifier.get_observable())); EXPECT_CALL(*pcs, onVerifiedProposal()) .WillRepeatedly(Return(verified_prop_notifier.get_observable())); @@ -61,6 +59,7 @@ class TransactionProcessorTest : public ::testing::Test { mst, status_bus, std::make_shared(), + commit_notifier.get_observable(), getTestLogger("TransactionProcessor")); } @@ -125,6 +124,10 @@ class TransactionProcessorTest : public ::testing::Test { mst_update_notifier; rxcpp::subjects::subject mst_prepared_notifier; rxcpp::subjects::subject mst_expired_notifier; + rxcpp::subjects::subject> + commit_notifier; + rxcpp::subjects::subject + verified_prop_notifier; std::shared_ptr pcs; std::shared_ptr status_bus; @@ -136,10 +139,6 @@ class TransactionProcessorTest : public ::testing::Test { shared_model::proto::TransactionStatusBuilder> status_builder; - rxcpp::subjects::subject commit_notifier; - rxcpp::subjects::subject - verified_prop_notifier; - consensus::Round round; const size_t proposal_size = 5; const size_t block_size = 3; @@ -317,13 +316,8 @@ TEST_F(TransactionProcessorTest, TransactionProcessorOnCommitTest) { auto block = TestBlockBuilder().transactions(txs).build(); // 2. Create block and notify transaction processor about it - SynchronizationEvent commit_event{ - rxcpp::observable<>::just( - std::shared_ptr(clone(block))), - SynchronizationOutcomeType::kCommit, - {}, - {}}; - commit_notifier.get_subscriber().on_next(commit_event); + commit_notifier.get_subscriber().on_next( + std::shared_ptr(clone(block))); SCOPED_TRACE("Committed status verification"); validateStatuses(txs); @@ -405,13 +399,8 @@ TEST_F(TransactionProcessorTest, TransactionProcessorInvalidTxsTest) { })) .build(); - SynchronizationEvent commit_event{ - rxcpp::observable<>::just( - std::shared_ptr(clone(block))), - SynchronizationOutcomeType::kCommit, - {}, - {}}; - commit_notifier.get_subscriber().on_next(commit_event); + commit_notifier.get_subscriber().on_next( + std::shared_ptr(clone(block))); { SCOPED_TRACE("Rejected status verification"); From 793124e1c17563732bd60a825c043b0ad3f99663 Mon Sep 17 00:00:00 2001 From: Sara Date: Thu, 7 Mar 2019 17:48:24 +0300 Subject: [PATCH 11/40] Markdown to RST docs configuration (#2146) * [ci skip] preparing files for md to rst documentation Signed-off-by: Sara * changes to allow md files be included into the rstdocs with an example Signed-off-by: Sara * adding automated deployment docs from .md Signed-off-by: Sara --- deploy/ansible/roles/iroha-docker/README.md | 22 ++++---- docs/source/conf.py | 7 ++- docs/source/contribution/index.rst | 14 +---- docs/source/guides/deployment.rst | 59 +-------------------- docs/source/requirements.txt | 1 + 5 files changed, 17 insertions(+), 86 deletions(-) diff --git a/deploy/ansible/roles/iroha-docker/README.md b/deploy/ansible/roles/iroha-docker/README.md index 4b0a92c3f9..7031632d8a 100644 --- a/deploy/ansible/roles/iroha-docker/README.md +++ b/deploy/ansible/roles/iroha-docker/README.md @@ -1,5 +1,5 @@ -## Description -This role deploys multiple replicas of Iroha containers (one Iroha peer per container) on remote hosts. Each Iroha peer can communicate with others in two ways: +##### Description +[This role](https://github.com/hyperledger/iroha/tree/master/deploy/ansible/roles/iroha-docker) deploys multiple replicas of Iroha containers (one Iroha peer per container) on remote hosts. Each Iroha peer can communicate with others in two ways: - using public IP addresses or hostnames set in inventory list OR - using private IP addresses of the Docker overlay network @@ -11,7 +11,7 @@ The second one can be used when there exists an overlay network between the host The second way is also suitable for local-only deployments. -## Requirements +##### Requirements Tested on Ubuntu 16.04, 18.04 - Local: - python3, python3-dev @@ -23,10 +23,10 @@ The second way is also suitable for local-only deployments. There is a role for setting up a remote part of the dependencies named `docker`. It works for Ubuntu OS only. Check `iroha-docker` playbook. -### Note: +**Note:** > `docker.io` package from Ubuntu repos will not work. Either use Ansible role or install Docker following official instructions for your OS flavor. -## Quick Start +##### Quick Start 1. Install Ansible ``` pip3 install ansible @@ -46,15 +46,15 @@ The second way is also suitable for local-only deployments. This will deploy 6 Iroha Docker containers along with 6 Postgres containers on the remote host specified in `iroha.list` file. Remote user is `ubuntu`. Torii port of each container is exposed on the host. Iroha peer can be communicated over port defined in `iroha_torii_port` variable (50051 by default). Overall, each host will listen the following port range: `iroha_torii_port` ... `iroha_torii_port` + *number-of-containers* - 1. It will also install Docker along with required python modules. If you want to skip this step, comment out `docker` role in the playbook (`playbooks/iroha-docker/main.yml`) -### Note: +**Note:** > This command escalates privileges on a remote host during the run. It is required to be able to spin up Docker containers. We recommend to run the playbook using a passwordless remote sudo user. -## Initial configuration +##### Initial configuration See `defaults/main.yml` file to get more details about available configuration options. -## Examples -### Example 1 +##### Examples +**Example 1** Deploying 6 Iroha peers on two remote hosts communicating using public IP addresses. With 2 and 4 replicas on each host respectively. @@ -90,10 +90,10 @@ Deploying 6 Iroha peers on two remote hosts communicating using public IP addres ansible-playbook -i inventory/iroha.list -b playbooks/iroha-docker/main.yml ``` -### Example 2 +**Example 2** Deploying 6 Iroha peers on two remote hosts communicating over overlay network (Calico) using custom hostnames. **TBD** -### Caveats +##### Caveats 1. If `/usr/bin/python` does not exist on a remote host, Ansible will fail with the misleading message: `... Make sure this host can be reached over ssh`. This usually happens when Ansible uses Python 3. On Ubuntu systems `/usr/bin/python3` is not symlinked to `/usr/bin/python` which Ansible expects to find. The problem can be solved by setting `ansible_python_interpreter` variable to `/usr/bin/python3`. diff --git a/docs/source/conf.py b/docs/source/conf.py index 621e3ddb26..dd7acd0f3e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -20,7 +20,8 @@ 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.ifconfig', - 'sphinx.ext.viewcode' + 'sphinx.ext.viewcode', + 'm2r' ] # Add any paths that contain templates here, relative to this directory. @@ -28,9 +29,7 @@ # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: -# -# source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ['.rst', '.md'] # The master toctree document. master_doc = 'index' diff --git a/docs/source/contribution/index.rst b/docs/source/contribution/index.rst index d8d8bbd75d..ebb306d27a 100644 --- a/docs/source/contribution/index.rst +++ b/docs/source/contribution/index.rst @@ -1,16 +1,4 @@ Contribution ============ -.. Attention:: Contents are missing for now. - -Code of conduct ---------------- - -Process -------- - -Communication -------------- - -Issue tracker -------------- +.. mdinclude:: ../../../CONTRIBUTING.md \ No newline at end of file diff --git a/docs/source/guides/deployment.rst b/docs/source/guides/deployment.rst index 1354767847..ee40b861df 100644 --- a/docs/source/guides/deployment.rst +++ b/docs/source/guides/deployment.rst @@ -159,64 +159,7 @@ In order to form a block, which includes more than a single peer, or requires cu Automated --------- -Anyone can reuse existing Ansible Playbook in order to create a network of peers running Iroha. -Currently, this is a solution for development and testing, in other words, a proof of concept, and cannot be used in production environment, due to some security flaws. -For production network, a manual composing of genesis block is required. - -Prerequisites -""""""""""""" - - * One ore more machines with a Linux distributive installed. - * SSH access to those machines - * Ansible installed on a local machine - -Step-by-step guide -"""""""""""""""""" - -1. Create peers.list file in $IROHA_HOME/deploy/ansible/data - -2. Write all peers IP addresses followed by the internal port 10001 (e.g 31.192.120.36:10001) - -3. Open $IROHA_HOME/deploy/ansible/hosts file - -4. Write all IP addresses in [hosts] group - -5. Open terminal - -6. Disable host key checking, because it can cause troubles due to interactive prompt - -.. code-block:: shell - - export ANSIBLE_HOST_KEY_CHECKING=False - -7. Go to ansible folder - -.. code-block:: shell - - cd $IROHA_HOME/deploy/ansible - -8. Run playbook, providing your private key and hosts file - -.. code-block:: shell - - ansible-playbook --private-key=~/.ssh/iroha -i hosts provisioning.yml - -9. Wait until playbook finishes and then Iroha network is ready and up. - -Checking Iroha peer status -"""""""""""""""""""""""""" - -1. SSH into any of your machines - -.. code-block:: shell - - ssh -i ~/.ssh/iroha iroha@35.205.142.238 - -2. Check Iroha container logs: - -.. code-block:: shell - - docker logs iroha +.. mdinclude:: ../../../deploy/ansible/roles/iroha-docker/README.md Dealing with troubles ^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/source/requirements.txt b/docs/source/requirements.txt index 645caa0034..dc8ac776a1 100644 --- a/docs/source/requirements.txt +++ b/docs/source/requirements.txt @@ -33,3 +33,4 @@ typing==3.6.2 urllib3==1.23 watchdog==0.8.3 yarg==0.1.9 +m2r From e389ee04c73ea251d09b7ce27aab862db8942763 Mon Sep 17 00:00:00 2001 From: Andrei Lebedev Date: Thu, 7 Mar 2019 20:32:58 +0300 Subject: [PATCH 12/40] Fix round number in SynchronizationEvent (#2147) Signed-off-by: Andrei Lebedev --- .../synchronizer/impl/synchronizer_impl.cpp | 12 ++++-- .../irohad/synchronizer/synchronizer_test.cpp | 43 ++++++++++++++++++- 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/irohad/synchronizer/impl/synchronizer_impl.cpp b/irohad/synchronizer/impl/synchronizer_impl.cpp index e5b2dd2e23..05109ac429 100644 --- a/irohad/synchronizer/impl/synchronizer_impl.cpp +++ b/irohad/synchronizer/impl/synchronizer_impl.cpp @@ -103,10 +103,14 @@ namespace iroha { auto ledger_state = mutable_factory_->commit(std::move(storage)); if (ledger_state) { - return SynchronizationEvent{chain, - SynchronizationOutcomeType::kCommit, - msg.round, - std::move(*ledger_state)}; + return SynchronizationEvent{ + chain, + SynchronizationOutcomeType::kCommit, + blocks.back()->height() > expected_height + // TODO 07.03.19 andrei: IR-387 Remove reject round + ? consensus::Round{blocks.back()->height(), 0} + : msg.round, + std::move(*ledger_state)}; } else { return boost::none; } diff --git a/test/module/irohad/synchronizer/synchronizer_test.cpp b/test/module/irohad/synchronizer/synchronizer_test.cpp index a9bdb797b2..eaa73353e5 100644 --- a/test/module/irohad/synchronizer/synchronizer_test.cpp +++ b/test/module/irohad/synchronizer/synchronizer_test.cpp @@ -85,9 +85,10 @@ class SynchronizerTest : public ::testing::Test { } std::shared_ptr makeCommit( + shared_model::interface::types::HeightType height = kHeight, size_t time = iroha::time::now()) const { auto block = TestUnsignedBlockBuilder() - .height(kHeight) + .height(height) .createdTime(time) .build() .signAndAddSignature( @@ -97,7 +98,7 @@ class SynchronizerTest : public ::testing::Test { return std::make_shared(std::move(block)); } - const shared_model::interface::types::HeightType kHeight{5}; + static const shared_model::interface::types::HeightType kHeight{5}; std::shared_ptr chain_validator; std::shared_ptr mutable_factory; @@ -220,6 +221,44 @@ TEST_F(SynchronizerTest, ValidWhenValidChain) { ASSERT_TRUE(wrapper.validate()); } +/** + * @given A commit from consensus and initialized components + * @when gate have voted for other block and multiple blocks are loaded + * @then Successful commit + */ +TEST_F(SynchronizerTest, ValidWhenValidChainMultipleBlocks) { + DefaultValue, std::string>>:: + SetFactory(&createMockMutableStorage); + + EXPECT_CALL(*mutable_factory, createMutableStorage()).Times(1); + + EXPECT_CALL(*mutable_factory, commit_(_)) + .WillOnce(Return(ByMove(std::make_unique(ledger_peers)))); + EXPECT_CALL(*chain_validator, validateAndApply(_, _)).WillOnce(Return(true)); + auto second_commit = makeCommit(kHeight + 1); + EXPECT_CALL(*block_loader, retrieveBlocks(_, _)) + .WillOnce(Return(rxcpp::observable<>::iterate( + std::vector>{ + commit_message, second_commit}))); + + auto wrapper = + make_test_subscriber(synchronizer->on_commit_chain(), 1); + wrapper.subscribe([this](auto commit_event) { + EXPECT_EQ(*this->ledger_peers, *commit_event.ledger_state->ledger_peers); + auto block_wrapper = + make_test_subscriber(commit_event.synced_blocks, 2); + block_wrapper.subscribe(); + ASSERT_EQ(commit_event.round.block_round, kHeight + 1); + ASSERT_EQ(commit_event.sync_outcome, SynchronizationOutcomeType::kCommit); + ASSERT_TRUE(block_wrapper.validate()); + }); + + gate_outcome.get_subscriber().on_next( + consensus::VoteOther{public_keys, hash, consensus::Round{kHeight, 1}}); + + ASSERT_TRUE(wrapper.validate()); +} + /** * @given A commit from consensus and initialized components * @when gate have voted for other block From 4e1b9cfcfaee2dc27be244b4182ae43fc6db9a2f Mon Sep 17 00:00:00 2001 From: Mikhail Boldyrev Date: Mon, 11 Mar 2019 11:52:05 +0300 Subject: [PATCH 13/40] Provide git version info (#2127) Signed-off-by: Mikhail Boldyrev --- irohad/main/CMakeLists.txt | 1 + irohad/main/irohad.cpp | 12 ++++-------- libs/common/CMakeLists.txt | 33 +++++++++++++++++++++++++++++++++ libs/common/irohad_version.cpp | 12 ++++++++++++ libs/common/irohad_version.hpp | 16 ++++++++++++++++ 5 files changed, 66 insertions(+), 8 deletions(-) create mode 100644 libs/common/irohad_version.cpp create mode 100644 libs/common/irohad_version.hpp diff --git a/irohad/main/CMakeLists.txt b/irohad/main/CMakeLists.txt index 8a7b0e1c85..62e5b2dbba 100644 --- a/irohad/main/CMakeLists.txt +++ b/irohad/main/CMakeLists.txt @@ -66,6 +66,7 @@ target_link_libraries(irohad iroha_conf_loader logger logger_manager + irohad_version ) add_library(iroha_conf_loader iroha_conf_loader.cpp) diff --git a/irohad/main/irohad.cpp b/irohad/main/irohad.cpp index f1f5f962de..ca4aaed959 100644 --- a/irohad/main/irohad.cpp +++ b/irohad/main/irohad.cpp @@ -10,6 +10,7 @@ #include #include #include "ametsuchi/storage.hpp" +#include "common/irohad_version.hpp" #include "common/result.hpp" #include "crypto/keys_manager_impl.hpp" #include "logger/logger.hpp" @@ -106,20 +107,14 @@ logger::LoggerManagerTreePtr getDefaultLogManager() { } int main(int argc, char *argv[]) { + gflags::SetVersionString(iroha::kGitPrettyVersion); + // Parsing command line arguments gflags::ParseCommandLineFlags(&argc, &argv, true); logger::LoggerManagerTreePtr log_manager; logger::LoggerPtr log; - // If the global log level override was set in the command line arguments, - // create a logger manager with the given log level for all subsystems: - if (FLAGS_verbosity != kLogSettingsFromConfigFile) { - logger::LoggerConfig cfg; - cfg.log_level = config_members::LogLevels.at(FLAGS_verbosity); - log_manager = std::make_shared(std::move(cfg)); - log = log_manager->getChild("Init")->getLogger(); - } // Check if validators are registered. if (not config_validator_registered @@ -137,6 +132,7 @@ int main(int argc, char *argv[]) { log_manager = config.logger_manager.value_or(getDefaultLogManager()); log = log_manager->getChild("Init")->getLogger(); } + log->info("Irohad version: {}", iroha::kGitPrettyVersion); log->info("config initialized"); // Reading public and private key files diff --git a/libs/common/CMakeLists.txt b/libs/common/CMakeLists.txt index 95853b7694..c7399afc4e 100644 --- a/libs/common/CMakeLists.txt +++ b/libs/common/CMakeLists.txt @@ -33,3 +33,36 @@ target_link_libraries(libs_timeout INTERFACE common rxcpp ) + +add_library(irohad_version irohad_version.cpp) + +# Get the git repo data +set(GIT_REPO_PRETTY_VER "version info unavailable") +if (EXISTS "${PROJECT_SOURCE_DIR}/.git") + if(NOT GIT_FOUND) + find_package(Git QUIET) + endif() + + if(GIT_EXECUTABLE) + # Get pretty version + execute_process(COMMAND + "${GIT_EXECUTABLE}" + describe --always + WORKING_DIRECTORY + "${PROJECT_SOURCE_DIR}" + RESULT_VARIABLE + res + OUTPUT_VARIABLE + GIT_REPO_PRETTY_VER + ERROR_QUIET + OUTPUT_STRIP_TRAILING_WHITESPACE) + if(NOT res EQUAL 0) + message(STATUS "Error running `git describe': ${res}") + endif() + message(STATUS "Git repo pretty version: ${GIT_REPO_PRETTY_VER}") + else() + message(STATUS "Git executable not found!") + endif() +endif() + +target_compile_definitions(irohad_version PRIVATE -DGIT_REPO_PRETTY_VER="${GIT_REPO_PRETTY_VER}") diff --git a/libs/common/irohad_version.cpp b/libs/common/irohad_version.cpp new file mode 100644 index 0000000000..626788d864 --- /dev/null +++ b/libs/common/irohad_version.cpp @@ -0,0 +1,12 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "common/irohad_version.hpp" + +namespace iroha { + + const char *kGitPrettyVersion = GIT_REPO_PRETTY_VER; + +} // namespace iroha diff --git a/libs/common/irohad_version.hpp b/libs/common/irohad_version.hpp new file mode 100644 index 0000000000..31fc8fbc33 --- /dev/null +++ b/libs/common/irohad_version.hpp @@ -0,0 +1,16 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef LIBS_COMMON_IROHAD_VERSION_HPP +#define LIBS_COMMON_IROHAD_VERSION_HPP + +namespace iroha { + + /// A string describing current git repository version in a human-readable way + extern const char *kGitPrettyVersion; + +} // namespace iroha + +#endif // LIBS_COMMON_IROHAD_VERSION_HPP From 4a76e8ca66e3d6c97ad64ba717a8aad72be18fc4 Mon Sep 17 00:00:00 2001 From: Konstantin Munichev Date: Mon, 11 Mar 2019 14:06:43 +0300 Subject: [PATCH 14/40] Cmake sanitizers (#2131) Signed-off-by: Konstantin Munichev --- CMakeLists.txt | 51 ++++++++++++++++++++++++-------- cmake/Modules/Finded25519.cmake | 5 +++- cmake/Modules/Findgflags.cmake | 2 ++ cmake/Modules/Findgrpc.cmake | 1 + cmake/Modules/Findgtest.cmake | 8 ++--- cmake/Modules/Findpq.cmake | 8 +++-- cmake/Modules/Findprotobuf.cmake | 7 ++++- cmake/Modules/Findsoci.cmake | 6 +++- cmake/Modules/Findtbb.cmake | 3 +- cmake/functions.cmake | 6 ++++ 10 files changed, 75 insertions(+), 22 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7eb90f995d..4d976ab3df 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -67,19 +67,43 @@ if(COVERAGE) include(cmake/analysis.cmake) endif() +include(cmake/functions.cmake) + +if(SANITIZE_THREAD) + append_build_flags(-fsanitize=thread) +endif() +if(SANITIZE_ADDRESS) + append_build_flags(-fsanitize=address -fno-omit-frame-pointer) +endif() +if(SANITIZE_MEMORY) + append_build_flags(-fsanitize=memory -fsanitize-memory-track-origins) +endif() +if(SANITIZE_UNDEFINED) + append_build_flags(-fsanitize=undefined) +endif() +set(DEPS_CMAKE_ARGS + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} + -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}) + SET(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules) -option(BENCHMARKING "Build benchmarks" OFF) -option(TESTING "Build tests" ON ) -option(USE_BTF "Build Binary Testing Framework" OFF) -option(COVERAGE "Enable coverage" OFF) -option(FUZZING "Build fuzzing binaries" OFF) -option(PACKAGE_ZIP "Create ZIP package" OFF) -option(PACKAGE_TGZ "Create TGZ package" OFF) -option(PACKAGE_RPM "Create RPM package" OFF) -option(PACKAGE_DEB "Create DEB package" OFF) -option(ENABLE_LIBS_PACKAGING "Enable libs packaging" ON) -option(USE_LIBIROHA "Use external model library" OFF) +option(BENCHMARKING "Build benchmarks" OFF) +option(TESTING "Build tests" ON ) +option(USE_BTF "Build Binary Testing Framework" OFF) +option(COVERAGE "Enable coverage" OFF) +option(FUZZING "Build fuzzing binaries" OFF) +option(PACKAGE_ZIP "Create ZIP package" OFF) +option(PACKAGE_TGZ "Create TGZ package" OFF) +option(PACKAGE_RPM "Create RPM package" OFF) +option(PACKAGE_DEB "Create DEB package" OFF) +option(ENABLE_LIBS_PACKAGING "Enable libs packaging" ON) +option(USE_LIBIROHA "Use external model library" OFF) +option(SANITIZE_THREAD "Build with thread sanitizer" OFF) +option(SANITIZE_ADDRESS "Build with address sanitizer" OFF) +option(SANITIZE_MEMORY "Build with memory sanitizer" OFF) +option(SANITIZE_UNDEFINED "Build with undefined behaviour sanitizer" OFF) if (NOT CMAKE_BUILD_TYPE) @@ -112,6 +136,10 @@ message(STATUS "-DPACKAGE_TGZ=${PACKAGE_TGZ}") message(STATUS "-DPACKAGE_RPM=${PACKAGE_RPM}") message(STATUS "-DPACKAGE_DEB=${PACKAGE_DEB}") message(STATUS "-DENABLE_LIBS_PACKAGING=${ENABLE_LIBS_PACKAGING}") +message(STATUS "-DSANITIZE_THREAD=${SANITIZE_THREAD}") +message(STATUS "-DSANITIZE_ADDRESS=${SANITIZE_ADDRESS}") +message(STATUS "-DSANITIZE_MEMORY=${SANITIZE_MEMORY}") +message(STATUS "-DSANITIZE_UNDEFINED=${SANITIZE_UNDEFINED}") set(IROHA_SCHEMA_DIR "${CMAKE_CURRENT_SOURCE_DIR}/schema") set(SM_SCHEMA_DIR "${PROJECT_SOURCE_DIR}/shared_model/schema") @@ -134,7 +162,6 @@ SET(IROHA_ROOT_PROJECT ON) add_definitions(-DBOOST_NO_RTTI) include(FeatureSummary) -include(cmake/functions.cmake) include(cmake/dependencies.cmake) include(cmake/clang-cxx-dev-tools.cmake) diff --git a/cmake/Modules/Finded25519.cmake b/cmake/Modules/Finded25519.cmake index e82fc30aca..a2d107e405 100644 --- a/cmake/Modules/Finded25519.cmake +++ b/cmake/Modules/Finded25519.cmake @@ -19,7 +19,10 @@ if (NOT ed25519_FOUND) externalproject_add(hyperledger_ed25519 GIT_REPOSITORY ${URL} GIT_TAG ${VERSION} - CMAKE_ARGS -DTESTING=OFF -DBUILD=STATIC + CMAKE_ARGS + -DTESTING=OFF + -DBUILD=STATIC + ${DEPS_CMAKE_ARGS} PATCH_COMMAND ${PATCH_RANDOM} BUILD_BYPRODUCTS ${EP_PREFIX}/src/hyperledger_ed25519-build/${CMAKE_STATIC_LIBRARY_PREFIX}ed25519${CMAKE_STATIC_LIBRARY_SUFFIX} INSTALL_COMMAND "" # remove install step diff --git a/cmake/Modules/Findgflags.cmake b/cmake/Modules/Findgflags.cmake index 7af7c3df12..45b91cc2fe 100644 --- a/cmake/Modules/Findgflags.cmake +++ b/cmake/Modules/Findgflags.cmake @@ -20,6 +20,8 @@ if (NOT gflags_FOUND) externalproject_add(gflags_gflags GIT_REPOSITORY ${URL} GIT_TAG ${VERSION} + CMAKE_ARGS + ${DEPS_CMAKE_ARGS} BUILD_BYPRODUCTS ${EP_PREFIX}/src/gflags_gflags-build/lib/libgflags.a INSTALL_COMMAND "" # remove install step TEST_COMMAND "" # remove test step diff --git a/cmake/Modules/Findgrpc.cmake b/cmake/Modules/Findgrpc.cmake index 0e0dd5b3f4..7f5efd7098 100644 --- a/cmake/Modules/Findgrpc.cmake +++ b/cmake/Modules/Findgrpc.cmake @@ -44,6 +44,7 @@ if (NOT grpc_FOUND) GIT_REPOSITORY ${URL} GIT_TAG ${VERSION} CMAKE_ARGS + ${DEPS_CMAKE_ARGS} -DgRPC_PROTOBUF_PROVIDER=package ${PROTO_DEP} -DgRPC_ZLIB_PROVIDER=package diff --git a/cmake/Modules/Findgtest.cmake b/cmake/Modules/Findgtest.cmake index daa6037cc1..9612799091 100644 --- a/cmake/Modules/Findgtest.cmake +++ b/cmake/Modules/Findgtest.cmake @@ -39,10 +39,10 @@ if (NOT gtest_FOUND) ExternalProject_Add(google_test GIT_REPOSITORY ${URL} GIT_TAG ${VERSION} - CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -Dgtest_force_shared_crt=ON - -Dgtest_disable_pthreads=OFF + CMAKE_ARGS + ${DEPS_CMAKE_ARGS} + -Dgtest_force_shared_crt=ON + -Dgtest_disable_pthreads=OFF BUILD_BYPRODUCTS ${EP_PREFIX}/src/google_test-build/googlemock/gtest/libgtest_main.a ${EP_PREFIX}/src/google_test-build/googlemock/gtest/libgtest.a ${EP_PREFIX}/src/google_test-build/googlemock/libgmock_main.a diff --git a/cmake/Modules/Findpq.cmake b/cmake/Modules/Findpq.cmake index b20f6c6c45..111dcba46a 100644 --- a/cmake/Modules/Findpq.cmake +++ b/cmake/Modules/Findpq.cmake @@ -40,9 +40,13 @@ if (NOT pq_FOUND) externalproject_add(postgres_postgres GIT_REPOSITORY ${URL} GIT_TAG ${VERSION} - CONFIGURE_COMMAND ./configure --without-readline + CONFIGURE_COMMAND + ./configure --without-readline + CC=${CMAKE_C_COMPILER} + BUILD_IN_SOURCE 1 - BUILD_COMMAND ${MAKE} -C ./src/bin/pg_config && ${MAKE} -C ./src/interfaces/libpq + BUILD_COMMAND ${MAKE} COPT=${CMAKE_C_FLAGS} -C ./src/bin/pg_config && + ${MAKE} COPT=${CMAKE_C_FLAGS} -C ./src/interfaces/libpq BUILD_BYPRODUCTS ${EP_PREFIX}/src/postgres_postgres/src/interfaces/libpq/libpq.a INSTALL_COMMAND "" # remove install step TEST_COMMAND "" # remove test step diff --git a/cmake/Modules/Findprotobuf.cmake b/cmake/Modules/Findprotobuf.cmake index 0b7d61e338..e09b3f87e4 100644 --- a/cmake/Modules/Findprotobuf.cmake +++ b/cmake/Modules/Findprotobuf.cmake @@ -26,7 +26,12 @@ if (NOT protobuf_FOUND) externalproject_add(google_protobuf GIT_REPOSITORY ${URL} GIT_TAG ${VERSION} - CONFIGURE_COMMAND ${CMAKE_COMMAND} -G${CMAKE_GENERATOR} -H${EP_PREFIX}/src/google_protobuf/cmake -B${EP_PREFIX}/src/google_protobuf-build -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_BUILD_SHARED_LIBS=ON + CONFIGURE_COMMAND ${CMAKE_COMMAND} + -G${CMAKE_GENERATOR} + -H${EP_PREFIX}/src/google_protobuf/cmake -B${EP_PREFIX}/src/google_protobuf-build + ${DEPS_CMAKE_ARGS} + -Dprotobuf_BUILD_TESTS=OFF + -Dprotobuf_BUILD_SHARED_LIBS=ON BUILD_BYPRODUCTS ${EP_PREFIX}/src/google_protobuf-build/protoc ${EP_PREFIX}/src/google_protobuf-build/${CMAKE_SHARED_LIBRARY_PREFIX}protobuf${CMAKE_SHARED_LIBRARY_SUFFIX} INSTALL_COMMAND "" diff --git a/cmake/Modules/Findsoci.cmake b/cmake/Modules/Findsoci.cmake index 897ff40d46..93138f0af6 100644 --- a/cmake/Modules/Findsoci.cmake +++ b/cmake/Modules/Findsoci.cmake @@ -37,6 +37,7 @@ set(VERSION 111b50af8c3876ea392367640b4bd83b4f903ab8) # 3.2.3 set_target_description(soci "The C++ Database Access Library" ${URL} ${VERSION}) if (NOT soci_FOUND) + SET(CMAKE_CXX_SOCI_FLAGS "${CMAKE_CXX_FLAGS} -I${postgres_INCLUDE_DIR}") externalproject_add(soci_soci GIT_REPOSITORY ${URL} GIT_TAG ${VERSION} @@ -47,7 +48,10 @@ if (NOT soci_FOUND) -DCMAKE_INCLUDE_PATH=${pq_INCLUDE_DIR} -DCMAKE_LIBRARY_PATH=${pq_INCLUDE_DIR} -DCMAKE_PROGRAM_PATH=${pg_config_EXECUTABLE_DIR} - -DCMAKE_CXX_FLAGS=-I${postgres_INCLUDE_DIR} + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} + -DCMAKE_CXX_FLAGS=${CMAKE_CXX_SOCI_FLAGS} -DCMAKE_INSTALL_PREFIX=${EP_PREFIX} -DWITH_BOOST=ON -DWITH_DB2=OFF diff --git a/cmake/Modules/Findtbb.cmake b/cmake/Modules/Findtbb.cmake index 2a55b1aa52..ba85efc52f 100644 --- a/cmake/Modules/Findtbb.cmake +++ b/cmake/Modules/Findtbb.cmake @@ -22,7 +22,8 @@ if (NOT tbb_FOUND) GIT_REPOSITORY ${URL} GIT_TAG ${VERSION} BUILD_IN_SOURCE 1 - BUILD_COMMAND ${MAKE} tbb_build_prefix=build + BUILD_COMMAND ${MAKE} + tbb_build_prefix=build BUILD_BYPRODUCTS ${EP_PREFIX}/src/01org_tbb/build/build_debug/${CMAKE_SHARED_LIBRARY_PREFIX}tbb_debug${CMAKE_SHARED_LIBRARY_SUFFIX} ${EP_PREFIX}/src/01org_tbb/build/build_release/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX} CONFIGURE_COMMAND "" # remove configure step diff --git a/cmake/functions.cmake b/cmake/functions.cmake index f9161fcdbb..5ecfc4009d 100644 --- a/cmake/functions.cmake +++ b/cmake/functions.cmake @@ -155,3 +155,9 @@ macro(get_git_revision commit) WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} ) endmacro() + +macro(append_build_flags) + string(REPLACE ";" " " SPACE_ARGS "${ARGN}") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SPACE_ARGS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SPACE_ARGS}") +endmacro() From dd879db520a488adf3a1a18dcc7b89062485297b Mon Sep 17 00:00:00 2001 From: Aidar Sabirov Date: Mon, 11 Mar 2019 17:46:51 +0300 Subject: [PATCH 15/40] Fix typo --- docs/source/guides/sec-install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/guides/sec-install.rst b/docs/source/guides/sec-install.rst index 2f73f894a3..baa48b3c6b 100644 --- a/docs/source/guides/sec-install.rst +++ b/docs/source/guides/sec-install.rst @@ -38,7 +38,7 @@ If IPv6 is not used, it might be a good idea to disable it. Updates ^^^^^^^ -Install latest operating system security patches and update it regularly. +Install the latest operating system security patches and update it regularly. If Iroha is running in Docker containers, update Docker regularly. While being optional, it is considered a good practice to test updates on a separate server before installing to production. From d0ded3cac3bdf2889ec280b16f4461b316b9b016 Mon Sep 17 00:00:00 2001 From: Mikhail Boldyrev Date: Tue, 12 Mar 2019 08:39:29 +0300 Subject: [PATCH 16/40] Llogger: fixed patterns and docs (#2152) Signed-off-by: Mikhail Boldyrev --- docs/source/guides/configuration.rst | 29 ++++++++++++++++++++++++---- irohad/main/iroha_conf_loader.cpp | 2 +- libs/logger/logger_manager.cpp | 3 ++- libs/logger/logger_spdlog.cpp | 19 +++++++++++++++--- libs/logger/logger_spdlog.hpp | 3 +++ 5 files changed, 47 insertions(+), 9 deletions(-) diff --git a/docs/source/guides/configuration.rst b/docs/source/guides/configuration.rst index 261bd87812..35a310a801 100644 --- a/docs/source/guides/configuration.rst +++ b/docs/source/guides/configuration.rst @@ -60,7 +60,11 @@ Environment-specific parameters Logging ------- -In Iroha logging can be adjusted as granularly as you want. Each component has its own logging configuration with properties inherited from its parent, able to be overriden through config file. This means all the component loggers are organized in a tree with a single root. The relevant section of the configuration file contains the overriden values: +In Iroha logging can be adjusted as granularly as you want. +Each component has its own logging configuration with properties inherited from +its parent, able to be overridden through config file. +This means all the component loggers are organized in a tree with a single root. +The relevant section of the configuration file contains the overriding values: .. code-block:: json :linenos: @@ -89,6 +93,23 @@ In Iroha logging can be adjusted as granularly as you want. Each component has i } Every part of this config section is optional. -- ``level`` sets the verbosity. Available valies are (in decreasing verbosity order): trace, debug, info, warning, error, critical. -- ``patterns`` controls the formatting of each log string for different verbosity levels. Each value overrides the less verbose levels too. So in the example above, the "don't panic" pattern also applies to info and warning levels, and the trace level pattern is the only one that is not initialized in the config (it will be set to default hardcoded value). -- ``children`` describes the overrides of child nodes. The keys are the names of the components, and the values have the same syntax and semantics as the root log configuration. + +- ``level`` sets the verbosity. + Available values are (in decreasing verbosity order): + + - ``trace`` - print everything + - ``debug`` + - ``info`` + - ``warning`` + - ``error`` + - ``critical`` - print only critical messages + +- ``patterns`` controls the formatting of each log string for different + verbosity levels. + Each value overrides the less verbose levels too. + So in the example above, the "don't panic" pattern also applies to info and + warning levels, and the trace level pattern is the only one that is not + initialized in the config (it will be set to default hardcoded value). +- ``children`` describes the overrides of child nodes. + The keys are the names of the components, and the values have the same syntax + and semantics as the root log configuration. diff --git a/irohad/main/iroha_conf_loader.cpp b/irohad/main/iroha_conf_loader.cpp index c6d19b9c63..c149fb6d6f 100644 --- a/irohad/main/iroha_conf_loader.cpp +++ b/irohad/main/iroha_conf_loader.cpp @@ -196,7 +196,7 @@ void getVal(const std::string &path, const rapidjson::Value &src) { assert_fatal(src.IsObject(), path + " must be a logger tree config"); logger::LoggerConfig root_config{logger::kDefaultLogLevel, - logger::getDefaultLogPatterns()}; + logger::LogPatterns{}}; updateLoggerConfig(path, root_config, src.GetObject()); dest = std::make_shared( std::make_shared(std::move(root_config))); diff --git a/libs/logger/logger_manager.cpp b/libs/logger/logger_manager.cpp index 04297f7de1..49db16fb4e 100644 --- a/libs/logger/logger_manager.cpp +++ b/libs/logger/logger_manager.cpp @@ -35,7 +35,8 @@ namespace logger { boost::optional patterns) { LoggerConfig child_config{ log_level.value_or(config_->log_level), - patterns ? LogPatterns{*std::move(patterns)} : config_->patterns}; + patterns ? std::move(patterns)->inherit(config_->patterns) + : config_->patterns}; // Operator new is employed due to private visibility of used constructor. LoggerManagerTreePtr child(new LoggerManagerTree( joinTags(full_tag_, tag), diff --git a/libs/logger/logger_spdlog.cpp b/libs/logger/logger_spdlog.cpp index fe5a19af46..9c1b0ee906 100644 --- a/libs/logger/logger_spdlog.cpp +++ b/libs/logger/logger_spdlog.cpp @@ -52,9 +52,9 @@ namespace logger { static LogPatterns default_patterns; if (not is_initialized.test_and_set()) { default_patterns.setPattern( - LogLevel::kTrace, R"([%Y-%m-%d %H:%M:%S.%F] [th:%t] [%5l] [%n]: %v)"); + LogLevel::kTrace, R"([%Y-%m-%d %H:%M:%S.%F][th:%t][%=8l][%n]: %v)"); default_patterns.setPattern(LogLevel::kInfo, - R"([%Y-%m-%d %H:%M:%S.%F] [%L] [%n]: %v)"); + R"([%Y-%m-%d %H:%M:%S.%F][%L][%n]: %v)"); } return default_patterns; } @@ -72,6 +72,19 @@ namespace logger { return getDefaultLogPatterns().getPattern(level); } + LogPatterns &LogPatterns::inherit(const LogPatterns &base) { + if (patterns_.empty()) { + patterns_ = base.patterns_; + } else { + for (auto it = base.patterns_.cbegin(); + it != base.patterns_.cend() and it->first < patterns_.begin()->first; + ++it) { + patterns_.emplace(*it); + } + } + return *this; + } + LoggerSpdlog::LoggerSpdlog(std::string tag, ConstLoggerConfigPtr config) : tag_(tag), config_(std::move(config)), logger_(getOrCreateLogger(tag)) { setupLogger(); @@ -83,7 +96,7 @@ namespace logger { } void LoggerSpdlog::logInternal(Level level, const std::string &s) const { - logger_->log(getSpdlogLogLevel(config_->log_level), s); + logger_->log(getSpdlogLogLevel(level), s); } bool LoggerSpdlog::shouldLog(Level level) const { diff --git a/libs/logger/logger_spdlog.hpp b/libs/logger/logger_spdlog.hpp index 9b38187ed4..305a6fb689 100644 --- a/libs/logger/logger_spdlog.hpp +++ b/libs/logger/logger_spdlog.hpp @@ -38,6 +38,9 @@ namespace logger { */ std::string getPattern(LogLevel level) const; + /// Inherit missing level overrides from another patterns + LogPatterns &inherit(const LogPatterns &base); + private: std::map patterns_; }; From 449ebbe68bcb66914bbe78005333f0f191cd1650 Mon Sep 17 00:00:00 2001 From: Fedor Muratov Date: Tue, 12 Mar 2019 09:14:02 +0300 Subject: [PATCH 17/40] Add fix for commit direct propagation (#2139) ### Description of the Change Add fix for direct propagation in Yac. If a peer sends a vote for the old round we will share last finalized state. ### Benefits Fix of synchronization in some cases. ### Possible Drawbacks No so elegant solution: YAC checks stateless invariants inside functions, vote storage responsibility is increased. Signed-off-by: Fedor Muratov --- irohad/consensus/yac/impl/yac.cpp | 122 +++++++++++------- .../yac/storage/impl/yac_vote_storage.cpp | 43 +++++- .../yac/storage/yac_vote_storage.hpp | 17 +++ irohad/consensus/yac/yac.hpp | 1 + .../irohad/consensus/yac/CMakeLists.txt | 6 + .../yac/yac_simple_cold_case_test.cpp | 4 +- .../yac/yac_synchronization_test.cpp | 115 +++++++++++++++++ .../irohad/consensus/yac/yac_test_util.hpp | 19 ++- 8 files changed, 266 insertions(+), 61 deletions(-) create mode 100644 test/module/irohad/consensus/yac/yac_synchronization_test.cpp diff --git a/irohad/consensus/yac/impl/yac.cpp b/irohad/consensus/yac/impl/yac.cpp index fb32a08013..38dd19dae0 100644 --- a/irohad/consensus/yac/impl/yac.cpp +++ b/irohad/consensus/yac/impl/yac.cpp @@ -18,6 +18,12 @@ #include "interfaces/common_objects/peer.hpp" #include "logger/logger.hpp" +// TODO: 2019-03-04 @muratovv refactor std::vector with a +// separate class IR-374 +auto &getRound(const std::vector &state) { + return state.at(0).hash.vote_round; +} + namespace iroha { namespace consensus { namespace yac { @@ -137,55 +143,77 @@ namespace iroha { // TODO 10.06.2018 andrei: IR-1407 move YAC propagation strategy to a // separate entity - answer | [&](const auto &answer) { - auto &proposal_round = state.at(0).hash.vote_round; - - /* - * It is possible that a new peer with an outdated peers list may - * collect an outcome from a smaller number of peers which are - * included in set of `f` peers in the system. The new peer will not - * accept our message with valid supermajority because he cannot apply - * votes from unknown peers. - */ - if (state.size() > 1) { - // some peer has already collected commit/reject, so it is sent - if (vote_storage_.getProcessingState(proposal_round) - == ProposalState::kNotSentNotProcessed) { - vote_storage_.nextProcessingState(proposal_round); - log_->info( - "Received supermajority of votes for {}, skip propagation", - proposal_round); - } - } + iroha::match_in_place( + answer, + [&](const auto &answer) { + auto &proposal_round = getRound(state); + + /* + * It is possible that a new peer with an outdated peers list may + * collect an outcome from a smaller number of peers which are + * included in set of `f` peers in the system. The new peer will + * not accept our message with valid supermajority because he + * cannot apply votes from unknown peers. + */ + if (state.size() > 1) { + // some peer has already collected commit/reject, so it is sent + if (vote_storage_.getProcessingState(proposal_round) + == ProposalState::kNotSentNotProcessed) { + vote_storage_.nextProcessingState(proposal_round); + log_->info( + "Received supermajority of votes for {}, skip " + "propagation", + proposal_round); + } + } - auto processing_state = - vote_storage_.getProcessingState(proposal_round); - - auto votes = [](const auto &state) { return state.votes; }; - - switch (processing_state) { - case ProposalState::kNotSentNotProcessed: - vote_storage_.nextProcessingState(proposal_round); - log_->info("Propagate state {} to whole network", proposal_round); - this->propagateState(visit_in_place(answer, votes)); - break; - case ProposalState::kSentNotProcessed: - vote_storage_.nextProcessingState(proposal_round); - log_->info("Pass outcome for {} to pipeline", proposal_round); - this->closeRound(); - notifier_.get_subscriber().on_next(answer); - break; - case ProposalState::kSentProcessed: - if (state.size() == 1) { - this->findPeer(state.at(0)) | [&](const auto &from) { - log_->info("Propagate state {} directly to {}", - proposal_round, - from->address()); - this->propagateStateDirectly(*from, - visit_in_place(answer, votes)); - }; + auto processing_state = + vote_storage_.getProcessingState(proposal_round); + + auto votes = [](const auto &state) { return state.votes; }; + + switch (processing_state) { + case ProposalState::kNotSentNotProcessed: + vote_storage_.nextProcessingState(proposal_round); + log_->info("Propagate state {} to whole network", + proposal_round); + this->propagateState(visit_in_place(answer, votes)); + break; + case ProposalState::kSentNotProcessed: + vote_storage_.nextProcessingState(proposal_round); + log_->info("Pass outcome for {} to pipeline", proposal_round); + this->closeRound(); + notifier_.get_subscriber().on_next(answer); + break; + case ProposalState::kSentProcessed: + this->tryPropagateBack(state); + break; } - break; + }, + // sent a state which didn't match with current one + [&]() { this->tryPropagateBack(state); }); + } + + void Yac::tryPropagateBack(const std::vector &state) { + // yac back propagation will work only if another peer is in + // propagation stage because if peer sends list of votes this means that + // state is already committed + if (state.size() != 1) { + return; + } + + vote_storage_.getLastFinalizedRound() | [&](const auto &last_round) { + if (getRound(state) <= last_round) { + vote_storage_.getState(last_round) | [&](const auto &last_state) { + this->findPeer(state.at(0)) | [&](const auto &from) { + log_->info("Propagate state {} directly to {}", + last_round, + from->address()); + auto votes = [](const auto &state) { return state.votes; }; + this->propagateStateDirectly(*from, + visit_in_place(last_state, votes)); + }; + }; } }; } diff --git a/irohad/consensus/yac/storage/impl/yac_vote_storage.cpp b/irohad/consensus/yac/storage/impl/yac_vote_storage.cpp index 47b041e91a..387e4963d0 100644 --- a/irohad/consensus/yac/storage/impl/yac_vote_storage.cpp +++ b/irohad/consensus/yac/storage/impl/yac_vote_storage.cpp @@ -19,12 +19,29 @@ namespace iroha { // --------| private api |-------- + namespace { + /** + * Find storage with corresponding key + * @tparam T - storage type + * @param storage - ref or const ref for the storage + * @param round - required round + * @return iterator for the storage + */ + template + auto findStorage(T &storage, const Round &round) { + return std::find_if( + storage.begin(), storage.end(), [&round](const auto &storage) { + return storage.getStorageKey() == round; + }); + } + } // namespace + auto YacVoteStorage::getProposalStorage(const Round &round) { - return std::find_if(proposal_storages_.begin(), - proposal_storages_.end(), - [&round](const auto &storage) { - return storage.getStorageKey() == round; - }); + return findStorage(proposal_storages_, round); + } + + auto YacVoteStorage::getProposalStorage(const Round &round) const { + return findStorage(proposal_storages_, round); } boost::optional::iterator> @@ -76,6 +93,8 @@ namespace iroha { return storage->insert(state) | [this, &round]( auto &&insert_outcome) -> boost::optional { + + last_round_ = std::max(last_round_.value_or(round), round); this->strategy_->finalize(round, insert_outcome) | [this](auto &&remove) { std::for_each( @@ -114,6 +133,20 @@ namespace iroha { } } + boost::optional YacVoteStorage::getLastFinalizedRound() const { + return last_round_; + } + + boost::optional YacVoteStorage::getState( + const Round &round) const { + auto proposal_storage = getProposalStorage(round); + if (proposal_storage != proposal_storages_.end()) { + return proposal_storage->getState(); + } else { + return boost::none; + } + } + } // namespace yac } // namespace consensus } // namespace iroha diff --git a/irohad/consensus/yac/storage/yac_vote_storage.hpp b/irohad/consensus/yac/storage/yac_vote_storage.hpp index 205ebe663f..fbaee797db 100644 --- a/irohad/consensus/yac/storage/yac_vote_storage.hpp +++ b/irohad/consensus/yac/storage/yac_vote_storage.hpp @@ -70,6 +70,7 @@ namespace iroha { * @return iterator to proposal storage */ auto getProposalStorage(const Round &round); + auto getProposalStorage(const Round &round) const; /** * Find existed proposal storage or create new if required @@ -136,6 +137,19 @@ namespace iroha { */ void nextProcessingState(const Round &round); + /** + * Get last by order finalized round + * @return round if it exists + */ + boost::optional getLastFinalizedRound() const; + + /** + * Get the state attached of a past round + * @param round - required round + * @return state if round exists and finalized + */ + boost::optional getState(const Round &round) const; + private: // --------| fields |-------- @@ -161,6 +175,9 @@ namespace iroha { */ std::shared_ptr strategy_; + /// last finalized round + boost::optional last_round_; + std::shared_ptr supermajority_checker_; logger::LoggerManagerTreePtr log_manager_; diff --git a/irohad/consensus/yac/yac.hpp b/irohad/consensus/yac/yac.hpp index 6eda74235a..95ed792925 100644 --- a/irohad/consensus/yac/yac.hpp +++ b/irohad/consensus/yac/yac.hpp @@ -85,6 +85,7 @@ namespace iroha { void propagateState(const std::vector &msg); void propagateStateDirectly(const shared_model::interface::Peer &to, const std::vector &msg); + void tryPropagateBack(const std::vector &state); // ------|Fields|------ YacVoteStorage vote_storage_; diff --git a/test/module/irohad/consensus/yac/CMakeLists.txt b/test/module/irohad/consensus/yac/CMakeLists.txt index 93309885bf..84c7c437e9 100644 --- a/test/module/irohad/consensus/yac/CMakeLists.txt +++ b/test/module/irohad/consensus/yac/CMakeLists.txt @@ -94,3 +94,9 @@ target_link_libraries(buffered_cleanup_strategy_test yac consensus_round ) + +addtest(yac_synchronization_test yac_synchronization_test.cpp) +target_link_libraries(yac_synchronization_test + yac + test_logger + ) diff --git a/test/module/irohad/consensus/yac/yac_simple_cold_case_test.cpp b/test/module/irohad/consensus/yac/yac_simple_cold_case_test.cpp index e46b935b20..45da565d28 100644 --- a/test/module/irohad/consensus/yac/yac_simple_cold_case_test.cpp +++ b/test/module/irohad/consensus/yac/yac_simple_cold_case_test.cpp @@ -137,7 +137,7 @@ TEST_F(YacTest, PropagateCommitBeforeNotifyingSubscribersApplyVote) { .WillRepeatedly(Return(true)); std::vector> messages; EXPECT_CALL(*network, sendState(_, _)) - .Times(default_peers.size()) + .Times(default_peers.size() + 1) .WillRepeatedly(Invoke( [&](const auto &, const auto &msg) { messages.push_back(msg); })); @@ -154,7 +154,7 @@ TEST_F(YacTest, PropagateCommitBeforeNotifyingSubscribersApplyVote) { } // verify that on_commit subscribers are notified - ASSERT_EQ(default_peers.size() + 1, messages.size()); + ASSERT_EQ(default_peers.size() + 2, messages.size()); } /** diff --git a/test/module/irohad/consensus/yac/yac_synchronization_test.cpp b/test/module/irohad/consensus/yac/yac_synchronization_test.cpp new file mode 100644 index 0000000000..c376cd44b6 --- /dev/null +++ b/test/module/irohad/consensus/yac/yac_synchronization_test.cpp @@ -0,0 +1,115 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "module/irohad/consensus/yac/yac_fixture.hpp" + +#include "common/hexutils.hpp" + +using namespace iroha::consensus::yac; + +using ::testing::_; +using ::testing::Return; + +/** + * The class helps to create fake network for unit testing of consensus + */ +class NetworkUtil { + public: + /// creates fake network of number_of_peers size + NetworkUtil(size_t number_of_peers) { + for (size_t i = 0; i < number_of_peers; ++i) { + peers_.push_back(makePeer(std::to_string(i))); + } + order_ = ClusterOrdering::create(peers_); + } + + auto createHash(const iroha::consensus::Round &r, + const std::string &block_hash = "default_block", + const std::string &proposal_hash = "default_proposal") const { + return YacHash(r, proposal_hash, block_hash); + } + + auto createVote(size_t from, const YacHash &yac_hash) const { + BOOST_ASSERT_MSG(from < peers_.size(), "Requested unknown index of peer"); + return iroha::consensus::yac::createVote( + yac_hash, + *iroha::hexstringToBytestring(peers_.at(from)->pubkey().hex())); + } + /// create votes of peers by their number + auto createVotes( + const std::vector &peers, + const iroha::consensus::Round &r, + const std::string &block_hash = "default_block", + const std::string &proposal_hash = "default_proposal") const { + return std::accumulate( + peers.begin(), + peers.end(), + std::vector(), + [&, this](auto vector, const auto &peer_number) { + vector.push_back(this->createVote( + peer_number, this->createHash(r, block_hash, proposal_hash))); + return std::move(vector); + }); + } + + std::vector> peers_; + boost::optional order_; +}; + +class YacSynchronizationTest : public YacTest { + public: + void SetUp() override { + YacTest::SetUp(); + + network_util_ = NetworkUtil(7); + initAndCommitState(network_util_); + } + + /// inits initial state and commits some rounds + void initAndCommitState(const NetworkUtil &network_util) { + size_t number_of_committed_rounds = 10; + + initYac(*network_util.order_); + EXPECT_CALL(*crypto, verify(_)).WillRepeatedly(Return(true)); + EXPECT_CALL(*timer, deny()).Times(number_of_committed_rounds); + + for (auto i = 0u; i < number_of_committed_rounds; i++) { + iroha::consensus::Round r{i, 0}; + yac->vote(network_util.createHash(r), *network_util.order_); + yac->onState(network_util.createVotes({1, 2, 3, 4, 5, 6}, r)); + } + EXPECT_CALL(*network, sendState(_, _)).Times(8); + yac->vote(network_util.createHash({10, 0}), *network_util.order_); + } + + NetworkUtil network_util_{1}; +}; + +/** + * @given Yac which stores commit + * @when Vote from known peer from old round which was presented in the cache + * @then Yac sends commit for the last round + */ +TEST_F(YacSynchronizationTest, SynchronizationOncommitInTheCahe) { + yac->onState(network_util_.createVotes({0}, iroha::consensus::Round{1, 0})); +} + +/** + * @given Yac which stores commit + * @when Vote from known peer from old round which presents in a cache + * @then Yac sends commit for the last round + */ +TEST_F(YacSynchronizationTest, SynchronizationOnCommitOutOfTheCahe) { + yac->onState(network_util_.createVotes({0}, iroha::consensus::Round{9, 0})); +} + +/** + * @given Yac received reject + * @when Vote from known peer from old round which doesn't present in the cache + * @then Yac sends last commit + */ +TEST_F(YacSynchronizationTest, SynchronizationRejectOutOfTheCahe) { + yac->onState(network_util_.createVotes({0}, iroha::consensus::Round{5, 5})); +} diff --git a/test/module/irohad/consensus/yac/yac_test_util.hpp b/test/module/irohad/consensus/yac/yac_test_util.hpp index 20f63e6d90..e451c553c0 100644 --- a/test/module/irohad/consensus/yac/yac_test_util.hpp +++ b/test/module/irohad/consensus/yac/yac_test_util.hpp @@ -20,21 +20,28 @@ namespace iroha { inline std::shared_ptr makePeer( const std::string &address) { - auto key = std::string(32, '0'); - std::copy(address.begin(), address.end(), key.begin()); auto peer = std::make_shared(); EXPECT_CALL(*peer, address()) .WillRepeatedly(::testing::ReturnRefOfCopy(address)); EXPECT_CALL(*peer, pubkey()) .WillRepeatedly(::testing::ReturnRefOfCopy( - shared_model::interface::types::PubkeyType(key))); - + shared_model::interface::types::PubkeyType(address))); return peer; } inline VoteMessage createVote(YacHash hash, const std::string &pub_key) { VoteMessage vote; + auto block_signature = std::make_shared(); + EXPECT_CALL(*block_signature, publicKey()) + .WillRepeatedly(::testing::ReturnRefOfCopy( + shared_model::crypto::PublicKey(pub_key))); + EXPECT_CALL(*block_signature, signedData()) + .WillRepeatedly(::testing::ReturnRefOfCopy( + shared_model::crypto::Signed(pub_key))); + hash.block_signature = block_signature; + vote.hash = std::move(hash); + auto signature = std::make_shared(); EXPECT_CALL(*signature, publicKey()) .WillRepeatedly(::testing::ReturnRefOfCopy( @@ -43,9 +50,7 @@ namespace iroha { .WillRepeatedly(::testing::ReturnRefOfCopy( shared_model::crypto::Signed(pub_key))); - hash.block_signature = signature; - vote.hash = std::move(hash); - vote.signature = createSig(pub_key); + vote.signature = signature; return vote; } From 9a6f0ef470c22dff6962094a5c9a06d7610dc7f6 Mon Sep 17 00:00:00 2001 From: Andrei Lebedev Date: Tue, 12 Mar 2019 11:44:56 +0300 Subject: [PATCH 18/40] Refactor ordering service queues (#2150) * Remove round from OdOsNotifications interface Signed-off-by: Igor Egorov Signed-off-by: Andrei Lebedev --- irohad/main/application.cpp | 1 - irohad/main/impl/on_demand_ordering_init.cpp | 4 - irohad/main/impl/on_demand_ordering_init.hpp | 5 - .../impl/on_demand_connection_manager.cpp | 20 +-- .../impl/on_demand_connection_manager.hpp | 2 +- .../ordering/impl/on_demand_ordering_gate.cpp | 48 +++---- .../ordering/impl/on_demand_ordering_gate.hpp | 7 +- .../impl/on_demand_ordering_service_impl.cpp | 136 ++++++++---------- .../impl/on_demand_ordering_service_impl.hpp | 10 +- .../impl/on_demand_os_client_grpc.cpp | 5 +- .../impl/on_demand_os_client_grpc.hpp | 2 +- .../impl/on_demand_os_server_grpc.cpp | 4 +- irohad/ordering/on_demand_os_transport.hpp | 4 +- schema/ordering.proto | 3 +- .../mock_on_demand_os_notification.hpp | 2 +- .../on_demand_connection_manager_test.cpp | 18 +-- .../ordering/on_demand_ordering_gate_test.cpp | 11 +- .../on_demand_os_client_grpc_test.cpp | 4 +- .../on_demand_os_server_grpc_test.cpp | 9 +- .../irohad/ordering/on_demand_os_test.cpp | 29 ++-- .../module/irohad/ordering/ordering_mocks.hpp | 2 +- 21 files changed, 129 insertions(+), 197 deletions(-) diff --git a/irohad/main/application.cpp b/irohad/main/application.cpp index 2dd8ee8e05..64852f6b53 100644 --- a/irohad/main/application.cpp +++ b/irohad/main/application.cpp @@ -362,7 +362,6 @@ void Irohad::initOrderingGate() { std::move(factory), proposal_factory, persistent_cache, - {blocks.back()->height(), 1}, delay, log_manager_->getChild("Ordering")); log_->info("[Init] => init ordering gate - [{}]", diff --git a/irohad/main/impl/on_demand_ordering_init.cpp b/irohad/main/impl/on_demand_ordering_init.cpp index 9bc559d649..6e463a4a29 100644 --- a/irohad/main/impl/on_demand_ordering_init.cpp +++ b/irohad/main/impl/on_demand_ordering_init.cpp @@ -200,7 +200,6 @@ namespace iroha { std::shared_ptr proposal_factory, std::shared_ptr tx_cache, - consensus::Round initial_round, std::function delay_func, const logger::LoggerManagerTreePtr &ordering_log_manager) { @@ -246,7 +245,6 @@ namespace iroha { std::move(cache), std::move(proposal_factory), std::move(tx_cache), - initial_round, ordering_log_manager->getChild("Gate")->getLogger()); } @@ -286,7 +284,6 @@ namespace iroha { proposal_factory, std::shared_ptr proposal_transport_factory, std::shared_ptr tx_cache, - consensus::Round initial_round, std::function delay_func, logger::LoggerManagerTreePtr ordering_log_manager) { @@ -309,7 +306,6 @@ namespace iroha { std::make_shared(), std::move(proposal_factory), std::move(tx_cache), - initial_round, std::move(delay_func), ordering_log_manager); } diff --git a/irohad/main/impl/on_demand_ordering_init.hpp b/irohad/main/impl/on_demand_ordering_init.hpp index 9a085d44cc..05ef366a90 100644 --- a/irohad/main/impl/on_demand_ordering_init.hpp +++ b/irohad/main/impl/on_demand_ordering_init.hpp @@ -72,7 +72,6 @@ namespace iroha { std::shared_ptr proposal_factory, std::shared_ptr tx_cache, - consensus::Round initial_round, std::function delay_func, const logger::LoggerManagerTreePtr &ordering_log_manager); @@ -89,7 +88,6 @@ namespace iroha { const logger::LoggerManagerTreePtr &ordering_log_manager); public: - /// Constructor. /// @param log - the logger to use for internal messages. OnDemandOrderingInit(logger::LoggerPtr log); @@ -115,8 +113,6 @@ namespace iroha { * requests to ordering service and processing responses * @param proposal_factory factory required by ordering service to produce * proposals - * @param initial_round initial value for current round used in - * OnDemandOrderingGate * @return initialized ordering gate */ std::shared_ptr initOrderingGate( @@ -137,7 +133,6 @@ namespace iroha { proposal_factory, std::shared_ptr proposal_transport_factory, std::shared_ptr tx_cache, - consensus::Round initial_round, std::function delay_func, logger::LoggerManagerTreePtr ordering_log_manager); diff --git a/irohad/ordering/impl/on_demand_connection_manager.cpp b/irohad/ordering/impl/on_demand_connection_manager.cpp index 5f3dc5eff3..b90e91e213 100644 --- a/irohad/ordering/impl/on_demand_connection_manager.cpp +++ b/irohad/ordering/impl/on_demand_connection_manager.cpp @@ -40,10 +40,8 @@ OnDemandConnectionManager::~OnDemandConnectionManager() { subscription_.unsubscribe(); } -void OnDemandConnectionManager::onBatches(consensus::Round round, - CollectionType batches) { +void OnDemandConnectionManager::onBatches(CollectionType batches) { std::shared_lock lock(mutex_); - /* * Transactions are always sent to the round after the next round (+2) * There are 3 possibilities - next reject in the current round, first reject @@ -57,19 +55,9 @@ void OnDemandConnectionManager::onBatches(consensus::Round round, * 2 v . . */ - auto propagate = [this, batches](PeerType type, consensus::Round round) { - log_->debug("onBatches, {}", round); - - connections_.peers[type]->onBatches(round, batches); - }; - - propagate( - kCurrentRoundRejectConsumer, - {round.block_round, currentRejectRoundConsumer(round.reject_round)}); - propagate(kNextRoundRejectConsumer, - {round.block_round + 1, kNextRejectRoundConsumer}); - propagate(kNextRoundCommitConsumer, - {round.block_round + 2, kNextCommitRoundConsumer}); + connections_.peers[kCurrentRoundRejectConsumer]->onBatches(batches); + connections_.peers[kNextRoundRejectConsumer]->onBatches(batches); + connections_.peers[kNextRoundCommitConsumer]->onBatches(batches); } boost::optional> diff --git a/irohad/ordering/impl/on_demand_connection_manager.hpp b/irohad/ordering/impl/on_demand_connection_manager.hpp index de4bfb9e12..5281b7b484 100644 --- a/irohad/ordering/impl/on_demand_connection_manager.hpp +++ b/irohad/ordering/impl/on_demand_connection_manager.hpp @@ -62,7 +62,7 @@ namespace iroha { ~OnDemandConnectionManager() override; - void onBatches(consensus::Round round, CollectionType batches) override; + void onBatches(CollectionType batches) override; boost::optional> onRequestProposal( consensus::Round round) override; diff --git a/irohad/ordering/impl/on_demand_ordering_gate.cpp b/irohad/ordering/impl/on_demand_ordering_gate.cpp index a10664c40f..a057ad86b7 100644 --- a/irohad/ordering/impl/on_demand_ordering_gate.cpp +++ b/irohad/ordering/impl/on_demand_ordering_gate.cpp @@ -18,6 +18,20 @@ using namespace iroha; using namespace iroha::ordering; +std::string OnDemandOrderingGate::BlockEvent::toString() const { + return shared_model::detail::PrettyStringBuilder() + .init("BlockEvent") + .append(round.toString()) + .finalize(); +} + +std::string OnDemandOrderingGate::EmptyEvent::toString() const { + return shared_model::detail::PrettyStringBuilder() + .init("EmptyEvent") + .append(round.toString()) + .finalize(); +} + OnDemandOrderingGate::OnDemandOrderingGate( std::shared_ptr ordering_service, std::shared_ptr network_client, @@ -25,27 +39,16 @@ OnDemandOrderingGate::OnDemandOrderingGate( std::shared_ptr cache, std::shared_ptr factory, std::shared_ptr tx_cache, - consensus::Round initial_round, logger::LoggerPtr log) : log_(std::move(log)), ordering_service_(std::move(ordering_service)), network_client_(std::move(network_client)), events_subscription_(events.subscribe([this](auto event) { - // exclusive lock - std::unique_lock lock(mutex_); - visit_in_place(event, - [this](const BlockEvent &block_event) { - // block committed, increment block round - log_->debug("BlockEvent. {}", block_event.round); - current_round_ = block_event.round; - }, - [this](const EmptyEvent &empty_event) { - // no blocks committed, increment reject round - log_->debug("EmptyEvent"); - current_round_ = empty_event.round; - }); - log_->debug("Current: {}", current_round_); - lock.unlock(); + consensus::Round current_round = + visit_in_place(event, [this, ¤t_round](const auto &event) { + log_->debug("{}", event); + return event.round; + }); visit_in_place(event, [this](const BlockEvent &block_event) { @@ -61,25 +64,23 @@ OnDemandOrderingGate::OnDemandOrderingGate( cache_->addToBack(batches); if (not batches.empty()) { network_client_->onBatches( - current_round_, transport::OdOsNotification::CollectionType{batches.begin(), batches.end()}); } // notify our ordering service about new round - ordering_service_->onCollaborationOutcome(current_round_); + ordering_service_->onCollaborationOutcome(current_round); // request proposal for the current round auto proposal = this->processProposalRequest( - network_client_->onRequestProposal(current_round_)); + network_client_->onRequestProposal(current_round)); // vote for the object received from the network proposal_notifier_.get_subscriber().on_next( - network::OrderingEvent{std::move(proposal), current_round_}); + network::OrderingEvent{std::move(proposal), current_round}); })), cache_(std::move(cache)), proposal_factory_(std::move(factory)), - tx_cache_(std::move(tx_cache)), - current_round_(initial_round) {} + tx_cache_(std::move(tx_cache)) {} OnDemandOrderingGate::~OnDemandOrderingGate() { events_subscription_.unsubscribe(); @@ -89,9 +90,8 @@ void OnDemandOrderingGate::propagateBatch( std::shared_ptr batch) { cache_->addToBack({batch}); - std::shared_lock lock(mutex_); network_client_->onBatches( - current_round_, transport::OdOsNotification::CollectionType{batch}); + transport::OdOsNotification::CollectionType{batch}); } rxcpp::observable OnDemandOrderingGate::onProposal() { diff --git a/irohad/ordering/impl/on_demand_ordering_gate.hpp b/irohad/ordering/impl/on_demand_ordering_gate.hpp index b23b643635..8306b6e2b6 100644 --- a/irohad/ordering/impl/on_demand_ordering_gate.hpp +++ b/irohad/ordering/impl/on_demand_ordering_gate.hpp @@ -40,6 +40,8 @@ namespace iroha { consensus::Round round; /// hashes of processed transactions cache::OrderingGateCache::HashesSetType hashes; + + std::string toString() const; }; /** @@ -48,6 +50,8 @@ namespace iroha { struct EmptyEvent { /// next round number consensus::Round round; + + std::string toString() const; }; using BlockRoundEventType = boost::variant; @@ -62,7 +66,6 @@ namespace iroha { std::shared_ptr factory, std::shared_ptr tx_cache, - consensus::Round initial_round, logger::LoggerPtr log); ~OnDemandOrderingGate() override; @@ -102,9 +105,7 @@ namespace iroha { proposal_factory_; std::shared_ptr tx_cache_; - consensus::Round current_round_; rxcpp::subjects::subject proposal_notifier_; - mutable std::shared_timed_mutex mutex_; }; } // namespace ordering diff --git a/irohad/ordering/impl/on_demand_ordering_service_impl.cpp b/irohad/ordering/impl/on_demand_ordering_service_impl.cpp index 14575f4175..b1f2a14576 100644 --- a/irohad/ordering/impl/on_demand_ordering_service_impl.cpp +++ b/irohad/ordering/impl/on_demand_ordering_service_impl.cpp @@ -57,11 +57,10 @@ void OnDemandOrderingServiceImpl::onCollaborationOutcome( // ----------------------------| OdOsNotification |----------------------------- -void OnDemandOrderingServiceImpl::onBatches(consensus::Round round, - CollectionType batches) { +void OnDemandOrderingServiceImpl::onBatches(CollectionType batches) { // read lock std::shared_lock guard(lock_); - log_->info("onBatches => collection size = {}, {}", batches.size(), round); + log_->info("onBatches => collection size = {}", batches.size()); auto unprocessed_batches = boost::adaptors::filter(batches, [this](const auto &batch) { @@ -69,27 +68,10 @@ void OnDemandOrderingServiceImpl::onBatches(consensus::Round round, batch->reducedHash().hex()); return not this->batchAlreadyProcessed(*batch); }); - auto it = current_proposals_.find(round); - if (it == current_proposals_.end()) { - it = - std::find_if(current_proposals_.begin(), - current_proposals_.end(), - [&round](const auto &p) { - auto request_reject_round = round.reject_round; - auto reject_round = p.first.reject_round; - return request_reject_round == reject_round - or (request_reject_round >= 2 and reject_round >= 2); - }); - if (it == current_proposals_.end()) { - log_->critical("No place to store the batches!"); - assert(false); // terminate if in debug build - return; - } - log_->debug("onBatches => collection will be inserted to {}", it->first); - } - std::for_each(unprocessed_batches.begin(), - unprocessed_batches.end(), - [&it](auto &obj) { it->second.push(std::move(obj)); }); + std::for_each( + unprocessed_batches.begin(), + unprocessed_batches.end(), + [this](auto &obj) { current_round_batches_.push(std::move(obj)); }); log_->debug("onBatches => collection is inserted"); } @@ -117,27 +99,32 @@ OnDemandOrderingServiceImpl::onRequestProposal(consensus::Round round) { * Get transactions from the given batches queue. Does not break batches - * continues getting all the transactions from the ongoing batch until the * required amount is collected. + * @tparam Lambda - type of side effect function for batches * @param requested_tx_amount - amount of transactions to get * @param tx_batches_queue - the queue to get transactions from * @param discarded_txs_amount - the amount of discarded txs - + * @param batch_operation - side effect function to be performed on each + * inserted batch. Passed pointer could be modified * @return transactions */ +template static std::vector> getTransactions(size_t requested_tx_amount, tbb::concurrent_queue &tx_batches_queue, - boost::optional discarded_txs_amount) { + boost::optional discarded_txs_amount, + Lambda batch_operation) { TransactionBatchType batch; std::vector> collection; std::unordered_set inserted; while (collection.size() < requested_tx_amount - and tx_batches_queue.try_pop(batch) - and inserted.insert(batch->reducedHash().hex()).second) { - collection.insert( - std::end(collection), - std::make_move_iterator(std::begin(batch->transactions())), - std::make_move_iterator(std::end(batch->transactions()))); + and tx_batches_queue.try_pop(batch)) { + if (inserted.insert(batch->reducedHash().hex()).second) { + collection.insert(std::end(collection), + std::begin(batch->transactions()), + std::end(batch->transactions())); + batch_operation(batch); + } } if (discarded_txs_amount) { @@ -152,40 +139,6 @@ getTransactions(size_t requested_tx_amount, void OnDemandOrderingServiceImpl::packNextProposals( const consensus::Round &round) { - auto close_round = [this](consensus::Round round) { - log_->debug("close {}", round); - - auto it = current_proposals_.find(round); - if (it != current_proposals_.end()) { - log_->debug("proposal found"); - if (not it->second.empty()) { - log_->debug("Mutable proposal generation for round {}", round); - size_t discarded_txs_amount; - auto txs = getTransactions(transaction_limit_, it->second, discarded_txs_amount); - if (not txs.empty()) { - log_->debug("Number of transactions in proposal = {}", txs.size()); - auto proposal = proposal_factory_->unsafeCreateProposal( - round.block_round, - iroha::time::now(), - std::move(txs) | boost::adaptors::indirected); - proposal_map_.emplace(round, std::move(proposal)); - log_->debug( - "packNextProposal: data has been fetched for {}. " - "Discarded {} transactions.", - round, - discarded_txs_amount); - round_queue_.push(round); - } - } - current_proposals_.erase(it); - } - }; - - auto open_round = [this](consensus::Round round) { - log_->debug("open {}", round); - current_proposals_[round]; - }; - /* * The possible cases can be visualised as a diagram, where: * o - current round, x - next round, v - target round @@ -217,23 +170,48 @@ void OnDemandOrderingServiceImpl::packNextProposals( * (1,0) - current round. The diagram is similar to the initial case. */ - // close next reject round - close_round({round.block_round, round.reject_round + 1}); + size_t discarded_txs_amount; + auto get_transactions = [this, &discarded_txs_amount](auto &queue, + auto lambda) { + return getTransactions( + transaction_limit_, queue, discarded_txs_amount, lambda); + }; + + auto now = iroha::time::now(); + auto generate_proposal = [this, now, &discarded_txs_amount]( + consensus::Round round, const auto &txs) { + auto proposal = proposal_factory_->unsafeCreateProposal( + round.block_round, now, txs | boost::adaptors::indirected); + proposal_map_.emplace(round, std::move(proposal)); + round_queue_.push(round); + log_->debug( + "packNextProposal: data has been fetched for {}. " + "Number of transactions in proposal = {}. Discarded {} " + "transactions.", + round, + txs.size(), + discarded_txs_amount); + }; - if (round.reject_round == kFirstRejectRound) { - // new block round - close_round({round.block_round + 1, round.reject_round}); + if (not current_round_batches_.empty()) { + auto txs = get_transactions(current_round_batches_, [this](auto &batch) { + next_round_batches_.push(std::move(batch)); + }); - // remove current queues - current_proposals_.clear(); - // initialize the 3 diagonal rounds from the commit case diagram - open_round({round.block_round + 1, kNextRejectRoundConsumer}); - open_round({round.block_round + 2, kNextCommitRoundConsumer}); + if (not txs.empty() and round.reject_round != kFirstRejectRound) { + generate_proposal({round.block_round, round.reject_round + 1}, txs); + } } - // new reject round - open_round( - {round.block_round, currentRejectRoundConsumer(round.reject_round)}); + if (not next_round_batches_.empty() + and round.reject_round == kFirstRejectRound) { + auto txs = get_transactions(next_round_batches_, [](auto &) {}); + + if (not txs.empty()) { + generate_proposal({round.block_round, kNextRejectRoundConsumer}, txs); + generate_proposal({round.block_round + 1, kNextCommitRoundConsumer}, txs); + } + } } void OnDemandOrderingServiceImpl::tryErase() { diff --git a/irohad/ordering/impl/on_demand_ordering_service_impl.hpp b/irohad/ordering/impl/on_demand_ordering_service_impl.hpp index 744886f0d1..9cb2fdb9c8 100644 --- a/irohad/ordering/impl/on_demand_ordering_service_impl.hpp +++ b/irohad/ordering/impl/on_demand_ordering_service_impl.hpp @@ -51,7 +51,7 @@ namespace iroha { // ----------------------- | OdOsNotification | -------------------------- - void onBatches(consensus::Round, CollectionType batches) override; + void onBatches(CollectionType batches) override; boost::optional> onRequestProposal( consensus::Round round) override; @@ -100,12 +100,10 @@ namespace iroha { proposal_map_; /** - * Proposals for current rounds + * Collections of batches for current and next rounds */ - std::unordered_map, - consensus::RoundTypeHasher> - current_proposals_; + tbb::concurrent_queue current_round_batches_, + next_round_batches_; /** * Read write mutex for public methods diff --git a/irohad/ordering/impl/on_demand_os_client_grpc.cpp b/irohad/ordering/impl/on_demand_os_client_grpc.cpp index 273b2cc31a..dc2a31f15a 100644 --- a/irohad/ordering/impl/on_demand_os_client_grpc.cpp +++ b/irohad/ordering/impl/on_demand_os_client_grpc.cpp @@ -31,11 +31,8 @@ OnDemandOsClientGrpc::OnDemandOsClientGrpc( time_provider_(std::move(time_provider)), proposal_request_timeout_(proposal_request_timeout) {} -void OnDemandOsClientGrpc::onBatches(consensus::Round round, - CollectionType batches) { +void OnDemandOsClientGrpc::onBatches(CollectionType batches) { proto::BatchesRequest request; - request.mutable_round()->set_block_round(round.block_round); - request.mutable_round()->set_reject_round(round.reject_round); for (auto &batch : batches) { for (auto &transaction : batch->transactions()) { *request.add_transactions() = std::move( diff --git a/irohad/ordering/impl/on_demand_os_client_grpc.hpp b/irohad/ordering/impl/on_demand_os_client_grpc.hpp index 6543d9e42f..31e506f158 100644 --- a/irohad/ordering/impl/on_demand_os_client_grpc.hpp +++ b/irohad/ordering/impl/on_demand_os_client_grpc.hpp @@ -42,7 +42,7 @@ namespace iroha { std::chrono::milliseconds proposal_request_timeout, logger::LoggerPtr log); - void onBatches(consensus::Round round, CollectionType batches) override; + void onBatches(CollectionType batches) override; boost::optional> onRequestProposal( consensus::Round round) override; diff --git a/irohad/ordering/impl/on_demand_os_server_grpc.cpp b/irohad/ordering/impl/on_demand_os_server_grpc.cpp index dc668b7433..d72392dc18 100644 --- a/irohad/ordering/impl/on_demand_os_server_grpc.cpp +++ b/irohad/ordering/impl/on_demand_os_server_grpc.cpp @@ -65,8 +65,6 @@ grpc::Status OnDemandOsServerGrpc::SendBatches( ::grpc::ServerContext *context, const proto::BatchesRequest *request, ::google::protobuf::Empty *response) { - consensus::Round round{request->round().block_round(), - request->round().reject_round()}; auto transactions = deserializeTransactions(request); auto batch_candidates = batch_parser_->parseBatches(std::move(transactions)); @@ -86,7 +84,7 @@ grpc::Status OnDemandOsServerGrpc::SendBatches( return acc; }); - ordering_service_->onBatches(round, std::move(batches)); + ordering_service_->onBatches(std::move(batches)); return ::grpc::Status::OK; } diff --git a/irohad/ordering/on_demand_os_transport.hpp b/irohad/ordering/on_demand_os_transport.hpp index b0f6021cae..e235c923df 100644 --- a/irohad/ordering/on_demand_os_transport.hpp +++ b/irohad/ordering/on_demand_os_transport.hpp @@ -48,11 +48,9 @@ namespace iroha { /** * Callback on receiving transactions - * @param round - expected proposal round * @param batches - vector of passed transaction batches */ - virtual void onBatches(consensus::Round round, - CollectionType batches) = 0; + virtual void onBatches(CollectionType batches) = 0; /** * Callback on request about proposal diff --git a/schema/ordering.proto b/schema/ordering.proto index ee75f9abb7..b9d0b2fb13 100644 --- a/schema/ordering.proto +++ b/schema/ordering.proto @@ -20,8 +20,7 @@ message ProposalRound { } message BatchesRequest { - ProposalRound round = 1; - repeated protocol.Transaction transactions = 2; + repeated protocol.Transaction transactions = 1; } message ProposalRequest { diff --git a/test/module/irohad/ordering/mock_on_demand_os_notification.hpp b/test/module/irohad/ordering/mock_on_demand_os_notification.hpp index 330125b7a8..8adb1e505e 100644 --- a/test/module/irohad/ordering/mock_on_demand_os_notification.hpp +++ b/test/module/irohad/ordering/mock_on_demand_os_notification.hpp @@ -15,7 +15,7 @@ namespace iroha { namespace transport { struct MockOdOsNotification : public OdOsNotification { - MOCK_METHOD2(onBatches, void(consensus::Round, CollectionType)); + MOCK_METHOD1(onBatches, void(CollectionType)); MOCK_METHOD1(onRequestProposal, boost::optional>( diff --git a/test/module/irohad/ordering/on_demand_connection_manager_test.cpp b/test/module/irohad/ordering/on_demand_connection_manager_test.cpp index 0525ae8fb1..5e987fae09 100644 --- a/test/module/irohad/ordering/on_demand_connection_manager_test.cpp +++ b/test/module/irohad/ordering/on_demand_connection_manager_test.cpp @@ -79,22 +79,16 @@ TEST_F(OnDemandConnectionManagerTest, FactoryUsed) { */ TEST_F(OnDemandConnectionManagerTest, onBatches) { OdOsNotification::CollectionType collection; - consensus::Round round{1, 2}; - auto set_expect = [&](OnDemandConnectionManager::PeerType type, - consensus::Round round) { - EXPECT_CALL(*connections[type], onBatches(round, collection)).Times(1); + auto set_expect = [&](OnDemandConnectionManager::PeerType type) { + EXPECT_CALL(*connections[type], onBatches(collection)).Times(1); }; - set_expect( - OnDemandConnectionManager::kCurrentRoundRejectConsumer, - {round.block_round, currentRejectRoundConsumer(round.reject_round)}); - set_expect(OnDemandConnectionManager::kNextRoundRejectConsumer, - {round.block_round + 1, kNextRejectRoundConsumer}); - set_expect(OnDemandConnectionManager::kNextRoundCommitConsumer, - {round.block_round + 2, kNextCommitRoundConsumer}); + set_expect(OnDemandConnectionManager::kCurrentRoundRejectConsumer); + set_expect(OnDemandConnectionManager::kNextRoundRejectConsumer); + set_expect(OnDemandConnectionManager::kNextRoundCommitConsumer); - manager->onBatches(round, collection); + manager->onBatches(collection); } /** diff --git a/test/module/irohad/ordering/on_demand_ordering_gate_test.cpp b/test/module/irohad/ordering/on_demand_ordering_gate_test.cpp index 6b4eb84343..254f9cebc2 100644 --- a/test/module/irohad/ordering/on_demand_ordering_gate_test.cpp +++ b/test/module/irohad/ordering/on_demand_ordering_gate_test.cpp @@ -53,7 +53,6 @@ class OnDemandOrderingGateTest : public ::testing::Test { cache, std::move(ufactory), tx_cache, - initial_round, getTestLogger("OrderingGate")); } @@ -66,8 +65,7 @@ class OnDemandOrderingGateTest : public ::testing::Test { std::shared_ptr cache; - const consensus::Round initial_round = {1, kFirstRejectRound}, - round = {2, kFirstRejectRound}; + const consensus::Round round = {2, kFirstRejectRound}; }; /** @@ -81,7 +79,7 @@ TEST_F(OnDemandOrderingGateTest, propagateBatch) { OdOsNotification::CollectionType collection{batch}; EXPECT_CALL(*cache, addToBack(UnorderedElementsAre(batch))).Times(1); - EXPECT_CALL(*notification, onBatches(initial_round, collection)).Times(1); + EXPECT_CALL(*notification, onBatches(collection)).Times(1); ordering_gate->propagateBatch(batch); } @@ -272,8 +270,7 @@ TEST_F(OnDemandOrderingGateTest, PopNonEmptyBatchesFromTheCache) { EXPECT_CALL(*cache, addToBack(UnorderedElementsAreArray(collection))) .Times(1); - EXPECT_CALL(*notification, - onBatches(round, UnorderedElementsAreArray(collection))) + EXPECT_CALL(*notification, onBatches(UnorderedElementsAreArray(collection))) .Times(1); rounds.get_subscriber().on_next(OnDemandOrderingGate::BlockEvent{round, {}}); @@ -291,7 +288,7 @@ TEST_F(OnDemandOrderingGateTest, PopEmptyBatchesFromTheCache) { EXPECT_CALL(*cache, pop()).WillOnce(Return(empty_collection)); EXPECT_CALL(*cache, addToBack(UnorderedElementsAreArray(empty_collection))) .Times(1); - EXPECT_CALL(*notification, onBatches(_, _)).Times(0); + EXPECT_CALL(*notification, onBatches(_)).Times(0); rounds.get_subscriber().on_next(OnDemandOrderingGate::BlockEvent{round, {}}); } diff --git a/test/module/irohad/ordering/on_demand_os_client_grpc_test.cpp b/test/module/irohad/ordering/on_demand_os_client_grpc_test.cpp index 8af2a79c4d..815d8ff32d 100644 --- a/test/module/irohad/ordering/on_demand_os_client_grpc_test.cpp +++ b/test/module/irohad/ordering/on_demand_os_client_grpc_test.cpp @@ -96,10 +96,8 @@ TEST_F(OnDemandOsClientGrpcTest, onBatches) { std::make_unique( shared_model::interface::types::SharedTxsCollectionType{ std::make_unique(tx)})); - client->onBatches(round, std::move(collection)); + client->onBatches(std::move(collection)); - ASSERT_EQ(request.round().block_round(), round.block_round); - ASSERT_EQ(request.round().reject_round(), round.reject_round); ASSERT_EQ(request.transactions() .Get(0) .payload() diff --git a/test/module/irohad/ordering/on_demand_os_server_grpc_test.cpp b/test/module/irohad/ordering/on_demand_os_server_grpc_test.cpp index d3109e5712..837115a742 100644 --- a/test/module/irohad/ordering/on_demand_os_server_grpc_test.cpp +++ b/test/module/irohad/ordering/on_demand_os_server_grpc_test.cpp @@ -64,8 +64,8 @@ struct OnDemandOsServerGrpcTest : public ::testing::Test { /** * Separate action required because CollectionType is non-copyable */ -ACTION_P(SaveArg1Move, var) { - *var = std::move(arg1); +ACTION_P(SaveArg0Move, var) { + *var = std::move(arg0); } /** @@ -93,11 +93,8 @@ TEST_F(OnDemandOsServerGrpcTest, SendBatches) { shared_model::interface::TransactionBatchImpl>( cand)); })); - EXPECT_CALL(*notification, onBatches(round, _)) - .WillOnce(SaveArg1Move(&collection)); + EXPECT_CALL(*notification, onBatches(_)).WillOnce(SaveArg0Move(&collection)); proto::BatchesRequest request; - request.mutable_round()->set_block_round(round.block_round); - request.mutable_round()->set_reject_round(round.reject_round); request.add_transactions() ->mutable_payload() ->mutable_reduced_payload() diff --git a/test/module/irohad/ordering/on_demand_os_test.cpp b/test/module/irohad/ordering/on_demand_os_test.cpp index 552c6ec0f8..6a4ea75c85 100644 --- a/test/module/irohad/ordering/on_demand_os_test.cpp +++ b/test/module/irohad/ordering/on_demand_os_test.cpp @@ -73,12 +73,10 @@ class OnDemandOsTest : public ::testing::Test { /** * Generate transactions with provided range - * @param os - ordering service for insertion * @param range - pair of [from, to) */ - void generateTransactionsAndInsert(consensus::Round round, - std::pair range) { - os->onBatches(round, generateTransactions(range)); + void generateTransactionsAndInsert(std::pair range) { + os->onBatches(generateTransactions(range)); } OnDemandOrderingService::CollectionType generateTransactions( @@ -135,7 +133,7 @@ TEST_F(OnDemandOsTest, EmptyRound) { * @then check that previous round has all transaction */ TEST_F(OnDemandOsTest, NormalRound) { - generateTransactionsAndInsert(target_round, {1, 2}); + generateTransactionsAndInsert({1, 2}); os->onCollaborationOutcome(commit_round); @@ -150,7 +148,7 @@ TEST_F(OnDemandOsTest, NormalRound) { * AND the rest of transactions isn't appeared in next after next round */ TEST_F(OnDemandOsTest, OverflowRound) { - generateTransactionsAndInsert(target_round, {1, transaction_limit * 2}); + generateTransactionsAndInsert({1, transaction_limit * 2}); os->onCollaborationOutcome(commit_round); @@ -181,7 +179,7 @@ TEST_F(OnDemandOsTest, DISABLED_ConcurrentInsert) { auto call = [this](auto bounds) { for (auto i = bounds.first; i < bounds.second; ++i) { - this->generateTransactionsAndInsert(target_round, {i, i + 1}); + this->generateTransactionsAndInsert({i, i + 1}); } }; @@ -203,7 +201,7 @@ TEST_F(OnDemandOsTest, Erase) { for (auto i = commit_round.block_round; i < commit_round.block_round + proposal_limit; ++i) { - generateTransactionsAndInsert({i + 1, commit_round.reject_round}, {1, 2}); + generateTransactionsAndInsert({1, 2}); os->onCollaborationOutcome({i, commit_round.reject_round}); ASSERT_TRUE(os->onRequestProposal({i + 1, commit_round.reject_round})); } @@ -211,7 +209,7 @@ TEST_F(OnDemandOsTest, Erase) { for (consensus::BlockRoundType i = commit_round.block_round + proposal_limit; i < commit_round.block_round + 2 * proposal_limit; ++i) { - generateTransactionsAndInsert({i + 1, commit_round.reject_round}, {1, 2}); + generateTransactionsAndInsert({1, 2}); os->onCollaborationOutcome({i, commit_round.reject_round}); ASSERT_FALSE(os->onRequestProposal( {i + 1 - proposal_limit, commit_round.reject_round})); @@ -228,7 +226,7 @@ TEST_F(OnDemandOsTest, EraseReject) { for (auto i = reject_round.reject_round; i < reject_round.reject_round + proposal_limit; ++i) { - generateTransactionsAndInsert({reject_round.block_round, i + 1}, {1, 2}); + generateTransactionsAndInsert({1, 2}); os->onCollaborationOutcome({reject_round.block_round, i}); ASSERT_TRUE(os->onRequestProposal({reject_round.block_round, i + 1})); } @@ -237,7 +235,7 @@ TEST_F(OnDemandOsTest, EraseReject) { reject_round.reject_round + proposal_limit; i < reject_round.reject_round + 2 * proposal_limit; ++i) { - generateTransactionsAndInsert({reject_round.block_round, i + 1}, {1, 2}); + generateTransactionsAndInsert({1, 2}); os->onCollaborationOutcome({reject_round.block_round, i}); ASSERT_FALSE(os->onRequestProposal( {reject_round.block_round, i + 1 - proposal_limit})); @@ -277,9 +275,10 @@ TEST_F(OnDemandOsTest, UseFactoryForProposal) { initial_round); EXPECT_CALL(*mock_factory, unsafeCreateProposal(_, _, _)) + .WillOnce(Return(ByMove(makeMockProposal()))) .WillOnce(Return(ByMove(makeMockProposal()))); - generateTransactionsAndInsert(target_round, {1, 2}); + generateTransactionsAndInsert({1, 2}); os->onCollaborationOutcome(commit_round); @@ -305,7 +304,7 @@ TEST_F(OnDemandOsTest, AlreadyProcessedProposalDiscarded) { .WillOnce(Return(std::vector{ iroha::ametsuchi::tx_cache_status_responses::Committed()})); - os->onBatches(initial_round, batches); + os->onBatches(batches); os->onCollaborationOutcome(commit_round); @@ -327,7 +326,7 @@ TEST_F(OnDemandOsTest, PassMissingTransaction) { .WillOnce(Return(std::vector{ iroha::ametsuchi::tx_cache_status_responses::Missing()})); - os->onBatches(target_round, batches); + os->onBatches(batches); os->onCollaborationOutcome(commit_round); @@ -359,7 +358,7 @@ TEST_F(OnDemandOsTest, SeveralTransactionsOneCommited) { .WillOnce(Return(std::vector{ iroha::ametsuchi::tx_cache_status_responses::Missing()})); - os->onBatches(target_round, batches); + os->onBatches(batches); os->onCollaborationOutcome(commit_round); diff --git a/test/module/irohad/ordering/ordering_mocks.hpp b/test/module/irohad/ordering/ordering_mocks.hpp index 5edae1828b..c867a72768 100644 --- a/test/module/irohad/ordering/ordering_mocks.hpp +++ b/test/module/irohad/ordering/ordering_mocks.hpp @@ -36,7 +36,7 @@ namespace iroha { } // namespace cache struct MockOnDemandOrderingService : public OnDemandOrderingService { - MOCK_METHOD2(onBatches, void(consensus::Round, CollectionType)); + MOCK_METHOD1(onBatches, void(CollectionType)); MOCK_METHOD1(onRequestProposal, boost::optional>( From 55fc323ea31d709b87d3356c86bdc8d379e73c0d Mon Sep 17 00:00:00 2001 From: Fedor Muratov Date: Tue, 12 Mar 2019 14:34:48 +0300 Subject: [PATCH 19/40] Fixes for OS/OG (#2149) ### Description of the Change * remove setPcs method from OG * minor issues with clang-format * add max transaction size to OG. Not it shares only a limited number of transactions. * OS queue replaced with an unordered set for checking not-unique transactions ### Benefits Performance improvements and DDoS prevention ### Possible Drawbacks The absence of tests. I think we have to discuss it during the review Signed-off-by: Fedor Muratov --- irohad/main/impl/on_demand_ordering_init.cpp | 15 +++-- irohad/main/impl/on_demand_ordering_init.hpp | 7 ++- irohad/network/ordering_gate.hpp | 9 --- irohad/ordering/CMakeLists.txt | 1 + .../ordering/impl/on_demand_ordering_gate.cpp | 63 ++++++++++++------- .../ordering/impl/on_demand_ordering_gate.hpp | 8 ++- .../impl/on_demand_ordering_service_impl.cpp | 33 +++++----- .../impl/on_demand_ordering_service_impl.hpp | 12 +++- libs/common/CMakeLists.txt | 2 +- .../ordering/on_demand_ordering_gate_test.cpp | 10 ++- 10 files changed, 92 insertions(+), 68 deletions(-) diff --git a/irohad/main/impl/on_demand_ordering_init.cpp b/irohad/main/impl/on_demand_ordering_init.cpp index 6e463a4a29..abd2cd413b 100644 --- a/irohad/main/impl/on_demand_ordering_init.cpp +++ b/irohad/main/impl/on_demand_ordering_init.cpp @@ -202,6 +202,7 @@ namespace iroha { std::shared_ptr tx_cache, std::function delay_func, + size_t max_number_of_transactions, const logger::LoggerManagerTreePtr &ordering_log_manager) { auto map = [](auto commit) { return matchEvent( @@ -245,17 +246,18 @@ namespace iroha { std::move(cache), std::move(proposal_factory), std::move(tx_cache), + max_number_of_transactions, ordering_log_manager->getChild("Gate")->getLogger()); } auto OnDemandOrderingInit::createService( - size_t max_size, + size_t max_number_of_transactions, std::shared_ptr proposal_factory, std::shared_ptr tx_cache, const logger::LoggerManagerTreePtr &ordering_log_manager) { return std::make_shared( - max_size, + max_number_of_transactions, std::move(proposal_factory), std::move(tx_cache), ordering_log_manager->getChild("Service")->getLogger()); @@ -267,7 +269,7 @@ namespace iroha { std::shared_ptr OnDemandOrderingInit::initOrderingGate( - size_t max_size, + size_t max_number_of_transactions, std::chrono::milliseconds delay, std::vector initial_hashes, std::shared_ptr peer_query_factory, @@ -287,8 +289,10 @@ namespace iroha { std::function delay_func, logger::LoggerManagerTreePtr ordering_log_manager) { - auto ordering_service = createService( - max_size, proposal_factory, tx_cache, ordering_log_manager); + auto ordering_service = createService(max_number_of_transactions, + proposal_factory, + tx_cache, + ordering_log_manager); service = std::make_shared( ordering_service, std::move(transaction_factory), @@ -307,6 +311,7 @@ namespace iroha { std::move(proposal_factory), std::move(tx_cache), std::move(delay_func), + max_number_of_transactions, ordering_log_manager); } diff --git a/irohad/main/impl/on_demand_ordering_init.hpp b/irohad/main/impl/on_demand_ordering_init.hpp index 05ef366a90..554ae4d7ef 100644 --- a/irohad/main/impl/on_demand_ordering_init.hpp +++ b/irohad/main/impl/on_demand_ordering_init.hpp @@ -74,6 +74,7 @@ namespace iroha { std::shared_ptr tx_cache, std::function delay_func, + size_t max_number_of_transactions, const logger::LoggerManagerTreePtr &ordering_log_manager); /** @@ -81,7 +82,7 @@ namespace iroha { * parameters */ auto createService( - size_t max_size, + size_t max_number_of_transactions, std::shared_ptr proposal_factory, std::shared_ptr tx_cache, @@ -97,7 +98,7 @@ namespace iroha { /** * Initializes on-demand ordering gate and ordering sevice components * - * @param max_size maximum number of transaction in a proposal + * @param max_number_of_transactions maximum number of transaction in a proposal * @param delay timeout for ordering service response on proposal request * @param initial_hashes seeds for peer list permutations for first k * rounds they are required since hash of block i defines round i + k @@ -116,7 +117,7 @@ namespace iroha { * @return initialized ordering gate */ std::shared_ptr initOrderingGate( - size_t max_size, + size_t max_number_of_transactions, std::chrono::milliseconds delay, std::vector initial_hashes, std::shared_ptr peer_query_factory, diff --git a/irohad/network/ordering_gate.hpp b/irohad/network/ordering_gate.hpp index b04ce481ed..553ce1e627 100644 --- a/irohad/network/ordering_gate.hpp +++ b/irohad/network/ordering_gate.hpp @@ -40,15 +40,6 @@ namespace iroha { */ virtual rxcpp::observable onProposal() = 0; - /** - * Set peer communication service for commit notification - * @param pcs - const reference for PeerCommunicationService - * design notes: pcs passed by const reference because of cyclic linking - * between OG and PCS in the implementation. Same reasons to move the pcs - * dependency not in ctor but make the setter method. - */ - virtual void setPcs(const PeerCommunicationService &pcs) = 0; - virtual ~OrderingGate() = default; }; } // namespace network diff --git a/irohad/ordering/CMakeLists.txt b/irohad/ordering/CMakeLists.txt index 9e1e6fcc3b..99316406e6 100644 --- a/irohad/ordering/CMakeLists.txt +++ b/irohad/ordering/CMakeLists.txt @@ -16,6 +16,7 @@ add_library(on_demand_ordering_service target_link_libraries(on_demand_ordering_service on_demand_common tbb + mst_hash shared_model_interfaces consensus_round logger diff --git a/irohad/ordering/impl/on_demand_ordering_gate.cpp b/irohad/ordering/impl/on_demand_ordering_gate.cpp index a057ad86b7..feeb64d8e7 100644 --- a/irohad/ordering/impl/on_demand_ordering_gate.cpp +++ b/irohad/ordering/impl/on_demand_ordering_gate.cpp @@ -5,12 +5,15 @@ #include "ordering/impl/on_demand_ordering_gate.hpp" +#include + #include #include #include #include #include "ametsuchi/tx_presence_cache.hpp" #include "common/visitor.hpp" +#include "interfaces/iroha_internal/transaction_batch.hpp" #include "interfaces/iroha_internal/transaction_batch_parser_impl.hpp" #include "logger/logger.hpp" #include "ordering/impl/on_demand_common.hpp" @@ -39,8 +42,10 @@ OnDemandOrderingGate::OnDemandOrderingGate( std::shared_ptr cache, std::shared_ptr factory, std::shared_ptr tx_cache, + size_t transaction_limit, logger::LoggerPtr log) : log_(std::move(log)), + transaction_limit_(transaction_limit), ordering_service_(std::move(ordering_service)), network_client_(std::move(network_client)), events_subscription_(events.subscribe([this](auto event) { @@ -50,27 +55,11 @@ OnDemandOrderingGate::OnDemandOrderingGate( return event.round; }); - visit_in_place(event, - [this](const BlockEvent &block_event) { - // block committed, remove transactions from cache - cache_->remove(block_event.hashes); - }, - [this](const EmptyEvent &) { - // no blocks committed, no transactions to remove - }); - - auto batches = cache_->pop(); - - cache_->addToBack(batches); - if (not batches.empty()) { - network_client_->onBatches( - transport::OdOsNotification::CollectionType{batches.begin(), - batches.end()}); - } - // notify our ordering service about new round ordering_service_->onCollaborationOutcome(current_round); + this->sendCachedTransactions(event); + // request proposal for the current round auto proposal = this->processProposalRequest( network_client_->onRequestProposal(current_round)); @@ -98,12 +87,6 @@ rxcpp::observable OnDemandOrderingGate::onProposal() { return proposal_notifier_.get_observable(); } -void OnDemandOrderingGate::setPcs( - const iroha::network::PeerCommunicationService &pcs) { - throw std::logic_error( - "Method is deprecated. PCS observable should be set in ctor"); -} - boost::optional> OnDemandOrderingGate::processProposalRequest( boost::optional< @@ -120,6 +103,38 @@ OnDemandOrderingGate::processProposalRequest( return proposal_without_replays; } +void OnDemandOrderingGate::sendCachedTransactions( + const BlockRoundEventType &event) { + visit_in_place(event, + [this](const BlockEvent &block_event) { + // block committed, remove transactions from cache + cache_->remove(block_event.hashes); + }, + [](const EmptyEvent &) { + // no blocks committed, no transactions to remove + }); + + auto batches = cache_->pop(); + cache_->addToBack(batches); + + // get only transactions which fit to next proposal + auto end_iterator = batches.begin(); + auto current_number_of_transactions = 0u; + for (; end_iterator != batches.end(); ++end_iterator) { + auto batch_size = (*end_iterator)->transactions().size(); + if (current_number_of_transactions + batch_size <= transaction_limit_) { + current_number_of_transactions += batch_size; + } else { + break; + } + } + + if (not batches.empty()) { + network_client_->onBatches(transport::OdOsNotification::CollectionType{ + batches.begin(), end_iterator}); + } +} + std::shared_ptr OnDemandOrderingGate::removeReplays( std::shared_ptr proposal) const { diff --git a/irohad/ordering/impl/on_demand_ordering_gate.hpp b/irohad/ordering/impl/on_demand_ordering_gate.hpp index 8306b6e2b6..dca6d15c39 100644 --- a/irohad/ordering/impl/on_demand_ordering_gate.hpp +++ b/irohad/ordering/impl/on_demand_ordering_gate.hpp @@ -66,6 +66,7 @@ namespace iroha { std::shared_ptr factory, std::shared_ptr tx_cache, + size_t transaction_limit, logger::LoggerPtr log); ~OnDemandOrderingGate() override; @@ -76,9 +77,6 @@ namespace iroha { rxcpp::observable onProposal() override; - [[deprecated("Use ctor")]] void setPcs( - const iroha::network::PeerCommunicationService &pcs) override; - private: /** * Handle an incoming proposal from ordering service @@ -89,6 +87,8 @@ namespace iroha { std::shared_ptr> proposal) const; + void sendCachedTransactions(const BlockRoundEventType &event); + /** * remove already processed transactions from proposal */ @@ -96,6 +96,8 @@ namespace iroha { std::shared_ptr proposal) const; + /// max number of transactions passed to one ordering service + size_t transaction_limit_; logger::LoggerPtr log_; std::shared_ptr ordering_service_; std::shared_ptr network_client_; diff --git a/irohad/ordering/impl/on_demand_ordering_service_impl.cpp b/irohad/ordering/impl/on_demand_ordering_service_impl.cpp index b1f2a14576..f9f5cbed3b 100644 --- a/irohad/ordering/impl/on_demand_ordering_service_impl.cpp +++ b/irohad/ordering/impl/on_demand_ordering_service_impl.cpp @@ -71,7 +71,7 @@ void OnDemandOrderingServiceImpl::onBatches(CollectionType batches) { std::for_each( unprocessed_batches.begin(), unprocessed_batches.end(), - [this](auto &obj) { current_round_batches_.push(std::move(obj)); }); + [this](auto &obj) { current_round_batches_.insert(std::move(obj)); }); log_->debug("onBatches => collection is inserted"); } @@ -101,7 +101,7 @@ OnDemandOrderingServiceImpl::onRequestProposal(consensus::Round round) { * required amount is collected. * @tparam Lambda - type of side effect function for batches * @param requested_tx_amount - amount of transactions to get - * @param tx_batches_queue - the queue to get transactions from + * @param batch_collection - the collection to get transactions from * @param discarded_txs_amount - the amount of discarded txs * @param batch_operation - side effect function to be performed on each * inserted batch. Passed pointer could be modified @@ -110,29 +110,28 @@ OnDemandOrderingServiceImpl::onRequestProposal(consensus::Round round) { template static std::vector> getTransactions(size_t requested_tx_amount, - tbb::concurrent_queue &tx_batches_queue, + detail::BatchSetType &batch_collection, boost::optional discarded_txs_amount, Lambda batch_operation) { - TransactionBatchType batch; std::vector> collection; - std::unordered_set inserted; - - while (collection.size() < requested_tx_amount - and tx_batches_queue.try_pop(batch)) { - if (inserted.insert(batch->reducedHash().hex()).second) { - collection.insert(std::end(collection), - std::begin(batch->transactions()), - std::end(batch->transactions())); - batch_operation(batch); - } + + auto it = batch_collection.begin(); + for (; + it != batch_collection.end() and collection.size() < requested_tx_amount; + ++it) { + collection.insert(std::end(collection), + std::begin((*it)->transactions()), + std::end((*it)->transactions())); + batch_operation(*it); } if (discarded_txs_amount) { *discarded_txs_amount = 0; - while (tx_batches_queue.try_pop(batch)) { - *discarded_txs_amount += boost::size(batch->transactions()); + for (; it != batch_collection.end(); ++it) { + *discarded_txs_amount += boost::size((*it)->transactions()); } } + batch_collection.clear(); return collection; } @@ -195,7 +194,7 @@ void OnDemandOrderingServiceImpl::packNextProposals( if (not current_round_batches_.empty()) { auto txs = get_transactions(current_round_batches_, [this](auto &batch) { - next_round_batches_.push(std::move(batch)); + next_round_batches_.insert(std::move(batch)); }); if (not txs.empty() and round.reject_round != kFirstRejectRound) { diff --git a/irohad/ordering/impl/on_demand_ordering_service_impl.hpp b/irohad/ordering/impl/on_demand_ordering_service_impl.hpp index 9cb2fdb9c8..a89c36a170 100644 --- a/irohad/ordering/impl/on_demand_ordering_service_impl.hpp +++ b/irohad/ordering/impl/on_demand_ordering_service_impl.hpp @@ -12,9 +12,10 @@ #include #include -#include +#include #include "interfaces/iroha_internal/unsafe_proposal_factory.hpp" #include "logger/logger_fwd.hpp" +#include "multi_sig_transactions/hash.hpp" #include "ordering/impl/on_demand_common.hpp" namespace iroha { @@ -22,6 +23,12 @@ namespace iroha { class TxPresenceCache; } namespace ordering { + namespace detail { + using BatchSetType = tbb::concurrent_unordered_set< + transport::OdOsNotification::TransactionBatchType, + model::PointerBatchHasher>; + } + class OnDemandOrderingServiceImpl : public OnDemandOrderingService { public: /** @@ -102,8 +109,7 @@ namespace iroha { /** * Collections of batches for current and next rounds */ - tbb::concurrent_queue current_round_batches_, - next_round_batches_; + detail::BatchSetType current_round_batches_, next_round_batches_; /** * Read write mutex for public methods diff --git a/libs/common/CMakeLists.txt b/libs/common/CMakeLists.txt index c7399afc4e..0c8f15d0be 100644 --- a/libs/common/CMakeLists.txt +++ b/libs/common/CMakeLists.txt @@ -47,7 +47,7 @@ if (EXISTS "${PROJECT_SOURCE_DIR}/.git") # Get pretty version execute_process(COMMAND "${GIT_EXECUTABLE}" - describe --always + describe --tags --always WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}" RESULT_VARIABLE diff --git a/test/module/irohad/ordering/on_demand_ordering_gate_test.cpp b/test/module/irohad/ordering/on_demand_ordering_gate_test.cpp index 254f9cebc2..e73a179559 100644 --- a/test/module/irohad/ordering/on_demand_ordering_gate_test.cpp +++ b/test/module/irohad/ordering/on_demand_ordering_gate_test.cpp @@ -53,6 +53,7 @@ class OnDemandOrderingGateTest : public ::testing::Test { cache, std::move(ufactory), tx_cache, + 1000, getTestLogger("OrderingGate")); } @@ -256,13 +257,16 @@ TEST_F(OnDemandOrderingGateTest, ReplayedTransactionInProposal) { * @then batch1 and batch2 are propagated to network */ TEST_F(OnDemandOrderingGateTest, PopNonEmptyBatchesFromTheCache) { - // prepare hashes for mock batches + // prepare internals of mock batches shared_model::interface::types::HashType hash1("hash1"); + auto tx1 = createMockTransactionWithHash(hash1); + shared_model::interface::types::HashType hash2("hash2"); + auto tx2 = createMockTransactionWithHash(hash2); // prepare batches - auto batch1 = createMockBatchWithHash(hash1); - auto batch2 = createMockBatchWithHash(hash2); + auto batch1 = createMockBatchWithTransactions({tx1}, "a"); + auto batch2 = createMockBatchWithTransactions({tx2}, "b"); cache::OrderingGateCache::BatchesSetType collection{batch1, batch2}; From 2a107c82411e8b6ce936965d070e17d901794062 Mon Sep 17 00:00:00 2001 From: Konstantin Munichev Date: Tue, 12 Mar 2019 16:20:40 +0300 Subject: [PATCH 20/40] Update ed25519 to 2.0.1 (#2157) Signed-off-by: Konstantin Munichev --- cmake/Modules/Finded25519.cmake | 4 +++- docker/dependencies/Dockerfile | 4 +++- docker/develop/Dockerfile | 4 +++- snap/snapcraft.yaml | 4 ++-- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/cmake/Modules/Finded25519.cmake b/cmake/Modules/Finded25519.cmake index a2d107e405..ddb02ffab3 100644 --- a/cmake/Modules/Finded25519.cmake +++ b/cmake/Modules/Finded25519.cmake @@ -12,7 +12,7 @@ find_package_handle_standard_args(ed25519 DEFAULT_MSG ) set(URL https://github.com/hyperledger/iroha-ed25519) -set(VERSION f42953c631fae93011612f6b1ee33f1f88c3f8af) +set(VERSION b61a1e77af5dc458ed6a5aee395d5b22775a4917) # v. 2.0.1 set_target_description(ed25519 "Digital signature algorithm" ${URL} ${VERSION}) if (NOT ed25519_FOUND) @@ -22,6 +22,8 @@ if (NOT ed25519_FOUND) CMAKE_ARGS -DTESTING=OFF -DBUILD=STATIC + -DHASH=sha3_brainhub + -DEDIMPL=ref10 ${DEPS_CMAKE_ARGS} PATCH_COMMAND ${PATCH_RANDOM} BUILD_BYPRODUCTS ${EP_PREFIX}/src/hyperledger_ed25519-build/${CMAKE_STATIC_LIBRARY_PREFIX}ed25519${CMAKE_STATIC_LIBRARY_SUFFIX} diff --git a/docker/dependencies/Dockerfile b/docker/dependencies/Dockerfile index d810283391..62be571199 100644 --- a/docker/dependencies/Dockerfile +++ b/docker/dependencies/Dockerfile @@ -237,10 +237,12 @@ RUN set -e; \ # install ed25519 RUN set -e; \ git clone git://github.com/hyperledger/iroha-ed25519.git /tmp/ed25519; \ - (cd /tmp/ed25519 ; git checkout f42953c631fae93011612f6b1ee33f1f88c3f8af); \ + (cd /tmp/ed25519 ; git checkout b61a1e77af5dc458ed6a5aee395d5b22775a4917); \ cmake \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ -DTESTING=OFF \ + -DHASH=sha3_brainhub \ + -DEDIMPL=ref10 \ -H/tmp/ed25519 \ -B/tmp/ed25519/build \ -DCMAKE_INSTALL_PREFIX=/opt/dependencies/ed25519; \ diff --git a/docker/develop/Dockerfile b/docker/develop/Dockerfile index a93089d654..3701b48817 100644 --- a/docker/develop/Dockerfile +++ b/docker/develop/Dockerfile @@ -233,10 +233,12 @@ RUN set -e; \ # install ed25519 RUN set -e; \ git clone git://github.com/hyperledger/iroha-ed25519.git /tmp/ed25519; \ - (cd /tmp/ed25519 ; git checkout f42953c631fae93011612f6b1ee33f1f88c3f8af); \ + (cd /tmp/ed25519 ; git checkout b61a1e77af5dc458ed6a5aee395d5b22775a4917); \ cmake \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ -DTESTING=OFF \ + -DHASH=sha3_brainhub \ + -DEDIMPL=ref10 \ -H/tmp/ed25519 \ -B/tmp/ed25519/build; \ cmake --build /tmp/ed25519/build --target install -- -j${PARALLELISM}; \ diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 51935bd5e8..156eab2367 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -151,7 +151,7 @@ parts: mv $SNAPCRAFT_PART_INSTALL/include $SNAPCRAFT_PART_INSTALL/usr/local/include ed25519: source: https://github.com/hyperledger/iroha-ed25519.git - source-commit: f42953c631fae93011612f6b1ee33f1f88c3f8af + source-commit: b61a1e77af5dc458ed6a5aee395d5b22775a4917 plugin: cmake - configflags: [-DTESTING=OFF] + configflags: [-DTESTING=OFF, -DHASH=sha3_brainhub, -DEDIMPL=ref10] after: [cmake] From dc5f9317e6cb3f766378796d3e78e2043ff6caaa Mon Sep 17 00:00:00 2001 From: Konstantin Munichev Date: Tue, 12 Mar 2019 21:57:12 +0300 Subject: [PATCH 21/40] Remove round queue (#2151) Signed-off-by: Konstantin Munichev --- .../impl/on_demand_ordering_service_impl.cpp | 27 +++++++--- .../impl/on_demand_ordering_service_impl.hpp | 12 ++--- test/framework/test_logger.cpp | 2 +- .../irohad/ordering/on_demand_os_test.cpp | 52 +++++-------------- 4 files changed, 35 insertions(+), 58 deletions(-) diff --git a/irohad/ordering/impl/on_demand_ordering_service_impl.cpp b/irohad/ordering/impl/on_demand_ordering_service_impl.cpp index f9f5cbed3b..c496aa48d1 100644 --- a/irohad/ordering/impl/on_demand_ordering_service_impl.cpp +++ b/irohad/ordering/impl/on_demand_ordering_service_impl.cpp @@ -52,7 +52,7 @@ void OnDemandOrderingServiceImpl::onCollaborationOutcome( log_->debug("onCollaborationOutcome => write lock is acquired"); packNextProposals(round); - tryErase(); + tryErase(round); } // ----------------------------| OdOsNotification |----------------------------- @@ -182,7 +182,6 @@ void OnDemandOrderingServiceImpl::packNextProposals( auto proposal = proposal_factory_->unsafeCreateProposal( round.block_round, now, txs | boost::adaptors::indirected); proposal_map_.emplace(round, std::move(proposal)); - round_queue_.push(round); log_->debug( "packNextProposal: data has been fetched for {}. " "Number of transactions in proposal = {}. Discarded {} " @@ -213,12 +212,24 @@ void OnDemandOrderingServiceImpl::packNextProposals( } } -void OnDemandOrderingServiceImpl::tryErase() { - while (round_queue_.size() > number_of_proposals_) { - auto &round = round_queue_.front(); - proposal_map_.erase(round); - log_->info("tryErase: erased {}", round); - round_queue_.pop(); +void OnDemandOrderingServiceImpl::tryErase( + const consensus::Round ¤t_round) { + auto current_proposal = + std::lower_bound(proposal_map_.begin(), + proposal_map_.end(), + current_round, + [](const auto &map_item, const auto &round) { + return map_item.first < round; + }); + + auto proposal_range_size = boost::size( + boost::make_iterator_range(proposal_map_.begin(), current_proposal)); + + while (proposal_range_size > number_of_proposals_) { + BOOST_ASSERT(proposal_map_.begin()->first < current_round); + log_->info("tryErase: erasing {}", proposal_map_.begin()->first); + proposal_map_.erase(proposal_map_.begin()); + --proposal_range_size; } } diff --git a/irohad/ordering/impl/on_demand_ordering_service_impl.hpp b/irohad/ordering/impl/on_demand_ordering_service_impl.hpp index a89c36a170..2b04d4b418 100644 --- a/irohad/ordering/impl/on_demand_ordering_service_impl.hpp +++ b/irohad/ordering/impl/on_demand_ordering_service_impl.hpp @@ -8,6 +8,7 @@ #include "ordering/on_demand_ordering_service.hpp" +#include #include #include #include @@ -75,7 +76,7 @@ namespace iroha { * Method removes the oldest commit or chain of the oldest rejects * Note: method is not thread-safe */ - void tryErase(); + void tryErase(const consensus::Round ¤t_round); /** * Check if batch was already processed by the peer @@ -93,17 +94,10 @@ namespace iroha { */ size_t number_of_proposals_; - /** - * Queue which holds all rounds in linear order - */ - std::queue round_queue_; - /** * Map of available proposals */ - std::unordered_map, - consensus::RoundTypeHasher> + std::map> proposal_map_; /** diff --git a/test/framework/test_logger.cpp b/test/framework/test_logger.cpp index 86de19c388..bac03dfb33 100644 --- a/test/framework/test_logger.cpp +++ b/test/framework/test_logger.cpp @@ -10,7 +10,7 @@ logger::LoggerManagerTreePtr getTestLoggerManager() { static logger::LoggerManagerTreePtr log_manager( std::make_shared(logger::LoggerConfig{ - logger::LogLevel::kInfo, logger::getDefaultLogPatterns()})); + logger::LogLevel::kDebug, logger::getDefaultLogPatterns()})); return log_manager->getChild("Test"); } diff --git a/test/module/irohad/ordering/on_demand_os_test.cpp b/test/module/irohad/ordering/on_demand_os_test.cpp index 6a4ea75c85..832b005a91 100644 --- a/test/module/irohad/ordering/on_demand_os_test.cpp +++ b/test/module/irohad/ordering/on_demand_os_test.cpp @@ -194,52 +194,24 @@ TEST_F(OnDemandOsTest, DISABLED_ConcurrentInsert) { /** * @given initialized on-demand OS - * @when insert proposal_limit rounds twice - * @then on second rounds check that old proposals are expired + * @when insert commit round and then proposal_limit reject rounds + * @then first proposal still not expired */ TEST_F(OnDemandOsTest, Erase) { - for (auto i = commit_round.block_round; - i < commit_round.block_round + proposal_limit; - ++i) { - generateTransactionsAndInsert({1, 2}); - os->onCollaborationOutcome({i, commit_round.reject_round}); - ASSERT_TRUE(os->onRequestProposal({i + 1, commit_round.reject_round})); - } - - for (consensus::BlockRoundType i = commit_round.block_round + proposal_limit; - i < commit_round.block_round + 2 * proposal_limit; - ++i) { - generateTransactionsAndInsert({1, 2}); - os->onCollaborationOutcome({i, commit_round.reject_round}); - ASSERT_FALSE(os->onRequestProposal( - {i + 1 - proposal_limit, commit_round.reject_round})); - } -} - -/** - * @given initialized on-demand OS - * @when insert proposal_limit rounds twice - * AND outcome is reject - * @then on second rounds check that old proposals are expired - */ -TEST_F(OnDemandOsTest, EraseReject) { - for (auto i = reject_round.reject_round; - i < reject_round.reject_round + proposal_limit; - ++i) { - generateTransactionsAndInsert({1, 2}); - os->onCollaborationOutcome({reject_round.block_round, i}); - ASSERT_TRUE(os->onRequestProposal({reject_round.block_round, i + 1})); - } + generateTransactionsAndInsert({1, 2}); + os->onCollaborationOutcome( + {commit_round.block_round, commit_round.reject_round}); + ASSERT_TRUE(os->onRequestProposal( + {commit_round.block_round + 1, commit_round.reject_round})); - for (consensus::RejectRoundType i = - reject_round.reject_round + proposal_limit; - i < reject_round.reject_round + 2 * proposal_limit; + for (auto i = commit_round.reject_round; + i < commit_round.reject_round + proposal_limit; ++i) { generateTransactionsAndInsert({1, 2}); - os->onCollaborationOutcome({reject_round.block_round, i}); - ASSERT_FALSE(os->onRequestProposal( - {reject_round.block_round, i + 1 - proposal_limit})); + os->onCollaborationOutcome({commit_round.block_round + 1, i}); } + ASSERT_TRUE(os->onRequestProposal( + {commit_round.block_round + 1, commit_round.reject_round})); } /** From c34c18c2e02230e22a177af0cca734ff19e4c65b Mon Sep 17 00:00:00 2001 From: Dumitru Date: Wed, 13 Mar 2019 14:06:58 +0700 Subject: [PATCH 22/40] Add docs (#2137) add error Signed-off-by: Dumitru --- docs/source/api/queries.rst | 54 +++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/docs/source/api/queries.rst b/docs/source/api/queries.rst index a7e446bd65..0d18a3b984 100644 --- a/docs/source/api/queries.rst +++ b/docs/source/api/queries.rst @@ -829,3 +829,57 @@ Possible Stateful Validation Errors .. [#f1] https://www.ietf.org/rfc/rfc1035.txt .. [#f2] https://www.ietf.org/rfc/rfc1123.txt + + +FetchCommits +^^^^^^^^^^^^^^^^^^^^ + +Purpose +------- + +To get new blocks as soon as they are committed, a user can invoke `FetchCommits` RPC call to Iroha network. + +Request Schema +-------------- + +No request arguments are needed + + +Response Schema +--------------- + +.. code-block:: proto + + message BlockQueryResponse { + oneof response { + BlockResponse block_response = 1; + BlockErrorResponse block_error_response = 2; + } + } + +Please note that it returns a stream of `BlockQueryResponse`. + +Response Structure +------------------ + +.. csv-table:: + :header: "Field", "Description", "Constraint", "Example" + :widths: 15, 30, 20, 15 + + "Block", "Iroha block", "only committed blocks", "{ 'block_v1': ....}" + +Possible Stateful Validation Errors +----------------------------------- + +.. csv-table:: + :header: "Code", "Error Name", "Description", "How to solve" + + "1", "Could not get block streaming", "Internal error happened", "Try again or contact developers" + "2", "No such permissions", "Query's creator does not have any of the permissions to get blocks", "Grant the necessary permission: individual, global or domain one" + "3", "Invalid signatures", "Signatures of this query did not pass validation", "Add more signatures and make sure query's signatures are a subset of account's signatories" + +Example +------- +You can check an example how to use this query here: +https://github.com/x3medima17/twitter + From ca478b4601e0fa65a564ae45664e5fc7268faa56 Mon Sep 17 00:00:00 2001 From: Mikhail Boldyrev Date: Thu, 14 Mar 2019 08:08:21 +0300 Subject: [PATCH 23/40] YAC: filter votes from unknown peers (#2134) Signed-off-by: Mikhail Boldyrev --- example/python/permissions/commons.py | 4 ++- irohad/consensus/yac/cluster_order.hpp | 4 +-- irohad/consensus/yac/impl/cluster_order.cpp | 5 +-- irohad/consensus/yac/impl/yac.cpp | 36 +++++++++++++++++++ .../yac/storage/impl/yac_vote_storage.cpp | 3 ++ .../yac/transport/impl/network_impl.cpp | 7 ++-- irohad/consensus/yac/yac.hpp | 3 ++ .../consensus/consensus_sunny_day.cpp | 9 +++-- .../yac/mock_yac_crypto_provider.hpp | 9 ++++- .../irohad/consensus/yac/yac_fixture.hpp | 2 +- .../yac/yac_simple_cold_case_test.cpp | 24 +++++++++---- .../consensus/yac/yac_unknown_peer_test.cpp | 24 ++++++++----- 12 files changed, 104 insertions(+), 26 deletions(-) diff --git a/example/python/permissions/commons.py b/example/python/permissions/commons.py index b1d950a419..89b57a06f2 100644 --- a/example/python/permissions/commons.py +++ b/example/python/permissions/commons.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # +import ed25519 import irohalib import primitive_pb2 import binascii @@ -76,7 +77,8 @@ def genesis_block(admin, alice, test_permissions, multidomain=False): """ peer = primitive_pb2.Peer() peer.address = '0.0.0.0:50541' - peer.peer_key = admin['key'] + # ed25519.publickey_unsafe takes and returns a bytes object, while we have hex strings + peer.peer_key = binascii.hexlify(ed25519.publickey_unsafe(binascii.unhexlify(admin['key']))) commands = [ command('AddPeer', peer=peer), command('CreateRole', role_name='admin_role', permissions=all_permissions()), diff --git a/irohad/consensus/yac/cluster_order.hpp b/irohad/consensus/yac/cluster_order.hpp index c9bcea35c6..cd3bd65089 100644 --- a/irohad/consensus/yac/cluster_order.hpp +++ b/irohad/consensus/yac/cluster_order.hpp @@ -52,8 +52,8 @@ namespace iroha { */ bool hasNext() const; - std::vector> getPeers() - const; + const std::vector> + &getPeers() const; PeersNumberType getNumberOfPeers() const; diff --git a/irohad/consensus/yac/impl/cluster_order.cpp b/irohad/consensus/yac/impl/cluster_order.cpp index ab928d5014..5e74b1aade 100644 --- a/irohad/consensus/yac/impl/cluster_order.cpp +++ b/irohad/consensus/yac/impl/cluster_order.cpp @@ -39,14 +39,15 @@ namespace iroha { return *this; } - std::vector> - ClusterOrdering::getPeers() const { + const std::vector> + &ClusterOrdering::getPeers() const { return order_; } size_t ClusterOrdering::getNumberOfPeers() const { return order_.size(); } + } // namespace yac } // namespace consensus } // namespace iroha diff --git a/irohad/consensus/yac/impl/yac.cpp b/irohad/consensus/yac/impl/yac.cpp index 38dd19dae0..c3d20d14fc 100644 --- a/irohad/consensus/yac/impl/yac.cpp +++ b/irohad/consensus/yac/impl/yac.cpp @@ -7,6 +7,7 @@ #include +#include #include "common/bind.hpp" #include "common/visitor.hpp" #include "consensus/yac/cluster_order.hpp" @@ -87,8 +88,43 @@ namespace iroha { // ------|Network notifications|------ + template + void removeMatching(std::vector &target, const P &predicate) { + target.erase(std::remove_if(target.begin(), target.end(), predicate), + target.end()); + } + + template + bool contains(const CollectionType &haystack, const ElementType &needle) { + return std::find(haystack.begin(), haystack.end(), needle) + != haystack.end(); + } + + /// moves the votes not present in known_keys from votes to return value + void Yac::removeUnknownPeersVotes(std::vector &votes) { + auto known_keys = cluster_order_.getPeers() + | boost::adaptors::transformed( + [](const auto &peer) { return peer->pubkey(); }); + removeMatching( + votes, + [known_keys = std::move(known_keys), this](VoteMessage &vote) { + if (not contains(known_keys, vote.signature->publicKey())) { + log_->warn("Got a vote from an unknown peer: {}", vote); + return true; + } + return false; + }); + } + void Yac::onState(std::vector state) { std::lock_guard guard(mutex_); + + removeUnknownPeersVotes(state); + if (state.empty()) { + log_->debug("No votes left in the message."); + return; + } + if (crypto_->verify(state)) { applyState(state); } else { diff --git a/irohad/consensus/yac/storage/impl/yac_vote_storage.cpp b/irohad/consensus/yac/storage/impl/yac_vote_storage.cpp index 387e4963d0..574ee31824 100644 --- a/irohad/consensus/yac/storage/impl/yac_vote_storage.cpp +++ b/irohad/consensus/yac/storage/impl/yac_vote_storage.cpp @@ -87,6 +87,9 @@ namespace iroha { boost::optional YacVoteStorage::store( std::vector state, PeersNumberType peers_in_round) { + if (state.empty()) { + return boost::none; + } return findProposalStorage(state.at(0), peers_in_round) | [this, &state](auto &&storage) { const auto &round = storage->getStorageKey(); diff --git a/irohad/consensus/yac/transport/impl/network_impl.cpp b/irohad/consensus/yac/transport/impl/network_impl.cpp index f49da08140..02771b8bc2 100644 --- a/irohad/consensus/yac/transport/impl/network_impl.cpp +++ b/irohad/consensus/yac/transport/impl/network_impl.cpp @@ -59,10 +59,13 @@ namespace iroha { auto vote = *PbConverters::deserializeVote(pb_vote, log_); state.push_back(vote); } + if (state.empty()) { + log_->info("Received an empty votes collection"); + return grpc::Status::CANCELLED; + } if (not sameKeys(state)) { log_->info( - "Votes are stateless invalid: proposals are different, or empty " - "collection"); + "Votes are statelessly invalid: proposal rounds are different"); return grpc::Status::CANCELLED; } diff --git a/irohad/consensus/yac/yac.hpp b/irohad/consensus/yac/yac.hpp index 95ed792925..7fcf3f23bc 100644 --- a/irohad/consensus/yac/yac.hpp +++ b/irohad/consensus/yac/yac.hpp @@ -78,6 +78,9 @@ namespace iroha { boost::optional> findPeer(const VoteMessage &vote); + /// Remove votes from unknown peers from given vector. + void removeUnknownPeersVotes(std::vector &votes); + // ------|Apply data|------ void applyState(const std::vector &state); diff --git a/test/integration/consensus/consensus_sunny_day.cpp b/test/integration/consensus/consensus_sunny_day.cpp index 6bba5df866..c35f5788ea 100644 --- a/test/integration/consensus/consensus_sunny_day.cpp +++ b/test/integration/consensus/consensus_sunny_day.cpp @@ -78,7 +78,9 @@ class ConsensusSunnyDayTest : public ::testing::Test { static const size_t port = 50541; - ConsensusSunnyDayTest() : my_peer(mk_local_peer(port + my_num)) { + ConsensusSunnyDayTest() + : my_peer(mk_local_peer(port + my_num)), + my_pub_key(shared_model::crypto::toBinaryString(my_peer->pubkey())) { for (decltype(num_peers) i = 0; i < num_peers; ++i) { default_peers.push_back(mk_local_peer(port + i)); } @@ -99,7 +101,7 @@ class ConsensusSunnyDayTest : public ::testing::Test { getTestLogger("AsyncCall")); network = std::make_shared(async_call, getTestLogger("YacNetwork")); - crypto = std::make_shared(std::to_string(my_num)); + crypto = std::make_shared(my_pub_key); timer = std::make_shared([this] { // static factory with a single thread // see YacInit::createTimer in consensus_init.cpp @@ -138,6 +140,7 @@ class ConsensusSunnyDayTest : public ::testing::Test { uint64_t delay_before, delay_after; std::shared_ptr my_peer; + const std::string my_pub_key; std::vector> default_peers; }; @@ -160,7 +163,7 @@ TEST_F(ConsensusSunnyDayTest, SunnyDayTest) { std::this_thread::sleep_for(std::chrono::milliseconds(delay_before)); YacHash my_hash(iroha::consensus::Round{1, 1}, "proposal_hash", "block_hash"); - my_hash.block_signature = createSig(""); + my_hash.block_signature = createSig(my_pub_key); auto order = ClusterOrdering::create(default_peers); ASSERT_TRUE(order); diff --git a/test/module/irohad/consensus/yac/mock_yac_crypto_provider.hpp b/test/module/irohad/consensus/yac/mock_yac_crypto_provider.hpp index 4e7eb24628..b68877d3a4 100644 --- a/test/module/irohad/consensus/yac/mock_yac_crypto_provider.hpp +++ b/test/module/irohad/consensus/yac/mock_yac_crypto_provider.hpp @@ -27,7 +27,7 @@ namespace iroha { auto tmp = shared_model::crypto::DefaultCryptoAlgorithmType::generateKeypair() .publicKey(); - std::string key(tmp.blob().size(), 0); + std::string key(tmp.blob().size(), '0'); std::copy(pub_key.begin(), pub_key.end(), key.begin()); auto sig = std::make_shared(); EXPECT_CALL(*sig, publicKey()) @@ -51,6 +51,13 @@ namespace iroha { return vote; } + VoteMessage getVote(YacHash hash, std::string pub_key) { + VoteMessage vote; + vote.hash = std::move(hash); + vote.signature = createSig(std::move(pub_key)); + return vote; + } + MockYacCryptoProvider() = default; MockYacCryptoProvider(const MockYacCryptoProvider &) {} diff --git a/test/module/irohad/consensus/yac/yac_fixture.hpp b/test/module/irohad/consensus/yac/yac_fixture.hpp index 3c49a19ff8..e7004fd366 100644 --- a/test/module/irohad/consensus/yac/yac_fixture.hpp +++ b/test/module/irohad/consensus/yac/yac_fixture.hpp @@ -40,7 +40,7 @@ namespace iroha { default_peers = [] { std::vector> result; - for (size_t i = 1; i <= 7; ++i) { + for (size_t i = 0; i < 7; ++i) { result.push_back(makePeer(std::to_string(i))); } return result; diff --git a/test/module/irohad/consensus/yac/yac_simple_cold_case_test.cpp b/test/module/irohad/consensus/yac/yac_simple_cold_case_test.cpp index 45da565d28..3064976df9 100644 --- a/test/module/irohad/consensus/yac/yac_simple_cold_case_test.cpp +++ b/test/module/irohad/consensus/yac/yac_simple_cold_case_test.cpp @@ -58,18 +58,23 @@ TEST_F(YacTest, YacWhenColdStartAndAchieveOneVote) { YacHash received_hash( iroha::consensus::Round{1, 1}, "my_proposal", "my_block"); - auto peer = default_peers.at(0); // assume that our peer receive message - network->notification->onState({crypto->getVote(received_hash)}); + network->notification->onState({crypto->getVote(received_hash, "0")}); ASSERT_TRUE(wrapper.validate()); } /** * Test provide scenario - * when yac cold started and achieve supermajority of votes + * when yac cold started and achieve supermajority of votes + * + * TODO 13.03.2019 mboldyrev IR-396: fix the test if needed + * the test passed successfully due to votes being equal and hence + * YacProposalStorage::checkPeerUniqueness(const VoteMessage &) + * returning `false'. This does not meet the `when' clause in this test + * description. */ -TEST_F(YacTest, YacWhenColdStartAndAchieveSupermajorityOfVotes) { +TEST_F(YacTest, DISABLED_YacWhenColdStartAndAchieveSupermajorityOfVotes) { cout << "----------|Start => receive supermajority of votes" "|----------" << endl; @@ -87,7 +92,8 @@ TEST_F(YacTest, YacWhenColdStartAndAchieveSupermajorityOfVotes) { YacHash received_hash( iroha::consensus::Round{1, 1}, "my_proposal", "my_block"); for (size_t i = 0; i < default_peers.size(); ++i) { - network->notification->onState({crypto->getVote(received_hash)}); + network->notification->onState( + {crypto->getVote(received_hash, std::to_string(i))}); } ASSERT_TRUE(wrapper.validate()); @@ -130,8 +136,14 @@ TEST_F(YacTest, YacWhenColdStartAndAchieveCommitMessage) { * @given initialized YAC * @when receive supermajority of votes for a hash * @then commit is sent to the network before notifying subscribers + * + * TODO 13.03.2019 mboldyrev IR-396: fix the test if needed + * the test passed successfully due to votes being equal and hence + * YacProposalStorage::checkPeerUniqueness(const VoteMessage &) + * returning `false'. This does not meet the `when' clause in this test + * description. */ -TEST_F(YacTest, PropagateCommitBeforeNotifyingSubscribersApplyVote) { +TEST_F(YacTest, DISABLED_PropagateCommitBeforeNotifyingSubscribersApplyVote) { EXPECT_CALL(*crypto, verify(_)) .Times(default_peers.size()) .WillRepeatedly(Return(true)); diff --git a/test/module/irohad/consensus/yac/yac_unknown_peer_test.cpp b/test/module/irohad/consensus/yac/yac_unknown_peer_test.cpp index 11086c2435..8b2582d7f0 100644 --- a/test/module/irohad/consensus/yac/yac_unknown_peer_test.cpp +++ b/test/module/irohad/consensus/yac/yac_unknown_peer_test.cpp @@ -23,21 +23,29 @@ using namespace std; * @then commit not emitted */ TEST_F(YacTest, UnknownVoteBeforeCommit) { + auto my_order = ClusterOrdering::create(default_peers); + ASSERT_TRUE(my_order); + initYac(my_order.value()); + // verify that commit not emitted auto wrapper = make_test_subscriber(yac->onOutcome(), 0); wrapper.subscribe(); EXPECT_CALL(*network, sendState(_, _)).Times(0); - EXPECT_CALL(*crypto, verify(_)).Times(1).WillRepeatedly(Return(true)); + EXPECT_CALL(*crypto, verify(_)) + .Times(testing::AnyNumber()) + .WillRepeatedly(Return(true)); - VoteMessage vote; - vote.hash = YacHash(iroha::consensus::Round{1, 1}, "my_proposal", "my_block"); - std::string unknown = "unknown"; - vote.signature = createSig(unknown); + YacHash my_hash{iroha::consensus::Round{1, 1}, "my_proposal", "my_block"}; - // assume that our peer receive message - network->notification->onState({vote}); + // send enough votes for next valid to make a commit + for (auto i = 0; i < 4; ++i) { + yac->onState({createVote(my_hash, std::to_string(i))}); + } + + // send a vote from unknown peer + yac->onState({createVote(my_hash, "unknown")}); ASSERT_TRUE(wrapper.validate()); } @@ -62,7 +70,7 @@ TEST_F(YacTest, UnknownVoteAfterCommit) { EXPECT_CALL(*timer, deny()).Times(AtLeast(1)); - EXPECT_CALL(*crypto, verify(_)).Times(2).WillRepeatedly(Return(true)); + EXPECT_CALL(*crypto, verify(_)).Times(1).WillRepeatedly(Return(true)); YacHash my_hash(iroha::consensus::Round{1, 1}, "proposal_hash", "block_hash"); From 6617c84850493c1d52a64ef9e09afdcd032041c2 Mon Sep 17 00:00:00 2001 From: Andrei Lebedev Date: Thu, 14 Mar 2019 13:24:32 +0300 Subject: [PATCH 24/40] Create block storage dependency for mutable storage (#1994) Signed-off-by: Andrei Lebedev --- irohad/ametsuchi/CMakeLists.txt | 2 + irohad/ametsuchi/block_storage.hpp | 67 ++++++++++ irohad/ametsuchi/block_storage_factory.hpp | 29 +++++ .../impl/in_memory_block_storage.cpp | 43 ++++++ .../impl/in_memory_block_storage.hpp | 44 +++++++ .../impl/in_memory_block_storage_factory.cpp | 14 ++ .../impl/in_memory_block_storage_factory.hpp | 22 ++++ .../ametsuchi/impl/mutable_storage_impl.cpp | 4 +- .../ametsuchi/impl/mutable_storage_impl.hpp | 9 +- irohad/ametsuchi/impl/storage_impl.cpp | 27 ++-- irohad/ametsuchi/impl/storage_impl.hpp | 8 +- irohad/main/application.cpp | 3 + test/module/irohad/ametsuchi/CMakeLists.txt | 5 + .../irohad/ametsuchi/ametsuchi_fixture.hpp | 5 +- .../in_memory_block_storage_test.cpp | 122 ++++++++++++++++++ .../irohad/ametsuchi/storage_init_test.cpp | 7 + 16 files changed, 389 insertions(+), 22 deletions(-) create mode 100644 irohad/ametsuchi/block_storage.hpp create mode 100644 irohad/ametsuchi/block_storage_factory.hpp create mode 100644 irohad/ametsuchi/impl/in_memory_block_storage.cpp create mode 100644 irohad/ametsuchi/impl/in_memory_block_storage.hpp create mode 100644 irohad/ametsuchi/impl/in_memory_block_storage_factory.cpp create mode 100644 irohad/ametsuchi/impl/in_memory_block_storage_factory.hpp create mode 100644 test/module/irohad/ametsuchi/in_memory_block_storage_test.cpp diff --git a/irohad/ametsuchi/CMakeLists.txt b/irohad/ametsuchi/CMakeLists.txt index 4695c96840..158f6ec988 100644 --- a/irohad/ametsuchi/CMakeLists.txt +++ b/irohad/ametsuchi/CMakeLists.txt @@ -18,6 +18,8 @@ add_library(ametsuchi impl/postgres_options.cpp impl/postgres_query_executor.cpp impl/tx_presence_cache_impl.cpp + impl/in_memory_block_storage.cpp + impl/in_memory_block_storage_factory.cpp ) target_link_libraries(ametsuchi diff --git a/irohad/ametsuchi/block_storage.hpp b/irohad/ametsuchi/block_storage.hpp new file mode 100644 index 0000000000..599e363885 --- /dev/null +++ b/irohad/ametsuchi/block_storage.hpp @@ -0,0 +1,67 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef IROHA_BLOCK_STORAGE_HPP +#define IROHA_BLOCK_STORAGE_HPP + +#include +#include +#include + +#include +#include "interfaces/iroha_internal/block.hpp" + +namespace iroha { + namespace ametsuchi { + + /** + * Append-only block storage interface + */ + class BlockStorage { + public: + /** + * Append block, if the storage doesn't already contain the same block + * @return true if inserted successfully, false otherwise + */ + virtual bool insert( + std::shared_ptr block) = 0; + + [[deprecated("Use shared_ptr")]] virtual bool insert( + const shared_model::interface::Block &block) = 0; + + /** + * Get block with given height + * @return block if exists, boost::none otherwise + */ + virtual boost::optional< + std::shared_ptr> + fetch(shared_model::interface::types::HeightType height) const = 0; + + /** + * Returns the size of the storage + */ + virtual size_t size() const = 0; + + /** + * Clears the contents of storage + */ + virtual void clear() = 0; + + /// type of function which can be applied to the elements of the storage + using FunctionType = std::function)>; + + /** + * Iterates through all the stored blocks + */ + virtual void forEach(FunctionType function) const = 0; + + virtual ~BlockStorage() = default; + }; + + } // namespace ametsuchi +} // namespace iroha + +#endif // IROHA_BLOCK_STORAGE_HPP diff --git a/irohad/ametsuchi/block_storage_factory.hpp b/irohad/ametsuchi/block_storage_factory.hpp new file mode 100644 index 0000000000..c0e7655407 --- /dev/null +++ b/irohad/ametsuchi/block_storage_factory.hpp @@ -0,0 +1,29 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef IROHA_BLOCK_STORAGE_FACTORY_HPP +#define IROHA_BLOCK_STORAGE_FACTORY_HPP + +#include + +#include "ametsuchi/block_storage.hpp" + +namespace iroha { + namespace ametsuchi { + + /** + * Creates a block storage + */ + class BlockStorageFactory { + public: + virtual std::unique_ptr create() = 0; + + virtual ~BlockStorageFactory() = default; + }; + + } // namespace ametsuchi +} // namespace iroha + +#endif // IROHA_BLOCK_STORAGE_FACTORY_HPP diff --git a/irohad/ametsuchi/impl/in_memory_block_storage.cpp b/irohad/ametsuchi/impl/in_memory_block_storage.cpp new file mode 100644 index 0000000000..b32929fc27 --- /dev/null +++ b/irohad/ametsuchi/impl/in_memory_block_storage.cpp @@ -0,0 +1,43 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "ametsuchi/impl/in_memory_block_storage.hpp" + +using namespace iroha::ametsuchi; + +bool InMemoryBlockStorage::insert( + std::shared_ptr block) { + auto height = block->height(); + return block_store_.emplace(height, std::move(block)).second; +} + +bool InMemoryBlockStorage::insert(const shared_model::interface::Block &block) { + return block_store_.emplace(block.height(), clone(block)).second; +} + +boost::optional> +InMemoryBlockStorage::fetch( + shared_model::interface::types::HeightType height) const { + auto it = block_store_.find(height); + if (it != block_store_.end()) { + return it->second; + } else { + return boost::none; + } +} + +size_t InMemoryBlockStorage::size() const { + return block_store_.size(); +} + +void InMemoryBlockStorage::clear() { + block_store_.clear(); +} + +void InMemoryBlockStorage::forEach(FunctionType function) const { + for (const auto &pair : block_store_) { + function(pair.second); + } +} diff --git a/irohad/ametsuchi/impl/in_memory_block_storage.hpp b/irohad/ametsuchi/impl/in_memory_block_storage.hpp new file mode 100644 index 0000000000..6f822729f2 --- /dev/null +++ b/irohad/ametsuchi/impl/in_memory_block_storage.hpp @@ -0,0 +1,44 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef IROHA_IN_MEMORY_BLOCK_STORAGE_HPP +#define IROHA_IN_MEMORY_BLOCK_STORAGE_HPP + +#include "ametsuchi/block_storage.hpp" + +#include + +namespace iroha { + namespace ametsuchi { + + /** + * Ordered map implementation of block storage + */ + class InMemoryBlockStorage : public BlockStorage { + public: + bool insert( + std::shared_ptr block) override; + + bool insert(const shared_model::interface::Block &block) override; + + boost::optional> + fetch(shared_model::interface::types::HeightType height) const override; + + size_t size() const override; + + void clear() override; + + void forEach(FunctionType function) const override; + + private: + std::map> + block_store_; + }; + + } // namespace ametsuchi +} // namespace iroha + +#endif // IROHA_IN_MEMORY_BLOCK_STORAGE_HPP diff --git a/irohad/ametsuchi/impl/in_memory_block_storage_factory.cpp b/irohad/ametsuchi/impl/in_memory_block_storage_factory.cpp new file mode 100644 index 0000000000..a3d22d4b79 --- /dev/null +++ b/irohad/ametsuchi/impl/in_memory_block_storage_factory.cpp @@ -0,0 +1,14 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "ametsuchi/impl/in_memory_block_storage_factory.hpp" + +#include "ametsuchi/impl/in_memory_block_storage.hpp" + +using namespace iroha::ametsuchi; + +std::unique_ptr InMemoryBlockStorageFactory::create() { + return std::make_unique(); +} diff --git a/irohad/ametsuchi/impl/in_memory_block_storage_factory.hpp b/irohad/ametsuchi/impl/in_memory_block_storage_factory.hpp new file mode 100644 index 0000000000..09a23a28b0 --- /dev/null +++ b/irohad/ametsuchi/impl/in_memory_block_storage_factory.hpp @@ -0,0 +1,22 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef IROHA_IN_MEMORY_BLOCK_STORAGE_FACTORY_HPP +#define IROHA_IN_MEMORY_BLOCK_STORAGE_FACTORY_HPP + +#include "ametsuchi/block_storage_factory.hpp" + +namespace iroha { + namespace ametsuchi { + + class InMemoryBlockStorageFactory : public BlockStorageFactory { + public: + std::unique_ptr create() override; + }; + + } // namespace ametsuchi +} // namespace iroha + +#endif // IROHA_IN_MEMORY_BLOCK_STORAGE_FACTORY_HPP diff --git a/irohad/ametsuchi/impl/mutable_storage_impl.cpp b/irohad/ametsuchi/impl/mutable_storage_impl.cpp index 944ab6aae3..e3b2504440 100644 --- a/irohad/ametsuchi/impl/mutable_storage_impl.cpp +++ b/irohad/ametsuchi/impl/mutable_storage_impl.cpp @@ -24,6 +24,7 @@ namespace iroha { std::shared_ptr cmd_executor, std::unique_ptr sql, std::shared_ptr factory, + std::unique_ptr block_storage, logger::LoggerManagerTreePtr log_manager) : top_hash_(top_hash), sql_(std::move(sql)), @@ -35,6 +36,7 @@ namespace iroha { block_index_(std::make_unique( *sql_, log_manager->getChild("PostgresBlockIndex")->getLogger())), command_executor_(std::move(cmd_executor)), + block_storage_(std::move(block_storage)), committed(false), log_(log_manager->getLogger()) { *sql_ << "BEGIN"; @@ -72,7 +74,7 @@ namespace iroha { block.transactions().end(), execute_transaction); if (block_applied) { - block_store_.insert(std::make_pair(block.height(), clone(block))); + block_storage_->insert(block); block_index_->index(block); top_hash_ = block.hash(); diff --git a/irohad/ametsuchi/impl/mutable_storage_impl.hpp b/irohad/ametsuchi/impl/mutable_storage_impl.hpp index 127bab928f..0a63527a17 100644 --- a/irohad/ametsuchi/impl/mutable_storage_impl.hpp +++ b/irohad/ametsuchi/impl/mutable_storage_impl.hpp @@ -8,9 +8,8 @@ #include "ametsuchi/mutable_storage.hpp" -#include - #include +#include "ametsuchi/block_storage.hpp" #include "ametsuchi/command_executor.hpp" #include "interfaces/common_objects/common_objects_factory.hpp" #include "logger/logger_fwd.hpp" @@ -31,6 +30,7 @@ namespace iroha { std::unique_ptr sql, std::shared_ptr factory, + std::unique_ptr block_storage, logger::LoggerManagerTreePtr log_manager); bool apply(const shared_model::interface::Block &block) override; @@ -58,15 +58,12 @@ namespace iroha { MutableStoragePredicate predicate); shared_model::interface::types::HashType top_hash_; - // ordered collection is used to enforce block insertion order in - // StorageImpl::commit - std::map> - block_store_; std::unique_ptr sql_; std::unique_ptr peer_query_; std::unique_ptr block_index_; std::shared_ptr command_executor_; + std::unique_ptr block_storage_; bool committed; diff --git a/irohad/ametsuchi/impl/storage_impl.cpp b/irohad/ametsuchi/impl/storage_impl.cpp index 7a6009da2c..a6d474dbd1 100644 --- a/irohad/ametsuchi/impl/storage_impl.cpp +++ b/irohad/ametsuchi/impl/storage_impl.cpp @@ -65,6 +65,7 @@ namespace iroha { std::shared_ptr converter, std::shared_ptr perm_converter, + std::unique_ptr block_storage_factory, size_t pool_size, bool enable_prepared_blocks, logger::LoggerManagerTreePtr log_manager) @@ -75,6 +76,7 @@ namespace iroha { factory_(std::move(factory)), converter_(std::move(converter)), perm_converter_(std::move(perm_converter)), + block_storage_factory_(std::move(block_storage_factory)), log_manager_(std::move(log_manager)), log_(log_manager_->getLogger()), pool_size_(pool_size), @@ -148,6 +150,7 @@ namespace iroha { std::make_shared(*sql, perm_converter_), std::move(sql), factory_, + block_storage_factory_->create(), log_manager_->getChild("MutableStorageImpl"))); } @@ -364,6 +367,7 @@ namespace iroha { std::shared_ptr converter, std::shared_ptr perm_converter, + std::unique_ptr block_storage_factory, logger::LoggerManagerTreePtr log_manager, size_t pool_size) { boost::optional string_res = boost::none; @@ -403,6 +407,7 @@ namespace iroha { factory, converter, perm_converter, + std::move(block_storage_factory), pool_size, enable_prepared_transactions, std::move(log_manager)))); @@ -414,12 +419,11 @@ namespace iroha { } boost::optional> StorageImpl::commit( - std::unique_ptr mutableStorage) { - auto storage_ptr = std::move(mutableStorage); // get ownership of storage - auto storage = static_cast(storage_ptr.get()); - for (const auto &block : storage->block_store_) { - storeBlock(*block.second); - } + std::unique_ptr mutable_storage) { + auto storage = static_cast(mutable_storage.get()); + + storage->block_storage_->forEach( + [this](const auto &block) { this->storeBlock(*block); }); try { *(storage->sql_) << "COMMIT"; storage->committed = true; @@ -427,12 +431,11 @@ namespace iroha { factory_, log_manager_->getChild("WsvQuery")->getLogger()) .getPeers() - | - [](auto &&peers) { - return boost::optional>( - std::make_unique( - std::make_shared(std::move(peers)))); - }; + | [](auto &&peers) { + return boost::optional>( + std::make_unique( + std::make_shared(std::move(peers)))); + }; } catch (std::exception &e) { storage->committed = false; log_->warn("Mutable storage is not committed. Reason: {}", e.what()); diff --git a/irohad/ametsuchi/impl/storage_impl.hpp b/irohad/ametsuchi/impl/storage_impl.hpp index f4e7f1e6d0..b1e5ef4d36 100644 --- a/irohad/ametsuchi/impl/storage_impl.hpp +++ b/irohad/ametsuchi/impl/storage_impl.hpp @@ -14,7 +14,7 @@ #include #include - +#include "ametsuchi/block_storage_factory.hpp" #include "ametsuchi/impl/postgres_options.hpp" #include "ametsuchi/key_value_storage.hpp" #include "interfaces/common_objects/common_objects_factory.hpp" @@ -57,6 +57,7 @@ namespace iroha { converter, std::shared_ptr perm_converter, + std::unique_ptr block_storage_factory, logger::LoggerManagerTreePtr log_manager, size_t pool_size = 10); @@ -100,7 +101,7 @@ namespace iroha { void freeConnections() override; boost::optional> commit( - std::unique_ptr mutableStorage) override; + std::unique_ptr mutable_storage) override; boost::optional> commitPrepared( const shared_model::interface::Block &block) override; @@ -127,6 +128,7 @@ namespace iroha { converter, std::shared_ptr perm_converter, + std::unique_ptr block_storage_factory, size_t pool_size, bool enable_prepared_blocks, logger::LoggerManagerTreePtr log_manager); @@ -164,6 +166,8 @@ namespace iroha { std::shared_ptr perm_converter_; + std::unique_ptr block_storage_factory_; + logger::LoggerManagerTreePtr log_manager_; logger::LoggerPtr log_; diff --git a/irohad/main/application.cpp b/irohad/main/application.cpp index 64852f6b53..9c8cdde8ef 100644 --- a/irohad/main/application.cpp +++ b/irohad/main/application.cpp @@ -5,6 +5,7 @@ #include "main/application.hpp" +#include "ametsuchi/impl/in_memory_block_storage_factory.hpp" #include "ametsuchi/impl/storage_impl.hpp" #include "ametsuchi/impl/tx_presence_cache_impl.hpp" #include "ametsuchi/impl/wsv_restorer_impl.hpp" @@ -160,11 +161,13 @@ void Irohad::initStorage() { std::make_shared(); auto block_converter = std::make_shared(); + auto block_storage_factory = std::make_unique(); auto storageResult = StorageImpl::create(block_store_dir_, pg_conn_, common_objects_factory_, std::move(block_converter), perm_converter, + std::move(block_storage_factory), log_manager_->getChild("Storage")); storageResult.match( [&](expected::Value> &_storage) { diff --git a/test/module/irohad/ametsuchi/CMakeLists.txt b/test/module/irohad/ametsuchi/CMakeLists.txt index 911eae73bf..09bed3e35d 100644 --- a/test/module/irohad/ametsuchi/CMakeLists.txt +++ b/test/module/irohad/ametsuchi/CMakeLists.txt @@ -69,6 +69,11 @@ target_link_libraries(tx_presence_cache_test shared_model_interfaces_factories ) +addtest(in_memory_block_storage_test in_memory_block_storage_test.cpp) +target_link_libraries(in_memory_block_storage_test + ametsuchi + ) + add_library(ametsuchi_fixture INTERFACE) target_link_libraries(ametsuchi_fixture INTERFACE integration_framework_config_helper diff --git a/test/module/irohad/ametsuchi/ametsuchi_fixture.hpp b/test/module/irohad/ametsuchi/ametsuchi_fixture.hpp index 3b945d0b4b..dc84f15687 100644 --- a/test/module/irohad/ametsuchi/ametsuchi_fixture.hpp +++ b/test/module/irohad/ametsuchi/ametsuchi_fixture.hpp @@ -12,6 +12,7 @@ #include #include #include +#include "ametsuchi/impl/in_memory_block_storage_factory.hpp" #include "ametsuchi/impl/storage_impl.hpp" #include "backend/protobuf/common_objects/proto_common_objects_factory.hpp" #include "backend/protobuf/proto_block_json_converter.hpp" @@ -42,12 +43,14 @@ namespace iroha { std::make_shared(); auto converter = std::make_shared(); - + auto block_storage_factory = + std::make_unique(); StorageImpl::create(block_store_path, pgopt_, factory, converter, perm_converter_, + std::move(block_storage_factory), getTestLoggerManager()->getChild("Storage")) .match([&](iroha::expected::Value> &_storage) { storage = _storage.value; }, diff --git a/test/module/irohad/ametsuchi/in_memory_block_storage_test.cpp b/test/module/irohad/ametsuchi/in_memory_block_storage_test.cpp new file mode 100644 index 0000000000..b30d927f97 --- /dev/null +++ b/test/module/irohad/ametsuchi/in_memory_block_storage_test.cpp @@ -0,0 +1,122 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "ametsuchi/impl/in_memory_block_storage.hpp" +#include "ametsuchi/impl/in_memory_block_storage_factory.hpp" + +#include +#include "module/shared_model/interface_mocks.hpp" + +using namespace iroha::ametsuchi; +using ::testing::Invoke; +using ::testing::NiceMock; +using ::testing::Return; + +class InMemoryBlockStorageTest : public ::testing::Test { + public: + InMemoryBlockStorageTest() { + ON_CALL(*block_, height()).WillByDefault(Return(height_)); + } + + protected: + void TearDown() override { + block_storage_.clear(); + } + + InMemoryBlockStorage block_storage_; + std::shared_ptr block_ = std::make_shared>(); + shared_model::interface::types::HeightType height_ = 1; +}; + +/** + * @given block storage factory + * @when create is called + * @then block storage is created + */ +TEST(InMemoryBlockStorageFactoryTest, Creation) { + InMemoryBlockStorageFactory factory; + + ASSERT_TRUE(factory.create()); +} + +/** + * @given initialized block storage, single block with height_ inserted + * @when another block with height_ is inserted + * @then second insertion fails + */ +TEST_F(InMemoryBlockStorageTest, Insert) { + ASSERT_TRUE(block_storage_.insert(block_)); + + ASSERT_FALSE(block_storage_.insert(block_)); +} + +/** + * @given initialized block storage, single block with height_ inserted + * @when block with height_ is fetched + * @then it is returned + */ +TEST_F(InMemoryBlockStorageTest, FetchExisting) { + ASSERT_TRUE(block_storage_.insert(block_)); + + auto block_var = block_storage_.fetch(height_); + + ASSERT_EQ(block_, *block_var); +} + +/** + * @given initialized block storage without blocks + * @when block with height_ is fetched + * @then nothing is returned + */ +TEST_F(InMemoryBlockStorageTest, FetchNonexistent) { + auto block_var = block_storage_.fetch(height_); + + ASSERT_FALSE(block_var); +} + +/** + * @given initialized block storage, single block with id_ inserted + * @when size is fetched + * @then 1 is returned + */ +TEST_F(InMemoryBlockStorageTest, Size) { + ASSERT_TRUE(block_storage_.insert(block_)); + + ASSERT_EQ(1, block_storage_.size()); +} + +/** + * @given initialized block storage, single block with id_ inserted + * @when storage is cleared with clear + * @then no blocks are left in storage + */ +TEST_F(InMemoryBlockStorageTest, Clear) { + ASSERT_TRUE(block_storage_.insert(block_)); + + block_storage_.clear(); + + auto block_var = block_storage_.fetch(height_); + + ASSERT_FALSE(block_var); +} + +/** + * @given initialized block storage, single block with id_ inserted + * @when forEach is called + * @then block with id_ is visited, lambda is invoked once + */ +TEST_F(InMemoryBlockStorageTest, ForEach) { + ASSERT_TRUE(block_storage_.insert(block_)); + + size_t count = 0; + + block_storage_.forEach([this, &count](const auto &block) { + ++count; + ASSERT_EQ(height_, block->height()); + ASSERT_EQ(block_, block); + }); + + ASSERT_EQ(1, count); +} diff --git a/test/module/irohad/ametsuchi/storage_init_test.cpp b/test/module/irohad/ametsuchi/storage_init_test.cpp index 5d75a77c4e..3179a257d2 100644 --- a/test/module/irohad/ametsuchi/storage_init_test.cpp +++ b/test/module/irohad/ametsuchi/storage_init_test.cpp @@ -11,6 +11,7 @@ #include #include #include +#include "ametsuchi/impl/in_memory_block_storage_factory.hpp" #include "backend/protobuf/common_objects/proto_common_objects_factory.hpp" #include "backend/protobuf/proto_block_json_converter.hpp" #include "backend/protobuf/proto_permission_to_string.hpp" @@ -52,6 +53,10 @@ class StorageInitTest : public ::testing::Test { std::shared_ptr perm_converter_ = std::make_shared(); + + std::unique_ptr block_storage_factory_ = + std::make_unique(); + void SetUp() override { ASSERT_FALSE(boost::filesystem::exists(block_store_path)) << "Temporary block store " << block_store_path @@ -81,6 +86,7 @@ TEST_F(StorageInitTest, CreateStorageWithDatabase) { factory, converter, perm_converter_, + std::move(block_storage_factory_), storage_log_manager_) .match( [&storage](const Value> &value) { @@ -110,6 +116,7 @@ TEST_F(StorageInitTest, CreateStorageWithInvalidPgOpt) { factory, converter, perm_converter_, + std::move(block_storage_factory_), storage_log_manager_) .match( [](const Value> &) { From deee7493db4a837485abe3ed581719d9c1c6ee9c Mon Sep 17 00:00:00 2001 From: Fedor Muratov Date: Thu, 14 Mar 2019 15:12:03 +0300 Subject: [PATCH 25/40] * Add checking of persistent cache on start status stream (#2169) ### Description of the Change Add fix for status stream initialization. If runtime cache doesn't contain any value we are going to check the persistent cache for the value. ### Benefits Fix misleading behavior of the status stream. Signed-off-by: Fedor Muratov --- irohad/torii/impl/command_service_impl.cpp | 25 ++++- irohad/torii/impl/command_service_impl.hpp | 1 + test/module/irohad/torii/CMakeLists.txt | 8 ++ .../irohad/torii/command_service_test.cpp | 96 +++++++++++++++++++ test/module/shared_model/interface_mocks.hpp | 39 +++++--- 5 files changed, 152 insertions(+), 17 deletions(-) create mode 100644 test/module/irohad/torii/command_service_test.cpp diff --git a/irohad/torii/impl/command_service_impl.cpp b/irohad/torii/impl/command_service_impl.cpp index 863c1d240b..ceb9a64f14 100644 --- a/irohad/torii/impl/command_service_impl.cpp +++ b/irohad/torii/impl/command_service_impl.cpp @@ -129,8 +129,29 @@ namespace iroha { using ResponsePtrType = std::shared_ptr; auto initial_status = cache_->findItem(hash).value_or([&] { - log_->debug("tx is not received: {}", hash); - return status_factory_->makeNotReceived(hash); + // if cache_ doesn't contain some status there is required to check + // persistent cache + + log_->debug("tx {} isn't present in cache", hash); + auto from_persistent_cache = tx_presence_cache_->check(hash); + if (not from_persistent_cache) { + // TODO andrei 30.11.18 IR-51 Handle database error + log_->warn("Check hash presence database error. {}", hash); + return status_factory_->makeNotReceived(hash); + } + return iroha::visit_in_place( + *from_persistent_cache, + [this, + &hash](const iroha::ametsuchi::tx_cache_status_responses::Committed + &) { return status_factory_->makeCommitted(hash); }, + [this, &hash]( + const iroha::ametsuchi::tx_cache_status_responses::Rejected &) { + return status_factory_->makeRejected(hash); + }, + [this, &hash]( + const iroha::ametsuchi::tx_cache_status_responses::Missing &) { + return status_factory_->makeNotReceived(hash); + }); }()); return status_bus_ ->statuses() diff --git a/irohad/torii/impl/command_service_impl.hpp b/irohad/torii/impl/command_service_impl.hpp index 52ffa9b1cd..fa08f3b1a2 100644 --- a/irohad/torii/impl/command_service_impl.hpp +++ b/irohad/torii/impl/command_service_impl.hpp @@ -24,6 +24,7 @@ namespace iroha { */ class CommandServiceImpl : public CommandService { public: + // TODO: 2019-03-13 @muratovv fix with abstract cache type IR-397 using CacheType = iroha::cache::Cache< shared_model::crypto::Hash, std::shared_ptr, diff --git a/test/module/irohad/torii/CMakeLists.txt b/test/module/irohad/torii/CMakeLists.txt index 37a6ec3a16..df4836ce37 100644 --- a/test/module/irohad/torii/CMakeLists.txt +++ b/test/module/irohad/torii/CMakeLists.txt @@ -56,3 +56,11 @@ target_link_libraries(command_service_replay_test torii_service test_logger ) + +addtest(command_service_test + command_service_test.cpp + ) +target_link_libraries(command_service_test + torii_service + test_logger + ) diff --git a/test/module/irohad/torii/command_service_test.cpp b/test/module/irohad/torii/command_service_test.cpp new file mode 100644 index 0000000000..19c800180f --- /dev/null +++ b/test/module/irohad/torii/command_service_test.cpp @@ -0,0 +1,96 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "torii/impl/command_service_impl.hpp" + +#include +#include "backend/protobuf/proto_tx_status_factory.hpp" +#include "cryptography/hash.hpp" +#include "cryptography/public_key.hpp" +#include "framework/test_logger.hpp" +#include "framework/test_subscriber.hpp" +#include "module/irohad/ametsuchi/ametsuchi_mocks.hpp" +#include "module/irohad/ametsuchi/mock_storage.hpp" +#include "module/irohad/ametsuchi/mock_tx_presence_cache.hpp" +#include "module/irohad/torii/torii_mocks.hpp" +#include "module/shared_model/interface_mocks.hpp" + +using namespace testing; + +class CommandServiceTest : public Test { + public: + void SetUp() override { + transaction_processor_ = + std::make_shared(); + storage_ = std::make_shared(); + + status_bus_ = std::make_shared(); + + tx_status_factory_ = + std::make_shared(); + cache_ = std::make_shared(); + tx_presence_cache_ = + std::make_shared(); + + log_ = getTestLogger("CommandServiceTest"); + } + + void initCommandService() { + command_service_ = std::make_shared( + transaction_processor_, + storage_, + status_bus_, + tx_status_factory_, + cache_, + tx_presence_cache_, + log_); + } + + std::shared_ptr + transaction_processor_; + std::shared_ptr storage_; + std::shared_ptr status_bus_; + std::shared_ptr tx_status_factory_; + std::shared_ptr tx_presence_cache_; + logger::LoggerPtr log_; + std::shared_ptr cache_; + std::shared_ptr command_service_; +}; + +/** + * @given intialized command service + * @and hash with passed consensus but not present in runtime cache + * @when invoke getStatusStream by hash + * @then verify that code checks run-time and persistent caches for the hash + * @and return CommittedTxResponse status + */ +TEST_F(CommandServiceTest, getStatusStreamWithAbsentHash) { + using HashType = shared_model::crypto::Hash; + auto hash = HashType("a"); + iroha::ametsuchi::TxCacheStatusType ret_value{ + iroha::ametsuchi::tx_cache_status_responses::Committed{hash}}; + + // TODO: 2019-03-13 @muratovv add expect call for runtime cache invocation + // IR-397 + EXPECT_CALL(*tx_presence_cache_, + check(Matcher(_))) + .Times(1) + .WillOnce(Return(ret_value)); + EXPECT_CALL(*status_bus_, statuses()) + .WillRepeatedly(Return( + rxcpp::observable<>::empty())); + + initCommandService(); + auto wrapper = framework::test_subscriber::make_test_subscriber< + framework::test_subscriber::CallExact>( + command_service_->getStatusStream(hash), 1); + wrapper.subscribe([](const auto &tx_response) { + return iroha::visit_in_place( + tx_response->get(), + [](const shared_model::interface::CommittedTxResponse &) {}, + [](const auto &a) { FAIL() << "Wrong response!"; }); + }); + ASSERT_TRUE(wrapper.validate()); +} diff --git a/test/module/shared_model/interface_mocks.hpp b/test/module/shared_model/interface_mocks.hpp index 5ac8f8abee..2d387c0a44 100644 --- a/test/module/shared_model/interface_mocks.hpp +++ b/test/module/shared_model/interface_mocks.hpp @@ -18,7 +18,6 @@ #include "interfaces/iroha_internal/unsafe_proposal_factory.hpp" #include "interfaces/transaction.hpp" - // TODO: 2019-01-18 @muratovv Separate file by classes IR-229 struct MockBlock : public shared_model::interface::Block { MOCK_CONST_METHOD0(txsNumber, @@ -112,6 +111,14 @@ struct MockTransactionBatch : public shared_model::interface::TransactionBatch { MOCK_CONST_METHOD0(clone, MockTransactionBatch *()); }; +namespace shared_model { + namespace interface { + std::ostream &operator<<(std::ostream &os, const TransactionBatch &resp) { + return os << resp.toString(); + } + } // namespace interface +} // namespace shared_model + /** * Creates mock batch with provided hash * @param hash -- const ref to reduced hash to be returned by the batch @@ -180,8 +187,10 @@ struct MockPeer : public shared_model::interface::Peer { inline auto makePeer(const std::string &address, const shared_model::crypto::PublicKey &pub_key) { auto peer = std::make_shared(); - EXPECT_CALL(*peer, address()).WillRepeatedly(testing::ReturnRefOfCopy(address)); - EXPECT_CALL(*peer, pubkey()).WillRepeatedly(testing::ReturnRefOfCopy(pub_key)); + EXPECT_CALL(*peer, address()) + .WillRepeatedly(testing::ReturnRefOfCopy(address)); + EXPECT_CALL(*peer, pubkey()) + .WillRepeatedly(testing::ReturnRefOfCopy(pub_key)); return peer; } @@ -234,21 +243,21 @@ struct MockCommonObjectsFactory }; struct MockDomain : public shared_model::interface::Domain { - MOCK_CONST_METHOD0(domainId, - shared_model::interface::types::DomainIdType &()); - MOCK_CONST_METHOD0(defaultRole, - shared_model::interface::types::RoleIdType &()); - MOCK_CONST_METHOD0(clone, MockDomain *()); + MOCK_CONST_METHOD0(domainId, + shared_model::interface::types::DomainIdType &()); + MOCK_CONST_METHOD0(defaultRole, + shared_model::interface::types::RoleIdType &()); + MOCK_CONST_METHOD0(clone, MockDomain *()); }; struct MockAccount : public shared_model::interface::Account { - MOCK_CONST_METHOD0(accountId, - shared_model::interface::types::AccountIdType &()); - MOCK_CONST_METHOD0(domainId, - shared_model::interface::types::DomainIdType &()); - MOCK_CONST_METHOD0(quorum, shared_model::interface::types::QuorumType()); - MOCK_CONST_METHOD0(jsonData, shared_model::interface::types::JsonType &()); - MOCK_CONST_METHOD0(clone, MockAccount *()); + MOCK_CONST_METHOD0(accountId, + shared_model::interface::types::AccountIdType &()); + MOCK_CONST_METHOD0(domainId, + shared_model::interface::types::DomainIdType &()); + MOCK_CONST_METHOD0(quorum, shared_model::interface::types::QuorumType()); + MOCK_CONST_METHOD0(jsonData, shared_model::interface::types::JsonType &()); + MOCK_CONST_METHOD0(clone, MockAccount *()); }; #endif // IROHA_SHARED_MODEL_INTERFACE_MOCKS_HPP From f6888739cb7f7ffc9416daacdb36a4822195df83 Mon Sep 17 00:00:00 2001 From: Vadim Rc Date: Thu, 14 Mar 2019 23:04:05 +0900 Subject: [PATCH 26/40] Add note for building and generating keypair documentation (#2162) Signed-off-by: Vadim Reutskiy --- docs/source/guides/build.rst | 7 +++++++ docs/source/guides/deployment.rst | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/docs/source/guides/build.rst b/docs/source/guides/build.rst index 3e36150b76..94e3f272a2 100644 --- a/docs/source/guides/build.rst +++ b/docs/source/guides/build.rst @@ -4,6 +4,10 @@ Building Iroha In this guide we will learn how to install all dependencies, required to build Iroha and how to build it. +.. note:: You don't need to build Iroha to start using it. + Instead, you can download prepared Docker image from the Hub, + this process explained in details in the :ref:`getting-started` page of this documentation. + Preparing the Environment ------------------------- @@ -57,11 +61,14 @@ After you execute this script, following things happen: 1. The script checks if you don't have containers with Iroha already running. Successful completion finishes with the new container shell. + 2. The script will download ``hyperledger/iroha:develop-build`` and ``postgres`` images. ``hyperledger/iroha:develop-build`` image contains all development dependencies and is based on top of ``ubuntu:16.04``. ``postgres`` image is required for starting and running Iroha. + 3. Two containers are created and launched. + 4. The user is attached to the interactive environment for development and testing with ``iroha`` folder mounted from the host machine. Iroha folder is mounted to ``/opt/iroha`` in Docker container. diff --git a/docs/source/guides/deployment.rst b/docs/source/guides/deployment.rst index ee40b861df..70e5ffdb75 100644 --- a/docs/source/guides/deployment.rst +++ b/docs/source/guides/deployment.rst @@ -185,3 +185,8 @@ Do not have a keypair for a peer -------------------------------- In order to create a keypair for an account or a peer, use iroha-cli binary by passing the name of the peer with `--new_account` option. +For example: + +.. code-block:: shell + + ./iroha-cli --account_name newuser@test --new_account From 0aa037f492fa7c1f16c1d8f0b4d84e9ef56cb38f Mon Sep 17 00:00:00 2001 From: Mikhail Boldyrev Date: Fri, 15 Mar 2019 13:26:34 +0300 Subject: [PATCH 27/40] Fix key padding in tests (#2172) Signed-off-by: Mikhail Boldyrev --- .../consensus/yac/mock_yac_crypto_provider.hpp | 17 +++++++++++------ .../irohad/consensus/yac/yac_test_util.hpp | 13 ++++++++----- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/test/module/irohad/consensus/yac/mock_yac_crypto_provider.hpp b/test/module/irohad/consensus/yac/mock_yac_crypto_provider.hpp index b68877d3a4..5131694907 100644 --- a/test/module/irohad/consensus/yac/mock_yac_crypto_provider.hpp +++ b/test/module/irohad/consensus/yac/mock_yac_crypto_provider.hpp @@ -17,6 +17,16 @@ namespace iroha { namespace consensus { namespace yac { + // TODO 15.03.2019 mboldyrev IR-402 + // fix the tests that impose requirements on mock public key format + std::string padPubKeyString(const std::string &str) { + using shared_model::crypto::DefaultCryptoAlgorithmType; + assert(str.size() <= DefaultCryptoAlgorithmType::kPublicKeyLength); + std::string padded(DefaultCryptoAlgorithmType::kPublicKeyLength, '0'); + std::copy(str.begin(), str.end(), padded.begin()); + return padded; + } + /** * Creates test signature with empty signed data, and provided pubkey * @param pub_key - public key to put in the signature @@ -24,15 +34,10 @@ namespace iroha { */ std::shared_ptr createSig( const std::string &pub_key) { - auto tmp = - shared_model::crypto::DefaultCryptoAlgorithmType::generateKeypair() - .publicKey(); - std::string key(tmp.blob().size(), '0'); - std::copy(pub_key.begin(), pub_key.end(), key.begin()); auto sig = std::make_shared(); EXPECT_CALL(*sig, publicKey()) .WillRepeatedly(::testing::ReturnRefOfCopy( - shared_model::crypto::PublicKey(key))); + shared_model::crypto::PublicKey(padPubKeyString(pub_key)))); EXPECT_CALL(*sig, signedData()) .WillRepeatedly( ::testing::ReturnRefOfCopy(shared_model::crypto::Signed(""))); diff --git a/test/module/irohad/consensus/yac/yac_test_util.hpp b/test/module/irohad/consensus/yac/yac_test_util.hpp index e451c553c0..aa7ad54727 100644 --- a/test/module/irohad/consensus/yac/yac_test_util.hpp +++ b/test/module/irohad/consensus/yac/yac_test_util.hpp @@ -25,30 +25,33 @@ namespace iroha { .WillRepeatedly(::testing::ReturnRefOfCopy(address)); EXPECT_CALL(*peer, pubkey()) .WillRepeatedly(::testing::ReturnRefOfCopy( - shared_model::interface::types::PubkeyType(address))); + shared_model::interface::types::PubkeyType( + padPubKeyString(address)))); + return peer; } inline VoteMessage createVote(YacHash hash, const std::string &pub_key) { VoteMessage vote; + std::string padded_pub_key = padPubKeyString(pub_key); auto block_signature = std::make_shared(); EXPECT_CALL(*block_signature, publicKey()) .WillRepeatedly(::testing::ReturnRefOfCopy( - shared_model::crypto::PublicKey(pub_key))); + shared_model::crypto::PublicKey(padded_pub_key))); EXPECT_CALL(*block_signature, signedData()) .WillRepeatedly(::testing::ReturnRefOfCopy( - shared_model::crypto::Signed(pub_key))); + shared_model::crypto::Signed(padded_pub_key))); hash.block_signature = block_signature; vote.hash = std::move(hash); auto signature = std::make_shared(); EXPECT_CALL(*signature, publicKey()) .WillRepeatedly(::testing::ReturnRefOfCopy( - shared_model::crypto::PublicKey(pub_key))); + shared_model::crypto::PublicKey(padded_pub_key))); EXPECT_CALL(*signature, signedData()) .WillRepeatedly(::testing::ReturnRefOfCopy( - shared_model::crypto::Signed(pub_key))); + shared_model::crypto::Signed(padded_pub_key))); vote.signature = signature; return vote; From a615ac0ffa5be1f05d88f2f27485bec1f645d12e Mon Sep 17 00:00:00 2001 From: Mikhail Boldyrev Date: Fri, 15 Mar 2019 13:51:41 +0300 Subject: [PATCH 28/40] Fix member init order warning (#2164) Signed-off-by: Mikhail Boldyrev --- irohad/ordering/impl/on_demand_ordering_gate.hpp | 3 ++- test/framework/integration_framework/iroha_instance.cpp | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/irohad/ordering/impl/on_demand_ordering_gate.hpp b/irohad/ordering/impl/on_demand_ordering_gate.hpp index dca6d15c39..d2a5592636 100644 --- a/irohad/ordering/impl/on_demand_ordering_gate.hpp +++ b/irohad/ordering/impl/on_demand_ordering_gate.hpp @@ -96,9 +96,10 @@ namespace iroha { std::shared_ptr proposal) const; + logger::LoggerPtr log_; + /// max number of transactions passed to one ordering service size_t transaction_limit_; - logger::LoggerPtr log_; std::shared_ptr ordering_service_; std::shared_ptr network_client_; rxcpp::composite_subscription events_subscription_; diff --git a/test/framework/integration_framework/iroha_instance.cpp b/test/framework/integration_framework/iroha_instance.cpp index 754216e042..663fd8198d 100644 --- a/test/framework/integration_framework/iroha_instance.cpp +++ b/test/framework/integration_framework/iroha_instance.cpp @@ -38,10 +38,10 @@ namespace integration_framework { vote_delay_(0ms), // amount of minutes in a day mst_expiration_time_(std::chrono::minutes(24 * 60)), - max_rounds_delay_(0ms), - stale_stream_max_rounds_(2), opt_mst_gossip_params_(boost::make_optional( mst_support, iroha::GossipPropagationStrategyParams{})), + max_rounds_delay_(0ms), + stale_stream_max_rounds_(2), irohad_log_manager_(std::move(irohad_log_manager)), log_(std::move(log)) {} From 8573d8d20ff28a341991a7afb53400e2edb30739 Mon Sep 17 00:00:00 2001 From: Fedor Muratov Date: Fri, 15 Mar 2019 15:43:48 +0300 Subject: [PATCH 29/40] Fix race bug in command service (#2167) ### Description of the Change Fix bug when not committed/rejected transaction will not pass to Iroha's pipeline because it is a "replay". ### Benefits Add fix for the problem. The idea of the fix to check only final statuses in the ram cache in command service. If there is a final status we will decline further checking of the transaction. Also, thanks @lebdron for cooperation in founding the bug. Signed-off-by: Fedor Muratov --- irohad/torii/impl/command_service_impl.cpp | 82 +++++++------------ .../irohad/torii/command_service_test.cpp | 26 ++++++ 2 files changed, 54 insertions(+), 54 deletions(-) diff --git a/irohad/torii/impl/command_service_impl.cpp b/irohad/torii/impl/command_service_impl.cpp index ceb9a64f14..4b25f275ea 100644 --- a/irohad/torii/impl/command_service_impl.cpp +++ b/irohad/torii/impl/command_service_impl.cpp @@ -195,62 +195,34 @@ namespace iroha { const auto status_issuer = "ToriiBatchProcessor"; const auto &txs = batch->transactions(); - bool cache_has_at_least_one_tx{false}; - bool batch_has_mst_pending_tx{false}; - std::tie(cache_has_at_least_one_tx, batch_has_mst_pending_tx) = - // this accumulate can be split on two parts to perform visit_in_place - // two times - one for cache lookup with booleans initialization and - // another for statuses pushing. That can allow to move a part of code - // to a separate method for simplification - std::accumulate( - txs.begin(), - txs.end(), - std::make_pair(false, false), - [this, &status_issuer](std::pair lookup_result, - const auto &tx) { - const auto &tx_hash = tx->hash(); - if (auto found = cache_->findItem(tx_hash)) { - iroha::visit_in_place( - (*found)->get(), - [this, &found, &lookup_result, &status_issuer]( - const shared_model::interface::MstPendingResponse &) { - this->pushStatus(status_issuer, *found); - lookup_result.second = true; - }, - [this, &tx_hash, &status_issuer]( - const shared_model::interface::NotReceivedTxResponse - &) { - // This branch covers an impossible case (this cache - // cannot contain NotReceivedTxResponse, because the tx - // has reached processBatch method, which means that - // the tx already has StatelessValid status). - // That is why we are not updating its status inside - // internal cache, but still pushing to status bus. - this->pushStatus( - status_issuer, - status_factory_->makeStatelessValid(tx_hash)); - }, - [this, &found, &status_issuer](const auto &status) { - this->pushStatus(status_issuer, *found); - }); - lookup_result.first = true; - } - return lookup_result; - }); + bool has_final_status{false}; - if (cache_has_at_least_one_tx and not batch_has_mst_pending_tx) { - // If a non-persistent cache says that a transaction has pending status - // that means we have to check persistent cache too. - // Non-persistent cache might be overflowed and mst replay become - // possible without checking persistent cache. + for (auto tx : txs) { + const auto &tx_hash = tx->hash(); + if (auto found = cache_->findItem(tx_hash)) { + log_->debug("Found in cache: {}", **found); + has_final_status = iroha::visit_in_place( + (*found)->get(), + [](const auto &final_responses) + -> std::enable_if_t< + FinalStatusValue, + bool> { return true; }, + [](const auto &rest_responses) + -> std::enable_if_t< + not FinalStatusValue, + bool> { return false; }); + } - // If there are no pending statuses and the transaction is found in - // non-persistent cache, then it is considered as a replay and prevented - // from further propagation. + if (has_final_status) { + break; + } + } - // If non-persistent cache does not contain any info about a - // transaction, then we just check persistent cache. - log_->warn("Replayed batch would not be served. {}", *batch); + if (has_final_status) { + // presence of the transaction or batch in the cache with final status + // guarantees that the transaction was passed to consensus before + log_->warn("Replayed batch would not be served - present in cache. {}", + *batch); return; } @@ -290,7 +262,9 @@ namespace iroha { }); }); if (is_replay) { - log_->warn("Replayed batch would not be served. {}", *batch); + log_->warn( + "Replayed batch would not be served - present in database. {}", + *batch); return; } diff --git a/test/module/irohad/torii/command_service_test.cpp b/test/module/irohad/torii/command_service_test.cpp index 19c800180f..e3ada835ab 100644 --- a/test/module/irohad/torii/command_service_test.cpp +++ b/test/module/irohad/torii/command_service_test.cpp @@ -94,3 +94,29 @@ TEST_F(CommandServiceTest, getStatusStreamWithAbsentHash) { }); ASSERT_TRUE(wrapper.validate()); } + +/** + * @given initialized command service + * @when invoke processBatch on batch which isn't present in runtime and + * persistent caches + * @then tx_processor batchHandle is invoked + */ +TEST_F(CommandServiceTest, ProcessBatchOn) { + auto hash = shared_model::crypto::Hash("a"); + auto batch = createMockBatchWithTransactions( + {createMockTransactionWithHash(hash)}, "a"); + EXPECT_CALL(*status_bus_, statuses()) + .WillRepeatedly(Return( + rxcpp::observable<>::empty())); + + EXPECT_CALL( + *tx_presence_cache_, + check(Matcher(_))) + .WillRepeatedly(Return(std::vector( + {iroha::ametsuchi::tx_cache_status_responses::Missing(hash)}))); + + EXPECT_CALL(*transaction_processor_, batchHandle(_)).Times(1); + + initCommandService(); + command_service_->handleTransactionBatch(batch); +} From 04684185ba208efc04bb6a6de9ca79743e2606b8 Mon Sep 17 00:00:00 2001 From: Andrei Lebedev Date: Fri, 15 Mar 2019 18:15:56 +0300 Subject: [PATCH 30/40] Fix hash set usage in OS. Fix block leaks in YacGate and Simulator (#2170) Signed-off-by: Andrei Lebedev --- irohad/consensus/yac/impl/yac_gate_impl.cpp | 5 +++ irohad/ordering/CMakeLists.txt | 1 + .../impl/on_demand_ordering_service_impl.hpp | 9 ++--- irohad/simulator/impl/simulator.cpp | 3 ++ .../irohad/consensus/yac/yac_gate_test.cpp | 36 ++++++++++++++++++- .../irohad/ordering/on_demand_os_test.cpp | 22 ++++++++++-- 6 files changed, 69 insertions(+), 7 deletions(-) diff --git a/irohad/consensus/yac/impl/yac_gate_impl.cpp b/irohad/consensus/yac/impl/yac_gate_impl.cpp index 199ed1cf9c..21957fd6e6 100644 --- a/irohad/consensus/yac/impl/yac_gate_impl.cpp +++ b/irohad/consensus/yac/impl/yac_gate_impl.cpp @@ -55,6 +55,11 @@ namespace iroha { if (not event.round_data) { current_block_ = boost::none; + // previous block is committed to block storage, it is safe to clear + // the cache + // TODO 2019-03-15 andrei: IR-405 Subscribe BlockLoaderService to + // BlockCreator::onBlock + consensus_result_cache_->release(); log_->debug("Agreed on nothing to commit"); } else { current_block_ = event.round_data->block; diff --git a/irohad/ordering/CMakeLists.txt b/irohad/ordering/CMakeLists.txt index 99316406e6..2efd5c8fab 100644 --- a/irohad/ordering/CMakeLists.txt +++ b/irohad/ordering/CMakeLists.txt @@ -17,6 +17,7 @@ target_link_libraries(on_demand_ordering_service on_demand_common tbb mst_hash + mst_state shared_model_interfaces consensus_round logger diff --git a/irohad/ordering/impl/on_demand_ordering_service_impl.hpp b/irohad/ordering/impl/on_demand_ordering_service_impl.hpp index 2b04d4b418..b2db7d24f6 100644 --- a/irohad/ordering/impl/on_demand_ordering_service_impl.hpp +++ b/irohad/ordering/impl/on_demand_ordering_service_impl.hpp @@ -9,14 +9,14 @@ #include "ordering/on_demand_ordering_service.hpp" #include -#include #include -#include #include #include "interfaces/iroha_internal/unsafe_proposal_factory.hpp" #include "logger/logger_fwd.hpp" #include "multi_sig_transactions/hash.hpp" +// TODO 2019-03-15 andrei: IR-403 Separate BatchHashEquality and MstState +#include "multi_sig_transactions/state/mst_state.hpp" #include "ordering/impl/on_demand_common.hpp" namespace iroha { @@ -27,8 +27,9 @@ namespace iroha { namespace detail { using BatchSetType = tbb::concurrent_unordered_set< transport::OdOsNotification::TransactionBatchType, - model::PointerBatchHasher>; - } + model::PointerBatchHasher, + BatchHashEquality>; + } // namespace detail class OnDemandOrderingServiceImpl : public OnDemandOrderingService { public: diff --git a/irohad/simulator/impl/simulator.cpp b/irohad/simulator/impl/simulator.cpp index 7e5c8ee362..2b0aacf7f8 100644 --- a/irohad/simulator/impl/simulator.cpp +++ b/irohad/simulator/impl/simulator.cpp @@ -149,6 +149,9 @@ namespace iroha { rejected_hashes); crypto_signer_->sign(*block); + // TODO 2019-03-15 andrei: IR-404 Make last_block an explicit dependency + last_block.reset(); + return block; } diff --git a/test/module/irohad/consensus/yac/yac_gate_test.cpp b/test/module/irohad/consensus/yac/yac_gate_test.cpp index 372a4c7e3f..4de5d93e3e 100644 --- a/test/module/irohad/consensus/yac/yac_gate_test.cpp +++ b/test/module/irohad/consensus/yac/yac_gate_test.cpp @@ -159,6 +159,39 @@ TEST_F(YacGateTest, YacGateSubscriptionTest) { ASSERT_TRUE(gate_wrapper.validate()); } +/** + * @given yac gate, voting for the block @and receiving it on commit + * @when voting for nothing + * @then block cache is released + */ +TEST_F(YacGateTest, CacheReleased) { + YacHash empty_hash; + + // yac consensus + EXPECT_CALL(*hash_gate, vote(expected_hash, _)).Times(1); + EXPECT_CALL(*hash_gate, vote(empty_hash, _)).Times(1); + + // generate order of peers + EXPECT_CALL(*peer_orderer, getOrdering(_)) + .Times(2) + .WillRepeatedly(Return(ClusterOrdering::create({makePeer("fake_node")}))); + + // make hash from block + EXPECT_CALL(*hash_provider, makeHash(_)) + .WillOnce(Return(expected_hash)) + .WillOnce(Return(empty_hash)); + + block_notifier.get_subscriber().on_next( + BlockCreatorEvent{RoundData{expected_proposal, expected_block}, round}); + + outcome_notifier.get_subscriber().on_next(expected_commit); + round.reject_round++; + + gate->vote({boost::none, round}); + + ASSERT_EQ(block_cache->get(), nullptr); +} + /** * @given yac gate * @when unsuccesfully trying to retrieve peers order @@ -259,7 +292,8 @@ class YacGateOlderTest : public YacGateTest { // generate order of peers ON_CALL(*peer_orderer, getOrdering(_)) - .WillByDefault(Return(ClusterOrdering::create({makePeer("fake_node")}))); + .WillByDefault( + Return(ClusterOrdering::create({makePeer("fake_node")}))); // make hash from block ON_CALL(*hash_provider, makeHash(_)).WillByDefault(Return(expected_hash)); diff --git a/test/module/irohad/ordering/on_demand_os_test.cpp b/test/module/irohad/ordering/on_demand_os_test.cpp index 832b005a91..f50226112c 100644 --- a/test/module/irohad/ordering/on_demand_os_test.cpp +++ b/test/module/irohad/ordering/on_demand_os_test.cpp @@ -80,8 +80,8 @@ class OnDemandOsTest : public ::testing::Test { } OnDemandOrderingService::CollectionType generateTransactions( - std::pair range) { - auto now = iroha::time::now(); + std::pair range, + shared_model::interface::types::TimestampType now = iroha::time::now()) { OnDemandOrderingService::CollectionType collection; for (auto i = range.first; i < range.second; ++i) { @@ -343,3 +343,21 @@ TEST_F(OnDemandOsTest, SeveralTransactionsOneCommited) { // already processed transaction is no present in the proposal EXPECT_TRUE(std::find(txs.begin(), txs.end(), batch2_tx) == txs.end()); } + +/** + * @given initialized on-demand OS with a batch in collection + * @when the same batch arrives, round is closed, proposal is requested + * @then the proposal contains the batch once + */ +TEST_F(OnDemandOsTest, DuplicateTxTest) { + auto now = iroha::time::now(); + auto txs1 = generateTransactions({1, 2}, now); + os->onBatches(txs1); + + auto txs2 = generateTransactions({1, 2}, now); + os->onBatches(txs2); + os->onCollaborationOutcome(commit_round); + auto proposal = os->onRequestProposal(target_round); + + ASSERT_EQ(1, boost::size((*proposal)->transactions())); +} From e2db4963df284f0e11c86d1416d4dfdd824baba3 Mon Sep 17 00:00:00 2001 From: Andrei Lebedev Date: Mon, 18 Mar 2019 13:56:46 +0300 Subject: [PATCH 31/40] Update config template for docker deployment (#2158) Signed-off-by: Andrei Lebedev --- deploy/ansible/playbooks/iroha-docker/main.yml | 2 +- deploy/ansible/roles/iroha-docker/defaults/main.yml | 6 ++++++ deploy/ansible/roles/iroha-docker/files/config_gen.sh | 1 + deploy/ansible/roles/iroha-docker/tasks/deploy.yml | 7 ++++++- .../roles/iroha-docker/templates/config.docker.j2 | 9 +++++++++ .../roles/iroha-docker/templates/docker-compose.yml.j2 | 3 +++ 6 files changed, 26 insertions(+), 2 deletions(-) diff --git a/deploy/ansible/playbooks/iroha-docker/main.yml b/deploy/ansible/playbooks/iroha-docker/main.yml index f25745dd85..ec52db060f 100644 --- a/deploy/ansible/playbooks/iroha-docker/main.yml +++ b/deploy/ansible/playbooks/iroha-docker/main.yml @@ -5,4 +5,4 @@ - { role: docker, tags: docker } # - { role: iroha-docker, tags: iroha-docker } vars: - hostnames: [] \ No newline at end of file + hostnames: [] diff --git a/deploy/ansible/roles/iroha-docker/defaults/main.yml b/deploy/ansible/roles/iroha-docker/defaults/main.yml index 168b0ed5ce..c301ca5142 100644 --- a/deploy/ansible/roles/iroha-docker/defaults/main.yml +++ b/deploy/ansible/roles/iroha-docker/defaults/main.yml @@ -47,6 +47,9 @@ postgres_docker_tag: '9.5' iroha_peer_port: 10001 # As above but for base Torii port iroha_torii_port: 50051 +# Enables 2 phase commit optimization. Postgres documentation suggests setting it to a max number of +# connections, which is 100 by default +iroha_postgres_max_prepared_transactions: 100 # Rest of the options affect Iroha configuration # See https://iroha.readthedocs.io/en/latest/guides/configuration.html#configuration @@ -60,3 +63,6 @@ iroha_postgres_password: postgres iroha_postgres_host: iroha-postgres iroha_postgres_port: 5432 iroha_blockstore_path: /tmp/block_store +#iroha_mst_expiration_time: 1440 +#iroha_max_rounds_delay: 3000 +#iroha_stale_stream_max_rounds: 2 diff --git a/deploy/ansible/roles/iroha-docker/files/config_gen.sh b/deploy/ansible/roles/iroha-docker/files/config_gen.sh index 7afa3f29c1..1888d0d0ab 100755 --- a/deploy/ansible/roles/iroha-docker/files/config_gen.sh +++ b/deploy/ansible/roles/iroha-docker/files/config_gen.sh @@ -1,4 +1,5 @@ #!/bin/bash +set -e FORCE_OVERWRITE=0 OUT_DIR='config' diff --git a/deploy/ansible/roles/iroha-docker/tasks/deploy.yml b/deploy/ansible/roles/iroha-docker/tasks/deploy.yml index d1965640fe..855e86d8fd 100644 --- a/deploy/ansible/roles/iroha-docker/tasks/deploy.yml +++ b/deploy/ansible/roles/iroha-docker/tasks/deploy.yml @@ -59,8 +59,13 @@ name: "{{ iroha_network_name }}" when: not overlay_network +- name: pull Iroha + command: docker-compose pull + args: + chdir: "{{ deploy_dir }}" + changed_when: False + - name: run Iroha docker_service: project_src: "{{ deploy_dir }}" project_name: ansible_iroha - pull: yes diff --git a/deploy/ansible/roles/iroha-docker/templates/config.docker.j2 b/deploy/ansible/roles/iroha-docker/templates/config.docker.j2 index e59a695935..4d292bd342 100644 --- a/deploy/ansible/roles/iroha-docker/templates/config.docker.j2 +++ b/deploy/ansible/roles/iroha-docker/templates/config.docker.j2 @@ -7,4 +7,13 @@ "proposal_delay" : {{ iroha_proposal_delay }}, "vote_delay" : {{ iroha_vote_delay }}, "mst_enable" : {{ iroha_mst_enable }} + {% if iroha_mst_expiration_time is defined %} + ,"mst_expiration_time": {{ iroha_mst_expiration_time }} + {% endif %} + {% if iroha_max_rounds_delay is defined %} + ,"max_rounds_delay": {{ iroha_max_rounds_delay }} + {% endif %} + {% if iroha_stale_stream_max_rounds is defined %} + ,"stale_stream_max_rounds": {{ iroha_stale_stream_max_rounds }} + {% endif %} } diff --git a/deploy/ansible/roles/iroha-docker/templates/docker-compose.yml.j2 b/deploy/ansible/roles/iroha-docker/templates/docker-compose.yml.j2 index 7fb2c462b4..2d26816a11 100644 --- a/deploy/ansible/roles/iroha-docker/templates/docker-compose.yml.j2 +++ b/deploy/ansible/roles/iroha-docker/templates/docker-compose.yml.j2 @@ -39,6 +39,9 @@ services: - psql_storage-{{ node.human_hostname }}:/var/lib/postgresql/data networks: - iroha-db-net +{% if iroha_postgres_max_prepared_transactions is defined %} + command: -c max_prepared_transactions={{ iroha_postgres_max_prepared_transactions }} +{% endif %} {% endfor %} From 089e455ebd3a36c7358dc213684ebff17cc862d0 Mon Sep 17 00:00:00 2001 From: Mikhail Boldyrev Date: Mon, 18 Mar 2019 18:18:21 +0300 Subject: [PATCH 32/40] Docs for new config file params (#2173) Signed-off-by: Mikhail Boldyrev --- docs/source/guides/configuration.rst | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/docs/source/guides/configuration.rst b/docs/source/guides/configuration.rst index 35a310a801..91a4aed53d 100644 --- a/docs/source/guides/configuration.rst +++ b/docs/source/guides/configuration.rst @@ -15,7 +15,10 @@ at ``example/config.sample`` "max_proposal_size": 10, "proposal_delay": 5000, "vote_delay": 5000, - "mst_enable" : false + "mst_enable" : false, + "mst_expiration_time" : 1440, + "max_rounds_delay": 3000, + "stale_stream_max_rounds": 2 } As you can see, configuration file is a valid ``json`` structure. Let's go @@ -53,9 +56,26 @@ Environment-specific parameters next peer. Optimal value depends heavily on the amount of Iroha peers in the network (higher amount of nodes requires longer ``vote_delay``). We recommend to start with 100-1000 milliseconds. -- ``mst_enable`` enables or disables multisignature transaction support in - Iroha. We recommend setting this parameter to ``false`` at the moment until - you really need it. +- ``mst_enable`` enables or disables multisignature transaction network + transport in Iroha. We recommend setting this parameter to ``false`` at the + moment until you really need it. + ``mst_expiration_time`` is the time period, in which a not fully signed + transaction is considered expired (in minutes). +- ``max_rounds_delay`` is an optional parameter specifying the maximum delay + between two consensus rounds (in milliseconds). + When Iroha is idle, it gradually increases the delay to reduce CPU, network + and logging load. + However too long delay may be unwanted when first transactions arrive after a + long idle time. + This parameter allows users to find an optimal value in a tradeoff between + resource consumption and the delay of getting back to work after an idle + period. +- ``stale_stream_max_rounds`` is the maximum amount of rounds to keep an open + status stream while no status update is reported. + Increasing this value reduces the amount of times a client must reconnect to + track a transaction if for some reason it is not updated with new rounds. + However large values increase the average number of connected clients during + each round. Logging ------- From 2cae43a307491425298e07291acde6af8df59bd7 Mon Sep 17 00:00:00 2001 From: Mikhail Boldyrev Date: Tue, 19 Mar 2019 14:10:52 +0300 Subject: [PATCH 33/40] removed block copy in BlocksQuery (#2177) Signed-off-by: Mikhail Boldyrev --- irohad/torii/processor/impl/query_processor_impl.cpp | 2 +- .../backend/protobuf/impl/proto_query_response_factory.cpp | 2 +- shared_model/backend/protobuf/proto_query_response_factory.hpp | 2 +- .../interfaces/iroha_internal/query_response_factory.hpp | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/irohad/torii/processor/impl/query_processor_impl.cpp b/irohad/torii/processor/impl/query_processor_impl.cpp index b52c23cc1b..d04f02b502 100644 --- a/irohad/torii/processor/impl/query_processor_impl.cpp +++ b/irohad/torii/processor/impl/query_processor_impl.cpp @@ -33,7 +33,7 @@ namespace iroha { storage_->on_commit().subscribe( [this](std::shared_ptr block) { auto block_response = - response_factory_->createBlockQueryResponse(clone(*block)); + response_factory_->createBlockQueryResponse(block); blocks_query_subject_.get_subscriber().on_next( std::move(block_response)); }); diff --git a/shared_model/backend/protobuf/impl/proto_query_response_factory.cpp b/shared_model/backend/protobuf/impl/proto_query_response_factory.cpp index 206c2db74a..fd15acbf07 100644 --- a/shared_model/backend/protobuf/impl/proto_query_response_factory.cpp +++ b/shared_model/backend/protobuf/impl/proto_query_response_factory.cpp @@ -323,7 +323,7 @@ shared_model::proto::ProtoQueryResponseFactory::createRolePermissionsResponse( std::unique_ptr shared_model::proto::ProtoQueryResponseFactory::createBlockQueryResponse( - std::unique_ptr block) const { + std::shared_ptr block) const { return createQueryResponse([block = std::move(block)]( iroha::protocol::BlockQueryResponse &protocol_query_response) { diff --git a/shared_model/backend/protobuf/proto_query_response_factory.hpp b/shared_model/backend/protobuf/proto_query_response_factory.hpp index c97e14c201..c43b2e23bf 100644 --- a/shared_model/backend/protobuf/proto_query_response_factory.hpp +++ b/shared_model/backend/protobuf/proto_query_response_factory.hpp @@ -78,7 +78,7 @@ namespace shared_model { const crypto::Hash &query_hash) const override; std::unique_ptr createBlockQueryResponse( - std::unique_ptr block) const override; + std::shared_ptr block) const override; std::unique_ptr createBlockQueryResponse( std::string error_message) const override; diff --git a/shared_model/interfaces/iroha_internal/query_response_factory.hpp b/shared_model/interfaces/iroha_internal/query_response_factory.hpp index 97b3160c07..b1bde689a8 100644 --- a/shared_model/interfaces/iroha_internal/query_response_factory.hpp +++ b/shared_model/interfaces/iroha_internal/query_response_factory.hpp @@ -205,7 +205,7 @@ namespace shared_model { * @return block query response with block */ virtual std::unique_ptr createBlockQueryResponse( - std::unique_ptr block) const = 0; + std::shared_ptr block) const = 0; /** * Create response for block query with error From 2d09de367751a38574f1b22ce16f80c4ab1ce7f6 Mon Sep 17 00:00:00 2001 From: Mikhail Boldyrev Date: Tue, 19 Mar 2019 14:14:56 +0300 Subject: [PATCH 34/40] StorageImpl: faster cleanup (#2174) Signed-off-by: Mikhail Boldyrev --- irohad/ametsuchi/impl/storage_impl.cpp | 37 +++++++++++++++----------- irohad/main/application.cpp | 2 +- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/irohad/ametsuchi/impl/storage_impl.cpp b/irohad/ametsuchi/impl/storage_impl.cpp index a6d474dbd1..de19d81ecd 100644 --- a/irohad/ametsuchi/impl/storage_impl.cpp +++ b/irohad/ametsuchi/impl/storage_impl.cpp @@ -272,6 +272,10 @@ namespace iroha { log_->warn("Drop database was failed. Reason: {}", e.what()); } } else { + // Clear all the tables first, as it takes much less time because the + // foreign key triggers are ignored. + soci::session(*connection_) << reset_; + // Empty tables can now be dropped very fast. soci::session(*connection_) << drop_; } @@ -583,25 +587,26 @@ DROP TABLE IF EXISTS tx_status_by_hash; DROP TABLE IF EXISTS height_by_account_set; DROP TABLE IF EXISTS index_by_creator_height; DROP TABLE IF EXISTS position_by_account_asset; +DROP TABLE IF EXISTS position_by_hash; )"; const std::string &StorageImpl::reset_ = R"( -DELETE FROM account_has_signatory; -DELETE FROM account_has_asset; -DELETE FROM role_has_permissions CASCADE; -DELETE FROM account_has_roles; -DELETE FROM account_has_grantable_permissions CASCADE; -DELETE FROM account; -DELETE FROM asset; -DELETE FROM domain; -DELETE FROM signatory; -DELETE FROM peer; -DELETE FROM role; -DELETE FROM position_by_hash; -DELETE FROM tx_status_by_hash; -DELETE FROM height_by_account_set; -DELETE FROM index_by_creator_height; -DELETE FROM position_by_account_asset; +TRUNCATE TABLE account_has_signatory RESTART IDENTITY CASCADE; +TRUNCATE TABLE account_has_asset RESTART IDENTITY CASCADE; +TRUNCATE TABLE role_has_permissions RESTART IDENTITY CASCADE; +TRUNCATE TABLE account_has_roles RESTART IDENTITY CASCADE; +TRUNCATE TABLE account_has_grantable_permissions RESTART IDENTITY CASCADE; +TRUNCATE TABLE account RESTART IDENTITY CASCADE; +TRUNCATE TABLE asset RESTART IDENTITY CASCADE; +TRUNCATE TABLE domain RESTART IDENTITY CASCADE; +TRUNCATE TABLE signatory RESTART IDENTITY CASCADE; +TRUNCATE TABLE peer RESTART IDENTITY CASCADE; +TRUNCATE TABLE role RESTART IDENTITY CASCADE; +TRUNCATE TABLE position_by_hash RESTART IDENTITY CASCADE; +TRUNCATE TABLE tx_status_by_hash RESTART IDENTITY CASCADE; +TRUNCATE TABLE height_by_account_set RESTART IDENTITY CASCADE; +TRUNCATE TABLE index_by_creator_height RESTART IDENTITY CASCADE; +TRUNCATE TABLE position_by_account_asset RESTART IDENTITY CASCADE; )"; const std::string &StorageImpl::init_ = diff --git a/irohad/main/application.cpp b/irohad/main/application.cpp index 9c8cdde8ef..751becedc9 100644 --- a/irohad/main/application.cpp +++ b/irohad/main/application.cpp @@ -175,7 +175,7 @@ void Irohad::initStorage() { }, [&](expected::Error &error) { log_->error(error.error); }); - log_->info("[Init] => storage", logger::logBool(storage)); + log_->info("[Init] => storage ({})", logger::logBool(storage)); } bool Irohad::restoreWsv() { From 68d6ef0d4c154dcb73c56a5f277cd5f9ae409d1b Mon Sep 17 00:00:00 2001 From: Mikhail Boldyrev Date: Tue, 19 Mar 2019 20:20:25 +0300 Subject: [PATCH 35/40] Storage: notify block commit after actual commit (#2179) Signed-off-by: Andrei Lebedev --- irohad/ametsuchi/impl/storage_impl.cpp | 6 +- .../irohad/ametsuchi/ametsuchi_test.cpp | 78 +++++++++++++++++++ 2 files changed, 82 insertions(+), 2 deletions(-) diff --git a/irohad/ametsuchi/impl/storage_impl.cpp b/irohad/ametsuchi/impl/storage_impl.cpp index de19d81ecd..e7374308d5 100644 --- a/irohad/ametsuchi/impl/storage_impl.cpp +++ b/irohad/ametsuchi/impl/storage_impl.cpp @@ -426,11 +426,13 @@ namespace iroha { std::unique_ptr mutable_storage) { auto storage = static_cast(mutable_storage.get()); - storage->block_storage_->forEach( - [this](const auto &block) { this->storeBlock(*block); }); try { *(storage->sql_) << "COMMIT"; storage->committed = true; + + storage->block_storage_->forEach( + [this](const auto &block) { this->storeBlock(*block); }); + return PostgresWsvQuery(*(storage->sql_), factory_, log_manager_->getChild("WsvQuery")->getLogger()) diff --git a/test/module/irohad/ametsuchi/ametsuchi_test.cpp b/test/module/irohad/ametsuchi/ametsuchi_test.cpp index 812199265f..a09fe2948f 100644 --- a/test/module/irohad/ametsuchi/ametsuchi_test.cpp +++ b/test/module/irohad/ametsuchi/ametsuchi_test.cpp @@ -511,6 +511,84 @@ TEST_F(AmetsuchiTest, TestRestoreWSV) { EXPECT_TRUE(res); } +/** + * @given created storage + * @and a subscribed observer on on_commit() event + * @when commit block + * @then the effect of transactions in the committed block can be verified with + * queries + */ +TEST_F(AmetsuchiTest, TestingWsvAfterCommitBlock) { + ASSERT_TRUE(storage); + + shared_model::crypto::Keypair key{ + shared_model::crypto::DefaultCryptoAlgorithmType::generateKeypair()}; + + auto genesis_tx = shared_model::proto::TransactionBuilder() + .creatorAccountId("admin@test") + .createdTime(iroha::time::now()) + .quorum(1) + .createRole("admin", + {Role::kCreateDomain, + Role::kCreateAccount, + Role::kAddAssetQty, + Role::kAddPeer, + Role::kReceive, + Role::kTransfer}) + .createDomain("test", "admin") + .createAccount("admin", "test", key.publicKey()) + .createAccount("receiver", "test", key.publicKey()) + .createAsset("coin", "test", 2) + .addAssetQuantity("coin#test", "20.00") + .build() + .signAndAddSignature(key) + .finish(); + auto genesis_block = + TestBlockBuilder() + .transactions( + std::vector({genesis_tx})) + .height(1) + .prevHash(shared_model::crypto::Sha3_256::makeHash( + shared_model::crypto::Blob(""))) + .createdTime(iroha::time::now()) + .build(); + apply(storage, genesis_block); + + auto add_ast_tx = + shared_model::proto::TransactionBuilder() + .creatorAccountId("admin@test") + .createdTime(iroha::time::now()) + .quorum(1) + .transferAsset( + "admin@test", "receiver@test", "coin#test", "deal", "10.00") + .build() + .signAndAddSignature(key) + .finish(); + + auto expected_block = + TestBlockBuilder() + .transactions( + std::vector({add_ast_tx})) + .height(1) + .prevHash(genesis_block.hash()) + .createdTime(iroha::time::now()) + .build(); + + static auto wrapper = + make_test_subscriber(storage->on_commit(), 1); + wrapper.subscribe([&](const auto &block) { + ASSERT_EQ(*block, expected_block); + shared_model::interface::Amount resultingAmount("10.00"); + validateAccountAsset( + sql_query, "receiver@test", "coin#test", resultingAmount); + }); + + apply(storage, expected_block); + + ASSERT_TRUE(wrapper.validate()); + wrapper.unsubscribe(); +} + class PreparedBlockTest : public AmetsuchiTest { public: PreparedBlockTest() From 0c8a0aaf2f682b9093d845cba88f339dc3c0ed3b Mon Sep 17 00:00:00 2001 From: Mikhail Boldyrev Date: Wed, 20 Mar 2019 12:28:42 +0300 Subject: [PATCH 36/40] timeout simple (#2160) Signed-off-by: Mikhail Boldyrev --- irohad/network/impl/block_loader_impl.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/irohad/network/impl/block_loader_impl.cpp b/irohad/network/impl/block_loader_impl.cpp index 99e15f3fb1..d641e2c529 100644 --- a/irohad/network/impl/block_loader_impl.cpp +++ b/irohad/network/impl/block_loader_impl.cpp @@ -6,6 +6,7 @@ #include "network/impl/block_loader_impl.hpp" #include +#include #include "backend/protobuf/block.hpp" #include "builders/protobuf/transport_builder.hpp" @@ -23,6 +24,7 @@ namespace { const char *kPeerNotFound = "Cannot find peer"; const char *kPeerRetrieveFail = "Failed to retrieve peers"; const char *kPeerFindFail = "Failed to find requested peer"; + const std::chrono::seconds kBlocksRequestTimeout{5}; } // namespace BlockLoaderImpl::BlockLoaderImpl( @@ -49,12 +51,16 @@ rxcpp::observable> BlockLoaderImpl::retrieveBlocks( grpc::ClientContext context; protocol::Block block; + // set a timeout to avoid being hung + context.set_deadline(std::chrono::system_clock::now() + + kBlocksRequestTimeout); + // request next block to our top request.set_height(height + 1); auto reader = this->getPeerStub(**peer).retrieveBlocks(&context, request); - while (reader->Read(&block)) { + while (subscriber.is_subscribed() and reader->Read(&block)) { auto proto_block = block_factory_.createBlock(std::move(block)); proto_block.match( [&subscriber]( From 88c4ad481cbe0f41de11060aabc79f9105f3961c Mon Sep 17 00:00:00 2001 From: Mikhail Boldyrev Date: Wed, 20 Mar 2019 16:53:35 +0300 Subject: [PATCH 37/40] Storage: avoid block copying (#2178) Signed-off-by: Mikhail Boldyrev --- irohad/ametsuchi/block_storage.hpp | 3 - .../impl/in_memory_block_storage.cpp | 4 - .../impl/in_memory_block_storage.hpp | 2 - .../ametsuchi/impl/mutable_storage_impl.cpp | 21 ++-- .../ametsuchi/impl/mutable_storage_impl.hpp | 5 +- irohad/ametsuchi/impl/storage_impl.cpp | 24 ++-- irohad/ametsuchi/impl/storage_impl.hpp | 13 +- irohad/ametsuchi/mutable_factory.hpp | 2 +- irohad/ametsuchi/mutable_storage.hpp | 11 +- irohad/ametsuchi/storage.hpp | 6 +- irohad/main/irohad.cpp | 2 +- .../synchronizer/impl/synchronizer_impl.cpp | 4 +- .../processor/impl/query_processor_impl.cpp | 2 +- .../impl/transaction_processor_impl.cpp | 2 +- .../processor/transaction_processor_impl.hpp | 4 +- .../validation/impl/chain_validator_impl.cpp | 12 +- .../validation/impl/chain_validator_impl.hpp | 2 +- .../impl/proto_query_response_factory.cpp | 5 +- .../protobuf/proto_query_response_factory.hpp | 2 +- .../iroha_internal/query_response_factory.hpp | 2 +- .../integration_test_framework.cpp | 4 +- .../integration_framework/iroha_instance.cpp | 5 +- .../integration_framework/iroha_instance.hpp | 6 +- .../chain_validator_storage_test.cpp | 25 ++-- .../irohad/ametsuchi/ametsuchi_test.cpp | 116 ++++-------------- .../irohad/ametsuchi/mock_mutable_factory.hpp | 2 +- .../irohad/ametsuchi/mock_mutable_storage.hpp | 8 +- test/module/irohad/ametsuchi/mock_storage.hpp | 10 +- .../postgres_query_executor_test.cpp | 40 ++---- .../processor/transaction_processor_test.cpp | 3 +- .../validation/chain_validation_test.cpp | 7 +- .../builders/protobuf/test_block_builder.hpp | 13 ++ 32 files changed, 154 insertions(+), 213 deletions(-) diff --git a/irohad/ametsuchi/block_storage.hpp b/irohad/ametsuchi/block_storage.hpp index 599e363885..17337147ac 100644 --- a/irohad/ametsuchi/block_storage.hpp +++ b/irohad/ametsuchi/block_storage.hpp @@ -28,9 +28,6 @@ namespace iroha { virtual bool insert( std::shared_ptr block) = 0; - [[deprecated("Use shared_ptr")]] virtual bool insert( - const shared_model::interface::Block &block) = 0; - /** * Get block with given height * @return block if exists, boost::none otherwise diff --git a/irohad/ametsuchi/impl/in_memory_block_storage.cpp b/irohad/ametsuchi/impl/in_memory_block_storage.cpp index b32929fc27..d79959623e 100644 --- a/irohad/ametsuchi/impl/in_memory_block_storage.cpp +++ b/irohad/ametsuchi/impl/in_memory_block_storage.cpp @@ -13,10 +13,6 @@ bool InMemoryBlockStorage::insert( return block_store_.emplace(height, std::move(block)).second; } -bool InMemoryBlockStorage::insert(const shared_model::interface::Block &block) { - return block_store_.emplace(block.height(), clone(block)).second; -} - boost::optional> InMemoryBlockStorage::fetch( shared_model::interface::types::HeightType height) const { diff --git a/irohad/ametsuchi/impl/in_memory_block_storage.hpp b/irohad/ametsuchi/impl/in_memory_block_storage.hpp index 6f822729f2..eaa0eb65a4 100644 --- a/irohad/ametsuchi/impl/in_memory_block_storage.hpp +++ b/irohad/ametsuchi/impl/in_memory_block_storage.hpp @@ -21,8 +21,6 @@ namespace iroha { bool insert( std::shared_ptr block) override; - bool insert(const shared_model::interface::Block &block) override; - boost::optional> fetch(shared_model::interface::types::HeightType height) const override; diff --git a/irohad/ametsuchi/impl/mutable_storage_impl.cpp b/irohad/ametsuchi/impl/mutable_storage_impl.cpp index e3b2504440..c4537622d5 100644 --- a/irohad/ametsuchi/impl/mutable_storage_impl.cpp +++ b/irohad/ametsuchi/impl/mutable_storage_impl.cpp @@ -42,8 +42,9 @@ namespace iroha { *sql_ << "BEGIN"; } - bool MutableStorageImpl::apply(const shared_model::interface::Block &block, - MutableStoragePredicate predicate) { + bool MutableStorageImpl::apply( + std::shared_ptr block, + MutableStoragePredicate predicate) { auto execute_transaction = [this](auto &transaction) { command_executor_->setCreatorAccountId(transaction.creatorAccountId()); command_executor_->doValidation(false); @@ -66,18 +67,18 @@ namespace iroha { }; log_->info("Applying block: height {}, hash {}", - block.height(), - block.hash().hex()); + block->height(), + block->hash().hex()); auto block_applied = predicate(block, *peer_query_, top_hash_) - and std::all_of(block.transactions().begin(), - block.transactions().end(), + and std::all_of(block->transactions().begin(), + block->transactions().end(), execute_transaction); if (block_applied) { block_storage_->insert(block); - block_index_->index(block); + block_index_->index(*block); - top_hash_ = block.hash(); + top_hash_ = block->hash(); } return block_applied; @@ -103,7 +104,7 @@ namespace iroha { } bool MutableStorageImpl::apply( - const shared_model::interface::Block &block) { + std::shared_ptr block) { return withSavepoint([&] { return this->apply( block, [](const auto &, auto &, const auto &) { return true; }); @@ -116,7 +117,7 @@ namespace iroha { MutableStoragePredicate predicate) { return withSavepoint([&] { return blocks - .all([&](auto block) { return this->apply(*block, predicate); }) + .all([&](auto block) { return this->apply(block, predicate); }) .as_blocking() .first(); }); diff --git a/irohad/ametsuchi/impl/mutable_storage_impl.hpp b/irohad/ametsuchi/impl/mutable_storage_impl.hpp index 0a63527a17..62166db3e6 100644 --- a/irohad/ametsuchi/impl/mutable_storage_impl.hpp +++ b/irohad/ametsuchi/impl/mutable_storage_impl.hpp @@ -33,7 +33,8 @@ namespace iroha { std::unique_ptr block_storage, logger::LoggerManagerTreePtr log_manager); - bool apply(const shared_model::interface::Block &block) override; + bool apply( + std::shared_ptr block) override; bool apply(rxcpp::observable< std::shared_ptr> blocks, @@ -54,7 +55,7 @@ namespace iroha { * Verifies whether the block is applicable using predicate, and applies * the block */ - bool apply(const shared_model::interface::Block &block, + bool apply(std::shared_ptr block, MutableStoragePredicate predicate); shared_model::interface::types::HashType top_hash_; diff --git a/irohad/ametsuchi/impl/storage_impl.cpp b/irohad/ametsuchi/impl/storage_impl.cpp index e7374308d5..accc43558f 100644 --- a/irohad/ametsuchi/impl/storage_impl.cpp +++ b/irohad/ametsuchi/impl/storage_impl.cpp @@ -194,7 +194,8 @@ namespace iroha { log_manager_->getChild("QueryExecutor"))); } - bool StorageImpl::insertBlock(const shared_model::interface::Block &block) { + bool StorageImpl::insertBlock( + std::shared_ptr block) { log_->info("create mutable storage"); auto storageResult = createMutableStorage(); bool inserted = false; @@ -222,7 +223,7 @@ namespace iroha { [&](iroha::expected::Value> &mutableStorage) { std::for_each(blocks.begin(), blocks.end(), [&](auto block) { - inserted &= mutableStorage.value->apply(*block); + inserted &= mutableStorage.value->apply(block); }); commit(std::move(mutableStorage.value)); }, @@ -431,7 +432,7 @@ namespace iroha { storage->committed = true; storage->block_storage_->forEach( - [this](const auto &block) { this->storeBlock(*block); }); + [this](const auto &block) { this->storeBlock(block); }); return PostgresWsvQuery(*(storage->sql_), factory_, @@ -450,7 +451,7 @@ namespace iroha { } boost::optional> StorageImpl::commitPrepared( - const shared_model::interface::Block &block) { + std::shared_ptr block) { if (not prepared_blocks_enabled_) { log_->warn("prepared blocks are not enabled"); return boost::none; @@ -472,7 +473,7 @@ namespace iroha { sql << "COMMIT PREPARED '" + prepared_block_name_ + "';"; PostgresBlockIndex block_index( sql, log_manager_->getChild("BlockIndex")->getLogger()); - block_index.index(block); + block_index.index(*block); block_is_prepared = false; return PostgresWsvQuery(sql, factory_, @@ -489,7 +490,7 @@ namespace iroha { }; } catch (const std::exception &e) { log_->warn("failed to apply prepared block {}: {}", - block.hash().hex(), + block->hash().hex(), e.what()); return boost::none; } @@ -520,7 +521,7 @@ namespace iroha { log_manager_->getChild("PostgresBlockQuery")->getLogger()); } - rxcpp::observable> + rxcpp::observable> StorageImpl::on_commit() { return notifier_.get_observable(); } @@ -557,12 +558,13 @@ namespace iroha { } } - bool StorageImpl::storeBlock(const shared_model::interface::Block &block) { - auto json_result = converter_->serialize(block); + bool StorageImpl::storeBlock( + std::shared_ptr block) { + auto json_result = converter_->serialize(*block); return json_result.match( [this, &block](const expected::Value &v) { - block_store_->add(block.height(), stringToBytes(v.value)); - notifier_.get_subscriber().on_next(clone(block)); + block_store_->add(block->height(), stringToBytes(v.value)); + notifier_.get_subscriber().on_next(block); return true; }, [this](const expected::Error &e) { diff --git a/irohad/ametsuchi/impl/storage_impl.hpp b/irohad/ametsuchi/impl/storage_impl.hpp index b1e5ef4d36..12b9d0170d 100644 --- a/irohad/ametsuchi/impl/storage_impl.hpp +++ b/irohad/ametsuchi/impl/storage_impl.hpp @@ -83,7 +83,8 @@ namespace iroha { * @param blocks - block for insertion * @return true if all blocks are inserted */ - bool insertBlock(const shared_model::interface::Block &block) override; + bool insertBlock( + std::shared_ptr block) override; /** * Insert blocks without validation @@ -104,13 +105,13 @@ namespace iroha { std::unique_ptr mutable_storage) override; boost::optional> commitPrepared( - const shared_model::interface::Block &block) override; + std::shared_ptr block) override; std::shared_ptr getWsvQuery() const override; std::shared_ptr getBlockQuery() const override; - rxcpp::observable> + rxcpp::observable> on_commit() override; void prepareBlock(std::unique_ptr wsv) override; @@ -150,7 +151,8 @@ namespace iroha { /** * add block to block storage */ - bool storeBlock(const shared_model::interface::Block &block); + bool storeBlock( + std::shared_ptr block); std::unique_ptr block_store_; @@ -158,7 +160,8 @@ namespace iroha { std::shared_ptr factory_; - rxcpp::subjects::subject> + rxcpp::subjects::subject< + std::shared_ptr> notifier_; std::shared_ptr converter_; diff --git a/irohad/ametsuchi/mutable_factory.hpp b/irohad/ametsuchi/mutable_factory.hpp index ece5a66ee7..0cd8738c65 100644 --- a/irohad/ametsuchi/mutable_factory.hpp +++ b/irohad/ametsuchi/mutable_factory.hpp @@ -49,7 +49,7 @@ namespace iroha { * prepared block failed to apply. WSV is not changed in this case. */ virtual boost::optional> commitPrepared( - const shared_model::interface::Block &block) = 0; + std::shared_ptr block) = 0; virtual ~MutableFactory() = default; }; diff --git a/irohad/ametsuchi/mutable_storage.hpp b/irohad/ametsuchi/mutable_storage.hpp index 0a1ad7577c..eb70a4dd2e 100644 --- a/irohad/ametsuchi/mutable_storage.hpp +++ b/irohad/ametsuchi/mutable_storage.hpp @@ -36,16 +36,17 @@ namespace iroha { * - PeerQuery - interface for ledger peers list retrieval * - HashType - hash of top block in blockchain */ - using MutableStoragePredicate = - std::function; + using MutableStoragePredicate = std::function, + PeerQuery &, + const shared_model::interface::types::HashType &)>; /** * Applies block without additional validation function * @see apply(block, function) */ - virtual bool apply(const shared_model::interface::Block &block) = 0; + virtual bool apply( + std::shared_ptr block) = 0; /** * Applies an observable of blocks to current mutable state using logic diff --git a/irohad/ametsuchi/storage.hpp b/irohad/ametsuchi/storage.hpp index b4f707c1b7..1089d2f631 100644 --- a/irohad/ametsuchi/storage.hpp +++ b/irohad/ametsuchi/storage.hpp @@ -48,7 +48,8 @@ namespace iroha { * @param block - block for insertion * @return true if inserted */ - virtual bool insertBlock(const shared_model::interface::Block &block) = 0; + virtual bool insertBlock( + std::shared_ptr block) = 0; /** * Raw insertion of blocks without validation @@ -63,7 +64,8 @@ namespace iroha { * method called when block is written to the storage * @return observable with the Block committed */ - virtual rxcpp::observable> + virtual rxcpp::observable< + std::shared_ptr> on_commit() = 0; /** diff --git a/irohad/main/irohad.cpp b/irohad/main/irohad.cpp index ca4aaed959..6ecf53c756 100644 --- a/irohad/main/irohad.cpp +++ b/irohad/main/irohad.cpp @@ -226,7 +226,7 @@ int main(int argc, char *argv[]) { // clear previous storage if any irohad.dropStorage(); - irohad.storage->insertBlock(*block.value()); + irohad.storage->insertBlock(block.value()); log->info("Genesis block inserted, number of transactions: {}", block.value()->transactions().size()); } diff --git a/irohad/synchronizer/impl/synchronizer_impl.cpp b/irohad/synchronizer/impl/synchronizer_impl.cpp index 05109ac429..4c76b9fcff 100644 --- a/irohad/synchronizer/impl/synchronizer_impl.cpp +++ b/irohad/synchronizer/impl/synchronizer_impl.cpp @@ -136,7 +136,7 @@ namespace iroha { void SynchronizerImpl::processNext(const consensus::PairValid &msg) { log_->info("at handleNext"); - auto ledger_state = mutable_factory_->commitPrepared(*msg.block); + auto ledger_state = mutable_factory_->commitPrepared(msg.block); if (ledger_state) { notifier_.get_subscriber().on_next( SynchronizationEvent{rxcpp::observable<>::just(msg.block), @@ -150,7 +150,7 @@ namespace iroha { } std::unique_ptr storage = std::move(opt_storage.value()); - if (storage->apply(*msg.block)) { + if (storage->apply(msg.block)) { ledger_state = mutable_factory_->commit(std::move(storage)); if (ledger_state) { notifier_.get_subscriber().on_next( diff --git a/irohad/torii/processor/impl/query_processor_impl.cpp b/irohad/torii/processor/impl/query_processor_impl.cpp index d04f02b502..7f259b4627 100644 --- a/irohad/torii/processor/impl/query_processor_impl.cpp +++ b/irohad/torii/processor/impl/query_processor_impl.cpp @@ -31,7 +31,7 @@ namespace iroha { response_factory_{std::move(response_factory)}, log_{std::move(log)} { storage_->on_commit().subscribe( - [this](std::shared_ptr block) { + [this](std::shared_ptr block) { auto block_response = response_factory_->createBlockQueryResponse(block); blocks_query_subject_.get_subscriber().on_next( diff --git a/irohad/torii/processor/impl/transaction_processor_impl.cpp b/irohad/torii/processor/impl/transaction_processor_impl.cpp index 3c714e8748..599e82695b 100644 --- a/irohad/torii/processor/impl/transaction_processor_impl.cpp +++ b/irohad/torii/processor/impl/transaction_processor_impl.cpp @@ -49,7 +49,7 @@ namespace iroha { std::shared_ptr status_bus, std::shared_ptr status_factory, - rxcpp::observable> + rxcpp::observable> commits, logger::LoggerPtr log) : pcs_(std::move(pcs)), diff --git a/irohad/torii/processor/transaction_processor_impl.hpp b/irohad/torii/processor/transaction_processor_impl.hpp index c92ba1acf3..2f173c2a19 100644 --- a/irohad/torii/processor/transaction_processor_impl.hpp +++ b/irohad/torii/processor/transaction_processor_impl.hpp @@ -37,8 +37,8 @@ namespace iroha { std::shared_ptr status_bus, std::shared_ptr status_factory, - rxcpp::observable> - commits, + rxcpp::observable< + std::shared_ptr> commits, logger::LoggerPtr log); void batchHandle( diff --git a/irohad/validation/impl/chain_validator_impl.cpp b/irohad/validation/impl/chain_validator_impl.cpp index e74ffbec0a..f985dbdf60 100644 --- a/irohad/validation/impl/chain_validator_impl.cpp +++ b/irohad/validation/impl/chain_validator_impl.cpp @@ -31,7 +31,7 @@ namespace iroha { return storage.apply( blocks, - [this](const auto &block, auto &queries, const auto &top_hash) { + [this](auto block, auto &queries, const auto &top_hash) { return this->validateBlock(block, queries, top_hash); }); } @@ -84,12 +84,12 @@ namespace iroha { } bool ChainValidatorImpl::validateBlock( - const shared_model::interface::Block &block, + std::shared_ptr block, ametsuchi::PeerQuery &queries, const shared_model::interface::types::HashType &top_hash) const { log_->info("validate block: height {}, hash {}", - block.height(), - block.hash().hex()); + block->height(), + block->hash().hex()); auto peers = queries.getLedgerPeers(); if (not peers) { @@ -97,8 +97,8 @@ namespace iroha { return false; } - return validatePreviousHash(block, top_hash) - and validatePeerSupermajority(block, *peers); + return validatePreviousHash(*block, top_hash) + and validatePeerSupermajority(*block, *peers); } } // namespace validation diff --git a/irohad/validation/impl/chain_validator_impl.hpp b/irohad/validation/impl/chain_validator_impl.hpp index af42fc8e3c..8f75b3db7c 100644 --- a/irohad/validation/impl/chain_validator_impl.hpp +++ b/irohad/validation/impl/chain_validator_impl.hpp @@ -60,7 +60,7 @@ namespace iroha { * of ledger peers */ bool validateBlock( - const shared_model::interface::Block &block, + std::shared_ptr block, ametsuchi::PeerQuery &queries, const shared_model::interface::types::HashType &top_hash) const; diff --git a/shared_model/backend/protobuf/impl/proto_query_response_factory.cpp b/shared_model/backend/protobuf/impl/proto_query_response_factory.cpp index fd15acbf07..c6faaa4f8d 100644 --- a/shared_model/backend/protobuf/impl/proto_query_response_factory.cpp +++ b/shared_model/backend/protobuf/impl/proto_query_response_factory.cpp @@ -323,14 +323,15 @@ shared_model::proto::ProtoQueryResponseFactory::createRolePermissionsResponse( std::unique_ptr shared_model::proto::ProtoQueryResponseFactory::createBlockQueryResponse( - std::shared_ptr block) const { + std::shared_ptr block) const { return createQueryResponse([block = std::move(block)]( iroha::protocol::BlockQueryResponse &protocol_query_response) { iroha::protocol::BlockResponse *protocol_specific_response = protocol_query_response.mutable_block_response(); *protocol_specific_response->mutable_block()->mutable_block_v1() = - static_cast(block.get())->getTransport(); + static_cast(block.get()) + ->getTransport(); }); } diff --git a/shared_model/backend/protobuf/proto_query_response_factory.hpp b/shared_model/backend/protobuf/proto_query_response_factory.hpp index c43b2e23bf..e97850a188 100644 --- a/shared_model/backend/protobuf/proto_query_response_factory.hpp +++ b/shared_model/backend/protobuf/proto_query_response_factory.hpp @@ -78,7 +78,7 @@ namespace shared_model { const crypto::Hash &query_hash) const override; std::unique_ptr createBlockQueryResponse( - std::shared_ptr block) const override; + std::shared_ptr block) const override; std::unique_ptr createBlockQueryResponse( std::string error_message) const override; diff --git a/shared_model/interfaces/iroha_internal/query_response_factory.hpp b/shared_model/interfaces/iroha_internal/query_response_factory.hpp index b1bde689a8..e72884f6e1 100644 --- a/shared_model/interfaces/iroha_internal/query_response_factory.hpp +++ b/shared_model/interfaces/iroha_internal/query_response_factory.hpp @@ -205,7 +205,7 @@ namespace shared_model { * @return block query response with block */ virtual std::unique_ptr createBlockQueryResponse( - std::shared_ptr block) const = 0; + std::shared_ptr block) const = 0; /** * Create response for block query with error diff --git a/test/framework/integration_framework/integration_test_framework.cpp b/test/framework/integration_framework/integration_test_framework.cpp index c9cf41bb25..682875cd32 100644 --- a/test/framework/integration_framework/integration_test_framework.cpp +++ b/test/framework/integration_framework/integration_test_framework.cpp @@ -227,7 +227,7 @@ namespace integration_framework { const Keypair &keypair) { initPipeline(keypair); iroha_instance_->makeGenesis( - IntegrationTestFramework::defaultBlock(keypair)); + clone(IntegrationTestFramework::defaultBlock(keypair))); log_->info("added genesis block"); subscribeQueuesAndRun(); return *this; @@ -244,7 +244,7 @@ namespace integration_framework { IntegrationTestFramework &IntegrationTestFramework::setInitialState( const Keypair &keypair, const shared_model::interface::Block &block) { initPipeline(keypair); - iroha_instance_->makeGenesis(block); + iroha_instance_->makeGenesis(clone(block)); log_->info("added genesis block"); subscribeQueuesAndRun(); return *this; diff --git a/test/framework/integration_framework/iroha_instance.cpp b/test/framework/integration_framework/iroha_instance.cpp index 663fd8198d..c23941b020 100644 --- a/test/framework/integration_framework/iroha_instance.cpp +++ b/test/framework/integration_framework/iroha_instance.cpp @@ -45,14 +45,15 @@ namespace integration_framework { irohad_log_manager_(std::move(irohad_log_manager)), log_(std::move(log)) {} - void IrohaInstance::makeGenesis(const shared_model::interface::Block &block) { + void IrohaInstance::makeGenesis( + std::shared_ptr block) { instance_->storage->reset(); rawInsertBlock(block); instance_->init(); } void IrohaInstance::rawInsertBlock( - const shared_model::interface::Block &block) { + std::shared_ptr block) { instance_->storage->insertBlock(block); } diff --git a/test/framework/integration_framework/iroha_instance.hpp b/test/framework/integration_framework/iroha_instance.hpp index 18be06d1c6..01eb0f8746 100644 --- a/test/framework/integration_framework/iroha_instance.hpp +++ b/test/framework/integration_framework/iroha_instance.hpp @@ -51,9 +51,11 @@ namespace integration_framework { logger::LoggerPtr log, const boost::optional &dbname = boost::none); - void makeGenesis(const shared_model::interface::Block &block); + void makeGenesis( + std::shared_ptr block); - void rawInsertBlock(const shared_model::interface::Block &block); + void rawInsertBlock( + std::shared_ptr block); void setMstGossipParams( std::chrono::milliseconds mst_gossip_emitting_period, diff --git a/test/integration/validation/chain_validator_storage_test.cpp b/test/integration/validation/chain_validator_storage_test.cpp index 229ea0d829..cc7eb74c23 100644 --- a/test/integration/validation/chain_validator_storage_test.cpp +++ b/test/integration/validation/chain_validator_storage_test.cpp @@ -65,10 +65,11 @@ namespace iroha { .build(); } - /// Complete wrapper and return a signed object + /// Complete wrapper and return a signed object through pointer template - auto completeBlock(Wrapper &&wrapper) { - return std::forward(wrapper).finish(); + std::shared_ptr completeBlock( + Wrapper &&wrapper) { + return clone(std::forward(wrapper).finish()); } /// Create mutable storage from initialized storage @@ -134,19 +135,19 @@ namespace iroha { auto add_peer = completeTx(baseTx().addPeer("0.0.0.0:50545", keys.at(4).publicKey())); - auto block2 = completeBlock(baseBlock({add_peer}, 2, block1.hash()) + auto block2 = completeBlock(baseBlock({add_peer}, 2, block1->hash()) .signAndAddSignature(keys.at(0)) .signAndAddSignature(keys.at(1)) .signAndAddSignature(keys.at(2))); - auto block3 = completeBlock(baseBlock({dummyTx(3)}, 3, block2.hash()) + auto block3 = completeBlock(baseBlock({dummyTx(3)}, 3, block2->hash()) .signAndAddSignature(keys.at(0)) .signAndAddSignature(keys.at(1)) .signAndAddSignature(keys.at(2)) .signAndAddSignature(keys.at(3)) .signAndAddSignature(keys.at(4))); - ASSERT_TRUE(createAndValidateChain({clone(block2), clone(block3)})); + ASSERT_TRUE(createAndValidateChain({block2, block3})); } /** @@ -160,18 +161,18 @@ namespace iroha { TEST_F(ChainValidatorStorageTest, NoPeerAdded) { auto block1 = generateAndApplyFirstBlock(); - auto block2 = completeBlock(baseBlock({dummyTx(2)}, 2, block1.hash()) + auto block2 = completeBlock(baseBlock({dummyTx(2)}, 2, block1->hash()) .signAndAddSignature(keys.at(0)) .signAndAddSignature(keys.at(1)) .signAndAddSignature(keys.at(2))); - auto block3 = completeBlock(baseBlock({dummyTx(3)}, 3, block2.hash()) + auto block3 = completeBlock(baseBlock({dummyTx(3)}, 3, block2->hash()) .signAndAddSignature(keys.at(0)) .signAndAddSignature(keys.at(1)) .signAndAddSignature(keys.at(2)) .signAndAddSignature(keys.at(3))); - ASSERT_TRUE(createAndValidateChain({clone(block2), clone(block3)})); + ASSERT_TRUE(createAndValidateChain({block2, block3})); } /** @@ -194,7 +195,7 @@ namespace iroha { .signAndAddSignature(keys.at(2)) .signAndAddSignature(keys.at(3))); - ASSERT_FALSE(createAndValidateChain({clone(block2)})); + ASSERT_FALSE(createAndValidateChain({block2})); } /** @@ -209,11 +210,11 @@ namespace iroha { ASSERT_FALSE(supermajority_checker->hasSupermajority(2, 4)) << "This test assumes that 2 out of 4 peers do not have supermajority!"; - auto block2 = completeBlock(baseBlock({dummyTx(2)}, 2, block1.hash()) + auto block2 = completeBlock(baseBlock({dummyTx(2)}, 2, block1->hash()) .signAndAddSignature(keys.at(0)) .signAndAddSignature(keys.at(1))); - ASSERT_FALSE(createAndValidateChain({clone(block2)})); + ASSERT_FALSE(createAndValidateChain({block2})); } } // namespace iroha diff --git a/test/module/irohad/ametsuchi/ametsuchi_test.cpp b/test/module/irohad/ametsuchi/ametsuchi_test.cpp index a09fe2948f..f58e867cd9 100644 --- a/test/module/irohad/ametsuchi/ametsuchi_test.cpp +++ b/test/module/irohad/ametsuchi/ametsuchi_test.cpp @@ -80,7 +80,8 @@ void validateAccount(W &&wsv, * @param block to apply */ template -void apply(S &&storage, const shared_model::interface::Block &block) { +void apply(S &&storage, + std::shared_ptr block) { std::unique_ptr ms; auto storageResult = storage->createMutableStorage(); storageResult.match( @@ -99,11 +100,11 @@ TEST_F(AmetsuchiTest, GetBlocksCompletedWhenCalled) { ASSERT_TRUE(storage); auto blocks = storage->getBlockQuery(); - auto block = TestBlockBuilder().height(1).prevHash(fake_hash).build(); + auto block = createBlock({}, 1, fake_hash); apply(storage, block); - ASSERT_EQ(*blocks->getBlocks(1, 1)[0], block); + ASSERT_EQ(*blocks->getBlocks(1, 1)[0], *block); } TEST_F(AmetsuchiTest, SampleTest) { @@ -125,11 +126,7 @@ TEST_F(AmetsuchiTest, SampleTest) { .createDomain(domain, "user") .createAccount(user1name, domain, fake_pubkey) .build()); - auto block1 = TestBlockBuilder() - .transactions(txs) - .height(1) - .prevHash(fake_hash) - .build(); + auto block1 = createBlock(txs, 1, fake_hash); apply(storage, block1); @@ -145,11 +142,7 @@ TEST_F(AmetsuchiTest, SampleTest) { .addAssetQuantity(assetid, "150.0") .transferAsset(user1id, user2id, assetid, "Transfer asset", "100.0") .build()); - auto block2 = TestBlockBuilder() - .transactions(txs) - .height(2) - .prevHash(block1.hash()) - .build(); + auto block2 = createBlock(txs, 2, block1->hash()); apply(storage, block2); validateAccountAsset( @@ -158,7 +151,7 @@ TEST_F(AmetsuchiTest, SampleTest) { sql_query, user2id, assetid, shared_model::interface::Amount("100.0")); // Block store tests - auto hashes = {block1.hash(), block2.hash()}; + auto hashes = {block1->hash(), block2->hash()}; auto stored_blocks = blocks->getBlocks(1, 2); ASSERT_EQ(2, stored_blocks.size()); @@ -175,7 +168,7 @@ TEST_F(AmetsuchiTest, PeerTest) { .addPeer("192.168.9.1:50051", fake_pubkey) .build()); - auto block = TestBlockBuilder().transactions(txs).prevHash(fake_hash).build(); + auto block = createBlock(txs, 1, fake_hash); apply(storage, block); @@ -207,11 +200,7 @@ TEST_F(AmetsuchiTest, AddSignatoryTest) { .createDomain("domain", "user") .createAccount("userone", "domain", pubkey1) .build()); - auto block1 = TestBlockBuilder() - .transactions(txs) - .height(1) - .prevHash(fake_hash) - .build(); + auto block1 = createBlock(txs, 1, fake_hash); apply(storage, block1); @@ -235,11 +224,7 @@ TEST_F(AmetsuchiTest, AddSignatoryTest) { .addSignatory(user1id, pubkey2) .build()); - auto block2 = TestBlockBuilder() - .transactions(txs) - .height(2) - .prevHash(block1.hash()) - .build(); + auto block2 = createBlock(txs, 2, block1->hash()); apply(storage, block2); @@ -261,11 +246,7 @@ TEST_F(AmetsuchiTest, AddSignatoryTest) { .createAccount("usertwo", "domain", pubkey1) .build()); - auto block3 = TestBlockBuilder() - .transactions(txs) - .height(3) - .prevHash(block2.hash()) - .build(); + auto block3 = createBlock(txs, 3, block2->hash()); apply(storage, block3); @@ -295,11 +276,7 @@ TEST_F(AmetsuchiTest, AddSignatoryTest) { .removeSignatory(user1id, pubkey1) .build()); - auto block4 = TestBlockBuilder() - .transactions(txs) - .height(4) - .prevHash(block3.hash()) - .build(); + auto block4 = createBlock(txs, 4, block3->hash()); apply(storage, block4); @@ -328,11 +305,7 @@ TEST_F(AmetsuchiTest, AddSignatoryTest) { .setAccountQuorum(user2id, 2) .build()); - auto block5 = TestBlockBuilder() - .transactions(txs) - .height(5) - .prevHash(block4.hash()) - .build(); + auto block5 = createBlock(txs, 5, block4->hash()); apply(storage, block5); @@ -358,11 +331,7 @@ TEST_F(AmetsuchiTest, AddSignatoryTest) { .setAccountQuorum(user2id, 2) .build()); - auto block6 = TestBlockBuilder() - .transactions(txs) - .height(6) - .prevHash(block5.hash()) - .build(); + auto block6 = createBlock(txs, 6, block5->hash()); apply(storage, block6); @@ -375,18 +344,14 @@ TEST_F(AmetsuchiTest, AddSignatoryTest) { } } -shared_model::proto::Block getBlock() { +std::shared_ptr getBlock() { std::vector txs; txs.push_back(TestTransactionBuilder() .creatorAccountId("adminone") .addPeer("192.168.0.0:10001", fake_pubkey) .build()); - auto block = TestBlockBuilder() - .transactions(txs) - .height(1) - .prevHash(fake_hash) - .build(); + auto block = createBlock(txs, 1, fake_hash); return block; } @@ -430,7 +395,7 @@ TEST_F(AmetsuchiTest, TestingStorageWhenCommitBlock) { static auto wrapper = make_test_subscriber(storage->on_commit(), 1); wrapper.subscribe([&expected_block](const auto &block) { - ASSERT_EQ(*block, expected_block); + ASSERT_EQ(block, expected_block); }); std::unique_ptr mutable_storage; @@ -479,13 +444,7 @@ TEST_F(AmetsuchiTest, TestRestoreWSV) { generateKeypair()) .finish()); - auto genesis_block = TestBlockBuilder() - .transactions(genesis_tx) - .height(1) - .prevHash(shared_model::crypto::Sha3_256::makeHash( - shared_model::crypto::Blob(""))) - .createdTime(iroha::time::now()) - .build(); + auto genesis_block = createBlock(genesis_tx); apply(storage, genesis_block); @@ -543,15 +502,8 @@ TEST_F(AmetsuchiTest, TestingWsvAfterCommitBlock) { .build() .signAndAddSignature(key) .finish(); - auto genesis_block = - TestBlockBuilder() - .transactions( - std::vector({genesis_tx})) - .height(1) - .prevHash(shared_model::crypto::Sha3_256::makeHash( - shared_model::crypto::Blob(""))) - .createdTime(iroha::time::now()) - .build(); + + auto genesis_block = createBlock({genesis_tx}); apply(storage, genesis_block); auto add_ast_tx = @@ -565,19 +517,12 @@ TEST_F(AmetsuchiTest, TestingWsvAfterCommitBlock) { .signAndAddSignature(key) .finish(); - auto expected_block = - TestBlockBuilder() - .transactions( - std::vector({add_ast_tx})) - .height(1) - .prevHash(genesis_block.hash()) - .createdTime(iroha::time::now()) - .build(); + auto expected_block = createBlock({add_ast_tx}, 1, genesis_block->hash()); static auto wrapper = make_test_subscriber(storage->on_commit(), 1); wrapper.subscribe([&](const auto &block) { - ASSERT_EQ(*block, expected_block); + ASSERT_EQ(*block, *expected_block); shared_model::interface::Amount resultingAmount("10.00"); validateAccountAsset( sql_query, "receiver@test", "coin#test", resultingAmount); @@ -606,17 +551,6 @@ class PreparedBlockTest : public AmetsuchiTest { .finish(); } - shared_model::proto::Block createBlock( - std::initializer_list txs) { - return TestBlockBuilder() - .transactions(std::vector(txs)) - .height(1) - .prevHash(shared_model::crypto::Sha3_256::makeHash( - shared_model::crypto::Blob(""))) - .createdTime(iroha::time::now()) - .build(); - } - void SetUp() override { AmetsuchiTest::SetUp(); genesis_tx = @@ -638,9 +572,9 @@ class PreparedBlockTest : public AmetsuchiTest { .build() .signAndAddSignature(key) .finish()); - genesis_block = clone(createBlock({*genesis_tx})); + genesis_block = createBlock({*genesis_tx}); initial_tx = clone(createAddAsset("5.00")); - apply(storage, *genesis_block); + apply(storage, genesis_block); using framework::expected::val; temp_wsv = std::move(val(storage->createTemporaryWsv())->value); } @@ -650,7 +584,7 @@ class PreparedBlockTest : public AmetsuchiTest { std::string default_role{"admin"}; std::unique_ptr genesis_tx; std::unique_ptr initial_tx; - std::unique_ptr genesis_block; + std::shared_ptr genesis_block; std::unique_ptr temp_wsv; shared_model::interface::Amount base_balance{"5.00"}; }; diff --git a/test/module/irohad/ametsuchi/mock_mutable_factory.hpp b/test/module/irohad/ametsuchi/mock_mutable_factory.hpp index 5f2f2325a2..8bb9df91a5 100644 --- a/test/module/irohad/ametsuchi/mock_mutable_factory.hpp +++ b/test/module/irohad/ametsuchi/mock_mutable_factory.hpp @@ -27,7 +27,7 @@ namespace iroha { MOCK_METHOD1(commitPrepared, boost::optional>( - const shared_model::interface::Block &)); + std::shared_ptr)); MOCK_METHOD1(commit_, boost::optional>( std::unique_ptr &)); diff --git a/test/module/irohad/ametsuchi/mock_mutable_storage.hpp b/test/module/irohad/ametsuchi/mock_mutable_storage.hpp index cd85e14191..bbbd4bad12 100644 --- a/test/module/irohad/ametsuchi/mock_mutable_storage.hpp +++ b/test/module/irohad/ametsuchi/mock_mutable_storage.hpp @@ -20,11 +20,13 @@ namespace iroha { bool(rxcpp::observable< std::shared_ptr>, std::function< - bool(const shared_model::interface::Block &, + bool(std::shared_ptr, PeerQuery &, const shared_model::interface::types::HashType &)>)); - MOCK_METHOD1(apply, bool(const shared_model::interface::Block &)); - MOCK_METHOD1(applyPrepared, bool(const shared_model::interface::Block &)); + MOCK_METHOD1(apply, + bool(std::shared_ptr)); + MOCK_METHOD1(applyPrepared, + bool(std::shared_ptr)); }; } // namespace ametsuchi diff --git a/test/module/irohad/ametsuchi/mock_storage.hpp b/test/module/irohad/ametsuchi/mock_storage.hpp index 92db4670c8..140bb30e6c 100644 --- a/test/module/irohad/ametsuchi/mock_storage.hpp +++ b/test/module/irohad/ametsuchi/mock_storage.hpp @@ -39,8 +39,9 @@ namespace iroha { MutableStorage *storage)); MOCK_METHOD1(commitPrepared, boost::optional>( - const shared_model::interface::Block &)); - MOCK_METHOD1(insertBlock, bool(const shared_model::interface::Block &)); + std::shared_ptr)); + MOCK_METHOD1(insertBlock, + bool(std::shared_ptr)); MOCK_METHOD1(insertBlocks, bool(const std::vector< std::shared_ptr> &)); @@ -54,7 +55,7 @@ namespace iroha { prepareBlock_(wsv); } - rxcpp::observable> + rxcpp::observable> on_commit() override { return notifier.get_observable(); } @@ -62,7 +63,8 @@ namespace iroha { std::unique_ptr storage) override { return doCommit(storage.get()); } - rxcpp::subjects::subject> + rxcpp::subjects::subject< + std::shared_ptr> notifier; }; diff --git a/test/module/irohad/ametsuchi/postgres_query_executor_test.cpp b/test/module/irohad/ametsuchi/postgres_query_executor_test.cpp index cddb73788e..e6f3ee1d9c 100644 --- a/test/module/irohad/ametsuchi/postgres_query_executor_test.cpp +++ b/test/module/irohad/ametsuchi/postgres_query_executor_test.cpp @@ -800,16 +800,13 @@ namespace iroha { auto prev_hash = shared_model::crypto::Hash(zero_string); for (decltype(number_of_blocks) i = 1; i < number_of_blocks; ++i) { auto block = - TestBlockBuilder() - .transactions(std::vector{ - TestTransactionBuilder() - .creatorAccountId(account_id) - .createAsset(std::to_string(i), domain_id, 1) - .build()}) - .height(i) - .prevHash(prev_hash) - .build(); - prev_hash = block.hash(); + createBlock({TestTransactionBuilder() + .creatorAccountId(account_id) + .createAsset(std::to_string(i), domain_id, 1) + .build()}, + i, + prev_hash); + prev_hash = block->hash(); if (not ms->apply(block)) { FAIL() << "could not apply block to the storage"; @@ -1066,7 +1063,8 @@ namespace iroha { * @param block to apply */ template - void apply(S &&storage, const shared_model::interface::Block &block) { + void apply(S &&storage, + std::shared_ptr block) { std::unique_ptr ms; auto storageResult = storage->createMutableStorage(); storageResult.match( @@ -1080,7 +1078,6 @@ namespace iroha { } void commitBlocks() { - auto fake_hash = shared_model::crypto::Hash(zero_string); auto fake_pubkey = shared_model::crypto::PublicKey(zero_string); std::vector txs1; @@ -1099,11 +1096,7 @@ namespace iroha { .createRole("user2", {}) .build()); - auto block1 = TestBlockBuilder() - .transactions(txs1) - .height(1) - .prevHash(fake_hash) - .build(); + auto block1 = createBlock(txs1, 1); apply(storage, block1); @@ -1118,11 +1111,7 @@ namespace iroha { .createRole("user3", {}) .build()); - auto block2 = TestBlockBuilder() - .transactions(txs2) - .height(2) - .prevHash(block1.hash()) - .build(); + auto block2 = createBlock(txs2, 2, block1->hash()); apply(storage, block2); @@ -1132,7 +1121,6 @@ namespace iroha { } const std::string asset_id = "coin#domain"; - shared_model::crypto::Hash fake_hash{zero_string}; shared_model::crypto::PublicKey fake_pubkey{zero_string}; shared_model::crypto::Hash hash1; shared_model::crypto::Hash hash2; @@ -1159,11 +1147,7 @@ namespace iroha { initial_txs.emplace_back(std::move(tx)); } - auto block = TestBlockBuilder() - .transactions(initial_txs) - .height(1) - .prevHash(fake_hash) - .build(); + auto block = createBlock(initial_txs, 1); apply(storage, block); } diff --git a/test/module/irohad/torii/processor/transaction_processor_test.cpp b/test/module/irohad/torii/processor/transaction_processor_test.cpp index b49bd7a7ee..492af01ef7 100644 --- a/test/module/irohad/torii/processor/transaction_processor_test.cpp +++ b/test/module/irohad/torii/processor/transaction_processor_test.cpp @@ -124,7 +124,8 @@ class TransactionProcessorTest : public ::testing::Test { mst_update_notifier; rxcpp::subjects::subject mst_prepared_notifier; rxcpp::subjects::subject mst_expired_notifier; - rxcpp::subjects::subject> + rxcpp::subjects::subject< + std::shared_ptr> commit_notifier; rxcpp::subjects::subject verified_prop_notifier; diff --git a/test/module/irohad/validation/chain_validation_test.cpp b/test/module/irohad/validation/chain_validation_test.cpp index 8126b931e7..29ef7dc8c0 100644 --- a/test/module/irohad/validation/chain_validation_test.cpp +++ b/test/module/irohad/validation/chain_validation_test.cpp @@ -85,7 +85,7 @@ TEST_F(ChainValidationTest, ValidCase) { EXPECT_CALL(*query, getLedgerPeers()).WillOnce(Return(peers)); EXPECT_CALL(*storage, apply(blocks, _)) - .WillOnce(InvokeArgument<1>(ByRef(*block), ByRef(*query), ByRef(hash))); + .WillOnce(InvokeArgument<1>(block, ByRef(*query), ByRef(hash))); ASSERT_TRUE(validator->validateAndApply(blocks, *storage)); ASSERT_EQ(boost::size(block->signatures()), block_signatures_amount); @@ -107,8 +107,7 @@ TEST_F(ChainValidationTest, FailWhenDifferentPrevHash) { EXPECT_CALL(*query, getLedgerPeers()).WillOnce(Return(peers)); EXPECT_CALL(*storage, apply(blocks, _)) - .WillOnce( - InvokeArgument<1>(ByRef(*block), ByRef(*query), ByRef(another_hash))); + .WillOnce(InvokeArgument<1>(block, ByRef(*query), ByRef(another_hash))); ASSERT_FALSE(validator->validateAndApply(blocks, *storage)); } @@ -127,7 +126,7 @@ TEST_F(ChainValidationTest, FailWhenNoSupermajority) { EXPECT_CALL(*query, getLedgerPeers()).WillOnce(Return(peers)); EXPECT_CALL(*storage, apply(blocks, _)) - .WillOnce(InvokeArgument<1>(ByRef(*block), ByRef(*query), ByRef(hash))); + .WillOnce(InvokeArgument<1>(block, ByRef(*query), ByRef(hash))); ASSERT_FALSE(validator->validateAndApply(blocks, *storage)); ASSERT_EQ(boost::size(block->signatures()), block_signatures_amount); diff --git a/test/module/shared_model/builders/protobuf/test_block_builder.hpp b/test/module/shared_model/builders/protobuf/test_block_builder.hpp index 8eba5c9b33..f5eac48d9a 100644 --- a/test/module/shared_model/builders/protobuf/test_block_builder.hpp +++ b/test/module/shared_model/builders/protobuf/test_block_builder.hpp @@ -27,4 +27,17 @@ using TestUnsignedBlockBuilder = shared_model::proto::TemplateBlockBuilder< shared_model::validation::AlwaysValidValidator, shared_model::proto::UnsignedWrapper>; +std::shared_ptr createBlock( + std::vector txs, + size_t height = 1, + shared_model::crypto::Hash prev_hash = shared_model::crypto::Hash("")) { + return clone( + TestBlockBuilder() + .transactions(std::vector(txs)) + .height(height) + .prevHash(prev_hash) + .createdTime(iroha::time::now()) + .build()); +} + #endif // IROHA_TEST_BLOCK_BUILDER_HPP From fafc0958eed6f5064f3e181849e7b47ab91927a1 Mon Sep 17 00:00:00 2001 From: Konstantin Munichev Date: Wed, 20 Mar 2019 21:54:14 +0300 Subject: [PATCH 38/40] Flat file block storage (#2180) Signed-off-by: Konstantin Munichev --- irohad/ametsuchi/CMakeLists.txt | 2 + irohad/ametsuchi/block_storage_factory.hpp | 1 - irohad/ametsuchi/impl/flat_file/flat_file.cpp | 88 ++++----- irohad/ametsuchi/impl/flat_file/flat_file.hpp | 42 ++-- .../impl/flat_file_block_storage.cpp | 85 +++++++++ .../impl/flat_file_block_storage.hpp | 50 +++++ .../impl/flat_file_block_storage_factory.cpp | 31 +++ .../impl/flat_file_block_storage_factory.hpp | 34 ++++ irohad/main/application.cpp | 13 +- .../block_json_deserializer.hpp | 2 +- .../iroha_internal/block_json_serializer.hpp | 2 +- test/module/irohad/ametsuchi/CMakeLists.txt | 6 + .../flat_file_block_storage_test.cpp | 180 ++++++++++++++++++ .../irohad/ametsuchi/flat_file_test.cpp | 62 +++++- .../in_memory_block_storage_test.cpp | 8 +- test/module/shared_model/interface_mocks.hpp | 15 ++ 16 files changed, 531 insertions(+), 90 deletions(-) create mode 100644 irohad/ametsuchi/impl/flat_file_block_storage.cpp create mode 100644 irohad/ametsuchi/impl/flat_file_block_storage.hpp create mode 100644 irohad/ametsuchi/impl/flat_file_block_storage_factory.cpp create mode 100644 irohad/ametsuchi/impl/flat_file_block_storage_factory.hpp create mode 100644 test/module/irohad/ametsuchi/flat_file_block_storage_test.cpp diff --git a/irohad/ametsuchi/CMakeLists.txt b/irohad/ametsuchi/CMakeLists.txt index 158f6ec988..cd9860493f 100644 --- a/irohad/ametsuchi/CMakeLists.txt +++ b/irohad/ametsuchi/CMakeLists.txt @@ -20,6 +20,8 @@ add_library(ametsuchi impl/tx_presence_cache_impl.cpp impl/in_memory_block_storage.cpp impl/in_memory_block_storage_factory.cpp + impl/flat_file_block_storage.cpp + impl/flat_file_block_storage_factory.cpp ) target_link_libraries(ametsuchi diff --git a/irohad/ametsuchi/block_storage_factory.hpp b/irohad/ametsuchi/block_storage_factory.hpp index c0e7655407..e738813d12 100644 --- a/irohad/ametsuchi/block_storage_factory.hpp +++ b/irohad/ametsuchi/block_storage_factory.hpp @@ -12,7 +12,6 @@ namespace iroha { namespace ametsuchi { - /** * Creates a block storage */ diff --git a/irohad/ametsuchi/impl/flat_file/flat_file.cpp b/irohad/ametsuchi/impl/flat_file/flat_file.cpp index b3b73643d2..5620205b0d 100644 --- a/irohad/ametsuchi/impl/flat_file/flat_file.cpp +++ b/irohad/ametsuchi/impl/flat_file/flat_file.cpp @@ -18,6 +18,7 @@ using namespace iroha::ametsuchi; using Identifier = FlatFile::Identifier; +using BlockIdCollectionType = FlatFile::BlockIdCollectionType; // ----------| public API |---------- @@ -27,9 +28,20 @@ std::string FlatFile::id_to_name(Identifier id) { return os.str(); } +boost::optional FlatFile::name_to_id(const std::string &name) { + if (name.size() != FlatFile::DIGIT_CAPACITY) { + return boost::none; + } + try { + auto id = std::stoul(name); + return boost::make_optional(id); + } catch (const std::exception &e) { + return boost::none; + } +} + boost::optional> FlatFile::create( - const std::string &path, - logger::LoggerPtr log) { + const std::string &path, logger::LoggerPtr log) { boost::system::error_code err; if (not boost::filesystem::is_directory(path, err) and not boost::filesystem::create_directory(path, err)) { @@ -37,19 +49,24 @@ boost::optional> FlatFile::create( return boost::none; } - auto res = FlatFile::check_consistency(path, log); - return std::make_unique(*res, path, private_tag{}, std::move(log)); + BlockIdCollectionType files_found; + for (auto it = boost::filesystem::directory_iterator{path}; + it != boost::filesystem::directory_iterator{}; + ++it) { + if (auto id = FlatFile::name_to_id(it->path().filename().string())) { + files_found.insert(*id); + } else { + boost::filesystem::remove(it->path()); + } + } + + return std::make_unique( + path, std::move(files_found), private_tag{}, std::move(log)); } bool FlatFile::add(Identifier id, const Bytes &block) { // TODO(x3medima17): Change bool to generic Result return type - if (id != current_id_ + 1) { - log_->warn("Cannot append non-consecutive block"); - return false; - } - - auto next_id = id; const auto file_name = boost::filesystem::path{dump_dir_} / id_to_name(id); // Write block to binary file @@ -71,8 +88,7 @@ bool FlatFile::add(Identifier id, const Bytes &block) { file.write(reinterpret_cast(block.data()), block.size() * val_size); - // Update internals, release lock - current_id_ = next_id; + available_blocks_.insert(id); return true; } @@ -100,50 +116,24 @@ std::string FlatFile::directory() const { } Identifier FlatFile::last_id() const { - return current_id_.load(); + return (available_blocks_.empty()) ? 0 : *available_blocks_.rbegin(); } void FlatFile::dropAll() { iroha::remove_dir_contents(dump_dir_, log_); - auto res = FlatFile::check_consistency(dump_dir_, log_); - current_id_.store(*res); + available_blocks_.clear(); +} + +const BlockIdCollectionType &FlatFile::blockIdentifiers() const { + return available_blocks_; } // ----------| private API |---------- -FlatFile::FlatFile(Identifier current_id, - const std::string &path, +FlatFile::FlatFile(std::string path, + BlockIdCollectionType existing_files, FlatFile::private_tag, logger::LoggerPtr log) - : dump_dir_(path), log_{std::move(log)} { - current_id_.store(current_id); -} - -boost::optional FlatFile::check_consistency( - const std::string &dump_dir, logger::LoggerPtr log) { - if (dump_dir.empty()) { - log->error("check_consistency({}), not directory", dump_dir); - return boost::none; - } - - auto const files = [&dump_dir] { - std::vector ps; - std::copy(boost::filesystem::directory_iterator{dump_dir}, - boost::filesystem::directory_iterator{}, - std::back_inserter(ps)); - std::sort(ps.begin(), ps.end(), std::less()); - return ps; - }(); - - auto const missing = boost::range::find_if( - files | boost::adaptors::indexed(1), [](const auto &it) { - return FlatFile::id_to_name(it.index()) != it.value().filename(); - }); - - std::for_each( - missing.get(), files.cend(), [](const boost::filesystem::path &p) { - boost::filesystem::remove(p); - }); - - return missing.get() - files.cbegin(); -} + : dump_dir_(std::move(path)), + available_blocks_(std::move(existing_files)), + log_{std::move(log)} {} diff --git a/irohad/ametsuchi/impl/flat_file/flat_file.hpp b/irohad/ametsuchi/impl/flat_file/flat_file.hpp index da20283075..59f7081848 100644 --- a/irohad/ametsuchi/impl/flat_file/flat_file.hpp +++ b/irohad/ametsuchi/impl/flat_file/flat_file.hpp @@ -8,8 +8,8 @@ #include "ametsuchi/key_value_storage.hpp" -#include #include +#include #include "logger/logger_fwd.hpp" @@ -29,6 +29,8 @@ namespace iroha { public: // ----------| public API |---------- + using BlockIdCollectionType = std::set; + static const uint32_t DIGIT_CAPACITY = 16; /** @@ -45,6 +47,13 @@ namespace iroha { */ static std::string id_to_name(Identifier id); + /** + * Converts aligned string (see above) to number. + * @param name - name to convert + * @return id or boost::none + */ + static boost::optional name_to_id(const std::string &name); + /** * Create storage in paths * @param path - target path for creating @@ -62,18 +71,12 @@ namespace iroha { Identifier last_id() const override; + void dropAll() override; + /** - * Checking consistency of storage for provided folder - * If some block in the middle is missing all blocks following it are - * deleted - * @param dump_dir - folder of storage - * @param log - log for local messages - * @return - last available identifier + * @return collection of available block ids */ - static boost::optional check_consistency( - const std::string &dump_dir, logger::LoggerPtr log); - - void dropAll() override; + const BlockIdCollectionType &blockIdentifiers() const; // ----------| modify operations |---------- @@ -88,29 +91,24 @@ namespace iroha { // ----------| private API |---------- /** - * Create storage in path with respect to last key - * @param last_id - maximal key written in storage + * Create storage in path * @param path - folder of storage + * @param existing_files - collection of existing files names * @param log to print progress */ - FlatFile(Identifier last_id, - const std::string &path, + FlatFile(std::string path, + BlockIdCollectionType existing_files, FlatFile::private_tag, logger::LoggerPtr log); private: - // ----------| private fields |---------- - - /** - * Last written key - */ - std::atomic current_id_; - /** * Folder of storage */ const std::string dump_dir_; + BlockIdCollectionType available_blocks_; + logger::LoggerPtr log_; public: diff --git a/irohad/ametsuchi/impl/flat_file_block_storage.cpp b/irohad/ametsuchi/impl/flat_file_block_storage.cpp new file mode 100644 index 0000000000..dba89a8a1e --- /dev/null +++ b/irohad/ametsuchi/impl/flat_file_block_storage.cpp @@ -0,0 +1,85 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "ametsuchi/impl/flat_file_block_storage.hpp" + +#include + +#include "backend/protobuf/block.hpp" +#include "common/byteutils.hpp" +#include "logger/logger.hpp" + +using namespace iroha::ametsuchi; + +FlatFileBlockStorage::FlatFileBlockStorage( + std::unique_ptr flat_file, + std::shared_ptr json_converter, + logger::LoggerPtr log) + : flat_file_storage_(std::move(flat_file)), + json_converter_(std::move(json_converter)), + log_(std::move(log)) {} + +FlatFileBlockStorage::~FlatFileBlockStorage() { + log_->info("Remove {} temp directory", flat_file_storage_->directory()); + boost::filesystem::remove_all(flat_file_storage_->directory()); +} + +bool FlatFileBlockStorage::insert( + std::shared_ptr block) { + return json_converter_->serialize(*block).match( + [&](const expected::Value &block_json) { + return flat_file_storage_->add(block->height(), + stringToBytes(block_json.value)); + }, + [this](const auto &error) { + log_->warn("Error while block serialization: {}", error.error); + return false; + }); +} + +bool FlatFileBlockStorage::insert(const shared_model::interface::Block &block) { + return insert(clone(block)); +} + +boost::optional> +FlatFileBlockStorage::fetch( + shared_model::interface::types::HeightType height) const { + auto storage_block = flat_file_storage_->get(height); + if (not storage_block) { + return boost::none; + } + + return json_converter_->deserialize(bytesToString(*storage_block)) + .match( + [&](expected::Value> + &block) { + return boost::make_optional< + std::shared_ptr>( + std::move(block.value)); + }, + [&](expected::Error &error) + -> boost::optional< + std::shared_ptr> { + log_->warn("Error while block deserialization: {}", error.error); + return boost::none; + }); +} + +size_t FlatFileBlockStorage::size() const { + return flat_file_storage_->blockIdentifiers().size(); +} + +void FlatFileBlockStorage::clear() { + flat_file_storage_->dropAll(); +} + +void FlatFileBlockStorage::forEach( + iroha::ametsuchi::BlockStorage::FunctionType function) const { + for (auto block_id : flat_file_storage_->blockIdentifiers()) { + auto block = fetch(block_id); + BOOST_ASSERT(block); + function(*block); + } +} diff --git a/irohad/ametsuchi/impl/flat_file_block_storage.hpp b/irohad/ametsuchi/impl/flat_file_block_storage.hpp new file mode 100644 index 0000000000..3cdb7be820 --- /dev/null +++ b/irohad/ametsuchi/impl/flat_file_block_storage.hpp @@ -0,0 +1,50 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef IROHA_FLAT_FILE_BLOCK_STORAGE_HPP +#define IROHA_FLAT_FILE_BLOCK_STORAGE_HPP + +#include "ametsuchi/block_storage.hpp" + +#include "ametsuchi/impl/flat_file/flat_file.hpp" +#include "interfaces/iroha_internal/block_json_converter.hpp" +#include "logger/logger_fwd.hpp" + +namespace iroha { + namespace ametsuchi { + class FlatFileBlockStorage : public BlockStorage { + public: + FlatFileBlockStorage( + std::unique_ptr flat_file, + std::shared_ptr + json_converter, + logger::LoggerPtr log); + + ~FlatFileBlockStorage() override; + + bool insert( + std::shared_ptr block) override; + + bool insert(const shared_model::interface::Block &block) override; + + boost::optional> + fetch(shared_model::interface::types::HeightType height) const override; + + size_t size() const override; + + void clear() override; + + void forEach(FunctionType function) const override; + + private: + std::unique_ptr flat_file_storage_; + std::shared_ptr + json_converter_; + logger::LoggerPtr log_; + }; + } // namespace ametsuchi +} // namespace iroha + +#endif // IROHA_FLAT_FILE_BLOCK_STORAGE_HPP diff --git a/irohad/ametsuchi/impl/flat_file_block_storage_factory.cpp b/irohad/ametsuchi/impl/flat_file_block_storage_factory.cpp new file mode 100644 index 0000000000..6b69e46f66 --- /dev/null +++ b/irohad/ametsuchi/impl/flat_file_block_storage_factory.cpp @@ -0,0 +1,31 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "ametsuchi/impl/flat_file_block_storage_factory.hpp" + +#include "ametsuchi/impl/flat_file_block_storage.hpp" + +using namespace iroha::ametsuchi; + +FlatFileBlockStorageFactory::FlatFileBlockStorageFactory( + std::function path_provider, + std::shared_ptr + json_block_converter, + logger::LoggerManagerTreePtr log_manager) + : path_provider_(std::move(path_provider)), + json_block_converter_(std::move(json_block_converter)), + log_manager_(std::move(log_manager)) {} + +std::unique_ptr FlatFileBlockStorageFactory::create() { + auto flat_file = FlatFile::create( + path_provider_(), log_manager_->getChild("FlatFile")->getLogger()); + if (not flat_file) { + return nullptr; + } + return std::make_unique( + std::move(flat_file.get()), + json_block_converter_, + log_manager_->getChild("FlatFileBlockFactory")->getLogger()); +} diff --git a/irohad/ametsuchi/impl/flat_file_block_storage_factory.hpp b/irohad/ametsuchi/impl/flat_file_block_storage_factory.hpp new file mode 100644 index 0000000000..30d9f288e5 --- /dev/null +++ b/irohad/ametsuchi/impl/flat_file_block_storage_factory.hpp @@ -0,0 +1,34 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef IROHA_FLAT_FILE_BLOCK_STORAGE_FACTORY_HPP +#define IROHA_FLAT_FILE_BLOCK_STORAGE_FACTORY_HPP + +#include "ametsuchi/block_storage_factory.hpp" + +#include "interfaces/iroha_internal/block_json_converter.hpp" +#include "logger/logger_manager.hpp" + +namespace iroha { + namespace ametsuchi { + class FlatFileBlockStorageFactory : public BlockStorageFactory { + public: + FlatFileBlockStorageFactory( + std::function path_provider, + std::shared_ptr + json_block_converter, + logger::LoggerManagerTreePtr log_manager); + std::unique_ptr create() override; + + private: + std::function path_provider_; + std::shared_ptr + json_block_converter_; + logger::LoggerManagerTreePtr log_manager_; + }; + } // namespace ametsuchi +} // namespace iroha + +#endif // IROHA_FLAT_FILE_BLOCK_STORAGE_FACTORY_HPP diff --git a/irohad/main/application.cpp b/irohad/main/application.cpp index 751becedc9..dc2085236b 100644 --- a/irohad/main/application.cpp +++ b/irohad/main/application.cpp @@ -5,7 +5,9 @@ #include "main/application.hpp" -#include "ametsuchi/impl/in_memory_block_storage_factory.hpp" +#include + +#include "ametsuchi/impl/flat_file_block_storage_factory.hpp" #include "ametsuchi/impl/storage_impl.hpp" #include "ametsuchi/impl/tx_presence_cache_impl.hpp" #include "ametsuchi/impl/wsv_restorer_impl.hpp" @@ -161,7 +163,14 @@ void Irohad::initStorage() { std::make_shared(); auto block_converter = std::make_shared(); - auto block_storage_factory = std::make_unique(); + auto block_storage_factory = std::make_unique( + []() { + return (boost::filesystem::temp_directory_path() + / boost::filesystem::unique_path()) + .string(); + }, + block_converter, + log_manager_); auto storageResult = StorageImpl::create(block_store_dir_, pg_conn_, common_objects_factory_, diff --git a/shared_model/interfaces/iroha_internal/block_json_deserializer.hpp b/shared_model/interfaces/iroha_internal/block_json_deserializer.hpp index 444bfaf5b4..d57273f71a 100644 --- a/shared_model/interfaces/iroha_internal/block_json_deserializer.hpp +++ b/shared_model/interfaces/iroha_internal/block_json_deserializer.hpp @@ -26,7 +26,7 @@ namespace shared_model { * @return pointer to a block if json was valid or an error */ virtual iroha::expected::Result, std::string> - deserialize(const types::JsonType &json) const noexcept = 0; + deserialize(const types::JsonType &json) const = 0; virtual ~BlockJsonDeserializer() = default; }; diff --git a/shared_model/interfaces/iroha_internal/block_json_serializer.hpp b/shared_model/interfaces/iroha_internal/block_json_serializer.hpp index f505e5321d..238528cb67 100644 --- a/shared_model/interfaces/iroha_internal/block_json_serializer.hpp +++ b/shared_model/interfaces/iroha_internal/block_json_serializer.hpp @@ -26,7 +26,7 @@ namespace shared_model { * @return json string or an error */ virtual iroha::expected::Result - serialize(const Block &block) const noexcept = 0; + serialize(const Block &block) const = 0; virtual ~BlockJsonSerializer() = default; }; diff --git a/test/module/irohad/ametsuchi/CMakeLists.txt b/test/module/irohad/ametsuchi/CMakeLists.txt index 09bed3e35d..63e12c49fc 100644 --- a/test/module/irohad/ametsuchi/CMakeLists.txt +++ b/test/module/irohad/ametsuchi/CMakeLists.txt @@ -74,6 +74,12 @@ target_link_libraries(in_memory_block_storage_test ametsuchi ) +addtest(flat_file_block_storage_test flat_file_block_storage_test.cpp) +target_link_libraries(flat_file_block_storage_test + ametsuchi + test_logger + ) + add_library(ametsuchi_fixture INTERFACE) target_link_libraries(ametsuchi_fixture INTERFACE integration_framework_config_helper diff --git a/test/module/irohad/ametsuchi/flat_file_block_storage_test.cpp b/test/module/irohad/ametsuchi/flat_file_block_storage_test.cpp new file mode 100644 index 0000000000..7ee0e6c5d8 --- /dev/null +++ b/test/module/irohad/ametsuchi/flat_file_block_storage_test.cpp @@ -0,0 +1,180 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "ametsuchi/impl/flat_file_block_storage.hpp" +#include "ametsuchi/impl/flat_file_block_storage_factory.hpp" + +#include +#include +#include "framework/test_logger.hpp" +#include "module/shared_model/interface_mocks.hpp" + +using namespace iroha::ametsuchi; +using namespace boost::filesystem; + +using ::testing::_; +using ::testing::Invoke; +using ::testing::NiceMock; +using ::testing::Return; + +class FlatFileBlockStorageTest : public ::testing::Test { + public: + FlatFileBlockStorageTest() { + ON_CALL(*block_, height()).WillByDefault(Return(height_)); + } + + protected: + void SetUp() override { + create_directory(path_provider_()); + } + + const std::string block_store_path_ = + (temp_directory_path() / unique_path()).string(); + + std::function path_provider_ = [&]() { + return block_store_path_; + }; + + std::shared_ptr converter_ = + std::make_shared>(); + logger::LoggerManagerTreePtr log_manager_ = getTestLoggerManager(); + std::shared_ptr block_ = std::make_shared>(); + shared_model::interface::types::HeightType height_ = 1; +}; + +/** + * @given block storage factory + * @when create is called + * @then block storage is created + */ +TEST_F(FlatFileBlockStorageTest, Creation) { + auto block_storage = + FlatFileBlockStorageFactory(path_provider_, converter_, log_manager_) + .create(); + ASSERT_TRUE(block_storage); +} + +/** + * @given initialized block storage, single block with height_ inserted + * @when another block with height_ is inserted + * @then second insertion fails + */ +TEST_F(FlatFileBlockStorageTest, Insert) { + auto block_storage = + FlatFileBlockStorageFactory(path_provider_, converter_, log_manager_) + .create(); + ASSERT_TRUE(block_storage->insert(block_)); + ASSERT_FALSE(block_storage->insert(block_)); +} + +/** + * @given initialized block storage, single block with height_ inserted + * @when block with height_ is fetched + * @then it is returned + */ +TEST_F(FlatFileBlockStorageTest, FetchExisting) { + auto block_storage = + FlatFileBlockStorageFactory(path_provider_, converter_, log_manager_) + .create(); + ASSERT_TRUE(block_storage->insert(block_)); + + shared_model::interface::Block *raw_block; + + EXPECT_CALL(*converter_, deserialize(_)) + .WillOnce(Invoke([&](const shared_model::interface::types::JsonType &) + -> iroha::expected::Result< + std::unique_ptr, + std::string> { + auto return_block = std::make_unique(); + raw_block = return_block.get(); + return iroha::expected::makeValue< + std::unique_ptr>( + std::move(return_block)); + })); + std::shared_ptr block_var = + *(block_storage->fetch(height_)); + + ASSERT_EQ(raw_block, block_var.get()); +} + +/** + * @given initialized block storage without blocks + * @when block with height_ is fetched + * @then nothing is returned + */ +TEST_F(FlatFileBlockStorageTest, FetchNonexistent) { + auto block_storage = + FlatFileBlockStorageFactory(path_provider_, converter_, log_manager_) + .create(); + auto block_var = block_storage->fetch(height_); + ASSERT_FALSE(block_var); +} + +/** + * @given initialized block storage, single block with height_ inserted + * @when size is fetched + * @then 1 is returned + */ +TEST_F(FlatFileBlockStorageTest, Size) { + auto block_storage = + FlatFileBlockStorageFactory(path_provider_, converter_, log_manager_) + .create(); + ASSERT_TRUE(block_storage->insert(block_)); + + ASSERT_EQ(1, block_storage->size()); +} + +/** + * @given initialized block storage, single block with height_ inserted + * @when storage is cleared with clear + * @then no blocks are left in storage + */ +TEST_F(FlatFileBlockStorageTest, Clear) { + auto block_storage = + FlatFileBlockStorageFactory(path_provider_, converter_, log_manager_) + .create(); + ASSERT_TRUE(block_storage->insert(block_)); + + block_storage->clear(); + + auto block_var = block_storage->fetch(height_); + + ASSERT_FALSE(block_var); +} + +/** + * @given initialized block storage, single block with height_ inserted + * @when forEach is called + * @then block with height_ is visited, lambda is invoked once + */ +TEST_F(FlatFileBlockStorageTest, ForEach) { + auto block_storage = + FlatFileBlockStorageFactory(path_provider_, converter_, log_manager_) + .create(); + ASSERT_TRUE(block_storage->insert(block_)); + + shared_model::interface::Block *raw_block; + + EXPECT_CALL(*converter_, deserialize(_)) + .WillOnce(Invoke([&](const shared_model::interface::types::JsonType &) + -> iroha::expected::Result< + std::unique_ptr, + std::string> { + auto return_block = std::make_unique(); + raw_block = return_block.get(); + return iroha::expected::makeValue< + std::unique_ptr>( + std::move(return_block)); + })); + + size_t count = 0; + + block_storage->forEach([&count, &raw_block](const auto &block) { + ++count; + ASSERT_EQ(raw_block, block.get()); + }); + + ASSERT_EQ(1, count); +} diff --git a/test/module/irohad/ametsuchi/flat_file_test.cpp b/test/module/irohad/ametsuchi/flat_file_test.cpp index e4d5a1647e..f6dc7f9ea5 100644 --- a/test/module/irohad/ametsuchi/flat_file_test.cpp +++ b/test/module/irohad/ametsuchi/flat_file_test.cpp @@ -52,6 +52,12 @@ TEST_F(BlStore_Test, Read_Write_Test) { ASSERT_EQ(*res, block); } +/** + * @given initialized FlatFile storage and 3 blocks are inserted into it + * @when storage removed, file for a second block removed and new storage is + * created on the same directory + * @then last block id is still 3 + */ TEST_F(BlStore_Test, BlockStoreWhenRemoveBlock) { log_->info("----------| Simulate removal of the block |----------"); // Remove file in the middle of the block store @@ -79,7 +85,7 @@ TEST_F(BlStore_Test, BlockStoreWhenRemoveBlock) { ASSERT_TRUE(store); auto bl_store = std::move(*store); auto res = bl_store->last_id(); - ASSERT_EQ(res, 1); + ASSERT_EQ(res, 3); } TEST_F(BlStore_Test, BlockStoreWhenAbsentFolder) { @@ -120,15 +126,6 @@ TEST_F(BlStore_Test, BlockStoreInitializationFromNonemptyFolder) { ASSERT_EQ(bl_store1->last_id(), bl_store2->last_id()); } -/** - * @given empty folder name - * @then check consistency fails - */ -TEST_F(BlStore_Test, EmptyDumpDir) { - auto res = FlatFile::check_consistency("", flat_file_log_); - ASSERT_FALSE(res); -} - /** * @given empty folder with block store * @when block id does not exist @@ -217,3 +214,48 @@ TEST_F(BlStore_Test, WriteDeniedFolder) { auto res = bl_store->add(id, block); ASSERT_FALSE(res); } + +/** + * @given initialized FlatFile storage + * @when several blocks with non-consecutive ids are inserted + * @then all inserted blocks are available, block 1 is not available + */ +TEST_F(BlStore_Test, RandomNumbers) { + auto store = FlatFile::create(block_store_path, flat_file_log_); + ASSERT_TRUE(store); + auto bl_store = std::move(*store); + bl_store->add(5, block); + bl_store->add(22, block); + bl_store->add(11, block); + ASSERT_TRUE(bl_store->get(5)); + ASSERT_TRUE(bl_store->get(22)); + ASSERT_TRUE(bl_store->get(11)); + ASSERT_FALSE(bl_store->get(1)); +} + +/** + * @given initialized FlatFile storage + * @when 3 blocks with non-consecutive ids are inserted, storage removed, and + * new storage is created on the same directory + * @then only blocks with inserted ids are available + */ +TEST_F(BlStore_Test, RemoveAndCreateNew) { + { + auto store = FlatFile::create(block_store_path, flat_file_log_); + ASSERT_TRUE(store); + auto bl_store = std::move(*store); + + bl_store->add(4, block); + bl_store->add(17, block); + bl_store->add(7, block); + } + + auto store = FlatFile::create(block_store_path, flat_file_log_); + ASSERT_TRUE(store); + auto bl_store = std::move(*store); + + ASSERT_TRUE(bl_store->get(4)); + ASSERT_TRUE(bl_store->get(17)); + ASSERT_TRUE(bl_store->get(7)); + ASSERT_FALSE(bl_store->get(1)); +} diff --git a/test/module/irohad/ametsuchi/in_memory_block_storage_test.cpp b/test/module/irohad/ametsuchi/in_memory_block_storage_test.cpp index b30d927f97..37cb983a11 100644 --- a/test/module/irohad/ametsuchi/in_memory_block_storage_test.cpp +++ b/test/module/irohad/ametsuchi/in_memory_block_storage_test.cpp @@ -77,7 +77,7 @@ TEST_F(InMemoryBlockStorageTest, FetchNonexistent) { } /** - * @given initialized block storage, single block with id_ inserted + * @given initialized block storage, single block with height_ inserted * @when size is fetched * @then 1 is returned */ @@ -88,7 +88,7 @@ TEST_F(InMemoryBlockStorageTest, Size) { } /** - * @given initialized block storage, single block with id_ inserted + * @given initialized block storage, single block with height_ inserted * @when storage is cleared with clear * @then no blocks are left in storage */ @@ -103,9 +103,9 @@ TEST_F(InMemoryBlockStorageTest, Clear) { } /** - * @given initialized block storage, single block with id_ inserted + * @given initialized block storage, single block with height_ inserted * @when forEach is called - * @then block with id_ is visited, lambda is invoked once + * @then block with height_ is visited, lambda is invoked once */ TEST_F(InMemoryBlockStorageTest, ForEach) { ASSERT_TRUE(block_storage_.insert(block_)); diff --git a/test/module/shared_model/interface_mocks.hpp b/test/module/shared_model/interface_mocks.hpp index 2d387c0a44..34fb5b5e50 100644 --- a/test/module/shared_model/interface_mocks.hpp +++ b/test/module/shared_model/interface_mocks.hpp @@ -13,6 +13,7 @@ #include "interfaces/common_objects/common_objects_factory.hpp" #include "interfaces/common_objects/peer.hpp" #include "interfaces/iroha_internal/block.hpp" +#include "interfaces/iroha_internal/block_json_converter.hpp" #include "interfaces/iroha_internal/proposal.hpp" #include "interfaces/iroha_internal/transaction_batch.hpp" #include "interfaces/iroha_internal/unsafe_proposal_factory.hpp" @@ -260,4 +261,18 @@ struct MockAccount : public shared_model::interface::Account { MOCK_CONST_METHOD0(clone, MockAccount *()); }; +struct MockBlockJsonConverter + : public shared_model::interface::BlockJsonConverter { + MOCK_CONST_METHOD1( + serialize, + iroha::expected::Result( + const shared_model::interface::Block &block)); + MOCK_CONST_METHOD1( + deserialize, + iroha::expected::Result, + std::string>( + const shared_model::interface::types::JsonType &json)); +}; + #endif // IROHA_SHARED_MODEL_INTERFACE_MOCKS_HPP From 5d1db98f4247c683ca074854833a619e9c104319 Mon Sep 17 00:00:00 2001 From: Fedor Muratov Date: Thu, 21 Mar 2019 13:01:13 +0300 Subject: [PATCH 39/40] Remove redefinitions of redundant method in flat file storage (#2190) ### Description of the Change Remove redundant definition of the insert in flat file block storage. ### Benefits Improve the stability of the project. Signed-off-by: Fedor Muratov --- irohad/ametsuchi/impl/flat_file_block_storage.cpp | 4 ---- irohad/ametsuchi/impl/flat_file_block_storage.hpp | 2 -- 2 files changed, 6 deletions(-) diff --git a/irohad/ametsuchi/impl/flat_file_block_storage.cpp b/irohad/ametsuchi/impl/flat_file_block_storage.cpp index dba89a8a1e..43970474d1 100644 --- a/irohad/ametsuchi/impl/flat_file_block_storage.cpp +++ b/irohad/ametsuchi/impl/flat_file_block_storage.cpp @@ -39,10 +39,6 @@ bool FlatFileBlockStorage::insert( }); } -bool FlatFileBlockStorage::insert(const shared_model::interface::Block &block) { - return insert(clone(block)); -} - boost::optional> FlatFileBlockStorage::fetch( shared_model::interface::types::HeightType height) const { diff --git a/irohad/ametsuchi/impl/flat_file_block_storage.hpp b/irohad/ametsuchi/impl/flat_file_block_storage.hpp index 3cdb7be820..feb502cb84 100644 --- a/irohad/ametsuchi/impl/flat_file_block_storage.hpp +++ b/irohad/ametsuchi/impl/flat_file_block_storage.hpp @@ -27,8 +27,6 @@ namespace iroha { bool insert( std::shared_ptr block) override; - bool insert(const shared_model::interface::Block &block) override; - boost::optional> fetch(shared_model::interface::types::HeightType height) const override; From f8efa83121120fc2ae8b6598e3074f956920d6f6 Mon Sep 17 00:00:00 2001 From: Igor Egorov Date: Thu, 21 Mar 2019 13:46:06 +0300 Subject: [PATCH 40/40] Refactor block streaming method (processing is done in grpc thread) (#2181) Signed-off-by: Igor Egorov --- .../impl/command_service_transport_grpc.cpp | 30 +---- irohad/torii/impl/query_service.cpp | 116 +++++++++++------- libs/common/run_loop_handler.hpp | 42 +++++++ 3 files changed, 114 insertions(+), 74 deletions(-) create mode 100644 libs/common/run_loop_handler.hpp diff --git a/irohad/torii/impl/command_service_transport_grpc.cpp b/irohad/torii/impl/command_service_transport_grpc.cpp index 91636ae534..f24d1e41b1 100644 --- a/irohad/torii/impl/command_service_transport_grpc.cpp +++ b/irohad/torii/impl/command_service_transport_grpc.cpp @@ -16,6 +16,7 @@ #include #include "backend/protobuf/transaction_responses/proto_tx_response.hpp" #include "common/combine_latest_until_first_completed.hpp" +#include "common/run_loop_handler.hpp" #include "interfaces/iroha_internal/transaction_batch.hpp" #include "interfaces/iroha_internal/transaction_batch_factory.hpp" #include "interfaces/iroha_internal/transaction_batch_parser.hpp" @@ -165,33 +166,6 @@ namespace iroha { return grpc::Status::OK; } - namespace { - void handleEvents(rxcpp::composite_subscription &subscription, - rxcpp::schedulers::run_loop &run_loop) { - std::condition_variable wait_cv; - - run_loop.set_notify_earlier_wakeup( - [&wait_cv](const auto &) { wait_cv.notify_one(); }); - - std::mutex wait_mutex; - std::unique_lock lock(wait_mutex); - while (subscription.is_subscribed() or not run_loop.empty()) { - while (not run_loop.empty() - and run_loop.peek().when <= run_loop.now()) { - run_loop.dispatch(); - } - - if (run_loop.empty()) { - wait_cv.wait(lock, [&run_loop, &subscription]() { - return not subscription.is_subscribed() or not run_loop.empty(); - }); - } else { - wait_cv.wait_until(lock, run_loop.peek().when); - } - } - } - } // namespace - grpc::Status CommandServiceTransportGrpc::StatusStream( grpc::ServerContext *context, const iroha::protocol::TxStatusRequest *request, @@ -271,7 +245,7 @@ namespace iroha { // run loop while subscription is active or there are pending events in // the queue - handleEvents(subscription, rl); + iroha::schedulers::handleEvents(subscription, rl); log_->debug("status stream done, {}", client_id); return grpc::Status::OK; diff --git a/irohad/torii/impl/query_service.cpp b/irohad/torii/impl/query_service.cpp index 8e6abf3d3b..1265ffb66d 100644 --- a/irohad/torii/impl/query_service.cpp +++ b/irohad/torii/impl/query_service.cpp @@ -7,6 +7,7 @@ #include "backend/protobuf/query_responses/proto_block_query_response.hpp" #include "backend/protobuf/query_responses/proto_query_response.hpp" +#include "common/run_loop_handler.hpp" #include "cryptography/default_hash_provider.hpp" #include "interfaces/iroha_internal/abstract_transport_factory.hpp" #include "logger/logger.hpp" @@ -18,7 +19,7 @@ namespace iroha { QueryService::QueryService( std::shared_ptr query_processor, std::shared_ptr query_factory, - logger::LoggerPtr log) + logger::LoggerPtr log) : query_processor_{std::move(query_processor)}, query_factory_{std::move(query_factory)}, log_{std::move(log)} {} @@ -70,59 +71,82 @@ namespace iroha { const iroha::protocol::BlocksQuery *request, grpc::ServerWriter *writer) { log_->debug("Fetching commits"); + + rxcpp::schedulers::run_loop run_loop; + auto current_thread = rxcpp::synchronize_in_one_worker( + rxcpp::schedulers::make_run_loop(run_loop)); + shared_model::proto::TransportBuilder< shared_model::proto::BlocksQuery, shared_model::validation::DefaultSignedBlocksQueryValidator>() .build(*request) .match( - [this, context, request, writer]( + [this, context, request, writer, ¤t_thread, &run_loop]( const iroha::expected::Value &query) { - rxcpp::composite_subscription sub; + rxcpp::composite_subscription subscription; + std::string client_id = + (boost::format("Peer: '%s'") % context->peer()).str(); query_processor_->blocksQueryHandle(query.value) - .as_blocking() - .subscribe( - sub, - [this, context, &sub, request, writer]( - const std::shared_ptr< - shared_model::interface::BlockQueryResponse> - response) { - if (context->IsCancelled()) { - log_->debug("Unsubscribed"); - sub.unsubscribe(); - } else { - iroha::visit_in_place( - response->get(), - [this, writer, request]( - const shared_model::interface::BlockResponse - &block_response) { - log_->debug( - "{} receives committed block", - request->meta().creator_account_id()); - auto proto_block_response = static_cast< - const shared_model::proto::BlockResponse - &>(block_response); - writer->Write( - proto_block_response.getTransport()); - }, - [this, writer, request]( - const shared_model::interface:: - BlockErrorResponse - &block_error_response) { - log_->debug( - "{} received error with message: {}", - request->meta().creator_account_id(), - block_error_response.message()); - auto proto_block_error_response = - static_cast( - block_error_response); - writer->WriteLast( - proto_block_error_response.getTransport(), - grpc::WriteOptions()); - }); - } - }); + .observe_on(current_thread) + .take_while([this, context, request, writer]( + const std::shared_ptr< + shared_model::interface:: + BlockQueryResponse> response) { + if (context->IsCancelled()) { + log_->debug("Unsubscribed from block stream"); + return false; + } else { + auto result = iroha::visit_in_place( + response->get(), + [this, writer, request]( + const shared_model::interface::BlockResponse + &block_response) { + log_->debug("{} receives committed block", + request->meta().creator_account_id()); + auto proto_block_response = static_cast< + const shared_model::proto::BlockResponse &>( + block_response); + bool written = writer->Write( + proto_block_response.getTransport()); + if (not written) { + log_->debug( + "Block stream appears to be closed"); + return false; + } + return true; + }, + [this, writer, request]( + const shared_model::interface:: + BlockErrorResponse &block_error_response) { + log_->debug("{} received error with message: {}", + request->meta().creator_account_id(), + block_error_response.message()); + auto proto_block_error_response = static_cast< + const shared_model::proto::BlockErrorResponse + &>(block_error_response); + writer->WriteLast( + proto_block_error_response.getTransport(), + grpc::WriteOptions()); + return false; + }); + return result; + } + }) + .subscribe(subscription, + [](const auto &) {}, + [&](std::exception_ptr ep) { + log_->error( + "something bad happened during block " + "streaming, client_id {}", + client_id); + }, + [&] { + log_->debug("block stream done, {}", + client_id); + }); + + iroha::schedulers::handleEvents(subscription, run_loop); }, [this, writer](const auto &error) { log_->debug("Stateless invalid: {}", error.error); diff --git a/libs/common/run_loop_handler.hpp b/libs/common/run_loop_handler.hpp new file mode 100644 index 0000000000..de971fc4c6 --- /dev/null +++ b/libs/common/run_loop_handler.hpp @@ -0,0 +1,42 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#ifndef IROHA_RUN_LOOP_HANDLER_HPP +#define IROHA_RUN_LOOP_HANDLER_HPP + +#include + +#include + +namespace iroha { + namespace schedulers { + + inline void handleEvents(rxcpp::composite_subscription &subscription, + rxcpp::schedulers::run_loop &run_loop) { + std::condition_variable wait_cv; + + run_loop.set_notify_earlier_wakeup( + [&wait_cv](const auto &) { wait_cv.notify_one(); }); + + std::mutex wait_mutex; + std::unique_lock lock(wait_mutex); + while (subscription.is_subscribed() or not run_loop.empty()) { + while (not run_loop.empty() + and run_loop.peek().when <= run_loop.now()) { + run_loop.dispatch(); + } + + if (run_loop.empty()) { + wait_cv.wait(lock, [&run_loop, &subscription]() { + return not subscription.is_subscribed() or not run_loop.empty(); + }); + } else { + wait_cv.wait_until(lock, run_loop.peek().when); + } + } + } + } // namespace schedulers +} // namespace iroha + +#endif // IROHA_RUN_LOOP_HANDLER_HPP