From 7db66284b5189e40685048c1d243041bb575d076 Mon Sep 17 00:00:00 2001 From: Yaron Date: Tue, 10 Oct 2023 16:15:26 +0300 Subject: [PATCH 1/5] Refactor Lookup by using common code. --- src/depends/libDatabase/LevelDB.cpp | 828 ++++++++++++---------------- src/depends/libDatabase/LevelDB.h | 241 ++++---- 2 files changed, 484 insertions(+), 585 deletions(-) diff --git a/src/depends/libDatabase/LevelDB.cpp b/src/depends/libDatabase/LevelDB.cpp index af002beefd..cff10ca6a7 100644 --- a/src/depends/libDatabase/LevelDB.cpp +++ b/src/depends/libDatabase/LevelDB.cpp @@ -15,9 +15,6 @@ * along with this program. If not, see . */ -#include - - #include "LevelDB.h" #include "common/Constants.h" #include "depends/common/Common.h" @@ -27,598 +24,483 @@ using namespace std; -void LevelDB::log_error(leveldb::Status status) const -{ - if(!status.IsNotFound()) - { - LOG_GENERAL(WARNING, "LevelDB " << m_dbName << " status is not OK - " << status.ToString()); - } +void LevelDB::log_error(leveldb::Status status) const { + if (!status.IsNotFound()) { + LOG_GENERAL(WARNING, "LevelDB " << m_dbName << " status is not OK - " + << status.ToString()); + } } -LevelDB::LevelDB(const string& dbName, const string& path, const string& subdirectory) -{ - this->m_subdirectory = subdirectory; - this->m_dbName = dbName; - this->m_db = NULL; - - if(!(std::filesystem::exists(path))) - { - LOG_GENERAL(WARNING, path + " does not exist"); - return; - } - - m_options.max_open_files = 256; - m_options.create_if_missing = true; +LevelDB::LevelDB(const string& dbName, const string& path, + const string& subdirectory) + : m_dbName{dbName}, m_subdirectory{subdirectory} { + if (!(std::filesystem::exists(path))) { + LOG_GENERAL(WARNING, "Can't open " << dbName << " since " << path + << " does not exist"); + return; + } - leveldb::DB* db; - leveldb::Status status; + m_options.max_open_files = 256; + m_options.create_if_missing = true; - if(m_subdirectory.empty()) - { - m_open_db_path = path + "/" + this->m_dbName; - status = leveldb::DB::Open(m_options, m_open_db_path, &db); - LOG_GENERAL(INFO, path + "/" + this->m_dbName); - } - else - { - if (!(std::filesystem::exists(path + "/" + this->m_subdirectory))) - { - std::filesystem::create_directories(path + "/" + this->m_subdirectory); - } - m_open_db_path = path + "/" + this->m_subdirectory + "/" + this->m_dbName; - status = leveldb::DB::Open(m_options, - m_open_db_path, - &db); - LOG_GENERAL(INFO, path + "/" + this->m_subdirectory + "/" + this->m_dbName); - } - - if(!status.ok()) - { - LOG_GENERAL(WARNING, "LevelDB " << m_dbName << " status is not OK - " << status.ToString()); - } + leveldb::DB* db = nullptr; + leveldb::Status status; - m_db.reset(db); -} - -LevelDB::LevelDB(const std::string & dbName, const std::string& subdirectory, bool diagnostic) -{ - this->m_subdirectory = subdirectory; - this->m_dbName = dbName; - - m_options.max_open_files = 256; - m_options.create_if_missing = true; - - leveldb::DB* db; - leveldb::Status status; - - // Diagnostic tool provides the option to pass the persistance db_path - // that might not be the current directory (case when 'diagnostic' is true). - // Its default value is false, and the non-diagnostic path is preserved - // from the original code. - string db_path = diagnostic ? (m_subdirectory + PERSISTENCE_PATH) : (STORAGE_PATH + PERSISTENCE_PATH + (m_subdirectory.empty() ? "" : "/" + m_subdirectory)); - if (!std::filesystem::exists(db_path)) - { - std::filesystem::create_directories(db_path); - } - - m_open_db_path = db_path + "/" + this->m_dbName; + if (m_subdirectory.empty()) { + m_open_db_path = path + "/" + this->m_dbName; status = leveldb::DB::Open(m_options, m_open_db_path, &db); - if(!status.ok()) - { - // throw exception(); - LOG_GENERAL(WARNING, "LevelDB " << dbName << " status is not OK - " << status.ToString()); + LOG_GENERAL(INFO, path + "/" + this->m_dbName); + } else { + if (!(std::filesystem::exists(path + "/" + this->m_subdirectory))) { + std::filesystem::create_directories(path + "/" + this->m_subdirectory); } - - m_db.reset(db); + m_open_db_path = path + "/" + this->m_subdirectory + "/" + this->m_dbName; + status = leveldb::DB::Open(m_options, m_open_db_path, &db); + LOG_GENERAL(INFO, path + "/" + this->m_subdirectory + "/" + this->m_dbName); + } + + if (!status.ok()) { + LOG_GENERAL(WARNING, "LevelDB " << m_dbName << " status is not OK - " + << status.ToString()); + } + + m_db.reset(db); +} + +LevelDB::LevelDB(const std::string& dbName, const std::string& subdirectory, + bool diagnostic) + : m_dbName{dbName}, m_subdirectory{subdirectory} { + m_options.max_open_files = 256; + m_options.create_if_missing = true; + + // Diagnostic tool provides the option to pass the persistance db_path + // that might not be the current directory (case when 'diagnostic' is true). + // Its default value is false, and the non-diagnostic path is preserved + // from the original code. + string db_path = diagnostic + ? (m_subdirectory + PERSISTENCE_PATH) + : (STORAGE_PATH + PERSISTENCE_PATH + + (m_subdirectory.empty() ? "" : "/" + m_subdirectory)); + if (!std::filesystem::exists(db_path)) { + std::filesystem::create_directories(db_path); + } + + m_open_db_path = db_path + "/" + this->m_dbName; + leveldb::DB* db = nullptr; + auto status = leveldb::DB::Open(m_options, m_open_db_path, &db); + if (!status.ok()) { + // throw exception(); + LOG_GENERAL(WARNING, "LevelDB " << dbName << " status is not OK - " + << status.ToString()); + } + + m_db.reset(db); } void LevelDB::Reopen() { - m_db.reset(); - leveldb::DB* db; - leveldb::Status status; + m_db.reset(); + leveldb::DB* db = nullptr; + leveldb::Status status; - status = leveldb::DB::Open(m_options, m_open_db_path, &db); - if (!status.ok()) - { - LOG_GENERAL(WARNING, "LevelDB " << m_dbName << " status is not OK - " << status.ToString()); - } - m_db.reset(db); + status = leveldb::DB::Open(m_options, m_open_db_path, &db); + if (!status.ok()) { + LOG_GENERAL(WARNING, "LevelDB " << m_dbName << " status is not OK - " + << status.ToString()); + } + m_db.reset(db); } -leveldb::Slice toSlice(boost::multiprecision::uint256_t num) -{ - dev::FixedHash<32> h; - dev::zbytesRef ref(h.data(), 32); - dev::toBigEndian(num, ref); - return (leveldb::Slice)h.ref(); -} - -string LevelDB::GetDBName() -{ - if (LOOKUP_NODE_MODE) - { - return m_dbName; - } - else - { - return m_dbName + (m_subdirectory.size() > 0 ? "/" : "") + m_subdirectory; - } -} - -string LevelDB::Lookup(const std::string & key) const -{ - string value; - leveldb::Status s = m_db->Get(leveldb::ReadOptions(), key, &value); - if (!s.ok()) - { - log_error(s); - return ""; - } - - return value; +leveldb::Slice toSlice(boost::multiprecision::uint256_t num) { + dev::FixedHash<32> h; + dev::zbytesRef ref(h.data(), 32); + dev::toBigEndian(num, ref); + return (leveldb::Slice)h.ref(); } -string LevelDB::Lookup(const vector& key) const -{ - string value; - leveldb::Status s = m_db->Get(leveldb::ReadOptions(), leveldb::Slice(vector_ref(&key[0], key.size())), &value); - if (!s.ok()) - { - log_error(s); - return ""; - } - - return value; +string LevelDB::GetDBName() { + if (LOOKUP_NODE_MODE) { + return m_dbName; + } else { + return m_dbName + (m_subdirectory.size() > 0 ? "/" : "") + m_subdirectory; + } } -string LevelDB::Lookup(const boost::multiprecision::uint256_t & blockNum) const -{ - string value; - leveldb::Status s = m_db->Get(leveldb::ReadOptions(), blockNum.convert_to(), &value); +string LevelDB::Lookup(const std::string& key) const { return LookupImpl(key); } - if (!s.ok()) - { - log_error(s); - return ""; - } - - return value; +string LevelDB::Lookup(const vector& key) const { + return LookupImpl(vector_ref(&key[0], key.size())); } -string LevelDB::Lookup(const boost::multiprecision::uint256_t & blockNum, bool &found) const -{ - string value; - leveldb::Status s = m_db->Get(leveldb::ReadOptions(), blockNum.convert_to(), &value); - - if (!s.ok()) - { - log_error(s); - found = false; - return ""; - } - found = true; - return value; -} - -string LevelDB::Lookup(const dev::h256 & key) const -{ - string value; - leveldb::Status s = m_db->Get(leveldb::ReadOptions(), leveldb::Slice(key.hex()), &value); - if (!s.ok()) - { - log_error(s); - return ""; - } - - return value; +string LevelDB::Lookup(const boost::multiprecision::uint256_t& blockNum) const { + return LookupImpl(blockNum.convert_to()); } -string LevelDB::Lookup(const dev::zbytesConstRef & key) const -{ - string value; - leveldb::Status s = m_db->Get(leveldb::ReadOptions(), ldb::Slice((char const*)key.data(), 32), - &value); - if (!s.ok()) - { - log_error(s); - return ""; - } +string LevelDB::Lookup(const boost::multiprecision::uint256_t& blockNum, + bool& found) const { + return LookupImpl(blockNum.convert_to(), &found); +} - return value; +string LevelDB::Lookup(const dev::h256& key) const { + return LookupImpl(key.hex()); } -std::shared_ptr LevelDB::GetDB() -{ - return this->m_db; +string LevelDB::Lookup(const dev::zbytesConstRef& key) const { + return LookupImpl(ldb::Slice((char const*)key.data(), 32)); } -int LevelDB::Insert(const dev::h256 & key, dev::zbytesConstRef value) -{ - return Insert(key, value.toString()); +int LevelDB::Insert(const dev::h256& key, dev::zbytesConstRef value) { + return Insert(key, value.toString()); } -int LevelDB::Insert(const vector& key, const vector& body) -{ - leveldb::Status s = m_db->Put(leveldb::WriteOptions(), - leveldb::Slice(vector_ref(&key[0], key.size())), - leveldb::Slice(vector_ref(&body[0], - body.size()))); +int LevelDB::Insert(const vector& key, + const vector& body) { + leveldb::Status s = m_db->Put( + leveldb::WriteOptions(), + leveldb::Slice(vector_ref(&key[0], key.size())), + leveldb::Slice(vector_ref(&body[0], body.size()))); - if (!s.ok()) - { - LOG_GENERAL(WARNING, "[Insert] Status: " << s.ToString()); - return -1; - } + if (!s.ok()) { + LOG_GENERAL(WARNING, "[Insert] Status: " << s.ToString()); + return -1; + } - return 0; + return 0; } -int LevelDB::Insert(const boost::multiprecision::uint256_t & blockNum, - const vector & body) -{ - leveldb::Status s = m_db->Put(leveldb::WriteOptions(), - leveldb::Slice(blockNum.convert_to()), - leveldb::Slice(vector_ref(&body[0], - body.size()))); +int LevelDB::Insert(const boost::multiprecision::uint256_t& blockNum, + const vector& body) { + leveldb::Status s = m_db->Put( + leveldb::WriteOptions(), leveldb::Slice(blockNum.convert_to()), + leveldb::Slice(vector_ref(&body[0], body.size()))); - if (!s.ok()) - { - LOG_GENERAL(WARNING, "[Insert] Status: " << s.ToString()); - return -1; - } + if (!s.ok()) { + LOG_GENERAL(WARNING, "[Insert] Status: " << s.ToString()); + return -1; + } - return 0; + return 0; } -int LevelDB::Insert(const boost::multiprecision::uint256_t & blockNum, - const std::string & body) -{ - leveldb::Status s = m_db->Put(leveldb::WriteOptions(), - leveldb::Slice(blockNum.convert_to()), - leveldb::Slice(body.c_str(), body.size())); +int LevelDB::Insert(const boost::multiprecision::uint256_t& blockNum, + const std::string& body) { + leveldb::Status s = m_db->Put(leveldb::WriteOptions(), + leveldb::Slice(blockNum.convert_to()), + leveldb::Slice(body.c_str(), body.size())); - if (!s.ok()) - { - LOG_GENERAL(WARNING, "[Insert] Status: " << s.ToString()); - return -1; - } + if (!s.ok()) { + LOG_GENERAL(WARNING, "[Insert] Status: " << s.ToString()); + return -1; + } - return 0; + return 0; } -int LevelDB::Insert(const string & key, const vector & body) -{ - return Insert(leveldb::Slice(key), leveldb::Slice(dev::zbytesConstRef(&body[0], body.size()))); +int LevelDB::Insert(const string& key, const vector& body) { + return Insert(leveldb::Slice(key), + leveldb::Slice(dev::zbytesConstRef(&body[0], body.size()))); } -int LevelDB::Insert(const leveldb::Slice & key, dev::zbytesConstRef value) -{ - leveldb::Status s = m_db->Put(leveldb::WriteOptions(), key, ldb::Slice(value)); - if (!s.ok()) - { - LOG_GENERAL(WARNING, "[Insert] Status: " << s.ToString()); - return -1; - } +int LevelDB::Insert(const leveldb::Slice& key, dev::zbytesConstRef value) { + leveldb::Status s = + m_db->Put(leveldb::WriteOptions(), key, ldb::Slice(value)); + if (!s.ok()) { + LOG_GENERAL(WARNING, "[Insert] Status: " << s.ToString()); + return -1; + } - return 0; + return 0; } -int LevelDB::Insert(const dev::h256 & key, const string & value) -{ - leveldb::Status s = m_db->Put(leveldb::WriteOptions(), - ldb::Slice((char const*)key.data(), key.size), - ldb::Slice(value.data(), value.size())); - if (!s.ok()) - { - LOG_GENERAL(WARNING, "[Insert] Status: " << s.ToString()); - return -1; - } +int LevelDB::Insert(const dev::h256& key, const string& value) { + leveldb::Status s = m_db->Put(leveldb::WriteOptions(), + ldb::Slice((char const*)key.data(), key.size), + ldb::Slice(value.data(), value.size())); + if (!s.ok()) { + LOG_GENERAL(WARNING, "[Insert] Status: " << s.ToString()); + return -1; + } - return 0; + return 0; } -int LevelDB::Insert(const dev::h256 & key, const vector & body) -{ - leveldb::Status s = m_db->Put(leveldb::WriteOptions(), leveldb::Slice(key.hex()), - leveldb::Slice(vector_ref(&body[0], - body.size()))); - if (!s.ok()) - { - LOG_GENERAL(WARNING, "[Insert] Status: " << s.ToString()); - return -1; - } +int LevelDB::Insert(const dev::h256& key, const vector& body) { + leveldb::Status s = m_db->Put( + leveldb::WriteOptions(), leveldb::Slice(key.hex()), + leveldb::Slice(vector_ref(&body[0], body.size()))); + if (!s.ok()) { + LOG_GENERAL(WARNING, "[Insert] Status: " << s.ToString()); + return -1; + } - return 0; + return 0; } -int LevelDB::Insert(const leveldb::Slice & key, const leveldb::Slice & value) -{ - leveldb::Status s = m_db->Put(leveldb::WriteOptions(), key, value); - - if (!s.ok()) - { - LOG_GENERAL(WARNING, "[Insert] Status: " << s.ToString()); - return -1; - } +int LevelDB::Insert(const leveldb::Slice& key, const leveldb::Slice& value) { + leveldb::Status s = m_db->Put(leveldb::WriteOptions(), key, value); - return 0; + if (!s.ok()) { + LOG_GENERAL(WARNING, "[Insert] Status: " << s.ToString()); + return -1; + } + + return 0; } -bool LevelDB::BatchInsert(const std::unordered_map> & m_main, - const std::unordered_map> & m_aux, unordered_set& inserted) -{ - ldb::WriteBatch batch; +bool LevelDB::BatchInsert( + const std::unordered_map>& + m_main, + const std::unordered_map>& m_aux, + unordered_set& inserted) { + ldb::WriteBatch batch; - for (const auto & i: m_main) { - if (i.second.second || (LOOKUP_NODE_MODE && KEEP_HISTORICAL_STATE)) { - batch.Put(leveldb::Slice(i.first.hex()), - leveldb::Slice(i.second.first.data(), i.second.first.size())); - if(i.second.second) - { - inserted.emplace(i.first); - } - } + for (const auto& i : m_main) { + if (i.second.second || (LOOKUP_NODE_MODE && KEEP_HISTORICAL_STATE)) { + batch.Put(leveldb::Slice(i.first.hex()), + leveldb::Slice(i.second.first.data(), i.second.first.size())); + if (i.second.second) { + inserted.emplace(i.first); + } } + } - for (const auto & i: m_aux) { - if (i.second.second) { - dev::zbytes b = i.first.asBytes(); - b.push_back(255); // for aux - batch.Put(dev::zbytesConstRef(&b), dev::zbytesConstRef(&i.second.first)); - inserted.emplace(i.first); - } + for (const auto& i : m_aux) { + if (i.second.second) { + dev::zbytes b = i.first.asBytes(); + b.push_back(255); // for aux + batch.Put(dev::zbytesConstRef(&b), dev::zbytesConstRef(&i.second.first)); + inserted.emplace(i.first); } + } - ldb::Status s = m_db->Write(leveldb::WriteOptions(), &batch); + ldb::Status s = m_db->Write(leveldb::WriteOptions(), &batch); - if (!s.ok()) { - LOG_GENERAL(WARNING, "[BatchInsert] Status: " << s.ToString()); - return false; - } + if (!s.ok()) { + LOG_GENERAL(WARNING, "[BatchInsert] Status: " << s.ToString()); + return false; + } - return true; + return true; } -bool LevelDB::BatchInsert(const std::unordered_map& kv_map) -{ - ldb::WriteBatch batch; +bool LevelDB::BatchInsert( + const std::unordered_map& kv_map) { + ldb::WriteBatch batch; - for (const auto & i: kv_map) { - if (!i.second.empty()) { - batch.Put(leveldb::Slice(i.first), - leveldb::Slice(i.second)); - } + for (const auto& i : kv_map) { + if (!i.second.empty()) { + batch.Put(leveldb::Slice(i.first), leveldb::Slice(i.second)); } + } - ldb::Status s = m_db->Write(leveldb::WriteOptions(), &batch); + ldb::Status s = m_db->Write(leveldb::WriteOptions(), &batch); - if (!s.ok()) { - LOG_GENERAL(WARNING, "[BatchInsert] Status: " << s.ToString()); - return false; - } + if (!s.ok()) { + LOG_GENERAL(WARNING, "[BatchInsert] Status: " << s.ToString()); + return false; + } - return true; + return true; } bool LevelDB::BatchDelete(const std::vector& toDelete) { - ldb::WriteBatch batch; - for (const auto& i : toDelete) { - batch.Delete(leveldb::Slice(i.hex())); - } + ldb::WriteBatch batch; + for (const auto& i : toDelete) { + batch.Delete(leveldb::Slice(i.hex())); + } - ldb::Status s = m_db->Write(leveldb::WriteOptions(), &batch); + ldb::Status s = m_db->Write(leveldb::WriteOptions(), &batch); - if (!s.ok()) { - LOG_GENERAL(WARNING, "[BatchDelete] Status: " << s.ToString()); - return false; - } + if (!s.ok()) { + LOG_GENERAL(WARNING, "[BatchDelete] Status: " << s.ToString()); + return false; + } - return true; + return true; } -bool LevelDB::Exists(const dev::h256 & key) const -{ - auto ret = Lookup(key); - return !ret.empty(); +bool LevelDB::Exists(const dev::h256& key) const { + auto ret = Lookup(key); + return !ret.empty(); } -bool LevelDB::Exists(const vector& key) const -{ - auto ret = Lookup(key); - return !ret.empty(); +bool LevelDB::Exists(const vector& key) const { + auto ret = Lookup(key); + return !ret.empty(); } -bool LevelDB::Exists(const boost::multiprecision::uint256_t & blockNum) const -{ - auto ret = Lookup(blockNum); - return !ret.empty(); +bool LevelDB::Exists(const boost::multiprecision::uint256_t& blockNum) const { + auto ret = Lookup(blockNum); + return !ret.empty(); } -bool LevelDB::Exists(const std::string & key) const -{ - auto ret = Lookup(key); - return !ret.empty(); +bool LevelDB::Exists(const std::string& key) const { + auto ret = Lookup(key); + return !ret.empty(); } -int LevelDB::DeleteKey(const dev::h256 & key) -{ - leveldb::Status s = m_db->Delete(leveldb::WriteOptions(), ldb::Slice(key.hex())); - if (!s.ok()) - { - LOG_GENERAL(WARNING, "[DeleteDB] Status: " << s.ToString()); - return -1; - } +int LevelDB::DeleteKey(const dev::h256& key) { + leveldb::Status s = + m_db->Delete(leveldb::WriteOptions(), ldb::Slice(key.hex())); + if (!s.ok()) { + LOG_GENERAL(WARNING, "[DeleteDB] Status: " << s.ToString()); + return -1; + } - return 0; + return 0; } -int LevelDB::DeleteKey(const boost::multiprecision::uint256_t & blockNum) -{ - leveldb::Status s = m_db->Delete(leveldb::WriteOptions(), ldb::Slice(blockNum.convert_to())); - if (!s.ok()) - { - LOG_GENERAL(WARNING, "[DeleteDB] Status: " << s.ToString()); - return -1; - } +int LevelDB::DeleteKey(const boost::multiprecision::uint256_t& blockNum) { + leveldb::Status s = m_db->Delete(leveldb::WriteOptions(), + ldb::Slice(blockNum.convert_to())); + if (!s.ok()) { + LOG_GENERAL(WARNING, "[DeleteDB] Status: " << s.ToString()); + return -1; + } - return 0; + return 0; } -int LevelDB::DeleteKey(const std::string & key) -{ - leveldb::Status s = m_db->Delete(leveldb::WriteOptions(), ldb::Slice(key)); - if(!s.ok()) - { - LOG_GENERAL(WARNING, "[DeleteKey] Status: " << s.ToString()); - return -1; - } +int LevelDB::DeleteKey(const std::string& key) { + leveldb::Status s = m_db->Delete(leveldb::WriteOptions(), ldb::Slice(key)); + if (!s.ok()) { + LOG_GENERAL(WARNING, "[DeleteKey] Status: " << s.ToString()); + return -1; + } - return 0; + return 0; } -int LevelDB::DeleteKey(const vector & key) -{ - leveldb::Status s = m_db->Delete(leveldb::WriteOptions(), leveldb::Slice(vector_ref(&key[0], key.size()))); - if(!s.ok()) - { - LOG_GENERAL(WARNING, "[DeleteKey] Status: " << s.ToString()); - return -1; - } +int LevelDB::DeleteKey(const vector& key) { + leveldb::Status s = m_db->Delete( + leveldb::WriteOptions(), + leveldb::Slice(vector_ref(&key[0], key.size()))); + if (!s.ok()) { + LOG_GENERAL(WARNING, "[DeleteKey] Status: " << s.ToString()); + return -1; + } - return 0; + return 0; } -int LevelDB::DeleteDB() -{ - if (LOOKUP_NODE_MODE) - { - return DeleteDBForLookupNode(); - } - else - { - return DeleteDBForNormalNode(); - } +int LevelDB::DeleteDB() { + if (LOOKUP_NODE_MODE) { + return DeleteDBForLookupNode(); + } else { + return DeleteDBForNormalNode(); + } } -bool LevelDB::ResetDB() -{ - if (LOOKUP_NODE_MODE) - { - return ResetDBForLookupNode(); - } - else - { - return ResetDBForNormalNode(); - } +bool LevelDB::ResetDB() { + if (LOOKUP_NODE_MODE) { + return ResetDBForLookupNode(); + } else { + return ResetDBForNormalNode(); + } } -bool LevelDB::RefreshDB() -{ - m_db.reset(); +bool LevelDB::RefreshDB() { + m_db.reset(); - leveldb::Options options; - options.max_open_files = 256; - options.create_if_missing = true; + leveldb::Options options; + options.max_open_files = 256; + options.create_if_missing = true; - leveldb::DB* db; + leveldb::DB* db = nullptr; - leveldb::Status status = leveldb::DB::Open(options, STORAGE_PATH + PERSISTENCE_PATH + "/" + this->m_dbName, &db); - if(!status.ok()) - { - // throw exception(); - LOG_GENERAL(WARNING, "LevelDB " << m_dbName << " status is not OK - " << status.ToString()); - return false; - } + leveldb::Status status = leveldb::DB::Open( + options, STORAGE_PATH + PERSISTENCE_PATH + "/" + this->m_dbName, &db); + if (!status.ok()) { + // throw exception(); + LOG_GENERAL(WARNING, "LevelDB " << m_dbName << " status is not OK - " + << status.ToString()); + return false; + } - m_db.reset(db); - return true; + m_db.reset(db); + return true; } -int LevelDB::DeleteDBForNormalNode() -{ - m_db.reset(); - leveldb::Status s = leveldb::DestroyDB(STORAGE_PATH + PERSISTENCE_PATH + - (this->m_subdirectory.size() ? "/" + this->m_subdirectory : "") + "/" + this->m_dbName, - leveldb::Options()); - if (!s.ok()) - { - LOG_GENERAL(WARNING, "[DeleteDB] Status: " << s.ToString()); - return -1; - } +int LevelDB::DeleteDBForNormalNode() { + m_db.reset(); + leveldb::Status s = leveldb::DestroyDB( + STORAGE_PATH + PERSISTENCE_PATH + + (this->m_subdirectory.size() ? "/" + this->m_subdirectory : "") + + "/" + this->m_dbName, + leveldb::Options()); + if (!s.ok()) { + LOG_GENERAL(WARNING, "[DeleteDB] Status: " << s.ToString()); + return -1; + } - if(this->m_subdirectory.size()) - { - std::filesystem::remove_all(STORAGE_PATH + PERSISTENCE_PATH + "/" + this->m_subdirectory + "/" + this->m_dbName); - } + if (this->m_subdirectory.size()) { + std::filesystem::remove_all(STORAGE_PATH + PERSISTENCE_PATH + "/" + + this->m_subdirectory + "/" + this->m_dbName); + } - return 0; + return 0; } -bool LevelDB::ResetDBForNormalNode() -{ - if(DeleteDBForNormalNode() == 0 && this->m_subdirectory.empty()) - { - std::filesystem::remove_all(STORAGE_PATH + PERSISTENCE_PATH + "/" + this->m_dbName); - - leveldb::Options options; - options.max_open_files = 256; - options.create_if_missing = true; +bool LevelDB::ResetDBForNormalNode() { + if (DeleteDBForNormalNode() == 0 && this->m_subdirectory.empty()) { + std::filesystem::remove_all(STORAGE_PATH + PERSISTENCE_PATH + "/" + + this->m_dbName); - leveldb::DB* db; + leveldb::Options options; + options.max_open_files = 256; + options.create_if_missing = true; - leveldb::Status status = leveldb::DB::Open(options, STORAGE_PATH + PERSISTENCE_PATH + "/" + this->m_dbName, &db); - if(!status.ok()) - { - // throw exception(); - LOG_GENERAL(WARNING, "LevelDB " << m_dbName << " status is not OK - " << status.ToString()); - } + leveldb::DB* db = nullptr; - m_db.reset(db); - return true; - } - else if(this->m_subdirectory.size()) - { - LOG_GENERAL(INFO, "DB in subdirectory cannot be reset"); + leveldb::Status status = leveldb::DB::Open( + options, STORAGE_PATH + PERSISTENCE_PATH + "/" + this->m_dbName, &db); + if (!status.ok()) { + // throw exception(); + LOG_GENERAL(WARNING, "LevelDB " << m_dbName << " status is not OK - " + << status.ToString()); } - LOG_GENERAL(WARNING, "Didn't reset DB, investigate why!"); - return false; + + m_db.reset(db); + return true; + } else if (this->m_subdirectory.size()) { + LOG_GENERAL(INFO, "DB in subdirectory cannot be reset"); + } + LOG_GENERAL(WARNING, "Didn't reset DB, investigate why!"); + return false; } -int LevelDB::DeleteDBForLookupNode() -{ - m_db.reset(); - leveldb::Status s = leveldb::DestroyDB(this->m_dbName, leveldb::Options()); - if (!s.ok()) - { - LOG_GENERAL(WARNING, "[DeleteDB] Status: " << s.ToString()); - return -1; - } +int LevelDB::DeleteDBForLookupNode() { + m_db.reset(); + leveldb::Status s = leveldb::DestroyDB(this->m_dbName, leveldb::Options()); + if (!s.ok()) { + LOG_GENERAL(WARNING, "[DeleteDB] Status: " << s.ToString()); + return -1; + } - return 0; + return 0; } -bool LevelDB::ResetDBForLookupNode() -{ - if(DeleteDBForLookupNode()==0) - { - std::filesystem::remove_all(STORAGE_PATH + PERSISTENCE_PATH + "/" + this->m_dbName); +bool LevelDB::ResetDBForLookupNode() { + if (DeleteDBForLookupNode() == 0) { + std::filesystem::remove_all(STORAGE_PATH + PERSISTENCE_PATH + "/" + + this->m_dbName); - leveldb::Options options; - options.max_open_files = 256; - options.create_if_missing = true; - - leveldb::DB* db; + leveldb::Options options; + options.max_open_files = 256; + options.create_if_missing = true; - leveldb::Status status = leveldb::DB::Open(options, STORAGE_PATH + PERSISTENCE_PATH + "/" + this->m_dbName, &db); - if(!status.ok()) - { - // throw exception(); - LOG_GENERAL(WARNING, "LevelDB " << m_dbName << " status is not OK - " << status.ToString()); - return false; - } + leveldb::DB* db = nullptr; - m_db.reset(db); - return true; + leveldb::Status status = leveldb::DB::Open( + options, STORAGE_PATH + PERSISTENCE_PATH + "/" + this->m_dbName, &db); + if (!status.ok()) { + // throw exception(); + LOG_GENERAL(WARNING, "LevelDB " << m_dbName << " status is not OK - " + << status.ToString()); + return false; } - return false; + + m_db.reset(db); + return true; + } + return false; } diff --git a/src/depends/libDatabase/LevelDB.h b/src/depends/libDatabase/LevelDB.h index 87db725d62..ef7e5528d2 100644 --- a/src/depends/libDatabase/LevelDB.h +++ b/src/depends/libDatabase/LevelDB.h @@ -15,12 +15,10 @@ * along with this program. If not, see . */ - #ifndef __LEVELDB_H__ #define __LEVELDB_H__ #include -#include #include #include @@ -29,134 +27,153 @@ #include "depends/common/Common.h" #include "depends/common/FixedHash.h" - leveldb::Slice toSlice(boost::multiprecision::uint256_t num); /// Utility class for providing database-type storage. -class LevelDB -{ - std::string m_dbName; - - std::string m_subdirectory; - - std::shared_ptr m_db; - - leveldb::Options m_options; - - std::string m_open_db_path; - - void log_error(leveldb::Status status) const; - -public: - - /// Constructor. - explicit LevelDB(const std::string & dbName, const std::string& subdirectory = "", bool diagnostic = false); - explicit LevelDB(const std::string& dbName, const std::string& path, const std::string& subdirectory = ""); - /// Destructor. - ~LevelDB() = default; - - /// manually compact, might be helpful in future - void compact () { - m_db->CompactRange(NULL, NULL); - } - - /// Reopen the leveldb object to trigger compact and cleaning of LOG/MANIFEST files - void Reopen(); - - /// Returns the reference to the leveldb database instance. - std::shared_ptr GetDB(); +class LevelDB { + std::string m_dbName; + std::string m_subdirectory; + std::shared_ptr m_db; + leveldb::Options m_options; + std::string m_open_db_path; - /// Returns the DB Name - std::string GetDBName(); + void log_error(leveldb::Status status) const; - /// Returns the value at the specified key. - std::string Lookup(const std::string & key) const; + public: + /// Constructor. + explicit LevelDB(const std::string& dbName, + const std::string& subdirectory = "", + bool diagnostic = false); + explicit LevelDB(const std::string& dbName, const std::string& path, + const std::string& subdirectory = ""); + /// Destructor. + ~LevelDB() = default; - /// Returns the value at the specified key. - std::string Lookup(const std::vector& key) const; + /// manually compact, might be helpful in future + void compact() { m_db->CompactRange(NULL, NULL); } - /// Returns the value at the specified key. - std::string Lookup(const boost::multiprecision::uint256_t & blockNum) const; + /// Reopen the leveldb object to trigger compact and cleaning of LOG/MANIFEST + /// files + void Reopen(); - /// Returns the value at the specified key and also mark if key was found or not - std::string Lookup(const boost::multiprecision::uint256_t & blockNum, bool &found) const; + /// Returns the reference to the leveldb database instance. + auto GetDB() { return m_db /*.get()*/; } - /// Returns the value at the specified key. - std::string Lookup(const dev::h256 & key) const; + /// Returns the DB Name + std::string GetDBName(); - /// Returns the value at the specified key. - std::string Lookup(const dev::zbytesConstRef & key) const; + /// Returns the value at the specified key. + std::string Lookup(const std::string& key) const; - /// Sets the value at the specified key. - int Insert(const dev::h256 & key, dev::zbytesConstRef value); + /// Returns the value at the specified key. + std::string Lookup(const std::vector& key) const; + + /// Returns the value at the specified key. + std::string Lookup(const boost::multiprecision::uint256_t& blockNum) const; + + /// Returns the value at the specified key and also mark if key was found or + /// not + std::string Lookup(const boost::multiprecision::uint256_t& blockNum, + bool& found) const; + + /// Returns the value at the specified key. + std::string Lookup(const dev::h256& key) const; + + /// Returns the value at the specified key. + std::string Lookup(const dev::zbytesConstRef& key) const; + + /// Sets the value at the specified key. + int Insert(const dev::h256& key, dev::zbytesConstRef value); + + /// Sets the value at the specified key. + int Insert(const std::vector& key, + const std::vector& body); + + /// Sets the value at the specified key. + int Insert(const boost::multiprecision::uint256_t& blockNum, + const std::vector& body); + + /// Sets the value at the specified key. + int Insert(const boost::multiprecision::uint256_t& blockNum, + const std::string& body); + + /// Sets the value at the specified key. + int Insert(const std::string& key, const std::vector& body); + + /// Sets the value at the specified key. + int Insert(const leveldb::Slice& key, dev::zbytesConstRef value); + + /// Sets the value at the specified key. + int Insert(const dev::h256& key, const std::string& value); + + /// Sets the value at the specified key. + int Insert(const dev::h256& key, const std::vector& body); + + /// Sets the value at the specified key. + int Insert(const leveldb::Slice& key, const leveldb::Slice& value); + + /// Sets the value at the specified key for multiple such pairs. + bool BatchInsert( + const std::unordered_map>& + m_main, + const std::unordered_map>& m_aux, + std::unordered_set& inserted); + bool BatchInsert(const std::unordered_map& kv_map); + + /// Remove the kv pair for multiple specified key. + bool BatchDelete(const std::vector& toDelete); + + /// Returns true if value corresponding to specified key exists. + bool Exists(const dev::h256& key) const; + bool Exists(const boost::multiprecision::uint256_t& blockNum) const; + bool Exists(const std::string& key) const; + bool Exists(const std::vector& key) const; + + /// Deletes the value at the specified key. + int DeleteKey(const dev::h256& key); + + /// Deletes the value at the specified key. + int DeleteKey(const boost::multiprecision::uint256_t& blockNum); - /// Sets the value at the specified key. - int Insert(const std::vector& key, - const std::vector& body); + /// Deletes the value at the specified key. + int DeleteKey(const std::string& key); - /// Sets the value at the specified key. - int Insert(const boost::multiprecision::uint256_t & blockNum, - const std::vector & body); + /// Deletes the value at the specified key. + int DeleteKey(const std::vector& key); - /// Sets the value at the specified key. - int Insert(const boost::multiprecision::uint256_t & blockNum, - const std::string & body); + /// Deletes the entire database. + int DeleteDB(); + int DeleteDBForNormalNode(); + int DeleteDBForLookupNode(); - /// Sets the value at the specified key. - int Insert(const std::string & key, const std::vector & body); + /// Reset the entire database. + bool ResetDB(); - /// Sets the value at the specified key. - int Insert(const leveldb::Slice & key, dev::zbytesConstRef value); + /// Refresh the entire database. + bool RefreshDB(); - /// Sets the value at the specified key. - int Insert(const dev::h256 & key, const std::string & value); + private: + bool ResetDBForNormalNode(); + bool ResetDBForLookupNode(); - /// Sets the value at the specified key. - int Insert(const dev::h256 & key, const std::vector & body); - - /// Sets the value at the specified key. - int Insert(const leveldb::Slice & key, const leveldb::Slice & value); - - /// Sets the value at the specified key for multiple such pairs. - bool BatchInsert(const std::unordered_map> & m_main, - const std::unordered_map> & m_aux, std::unordered_set& inserted); - bool BatchInsert(const std::unordered_map& kv_map); - - /// Remove the kv pair for multiple specified key. - bool BatchDelete(const std::vector& toDelete); - - /// Returns true if value corresponding to specified key exists. - bool Exists(const dev::h256 & key) const; - bool Exists(const boost::multiprecision::uint256_t & blockNum) const; - bool Exists(const std::string & key) const; - bool Exists(const std::vector & key) const; - - /// Deletes the value at the specified key. - int DeleteKey(const dev::h256 & key); - - /// Deletes the value at the specified key. - int DeleteKey(const boost::multiprecision::uint256_t & blockNum); - - /// Deletes the value at the specified key. - int DeleteKey(const std::string & key); - - /// Deletes the value at the specified key. - int DeleteKey(const std::vector & key); - - /// Deletes the entire database. - int DeleteDB(); - int DeleteDBForNormalNode(); - int DeleteDBForLookupNode(); - - /// Reset the entire database. - bool ResetDB(); - - /// Refresh the entire database. - bool RefreshDB(); + template + std::string LookupImpl(ArgT&& arg, bool* found = nullptr) const { + std::string value; + leveldb::Status s = + m_db->Get(leveldb::ReadOptions(), std::forward(arg), &value); + if (!s.ok()) { + log_error(s); + if (found) { + *found = false; + } + return ""; + } -private: - bool ResetDBForNormalNode(); - bool ResetDBForLookupNode(); + if (found) { + *found = true; + } + return value; + } }; -#endif // __LEVELDB_H__ +#endif // __LEVELDB_H__ From cf18f7de9186ece0cb76d6b4e7d06afc2671082d Mon Sep 17 00:00:00 2001 From: Yaron Date: Tue, 10 Oct 2023 17:42:41 +0300 Subject: [PATCH 2/5] Improve logging and refactor to use common code. --- src/depends/libDatabase/LevelDB.cpp | 223 ++++++++++++++-------------- src/depends/libDatabase/LevelDB.h | 21 --- 2 files changed, 112 insertions(+), 132 deletions(-) diff --git a/src/depends/libDatabase/LevelDB.cpp b/src/depends/libDatabase/LevelDB.cpp index cff10ca6a7..aec86e33af 100644 --- a/src/depends/libDatabase/LevelDB.cpp +++ b/src/depends/libDatabase/LevelDB.cpp @@ -17,20 +17,81 @@ #include "LevelDB.h" #include "common/Constants.h" -#include "depends/common/Common.h" #include "depends/common/CommonData.h" -#include "depends/common/FixedHash.h" #include "libUtils/Logger.h" using namespace std; -void LevelDB::log_error(leveldb::Status status) const { - if (!status.IsNotFound()) { - LOG_GENERAL(WARNING, "LevelDB " << m_dbName << " status is not OK - " - << status.ToString()); +namespace { + +template +std::string LookupImpl(const std::shared_ptr db, + const std::string& dbName, ArgT&& arg, + bool* found = nullptr) { + std::string value; + if (!db) { + LOG_GENERAL(WARNING, "LevelDB " << dbName << " isn't available"); + if (found) { + *found = false; + } + return value; + } + + auto s = db->Get(leveldb::ReadOptions(), std::forward(arg), &value); + if (!s.ok()) { + if (!s.IsNotFound()) { + LOG_GENERAL(WARNING, "LevelDB " << dbName << " status is not OK - " + << s.ToString()); + } + if (found) { + *found = false; + } + } else if (found) { + *found = true; + } + + return value; +} + +template +int InsertImpl(const std::shared_ptr db, const std::string& dbName, + KeyT&& key, BodyT&& body) { + if (!db) { + LOG_GENERAL(WARNING, "LevelDB " << dbName << " isn't available"); + return -1; + } + + auto s = + db->Put(leveldb::WriteOptions(), leveldb::Slice(std::forward(key)), + leveldb::Slice(std::forward(body))); + + if (!s.ok()) { + LOG_GENERAL(WARNING, "[Insert] Status: " << s.ToString()); + return -1; + } + + return 0; +} + +template +int DeleteImpl(const std::shared_ptr db, const std::string& dbName, + ArgT&& arg) { + if (!db) { + LOG_GENERAL(WARNING, "LevelDB " << dbName << " isn't available"); + return -1; + } + + auto s = db->Delete(leveldb::WriteOptions(), std::forward(arg)); + if (!s.ok()) { + LOG_GENERAL(WARNING, "[DeleteDB] Status: " << s.ToString()); + return -1; } + + return 0; } +} // namespace + LevelDB::LevelDB(const string& dbName, const string& path, const string& subdirectory) : m_dbName{dbName}, m_subdirectory{subdirectory} { @@ -125,27 +186,30 @@ string LevelDB::GetDBName() { } } -string LevelDB::Lookup(const std::string& key) const { return LookupImpl(key); } +string LevelDB::Lookup(const std::string& key) const { + return LookupImpl(m_db, m_dbName, key); +} string LevelDB::Lookup(const vector& key) const { - return LookupImpl(vector_ref(&key[0], key.size())); + return LookupImpl(m_db, m_dbName, + vector_ref(&key[0], key.size())); } string LevelDB::Lookup(const boost::multiprecision::uint256_t& blockNum) const { - return LookupImpl(blockNum.convert_to()); + return LookupImpl(m_db, m_dbName, blockNum.convert_to()); } string LevelDB::Lookup(const boost::multiprecision::uint256_t& blockNum, bool& found) const { - return LookupImpl(blockNum.convert_to(), &found); + return LookupImpl(m_db, m_dbName, blockNum.convert_to(), &found); } string LevelDB::Lookup(const dev::h256& key) const { - return LookupImpl(key.hex()); + return LookupImpl(m_db, m_dbName, key.hex()); } string LevelDB::Lookup(const dev::zbytesConstRef& key) const { - return LookupImpl(ldb::Slice((char const*)key.data(), 32)); + return LookupImpl(m_db, m_dbName, ldb::Slice((char const*)key.data(), 32)); } int LevelDB::Insert(const dev::h256& key, dev::zbytesConstRef value) { @@ -154,96 +218,45 @@ int LevelDB::Insert(const dev::h256& key, dev::zbytesConstRef value) { int LevelDB::Insert(const vector& key, const vector& body) { - leveldb::Status s = m_db->Put( - leveldb::WriteOptions(), - leveldb::Slice(vector_ref(&key[0], key.size())), - leveldb::Slice(vector_ref(&body[0], body.size()))); - - if (!s.ok()) { - LOG_GENERAL(WARNING, "[Insert] Status: " << s.ToString()); - return -1; - } - - return 0; + return InsertImpl(m_db, m_dbName, + vector_ref(&key[0], key.size()), + vector_ref(&body[0], body.size())); } int LevelDB::Insert(const boost::multiprecision::uint256_t& blockNum, const vector& body) { - leveldb::Status s = m_db->Put( - leveldb::WriteOptions(), leveldb::Slice(blockNum.convert_to()), - leveldb::Slice(vector_ref(&body[0], body.size()))); - - if (!s.ok()) { - LOG_GENERAL(WARNING, "[Insert] Status: " << s.ToString()); - return -1; - } - - return 0; + return InsertImpl(m_db, m_dbName, blockNum.convert_to(), + vector_ref(&body[0], body.size())); } int LevelDB::Insert(const boost::multiprecision::uint256_t& blockNum, const std::string& body) { - leveldb::Status s = m_db->Put(leveldb::WriteOptions(), - leveldb::Slice(blockNum.convert_to()), - leveldb::Slice(body.c_str(), body.size())); - - if (!s.ok()) { - LOG_GENERAL(WARNING, "[Insert] Status: " << s.ToString()); - return -1; - } - - return 0; + return InsertImpl(m_db, m_dbName, blockNum.convert_to(), + leveldb::Slice{body.c_str(), body.size()}); } int LevelDB::Insert(const string& key, const vector& body) { - return Insert(leveldb::Slice(key), - leveldb::Slice(dev::zbytesConstRef(&body[0], body.size()))); + return InsertImpl(m_db, m_dbName, key, + dev::zbytesConstRef(&body[0], body.size())); } int LevelDB::Insert(const leveldb::Slice& key, dev::zbytesConstRef value) { - leveldb::Status s = - m_db->Put(leveldb::WriteOptions(), key, ldb::Slice(value)); - if (!s.ok()) { - LOG_GENERAL(WARNING, "[Insert] Status: " << s.ToString()); - return -1; - } - - return 0; + return InsertImpl(m_db, m_dbName, key, value); } int LevelDB::Insert(const dev::h256& key, const string& value) { - leveldb::Status s = m_db->Put(leveldb::WriteOptions(), - ldb::Slice((char const*)key.data(), key.size), - ldb::Slice(value.data(), value.size())); - if (!s.ok()) { - LOG_GENERAL(WARNING, "[Insert] Status: " << s.ToString()); - return -1; - } - - return 0; + return InsertImpl(m_db, m_dbName, + ldb::Slice((char const*)key.data(), key.size), + ldb::Slice(value.data(), value.size())); } int LevelDB::Insert(const dev::h256& key, const vector& body) { - leveldb::Status s = m_db->Put( - leveldb::WriteOptions(), leveldb::Slice(key.hex()), - leveldb::Slice(vector_ref(&body[0], body.size()))); - if (!s.ok()) { - LOG_GENERAL(WARNING, "[Insert] Status: " << s.ToString()); - return -1; - } - - return 0; + return InsertImpl(m_db, m_dbName, key.hex(), + vector_ref(&body[0], body.size())); } int LevelDB::Insert(const leveldb::Slice& key, const leveldb::Slice& value) { - leveldb::Status s = m_db->Put(leveldb::WriteOptions(), key, value); - - if (!s.ok()) { - LOG_GENERAL(WARNING, "[Insert] Status: " << s.ToString()); - return -1; - } - - return 0; + return InsertImpl(m_db, m_dbName, key, value); } bool LevelDB::BatchInsert( @@ -272,6 +285,11 @@ bool LevelDB::BatchInsert( } } + if (!m_db) { + LOG_GENERAL(WARNING, "LevelDB " << m_dbName << " isn't available"); + return false; + } + ldb::Status s = m_db->Write(leveldb::WriteOptions(), &batch); if (!s.ok()) { @@ -292,6 +310,11 @@ bool LevelDB::BatchInsert( } } + if (!m_db) { + LOG_GENERAL(WARNING, "LevelDB " << m_dbName << " isn't available"); + return false; + } + ldb::Status s = m_db->Write(leveldb::WriteOptions(), &batch); if (!s.ok()) { @@ -308,6 +331,11 @@ bool LevelDB::BatchDelete(const std::vector& toDelete) { batch.Delete(leveldb::Slice(i.hex())); } + if (!m_db) { + LOG_GENERAL(WARNING, "LevelDB " << m_dbName << " isn't available"); + return false; + } + ldb::Status s = m_db->Write(leveldb::WriteOptions(), &batch); if (!s.ok()) { @@ -339,47 +367,20 @@ bool LevelDB::Exists(const std::string& key) const { } int LevelDB::DeleteKey(const dev::h256& key) { - leveldb::Status s = - m_db->Delete(leveldb::WriteOptions(), ldb::Slice(key.hex())); - if (!s.ok()) { - LOG_GENERAL(WARNING, "[DeleteDB] Status: " << s.ToString()); - return -1; - } - - return 0; + return DeleteImpl(m_db, m_dbName, key.hex()); } int LevelDB::DeleteKey(const boost::multiprecision::uint256_t& blockNum) { - leveldb::Status s = m_db->Delete(leveldb::WriteOptions(), - ldb::Slice(blockNum.convert_to())); - if (!s.ok()) { - LOG_GENERAL(WARNING, "[DeleteDB] Status: " << s.ToString()); - return -1; - } - - return 0; + return DeleteImpl(m_db, m_dbName, blockNum.convert_to()); } int LevelDB::DeleteKey(const std::string& key) { - leveldb::Status s = m_db->Delete(leveldb::WriteOptions(), ldb::Slice(key)); - if (!s.ok()) { - LOG_GENERAL(WARNING, "[DeleteKey] Status: " << s.ToString()); - return -1; - } - - return 0; + return DeleteImpl(m_db, m_dbName, key); } int LevelDB::DeleteKey(const vector& key) { - leveldb::Status s = m_db->Delete( - leveldb::WriteOptions(), - leveldb::Slice(vector_ref(&key[0], key.size()))); - if (!s.ok()) { - LOG_GENERAL(WARNING, "[DeleteKey] Status: " << s.ToString()); - return -1; - } - - return 0; + return DeleteImpl(m_db, m_dbName, + vector_ref(&key[0], key.size())); } int LevelDB::DeleteDB() { diff --git a/src/depends/libDatabase/LevelDB.h b/src/depends/libDatabase/LevelDB.h index ef7e5528d2..ac7db41761 100644 --- a/src/depends/libDatabase/LevelDB.h +++ b/src/depends/libDatabase/LevelDB.h @@ -37,8 +37,6 @@ class LevelDB { leveldb::Options m_options; std::string m_open_db_path; - void log_error(leveldb::Status status) const; - public: /// Constructor. explicit LevelDB(const std::string& dbName, @@ -155,25 +153,6 @@ class LevelDB { private: bool ResetDBForNormalNode(); bool ResetDBForLookupNode(); - - template - std::string LookupImpl(ArgT&& arg, bool* found = nullptr) const { - std::string value; - leveldb::Status s = - m_db->Get(leveldb::ReadOptions(), std::forward(arg), &value); - if (!s.ok()) { - log_error(s); - if (found) { - *found = false; - } - return ""; - } - - if (found) { - *found = true; - } - return value; - } }; #endif // __LEVELDB_H__ From c77b94fa4ae82a6a8f3e057054d893b47f5fff33 Mon Sep 17 00:00:00 2001 From: Yaron Date: Wed, 11 Oct 2023 15:11:26 +0300 Subject: [PATCH 3/5] Add missing conditions for otterscan database. --- src/libPersistence/BlockStorage.cpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/libPersistence/BlockStorage.cpp b/src/libPersistence/BlockStorage.cpp index c01b061977..023e5659ba 100644 --- a/src/libPersistence/BlockStorage.cpp +++ b/src/libPersistence/BlockStorage.cpp @@ -2149,7 +2149,7 @@ bool BlockStorage::GetOtterTrace(const dev::h256& key, std::string& trace) { bool BlockStorage::PutOtterTxAddressMapping( const dev::h256& txId, const std::set& addresses, const uint64_t& blocknum) { - if (!ARCHIVAL_LOOKUP_WITH_TX_TRACES) { + if (!ARCHIVAL_LOOKUP_WITH_TX_TRACES || !LOOKUP_NODE_MODE) { LOG_GENERAL( WARNING, "This should only be triggered when archival lookup is enabled!."); @@ -2163,6 +2163,13 @@ bool BlockStorage::PutOtterTxAddressMapping( lock_guard g(m_mutexTxBody); + if (!m_otterAddressNonceLookup) { + LOG_GENERAL( + WARNING, + "Attempt to access non initialized DB! Are you in lookup mode? "); + return false; + } + // for each address, add to the tx hashes and block number that touched them for (auto address : addresses) { // lowercase address @@ -2289,7 +2296,7 @@ std::vector BlockStorage::GetOtterTxAddressMapping( bool BlockStorage::PutOtterAddressNonceLookup(const dev::h256& txId, uint64_t nonce, std::string address) { - if (!ARCHIVAL_LOOKUP_WITH_TX_TRACES) { + if (!ARCHIVAL_LOOKUP_WITH_TX_TRACES || !LOOKUP_NODE_MODE) { LOG_GENERAL( WARNING, "This should only be triggered when archival lookup is enabled!."); @@ -2318,6 +2325,13 @@ bool BlockStorage::PutOtterAddressNonceLookup(const dev::h256& txId, lock_guard g(m_mutexTxBody); + if (!m_otterAddressNonceLookup) { + LOG_GENERAL( + WARNING, + "Attempt to access non initialized DB! Are you in lookup mode? "); + return {}; + } + ZilliqaMessage::OtterscanAddressNonceLookup insert; insert.set_hash("0x" + txId.hex()); From dbc584e1ba8987db29bdf01ec25844ae6bdef9cc Mon Sep 17 00:00:00 2001 From: Yaron Date: Wed, 11 Oct 2023 17:56:10 +0300 Subject: [PATCH 4/5] Further checks. --- src/libPersistence/BlockStorage.cpp | 71 +++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 20 deletions(-) diff --git a/src/libPersistence/BlockStorage.cpp b/src/libPersistence/BlockStorage.cpp index 023e5659ba..39876278d4 100644 --- a/src/libPersistence/BlockStorage.cpp +++ b/src/libPersistence/BlockStorage.cpp @@ -176,7 +176,7 @@ bool BlockStorage::PutTxBody(const uint64_t& epochNum, const dev::h256& key, if (!m_txEpochDB) { LOG_GENERAL( WARNING, - "Attempt to access non initialized DB! Are you in lookup mode? "); + "Attempt to access non initialized DB! Are you in lookup mode?"); return false; } @@ -530,6 +530,11 @@ bool BlockStorage::GetLatestTxBlock(TxBlockSharedPtr& block) { } bool BlockStorage::GetTxBody(const dev::h256& key, TxBodySharedPtr& body) { + if (!LOOKUP_NODE_MODE) { + LOG_GENERAL(WARNING, "Non lookup node should not trigger this."); + return false; + } + const zbytes& keyBytes = key.asBytes(); lock_guard g(m_mutexTxBody); @@ -537,7 +542,7 @@ bool BlockStorage::GetTxBody(const dev::h256& key, TxBodySharedPtr& body) { if (!m_txEpochDB) { LOG_GENERAL( WARNING, - "Attempt to access non initialized DB! Are you in lookup mode? "); + "Attempt to access non initialized DB! Are you in lookup mode?"); return false; } @@ -572,7 +577,7 @@ bool BlockStorage::CheckTxBody(const dev::h256& key) { if (!m_txEpochDB) { LOG_GENERAL( WARNING, - "Attempt to access non initialized DB! Are you in lookup mode? "); + "Attempt to access non initialized DB! Are you in lookup mode?"); return false; } @@ -687,11 +692,22 @@ bool BlockStorage::PutTxTrace(const dev::h256& key, const std::string& trace) { return false; } + if (!LOOKUP_NODE_MODE) { + LOG_GENERAL(WARNING, "Non lookup node should not trigger this."); + return false; + } + if (!key) { LOG_GENERAL(WARNING, "Setting with a zero hash is not allowed"); return false; } + if (!m_txTraceDB) { + LOG_GENERAL( + WARNING, + "Attempt to access non initialized DB! Are you in lookup mode?"); + return false; + } // First we retrieve the struct at the null location as it tells us what to // delete, and we need to add this hash auto baseStruct = GetTxTraceInfoStruct(*this); @@ -712,7 +728,7 @@ bool BlockStorage::PutTxTrace(const dev::h256& key, const std::string& trace) { if (!m_txTraceDB) { LOG_GENERAL( WARNING, - "Attempt to access non initialized DB! Are you in lookup mode? "); + "Attempt to access non initialized DB! Are you in lookup mode?"); return false; } @@ -734,7 +750,7 @@ bool BlockStorage::GetTxTrace(const dev::h256& key, std::string& trace) { if (!m_txTraceDB) { LOG_GENERAL( WARNING, - "Attempt to access non initialized DB! Are you in lookup mode? "); + "Attempt to access non initialized DB! Are you in lookup mode?"); return false; } @@ -803,7 +819,7 @@ bool BlockStorage::PutExtSeedPubKey(const PubKey& pubK) { if (!m_extSeedPubKeysDB) { LOG_GENERAL( WARNING, - "Attempt to access non initialized DB! Are you in lookup mode? "); + "Attempt to access non initialized DB! Are you in lookup mode?"); return false; } @@ -840,7 +856,7 @@ bool BlockStorage::DeleteExtSeedPubKey(const PubKey& pubK) { if (!m_extSeedPubKeysDB) { LOG_GENERAL( WARNING, - "Attempt to access non initialized DB! Are you in lookup mode? "); + "Attempt to access non initialized DB! Are you in lookup mode?"); return false; } @@ -871,7 +887,7 @@ bool BlockStorage::GetAllExtSeedPubKeys(unordered_set& pubKeys) { if (!m_extSeedPubKeysDB) { LOG_GENERAL( WARNING, - "Attempt to access non initialized DB! Are you in lookup mode? "); + "Attempt to access non initialized DB! Are you in lookup mode?"); return false; } @@ -1560,7 +1576,7 @@ bool BlockStorage::PutMinerInfoDSComm(const uint64_t& dsBlockNum, if (!m_minerInfoDSCommDB) { LOG_GENERAL( WARNING, - "Attempt to access non initialized DB! Are you in lookup mode? "); + "Attempt to access non initialized DB! Are you in lookup mode?"); return false; } @@ -1580,7 +1596,7 @@ bool BlockStorage::GetMinerInfoDSComm(const uint64_t& dsBlockNum, if (!m_minerInfoDSCommDB) { LOG_GENERAL( WARNING, - "Attempt to access non initialized DB! Are you in lookup mode? "); + "Attempt to access non initialized DB! Are you in lookup mode?"); return false; } @@ -1616,7 +1632,7 @@ bool BlockStorage::PutMinerInfoShards(const uint64_t& dsBlockNum, if (!m_minerInfoShardsDB) { LOG_GENERAL( WARNING, - "Attempt to access non initialized DB! Are you in lookup mode? "); + "Attempt to access non initialized DB! Are you in lookup mode?"); return false; } @@ -1636,7 +1652,7 @@ bool BlockStorage::GetMinerInfoShards(const uint64_t& dsBlockNum, if (!m_minerInfoShardsDB) { LOG_GENERAL( WARNING, - "Attempt to access non initialized DB! Are you in lookup mode? "); + "Attempt to access non initialized DB! Are you in lookup mode?"); return false; } @@ -2092,6 +2108,11 @@ bool BlockStorage::PutOtterTrace(const dev::h256& key, return false; } + if (!LOOKUP_NODE_MODE) { + LOG_GENERAL(WARNING, "Non lookup node should not trigger this."); + return false; + } + if (!key) { LOG_GENERAL(WARNING, "Setting with a zero hash is not allowed"); return false; @@ -2107,7 +2128,7 @@ bool BlockStorage::PutOtterTrace(const dev::h256& key, if (!m_otterTraceDB) { LOG_GENERAL( WARNING, - "Attempt to access non initialized DB! Are you in lookup mode? "); + "Attempt to access non initialized DB! Are you in lookup mode?"); return false; } @@ -2129,7 +2150,7 @@ bool BlockStorage::GetOtterTrace(const dev::h256& key, std::string& trace) { if (!m_otterTraceDB) { LOG_GENERAL( WARNING, - "Attempt to access non initialized DB! Are you in lookup mode? "); + "Attempt to access non initialized DB! Are you in lookup mode?"); return false; } @@ -2149,13 +2170,18 @@ bool BlockStorage::GetOtterTrace(const dev::h256& key, std::string& trace) { bool BlockStorage::PutOtterTxAddressMapping( const dev::h256& txId, const std::set& addresses, const uint64_t& blocknum) { - if (!ARCHIVAL_LOOKUP_WITH_TX_TRACES || !LOOKUP_NODE_MODE) { + if (!ARCHIVAL_LOOKUP_WITH_TX_TRACES) { LOG_GENERAL( WARNING, "This should only be triggered when archival lookup is enabled!."); return false; } + if (!LOOKUP_NODE_MODE) { + LOG_GENERAL(WARNING, "Non lookup node should not trigger this."); + return false; + } + if (!txId) { LOG_GENERAL(WARNING, "Setting with a zero txid is not allowed"); return false; @@ -2166,7 +2192,7 @@ bool BlockStorage::PutOtterTxAddressMapping( if (!m_otterAddressNonceLookup) { LOG_GENERAL( WARNING, - "Attempt to access non initialized DB! Are you in lookup mode? "); + "Attempt to access non initialized DB! Are you in lookup mode?"); return false; } @@ -2223,7 +2249,7 @@ std::vector BlockStorage::GetOtterTxAddressMapping( if (!m_otterTxAddressMappingDB) { LOG_GENERAL( WARNING, - "Attempt to access non initialized DB! Are you in lookup mode? "); + "Attempt to access non initialized DB! Are you in lookup mode?"); return {}; } @@ -2296,13 +2322,18 @@ std::vector BlockStorage::GetOtterTxAddressMapping( bool BlockStorage::PutOtterAddressNonceLookup(const dev::h256& txId, uint64_t nonce, std::string address) { - if (!ARCHIVAL_LOOKUP_WITH_TX_TRACES || !LOOKUP_NODE_MODE) { + if (!ARCHIVAL_LOOKUP_WITH_TX_TRACES) { LOG_GENERAL( WARNING, "This should only be triggered when archival lookup is enabled!."); return false; } + if (!LOOKUP_NODE_MODE) { + LOG_GENERAL(WARNING, "Non lookup node should not trigger this."); + return false; + } + if (!txId) { LOG_GENERAL(WARNING, "Setting with a zero txid is not allowed"); return false; @@ -2328,7 +2359,7 @@ bool BlockStorage::PutOtterAddressNonceLookup(const dev::h256& txId, if (!m_otterAddressNonceLookup) { LOG_GENERAL( WARNING, - "Attempt to access non initialized DB! Are you in lookup mode? "); + "Attempt to access non initialized DB! Are you in lookup mode?"); return {}; } @@ -2348,7 +2379,7 @@ std::string BlockStorage::GetOtterAddressNonceLookup(std::string address, if (!m_otterAddressNonceLookup) { LOG_GENERAL( WARNING, - "Attempt to access non initialized DB! Are you in lookup mode? "); + "Attempt to access non initialized DB! Are you in lookup mode?"); return {}; } From 71a6ee53cae526410ff8d4027effb6243519ba26 Mon Sep 17 00:00:00 2001 From: Yaron Date: Wed, 18 Oct 2023 12:15:21 +0300 Subject: [PATCH 5/5] Make the bucket name in localdev.py configurable. --- scripts/localdev.py | 88 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 71 insertions(+), 17 deletions(-) diff --git a/scripts/localdev.py b/scripts/localdev.py index f53a6788a3..5b7d4e0bd0 100755 --- a/scripts/localdev.py +++ b/scripts/localdev.py @@ -343,7 +343,7 @@ def wait_for_helm_pod(config, pod_partial_name): return pod_name -def localstack_up(config): +def localstack_up(config, bucket_name): """ Let helm deploy localstack """ run_or_die(config, ["helm", "upgrade", "--install", "localstack", "localstack/localstack"]) localstack_pod_name = wait_for_helm_pod(config, "localstack-") @@ -351,7 +351,6 @@ def localstack_up(config): # Port forward localstack so we can talk to it run_or_die(config, ["kubectl", "port-forward", "deployment/localstack", "4566:4566"], in_background=True) - bucket_name = 'zilliqa-devnet' run_or_die(config, ['kubectl', 'exec', '-it', localstack_pod_name, '--', 'awslocal', 's3', 'mb', f's3://{bucket_name}']) run_or_die(config, ['kubectl', 'exec', '-it', localstack_pod_name, '--', 'awslocal', 's3', 'mb', f's3://tempo']) @@ -427,6 +426,10 @@ def prometheus_up(config, testnet_name, count = 8): ips.append(pod_ip) if len(ips) != count: + # If for some reason you see the following line repeating indefinitely when + # bringing up a devnet, it's most likely because the count is incorrect either + # due to a change in write_testnet_configuration or in the testnet repo (perhaps + # someone added/removed pods?). print(f"Waiting for all pods to be assigned an IP...") time.sleep(2) else: @@ -507,12 +510,13 @@ def tempo_down(config): default=True, show_default=True, help="Use isolated_server_accounts.json to create accounts when zilliqa is up") +@click.option("--bucket-name", help="The name of the bucket", default="zilliqa-devnet") @click.option("--persistence", help="A persistence directory to start the network with. Has no effect without also passing `--key-file`.") @click.option("--key-file", help="A `.tar.gz` generated by `./testnet.sh back-up auto` containing the keys used to start this network. Has no effect without also passing `--persistence`.") @click.option("--monitoring", help="Start monitoring - when set to false, skips grafana, prometheus, and tempo", default=True,show_default=True) @click.option("--desk", is_flag=True, default=False, help="use a small testnet configuration") @click.option("--chain-id", help="Set the chain id", default=None) -def up_cmd(ctx, driver, zilliqa_image, testnet_name, isolated_server_accounts, persistence, key_file, monitoring, chain_id, desk): +def up_cmd(ctx, driver, zilliqa_image, testnet_name, isolated_server_accounts, bucket_name, persistence, key_file, monitoring, chain_id, desk): """ Build Zilliqa (via a process equivalent to the build-zilliqa & build-scilla commands), write configuration files for a testnet named localdev, run `localdev/config.sh up`, and start a proxy to allow the user to monitor traffic @@ -524,15 +528,15 @@ def up_cmd(ctx, driver, zilliqa_image, testnet_name, isolated_server_accounts, p else: adjust_config(config, driver) - up(config, zilliqa_image, testnet_name, isolated_server_accounts, persistence, key_file, monitoring, chain_id = chain_id,desk=desk) + up(config, zilliqa_image, testnet_name, isolated_server_accounts, bucket_name, persistence, key_file, monitoring, chain_id = chain_id, desk=desk) -def up(config, zilliqa_image, testnet_name, isolated_server_accounts, persistence, key_file, monitoring, chain_id, desk): +def up(config, zilliqa_image, testnet_name, isolated_server_accounts, bucket_name, persistence, key_file, monitoring, chain_id, desk): minikube = get_minikube_ip(config) - write_testnet_configuration(config, zilliqa_image, testnet_name, isolated_server_accounts, persistence, key_file, chain_id, desk) - localstack_up(config) + write_testnet_configuration(config, zilliqa_image, testnet_name, isolated_server_accounts, bucket_name, persistence, key_file, chain_id, desk) + localstack_up(config, bucket_name) if monitoring: grafana_up(config, testnet_name) - start_testnet(config, testnet_name, persistence) + start_testnet(config, testnet_name, persistence, bucket_name) if monitoring: prometheus_up(config, testnet_name) tempo_up(config, testnet_name) @@ -657,7 +661,7 @@ def isolated(config, enable_evm = True, block_time_ms = None, chain_id = None): # except: # pass -def start_testnet(config, testnet_name, persistence): +def start_testnet(config, testnet_name, persistence, bucket_name): run_or_die(config, ["./testnet.sh", "up"], in_dir=os.path.join(TESTNET_DIR, testnet_name)) if persistence is not None: @@ -675,7 +679,6 @@ def aws(cmd): # Copy persistence to S3 in localstack. Each of the subdirectories in S3 are meant to contain only a subset of # persistence, but we choose to just copy everything into everywhere. - bucket_name = "zilliqa-devnet" aws(["s3", "sync", f"{persistence}/", f"s3://{bucket_name}/blockchain-data/{testnet_name}/"]) aws(["s3", "sync", f"{persistence}/", f"s3://{bucket_name}/incremental/localdev/persistence/"]) aws(["s3", "cp", f"_{testnet_name}/.currentTxBlk", f"s3://{bucket_name}/incremental/{testnet_name}/"]) @@ -731,7 +734,7 @@ def stop_testnet(config, testnet_name): # tediously, testnet.sh has a habit of returning non-zero error codes when eg. the testnet has already been destroyed :-( run_or_die(config, ["sh", "-c", "echo localdev | ./testnet.sh down"], in_dir=os.path.join(TESTNET_DIR, testnet_name), allow_failure = True) -def write_testnet_configuration(config, zilliqa_image, testnet_name, isolated_server_accounts, persistence, key_file, chain_id,desk): +def write_testnet_configuration(config, zilliqa_image, testnet_name, isolated_server_accounts, bucket_name, persistence, key_file, chain_id, desk): instance_dir = os.path.join(TESTNET_DIR, testnet_name) minikube_ip = get_minikube_ip(config) @@ -739,6 +742,11 @@ def write_testnet_configuration(config, zilliqa_image, testnet_name, isolated_se print(f"Removing old testnet configuration ..") shutil.rmtree(instance_dir) print(f"Generating testnet configuration .. ") + + # IMPORTANT: if you change any of the flags make sure to update the count returned + # by this function so that prometheus_up knows how many pods to wait for. + # 20 normal pods (i.e. 5 ds + 15 non-ds), 1 lookup, 2 seedpubs, 1 seedprv. + count = 20 + 1 + 2 + 1 if desk: cmd = ["./bootstrap.py", testnet_name, "--clusters", "minikube", "--constants-from-file", os.path.join(ZILLIQA_DIR, "constants.xml"), @@ -752,6 +760,7 @@ def write_testnet_configuration(config, zilliqa_image, testnet_name, isolated_se "--host-network", "false", "--https", "localdomain", "--seed-multiplier", "true", + "--bucket", bucket_name, "--localstack", "true"] else: cmd = ["./bootstrap.py", testnet_name, "--clusters", "minikube", "--constants-from-file", @@ -772,9 +781,7 @@ def write_testnet_configuration(config, zilliqa_image, testnet_name, isolated_se cmd = cmd + ([ "--isolated-server-accounts", os.path.join(ZILLIQA_DIR, "isolated-server-accounts.json") ] if isolated_server_accounts else []) cmd = cmd + [ "-f" ] if persistence is not None and key_file is not None: - bucket_name = "zilliqa-devnet" cmd.extend([ - "--bucket", bucket_name, "--recover-from-s3", f"s3://{bucket_name}/persistence/{testnet_name}-persistence.tar.gz", "--recover-key-files", key_file, ]) @@ -796,6 +803,8 @@ def write_testnet_configuration(config, zilliqa_image, testnet_name, isolated_se with open(constants_xml_target_path, 'w') as f: f.write(output_config) + return count + def kill_mitmweb(config, pidfile_name): pidfile = Pidfile(config, pidfile_name) pid = pidfile.get() @@ -891,7 +900,7 @@ def build_native_to_workspace(config): build_env['SCILLA_REPO_ROOT'] = SCILLA_DIR # Let's start off by building Scilla, in case it breaks. run_or_die(config, ["make"], in_dir = SCILLA_DIR, env = build_env) - run_or_die(config, ["./build.sh"], in_dir = ZILLIQA_DIR) + run_or_die(config, ["./build.sh", "ninja", "debug"], in_dir = ZILLIQA_DIR) run_or_die(config, ["cargo", "build", "--release", "--package", "evm-ds"], in_dir = os.path.join(ZILLIQA_DIR, "evm-ds")) # OK. That worked. Now copy the relevant bits to our workspace @@ -1114,6 +1123,29 @@ def build_zilliqa(config, driver, scilla_image, tag): capture_output = False) return image_name +def build_native_zilliqa(config): + build_env = os.environ.copy() + build_env.update(config.default_env) + + if not build_env.get("VCPKG_ROOT"): + raise GiveUp("Environment variable VCPKG_ROOT must be defined to point to vcpkg") + + # vcpkg_triplet = run_or_die(config, ["scripts/vcpkg_triplet.sh"], in_dir = ZILLIQA_DIR, env = build_env, + # capture_output = True).decode('utf-8') + + # Cleanup + build_dir = os.path.join(ZILLIQA_DIR, ".build.vcpkg") + shutil.rmtree(os.path.join(build_dir, "vcpkg_installed"), ignore_errors = True) + shutil.rmtree(os.path.join(build_dir, "CMakeFiles"), ignore_errors = True) + try: + os.remove(os.path.join(build_dir, "CMakeCache.txt")) + except: + pass + + run_or_die(config, ["./build.sh", "ninja", "debug"], in_dir = ZILLIQA_DIR) + run_or_die(config, ["cargo", "build", "--release", "--package", "evm-ds"], in_dir = + os.path.join(ZILLIQA_DIR, "evm-ds")) + @click.command("build-zilliqa") @click.option("--driver", required=True, @@ -1121,17 +1153,39 @@ def build_zilliqa(config, driver, scilla_image, tag): show_default=True, help="The minikube driver to use") @click.option("--scilla-image", - required=True, + required=False, help="the scilla image to use when building the zilliqa image (i.e. scilla:)") @click.option("--tag", help="The zilliqa image tag. Will be generated if not given.") +@click.option("--native", + is_flag=True, + default=False, + show_default=True, + help="Build Scilla natively.") @click.pass_context -def build_zilliqa_cmd(ctx, driver, scilla_image, tag): +def build_zilliqa_cmd(ctx, driver, scilla_image, tag, native): """ Builds a zilliqa image. """ + if native: + if tag: + raise GiveUp("--native and --tag can't be specified together") + elif ctx.params["driver"]: + print("ignoring --driver since --native is specified; building natively...") + else: + if not scilla_image: + raise GiveUp("--scilla-image must be specified") + + if not driver: + driver = default_driver() + ctx.params["driver"] = driver + + config = get_config(ctx) - build_zilliqa(config, driver, scilla_image, tag) + if native: + build_native_zilliqa(config) + else: + build_zilliqa(config, driver, scilla_image, tag) def get_pod_names(config, node_type = None): cmd = ["kubectl",