diff --git a/hub/db/db.py b/hub/db/db.py index 5f289b7..69806d1 100644 --- a/hub/db/db.py +++ b/hub/db/db.py @@ -34,7 +34,7 @@ NAMESPACE = f"{PROMETHEUS_NAMESPACE}_db" class SecondaryDB: - DB_VERSIONS = [7, 8, 9, 10, 11] + DB_VERSIONS = [7, 8, 9, 10, 11, 12] def __init__(self, coin, db_dir: str, secondary_name: str, max_open_files: int = -1, reorg_limit: int = 200, cache_all_claim_txos: bool = False, cache_all_tx_hashes: bool = False, diff --git a/hub/db/migrators/migrate11to12.py b/hub/db/migrators/migrate11to12.py new file mode 100644 index 0000000..d76c795 --- /dev/null +++ b/hub/db/migrators/migrate11to12.py @@ -0,0 +1,57 @@ +import logging +from collections import defaultdict + +FROM_VERSION = 11 +TO_VERSION = 12 + + +def migrate(db): + log = logging.getLogger(__name__) + prefix_db = db.prefix_db + + log.info("migrating the db to version 12") + + effective_amounts = defaultdict(int) + + log.info("deleting any existing future effective amounts") + + to_delete = list(prefix_db.future_effective_amount.iterate(deserialize_key=False, deserialize_value=False)) + while to_delete: + batch, to_delete = to_delete[:100000], to_delete[100000:] + if batch: + prefix_db.multi_delete(batch) + prefix_db.unsafe_commit() + + log.info("calculating future claim effective amounts for the new index at block %i", db.db_height) + cnt = 0 + for k, v in prefix_db.active_amount.iterate(): + cnt += 1 + effective_amounts[k.claim_hash] += v.amount + if cnt % 1000000 == 0: + log.info("scanned %i amounts for %i claims", cnt, len(effective_amounts)) + log.info("preparing to insert future effective amounts") + + effective_amounts_to_put = [ + prefix_db.future_effective_amount.pack_item(claim_hash, effective_amount) + for claim_hash, effective_amount in effective_amounts.items() + ] + + log.info("inserting %i future effective amounts", len(effective_amounts_to_put)) + + cnt = 0 + + while effective_amounts_to_put: + batch, effective_amounts_to_put = effective_amounts_to_put[:100000], effective_amounts_to_put[100000:] + if batch: + prefix_db.multi_put(batch) + prefix_db.unsafe_commit() + cnt += len(batch) + if cnt % 1000000 == 0: + log.info("inserted effective amounts for %i claims", cnt) + + log.info("finished building the effective amount index") + + db.db_version = 12 + db.write_db_state() + db.prefix_db.unsafe_commit() + log.info("finished migration to version 12") diff --git a/hub/scribe/service.py b/hub/scribe/service.py index 9c3561d..abe537d 100644 --- a/hub/scribe/service.py +++ b/hub/scribe/service.py @@ -2145,6 +2145,9 @@ class BlockchainProcessorService(BlockchainService): elif self.db.db_version == 10: from hub.db.migrators.migrate10to11 import migrate, FROM_VERSION, TO_VERSION self.db._index_address_status = self.env.index_address_status + elif self.db.db_version == 11: + from hub.db.migrators.migrate11to12 import migrate, FROM_VERSION, TO_VERSION + self.db._index_address_status = self.env.index_address_status else: raise RuntimeError("unknown db version") self.log.warning(f"migrating database from version {FROM_VERSION} to version {TO_VERSION}")