From af0e9368d431e85a5b99c02d6c4ede3464e2db94 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 16 Mar 2020 11:28:11 -0300 Subject: [PATCH 01/10] headers get now async --- lbry/extras/daemon/json_response_encoder.py | 7 ++++--- lbry/stream/stream_manager.py | 2 +- lbry/wallet/header.py | 17 ++++++++++++----- lbry/wallet/ledger.py | 4 ++-- .../blockchain/test_resolve_command.py | 4 ++-- tests/unit/wallet/test_headers.py | 16 ++++++++-------- 6 files changed, 29 insertions(+), 21 deletions(-) diff --git a/lbry/extras/daemon/json_response_encoder.py b/lbry/extras/daemon/json_response_encoder.py index 997e2019e..92a43092c 100644 --- a/lbry/extras/daemon/json_response_encoder.py +++ b/lbry/extras/daemon/json_response_encoder.py @@ -159,6 +159,7 @@ class JSONResponseEncoder(JSONEncoder): return tx_height = txo.tx_ref.height best_height = self.ledger.headers.height + timestamp = self.ledger.headers.synchronous_get(tx_height)['timestamp'] if 0 < tx_height <= best_height else None output = { 'txid': txo.tx_ref.id, 'nout': txo.position, @@ -166,7 +167,7 @@ class JSONResponseEncoder(JSONEncoder): 'amount': dewies_to_lbc(txo.amount), 'address': txo.get_address(self.ledger) if txo.has_address else None, 'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height, - 'timestamp': self.ledger.headers[tx_height]['timestamp'] if 0 < tx_height <= best_height else None + 'timestamp': timestamp } if txo.is_spent is not None: output['is_spent'] = txo.is_spent @@ -244,7 +245,7 @@ class JSONResponseEncoder(JSONEncoder): if isinstance(value, int): meta[key] = dewies_to_lbc(value) if 0 < meta.get('creation_height', 0) <= self.ledger.headers.height: - meta['creation_timestamp'] = self.ledger.headers[meta['creation_height']]['timestamp'] + meta['creation_timestamp'] = self.ledger.headers.synchronous_get(meta['creation_height'])['timestamp'] return meta def encode_input(self, txi): @@ -306,7 +307,7 @@ class JSONResponseEncoder(JSONEncoder): 'added_on': managed_stream.added_on, 'height': tx_height, 'confirmations': (best_height + 1) - tx_height if tx_height > 0 else tx_height, - 'timestamp': self.ledger.headers[tx_height]['timestamp'] if 0 < tx_height <= best_height else None, + 'timestamp': self.ledger.headers.synchronous_get(tx_height)['timestamp'] if 0 < tx_height <= best_height else None, 'is_fully_reflected': managed_stream.is_fully_reflected } diff --git a/lbry/stream/stream_manager.py b/lbry/stream/stream_manager.py index 92983b368..891af7ddb 100644 --- a/lbry/stream/stream_manager.py +++ b/lbry/stream/stream_manager.py @@ -338,7 +338,7 @@ class StreamManager: 'claim_sequence': -1, 'address': txo.get_address(wallet_manager.ledger), 'valid_at_height': txo.meta.get('activation_height', None), - 'timestamp': wallet_manager.ledger.headers[tx_height]['timestamp'], + 'timestamp': wallet_manager.ledger.headers.synchronous_get(tx_height)['timestamp'], 'supports': [] } else: diff --git a/lbry/wallet/header.py b/lbry/wallet/header.py index 43c7b7743..2adbbff02 100644 --- a/lbry/wallet/header.py +++ b/lbry/wallet/header.py @@ -104,7 +104,14 @@ class Headers: def __bool__(self): return True - def __getitem__(self, height) -> dict: + async def get(self, height) -> dict: + if isinstance(height, slice): + raise NotImplementedError("Slicing of header chain has not been implemented yet.") + if not 0 <= height <= self.height: + raise IndexError(f"{height} is out of bounds, current height: {self.height}") + return self.deserialize(height, self.get_raw_header(height)) + + def synchronous_get(self, height): if isinstance(height, slice): raise NotImplementedError("Slicing of header chain has not been implemented yet.") if not 0 <= height <= self.height: @@ -167,7 +174,7 @@ class Headers: for height, chunk in self._iterate_chunks(start, headers): try: # validate_chunk() is CPU bound and reads previous chunks from file system - self.validate_chunk(height, chunk) + await self.validate_chunk(height, chunk) except InvalidHeader as e: bail = True chunk = chunk[:(height-e.height)*self.header_size] @@ -186,13 +193,13 @@ class Headers: self._size = self.io.tell() // self.header_size return written - def validate_chunk(self, height, chunk): + async def validate_chunk(self, height, chunk): previous_hash, previous_header, previous_previous_header = None, None, None if height > 0: - previous_header = self[height-1] + previous_header = await self.get(height-1) previous_hash = self.hash(height-1) if height > 1: - previous_previous_header = self[height-2] + previous_previous_header = await self.get(height-2) chunk_target = self.get_next_chunk_target(height // 2016 - 1) for current_hash, current_header in self._iterate_headers(height, chunk): block_target = self.get_next_block_target(chunk_target, previous_previous_header, previous_header) diff --git a/lbry/wallet/ledger.py b/lbry/wallet/ledger.py index b6870d636..abe407453 100644 --- a/lbry/wallet/ledger.py +++ b/lbry/wallet/ledger.py @@ -601,7 +601,7 @@ class Ledger(metaclass=LedgerRegistry): if 0 < remote_height < len(self.headers): merkle = await self.network.retriable_call(self.network.get_merkle, tx.id, remote_height) merkle_root = self.get_root_of_merkle_tree(merkle['merkle'], merkle['pos'], tx.hash) - header = self.headers[remote_height] + header = await self.headers.get(remote_height) tx.position = merkle['pos'] tx.is_verified = merkle_root == header['merkle_root'] @@ -899,7 +899,7 @@ class Ledger(metaclass=LedgerRegistry): headers = self.headers history = [] for tx in txs: # pylint: disable=too-many-nested-blocks - ts = headers[tx.height]['timestamp'] if tx.height > 0 else None + ts = headers.synchronous_get(tx.height)['timestamp'] if tx.height > 0 else None item = { 'txid': tx.id, 'timestamp': ts, diff --git a/tests/integration/blockchain/test_resolve_command.py b/tests/integration/blockchain/test_resolve_command.py index 2681e3826..2a9a1fe08 100644 --- a/tests/integration/blockchain/test_resolve_command.py +++ b/tests/integration/blockchain/test_resolve_command.py @@ -49,11 +49,11 @@ class ResolveCommand(BaseResolveTestCase): self.assertTrue(claim['is_channel_signature_valid']) self.assertEqual( claim['timestamp'], - self.ledger.headers[claim['height']]['timestamp'] + self.ledger.headers.synchronous_get(claim['height'])['timestamp'] ) self.assertEqual( claim['signing_channel']['timestamp'], - self.ledger.headers[claim['signing_channel']['height']]['timestamp'] + self.ledger.headers.synchronous_get(claim['signing_channel']['height'])['timestamp'] ) # resolving claim foo by itself diff --git a/tests/unit/wallet/test_headers.py b/tests/unit/wallet/test_headers.py index 456529e75..8499a2b99 100644 --- a/tests/unit/wallet/test_headers.py +++ b/tests/unit/wallet/test_headers.py @@ -15,11 +15,11 @@ def block_bytes(blocks): class TestHeaders(AsyncioTestCase): - def test_deserialize(self): + async def test_deserialize(self): self.maxDiff = None h = Headers(':memory:') h.io.write(HEADERS) - self.assertEqual(h[0], { + self.assertEqual(await h.get(0), { 'bits': 520159231, 'block_height': 0, 'claim_trie_root': b'0000000000000000000000000000000000000000000000000000000000000001', @@ -29,7 +29,7 @@ class TestHeaders(AsyncioTestCase): 'timestamp': 1446058291, 'version': 1 }) - self.assertEqual(h[10], { + self.assertEqual(await h.get(10), { 'bits': 509349720, 'block_height': 10, 'merkle_root': b'f4d8fded6a181d4a8a2817a0eb423cc0f414af29490004a620e66c35c498a554', @@ -112,11 +112,11 @@ class TestHeaders(AsyncioTestCase): await headers.connect(0, HEADERS) self.assertEqual(19, headers.height) with self.assertRaises(IndexError): - _ = headers[3001] + _ = await headers.get(3001) with self.assertRaises(IndexError): - _ = headers[-1] - self.assertIsNotNone(headers[19]) - self.assertIsNotNone(headers[0]) + _ = await headers.get(-1) + self.assertIsNotNone(await headers.get(19)) + self.assertIsNotNone(await headers.get(0)) async def test_repair(self): headers = Headers(':memory:') @@ -166,7 +166,7 @@ class TestHeaders(AsyncioTestCase): for block_index in range(BLOCKS): while len(headers) < block_index: await asyncio.sleep(0.000001) - assert headers[block_index]['block_height'] == block_index + assert (await headers.get(block_index))['block_height'] == block_index reader_task = asyncio.create_task(reader()) await writer() await reader_task From ec8e24332309e33b2788dd493f7946954aa8934d Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 18 Mar 2020 00:45:44 -0300 Subject: [PATCH 02/10] estimate timestamps instead of using block headers --- lbry/extras/daemon/json_response_encoder.py | 7 +++---- lbry/stream/stream_manager.py | 2 +- lbry/wallet/header.py | 14 ++++---------- lbry/wallet/ledger.py | 2 +- .../integration/blockchain/test_resolve_command.py | 4 ++-- tests/unit/stream/test_stream_manager.py | 3 +++ 6 files changed, 14 insertions(+), 18 deletions(-) diff --git a/lbry/extras/daemon/json_response_encoder.py b/lbry/extras/daemon/json_response_encoder.py index 92a43092c..d2080db8c 100644 --- a/lbry/extras/daemon/json_response_encoder.py +++ b/lbry/extras/daemon/json_response_encoder.py @@ -159,7 +159,6 @@ class JSONResponseEncoder(JSONEncoder): return tx_height = txo.tx_ref.height best_height = self.ledger.headers.height - timestamp = self.ledger.headers.synchronous_get(tx_height)['timestamp'] if 0 < tx_height <= best_height else None output = { 'txid': txo.tx_ref.id, 'nout': txo.position, @@ -167,7 +166,7 @@ class JSONResponseEncoder(JSONEncoder): 'amount': dewies_to_lbc(txo.amount), 'address': txo.get_address(self.ledger) if txo.has_address else None, 'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height, - 'timestamp': timestamp + 'timestamp': self.ledger.headers.estimated_timestamp(tx_height) } if txo.is_spent is not None: output['is_spent'] = txo.is_spent @@ -245,7 +244,7 @@ class JSONResponseEncoder(JSONEncoder): if isinstance(value, int): meta[key] = dewies_to_lbc(value) if 0 < meta.get('creation_height', 0) <= self.ledger.headers.height: - meta['creation_timestamp'] = self.ledger.headers.synchronous_get(meta['creation_height'])['timestamp'] + meta['creation_timestamp'] = self.ledger.headers.estimated_timestamp(meta['creation_height']) return meta def encode_input(self, txi): @@ -307,7 +306,7 @@ class JSONResponseEncoder(JSONEncoder): 'added_on': managed_stream.added_on, 'height': tx_height, 'confirmations': (best_height + 1) - tx_height if tx_height > 0 else tx_height, - 'timestamp': self.ledger.headers.synchronous_get(tx_height)['timestamp'] if 0 < tx_height <= best_height else None, + 'timestamp': self.ledger.headers.estimated_timestamp(tx_height), 'is_fully_reflected': managed_stream.is_fully_reflected } diff --git a/lbry/stream/stream_manager.py b/lbry/stream/stream_manager.py index 891af7ddb..0ff206936 100644 --- a/lbry/stream/stream_manager.py +++ b/lbry/stream/stream_manager.py @@ -338,7 +338,7 @@ class StreamManager: 'claim_sequence': -1, 'address': txo.get_address(wallet_manager.ledger), 'valid_at_height': txo.meta.get('activation_height', None), - 'timestamp': wallet_manager.ledger.headers.synchronous_get(tx_height)['timestamp'], + 'timestamp': wallet_manager.ledger.headers.estimated_timestamp(tx_height), 'supports': [] } else: diff --git a/lbry/wallet/header.py b/lbry/wallet/header.py index 2adbbff02..4eae54a87 100644 --- a/lbry/wallet/header.py +++ b/lbry/wallet/header.py @@ -33,6 +33,8 @@ class Headers: genesis_hash = b'9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463' target_timespan = 150 checkpoint = (600_000, b'100b33ca3d0b86a48f0d6d6f30458a130ecb89d5affefe4afccb134d5a40f4c2') + first_block_timestamp = 1466646588 # block 1, as 0 is off by a lot + timestamp_average_offset = 160.6855883050695 # calculated at 733447 validate_difficulty: bool = True @@ -111,12 +113,8 @@ class Headers: raise IndexError(f"{height} is out of bounds, current height: {self.height}") return self.deserialize(height, self.get_raw_header(height)) - def synchronous_get(self, height): - if isinstance(height, slice): - raise NotImplementedError("Slicing of header chain has not been implemented yet.") - if not 0 <= height <= self.height: - raise IndexError(f"{height} is out of bounds, current height: {self.height}") - return self.deserialize(height, self.get_raw_header(height)) + def estimated_timestamp(self, height): + return self.first_block_timestamp + (height * self.timestamp_average_offset) def get_raw_header(self, height) -> bytes: self.io.seek(height * self.header_size, os.SEEK_SET) @@ -283,10 +281,6 @@ class Headers: header = headers[start:end] yield self.hash_header(header), self.deserialize(height+idx, header) - @property - def claim_trie_root(self): - return self[self.height]['claim_trie_root'] - @staticmethod def header_hash_to_pow_hash(header_hash: bytes): header_hash_bytes = unhexlify(header_hash)[::-1] diff --git a/lbry/wallet/ledger.py b/lbry/wallet/ledger.py index abe407453..4d1207bee 100644 --- a/lbry/wallet/ledger.py +++ b/lbry/wallet/ledger.py @@ -899,7 +899,7 @@ class Ledger(metaclass=LedgerRegistry): headers = self.headers history = [] for tx in txs: # pylint: disable=too-many-nested-blocks - ts = headers.synchronous_get(tx.height)['timestamp'] if tx.height > 0 else None + ts = headers.estimated_timestamp(tx.height)['timestamp'] item = { 'txid': tx.id, 'timestamp': ts, diff --git a/tests/integration/blockchain/test_resolve_command.py b/tests/integration/blockchain/test_resolve_command.py index 2a9a1fe08..ea6874cfd 100644 --- a/tests/integration/blockchain/test_resolve_command.py +++ b/tests/integration/blockchain/test_resolve_command.py @@ -49,11 +49,11 @@ class ResolveCommand(BaseResolveTestCase): self.assertTrue(claim['is_channel_signature_valid']) self.assertEqual( claim['timestamp'], - self.ledger.headers.synchronous_get(claim['height'])['timestamp'] + self.ledger.headers.estimated_timestamp(claim['height']) ) self.assertEqual( claim['signing_channel']['timestamp'], - self.ledger.headers.synchronous_get(claim['signing_channel']['height'])['timestamp'] + self.ledger.headers.estimated_timestamp(claim['signing_channel']['height']) ) # resolving claim foo by itself diff --git a/tests/unit/stream/test_stream_manager.py b/tests/unit/stream/test_stream_manager.py index 4dfb76b35..e33064503 100644 --- a/tests/unit/stream/test_stream_manager.py +++ b/tests/unit/stream/test_stream_manager.py @@ -84,6 +84,9 @@ async def get_mock_wallet(sd_hash, storage, balance=10.0, fee=None): }) class FakeHeaders: + def estimated_timestamp(self, height): + return 1984 + def __init__(self, height): self.height = height From e45375dc266f238f538f3334e0152eeeece128aa Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 18 Mar 2020 00:59:03 -0300 Subject: [PATCH 03/10] more async parts --- lbry/wallet/header.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lbry/wallet/header.py b/lbry/wallet/header.py index 4eae54a87..01694e735 100644 --- a/lbry/wallet/header.py +++ b/lbry/wallet/header.py @@ -111,12 +111,12 @@ class Headers: raise NotImplementedError("Slicing of header chain has not been implemented yet.") if not 0 <= height <= self.height: raise IndexError(f"{height} is out of bounds, current height: {self.height}") - return self.deserialize(height, self.get_raw_header(height)) + return self.deserialize(height, await self.get_raw_header(height)) def estimated_timestamp(self, height): return self.first_block_timestamp + (height * self.timestamp_average_offset) - def get_raw_header(self, height) -> bytes: + async def get_raw_header(self, height) -> bytes: self.io.seek(height * self.header_size, os.SEEK_SET) return self.io.read(self.header_size) @@ -128,9 +128,9 @@ class Headers: def bytes_size(self): return len(self) * self.header_size - def hash(self, height=None) -> bytes: + async def hash(self, height=None) -> bytes: return self.hash_header( - self.get_raw_header(height if height is not None else self.height) + await self.get_raw_header(height if height is not None else self.height) ) @staticmethod @@ -195,7 +195,7 @@ class Headers: previous_hash, previous_header, previous_previous_header = None, None, None if height > 0: previous_header = await self.get(height-1) - previous_hash = self.hash(height-1) + previous_hash = await self.hash(height-1) if height > 1: previous_previous_header = await self.get(height-2) chunk_target = self.get_next_chunk_target(height // 2016 - 1) From 241e946d917941d48b63cf469ba67c0efe469810 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 18 Mar 2020 04:10:55 -0300 Subject: [PATCH 04/10] first experimental version --- lbry/wallet/checkpoints/__init__.py | 735 ++++++++++++++++++++++++++++ lbry/wallet/header.py | 38 +- lbry/wallet/ledger.py | 26 +- 3 files changed, 783 insertions(+), 16 deletions(-) create mode 100644 lbry/wallet/checkpoints/__init__.py diff --git a/lbry/wallet/checkpoints/__init__.py b/lbry/wallet/checkpoints/__init__.py new file mode 100644 index 000000000..3c103be4e --- /dev/null +++ b/lbry/wallet/checkpoints/__init__.py @@ -0,0 +1,735 @@ +HASHES = { + 0: 'bf3ff54138625c56737509f080e7e7f3c55972f0f80e684f8e25d2ad83bedbe2', + 1000: '4ec1f9aebc8f7f75d5d05430d1512e38598188d56a8b51510c9e47656c4ffad9', + 2000: '5d5965e43d187b6f8b35b4be51099d9ec6f7b0446dac778f30040b8371b621a2', + 3000: 'd429f69a9dd890f7d9827232a348fe5120371ed402baf53c383e860081f6706e', + 4000: 'd18fef650f23032e4e21299b8b71127f53ff6d2114c3eac4592a71f550dc5071', + 5000: '12c1dd1b9cda29eb4448e619a4099c7bb6612159e1cd9765fbbabeee50f1daff', + 6000: 'ebda54cab7979bc60f46cab8d36b6fcea31a44c38d6e59bdcb582054b6885cf8', + 7000: 'd2be08791053336ae9bd322ee5b97e9920764858f173a9e0e8fb54ee5d8bad0e', + 8000: '94583639d9e2ce5eac78c08ef662190830263ea13468453b0a8492c3aca562fe', + 9000: 'a6ebb24a3ec9bb3fbdb9d9525729513dd705fe85093c33a07bbfe34ba1f1eae0', + 10000: 'a5d32c6cd43097f3725e36dd55d08ec04cbe62e40d1fb3b798167cc9c1fa1ce4', + 11000: 'b5cf8df354bc26e8b0315e8038564a9a7fc1faa6ad75e67819b7eafbcfdcb145', + 12000: 'ae7b0e5106299e43b78f337fe15e85c27f88d5bcf618a21c03ce43c094dde909', + 13000: 'b57dfe631571b654e11fb4295aecfc11f612664d22d64a9690f64c0b3ec128bf', + 14000: '76e55cc1c38d9a6a349c9a42ae0b748d0d8b27ebb30539be8bb92e71e7abca17', + 15000: '5d33cbc872acb824a89ef601aa3c918e7f4dbb8d5be509a400fa41b03f7fe103', + 16000: '25760f6a6bfb58104d02e60efa62ba5ae141bbac1f65d944a667ad66fbef15a9', + 17000: 'ad37d98bb272aee3a950679cdd4cc203b1c129bb3a03fe74223ff858badfcc50', + 18000: '9eac34fc453b97b1c62228740efcae4ef910b0bea26c322e17e3474b81713b0d', + 19000: '06783429c8f50cc6495da29d38c86c3873800cf1eceffedfffe2a73f116a39d3', + 20000: '0237c620f7f68d9ae9852d00551c98a938352d2e3d9336c536f1cbf5246fade6', + 21000: 'c4dbb4eea7195a2cf73de2d49b32447cec9c063051acbd300b26399c9242b0ed', + 22000: 'd18fea80b659d9690830c2f37c7c9d53f79fb919debb27cc2131044df26d7ff5', + 23000: '0371254f4698d3c46596c1fd861e2e1e75053beb69c3c4d8ee6702f0992f0026', + 24000: 'dbc97c3986bb9bb6230321d1fc8b7ca13369033f57b08ac01bb775f7241861fe', + 25000: 'cfb9baa125e91d02c060940b5b51cce52b495a0b0bd081cd869a7ca45a0e0585', + 26000: '6e07fecd3d0f72b8cd50a3cdc1e9f624beb3dac9f6356308a1ec295f47d68f3b', + 27000: '8ad248da3aac7c1db958d0ce8a2d50fde461bbb96a628efa215116d3b74b74e5', + 28000: '4e8231ede23b397c49f1a97020ab5fe45465936cb6c8cf1c39ade1b38a80547f', + 29000: '1453b7a4be4076e2c77b94bc35f639c571dc05b225a5f8018a9a60337fb3ef66', + 30000: '43d779c83032d75e8df09aeb6ab103ad41618777565fabc2e56bf31c3a20285a', + 31000: 'ea1426f57f75ee781a6e569772944119f5e65c8b110b31a32b61d951c3a1900f', + 32000: '43d1a3941b2c3033f78040830d39e657f74958d71b3cd0bc728550469cdb5fde', + 33000: '3b0c0eac231476efed2b9bb9dff206a6438a14946c3b4238bf0af6d03f9a350f', + 34000: 'a8136e9d6f3f5048ef8f9f56628009f5a87bb28664ba8028ecae0797e5328309', + 35000: 'd8136e638995aaea03c6b50f1cc0f1891bd107290f12a6ee57e3cae36b7e3374', + 36000: '06a7692cf5ce9bb71461e5427e6f55bce2904f9d3af4c28e7343fecbf0e336da', + 37000: '12a1a49c237497295150f99c76d79d8affe9fd720e0cbffa62be80e2bda7f832', + 38000: 'fcf3e526931b7f7cead47652f12cdf76641032bc545e22aa0dc83bef46a085fa', + 39000: 'ffc34a198ab1cc13819c0c90b26a8455627ff12868b49ac1e6b5e1a086ed011f', + 40000: 'ac71a9895999531350cf77a701035b0b59720a39994c990354b9cb1f6ea8eb49', + 41000: '4fa753f5f2de41ad69ab32c6802eeb352f070d8684c3bd154eeb8a3c17aa6363', + 42000: '6f0febbddf9248e3df2254a7196ec13f353c7a8049e098f2fcdf920580abffa4', + 43000: 'e3e98488cb203be9b531e3b5ce2200f4e5655297a4f4534bcf486ad5e26687bf', + 44000: 'c7c6078e30163204e49d493d506f72f71d68d2a591874328342ca3cca81c87bd', + 45000: '2a65629529f69c15a6e404ba54b83e024acba9d6c786be554fb14d28951ddf9b', + 46000: 'cbc148ad661e6fc390746e94cf21df71edcb222b4e8d8f0771237a8e03412f9c', + 47000: 'd6977f770749b7ce1a624a61f7b2d26d817a27c50a895466ee10f8bd758c029a', + 48000: 'df05fddcd5ccfab7811e5d7b6cf7b49df71c7e0f77a57ce384936948114bb93c', + 49000: '9d26a67c06f066229bcdafd55cbe71e2b2518465d6deb618c418b2f229ef513f', + 50000: '4ceb896d901315e0f3e10f3b1ed8472f5bb5230d4ed6377214b07e425e515f6a', + 51000: '218019bd8440a8c242e84940119d4eccccd26134a208cbb92cdea3480069f482', + 52000: 'c15f1538280e71abc1cf3e117c79b0b671a9f6fb46102fea7b9a1fbcad418011', + 53000: '0ad4d3812a42e23fc0defe7c62dad451266ee21699ade625a5f00b91993850d3', + 54000: 'f994cff7b8166b0c39a310bb686f043fe9355678ae406a7fa1a74d4bf6f9b849', + 55000: '6723d30c99fb45513a42a3ffaa50d1b8350e3c8b52c6dc6894303147eba95d55', + 56000: '947cbea79aabe2051522d9e957d92b86dee2c420c32a83cf6f6d655136731926', + 57000: '8a153fa31a971e7e304650257bfdbaac230b36a9beebc01a85a7aab9a357a5f2', + 58000: 'd19a5234ab995cb851d691b2c2c84ae665890621c52794e54c16d6a87f516c7b', + 59000: 'f9c7a353e62ce57a5e3f983362b08fdbff5e2aaadce325bba7680104a6f51cb7', + 60000: '9e1d535bd31525803e93e4b4c2346fcdfee7c5cdc4b9fe3032df9295aa85c754', + 61000: '640387271e7eb0816b9e290dd157f58d1eb54cfc39bc7ecc4486099af2a22b76', + 62000: '3d36acc4169a238657729df674277c6cf236e21c2bed4e99b8f1a6e82d00bbd6', + 63000: '6d4201e45595ea160773d63bb1481f78664156de1f4564d9a8c3f61b3cc1a6c9', + 64000: 'b2cadf2ba39b2d5e688029cbbc68cc6d71337b29861dee24fe941b31be1198a3', + 65000: 'd8a364d385ef42e53ce88c82355235ee7a27dd3e642849dc22d08a4041695dfb', + 66000: '7025d4b2d9537002ce2b9ec42c596eb864e30f41c06f02413997b5629d6dce3b', + 67000: '852a3fecfb3a77bcd6d2604170cbe9646d9b889c90d234eddc8f9d112777fe41', + 68000: '38956213012f66a89377f4914db838faa6738732f7a7a865ebc5777f2f126b73', + 69000: '4c89b07d23727d4bd1a080bcb9a22dcdb45c904f4c9866153a000056d7ec1546', + 70000: 'e4e7fbf0d17b5f933deba12e045ea23eabfa6f7dbdc633e5c3d3ef8ddd066b47', + 71000: '1f07c36e82f8e7582ec9df23ddc90715df45978b6aa0a2f17c08ee14cbb23000', + 72000: 'c59c660b5aca2fa3364524ef43a4fa93fc3f16df777427cf1d810f65ce42e4d2', + 73000: '83a05ac2fcae8c5d50a90df308f641616fa9b9fb553763c874b764a73def2f95', + 74000: '448d209c032a2c6f5330481bf34fcf0bf1ea2c47f43fa5f82cf42e0d8f4d2b20', + 75000: '5627eb3491e1154ad2c90094da85dc7dbfaf89d3765848db1ed915b7e26dde58', + 76000: 'b1417b1b2360e24fe42345508ba0a333f4bd76f48c959b8bb40a99bf34ec5ba0', + 77000: '250aca2bc4e6da7a8e1cb2e4deb0396aee7c317177b099d7dab9731b4c0f7573', + 78000: 'dbfb1dbe80a5db06c9dcc645c5279eaf40559f5e04df0e64b696e80aba5659e0', + 79000: 'a7183a545925164b5a9882810ae117d0f3df9b3ed55885fe74ebea284bec484f', + 80000: '5b4a7bfd9b5394e2759daecfe472ea6e92b281a7821b106314b2b2e0facbaab8', + 81000: 'b7075fe530b44e4f5895521b3cf99a79fe23d8bedde0027d6e7924ca93793eab', + 82000: '6a1fa75b896d799ae84046cba7cd942dad47ec04bf50a85cd1b2bb15861995e1', + 83000: 'aa5f4caf970433e90e2e45f9524ab9a6e22281505ada4c206ecf4672486240e3', + 84000: '103732d12ef792a1adbe8f295ca6abd008867323ee996698b2650bdc1bb8d06b', + 85000: 'a214c16ffdc505d6caba7bd0c2bae766bb19f7eb4d436cfc037e93783afae7e8', + 86000: 'f54226ae9a6814a345968b5c2982adf638b995ff50e27de1890b3760ead12158', + 87000: '26546212a0720cab988194432f4fa7c3e47caf0fb8e31efdd1ee93c3ad056868', + 88000: '48f7d47bc443bdbc6a37ffe8f1d0d91de16d4a470d03655692de8a04f84c2561', + 89000: '005bac455ff093a39b907af0441897e7a9ceb19eaf51d9dd9ec2b5ea43ff6d57', + 90000: 'b482f10315170e6bd280325e180ce37b0423b685baa06d33586ee659649b71a7', + 91000: '56db4d5d6c460fa76d492d9900c02e8fa99bfe97c7d9fa2a75bf115128ea0da7', + 92000: '72ac917c4f531f91f65fbe33e78b4d4e2bef18aab3d2ff584d59fed1ba2cf398', + 93000: '1100df2fdb03d44530723d131dd7d687053db6a317ac9181cfdd51925152df02', + 94000: '6d50f66efb571136501340fc00b52105e0011f0f11fd68c9b185717bd42f9307', + 95000: 'bd0c996d9ec40d5e91c2b5f278a5c9c9ea0b839012a7d135e1afc73a725bd8d6', + 96000: '4ebfe8a3e1a1625632896d5d2970d8b080f08c5948f0ac38e81f3fc85db5af5e', + 97000: 'cbc03464aa3513aa33b540d0f026b93db7607c4bc245af8fd88c3b476ea5e394', + 98000: 'c9a185ffa55d01fc73c2e60004b0c8dee60ffbac7784cd16046effd7a0a51a86', + 99000: '1eba97d5ee69229f4cd697c934d60cf3262ebcdf4c79526cda71e07b52fa22f7', + 100000: '0b1650207a55c64e7a6cec62ddc3cc190ed3796507bbd48358a0b7d6186bc3ce', + 101000: '1f47841b0034d3cf1046660c2246902f679445b5ef621df2762bbea33b7d685a', + 102000: '42f63b1b622d08e120b5405ba65d25e1a9777312cca77248b7827d7084e6d482', + 103000: 'ef4861fbd4e578ff80e0d5e2afc62fb113aa203d3933f74efd3e2400d73de922', + 104000: 'e97a3cb78c09eef732757e81c6a7135beb33d95398054b45df738853acedaaad', + 105000: 'd12e8feec15893ccab662a1ad0754b2ccbc18b078ff3894eb2214ee4ac2f7af9', + 106000: '83ae84103b37c93b2d1535fc47b053194270b3c186b1b25d69cd8a1540caaa1a', + 107000: 'a102c6dce88bc4695250c24a0ebd2913b2b85c211b7d5dbbd18ecd95bea63144', + 108000: 'a0c20b36140d55288dc4dfeb6c16a6ece3d43efb57b1728bc872c35d6660c704', + 109000: '412fe428b2929c3cf5c6991a2657e5414a569b30e77a3f4fecef14239d89caca', + 110000: '8518706e957f3ef892d1d2d7f65c5f4588620be994cda9cb7d81fb600554a456', + 111000: '66f4e25f40d24360299e5358f954d8a79b245399c46dbc4e8a3db0c10fa14e18', + 112000: '81c0ded5c5e30f92eec84ca293abb0165068d917bdcf436754a0e9b0ef1e325b', + 113000: 'ebd5e034a45517e59e96e986b55e38051a5d2a3120a7ed92d4f3e01e18e73972', + 114000: '681b85720d71ab660c7443392883ed4bdcf9aceffe202cc5ead94664e315a744', + 115000: '3fbda839115bc6e6d451a05f78b27b35b137c94374fdaa42235f3035980e85e5', + 116000: '7bfb88d39fe0ee7ce046648676b226997a812af1f1ed79040750045536c76067', + 117000: '5853c6d48fbf99b06d3e1f474e1b667ca598712f9feaff8cda48a95935cfc498', + 118000: '6a23987731379799289d2a527dd40fde8aaca8398625d2a2e3e646372ffc99e1', + 119000: 'af73cbe957865b60c3c3139dd1e2bf8bf1cc504dd167545463105996609750c4', + 120000: 'b309a20170421fc4ddfedfd7611716e1cf5a802a1db09b66b9ae448fdf958792', + 121000: '624cc893801e4da093ca973e2135352bb428dc278e47f5308412d05acbbdeacc', + 122000: '2f20c67ac3485b6e5407762a23e8a7df1bb380f1607f538cb5e2e1ffaad578ef', + 123000: '89c73ed65ab8f729cdcd678dc52e876969907f1560898220702a696893744e00', + 124000: '17d8155d97fa4e8705aa55969709f9cbc97f87815d40f5a2b5cf4d7b85dfe4b3', + 125000: 'd3cbbdb7f51c8a1f5794bb3d1f40c413b313f91bd3f3dec0d8ebb6a02107e52a', + 126000: 'bfd02d760eb5de2e792ea74215ae37d13174c3fc0aff08587281f0f39e427caf', + 127000: '937dcc09a14d1de79f5eb679996225995cdce69497cff5f12c39c098c42656a0', + 128000: '81d372b07e2467bfa62bb4b974d0eb75ea313e547b9d4e62f3c617f939bcf67d', + 129000: '9b4b315738ab8b5266b5accada140dcf487b2f509f1d758b9b972b93ecd03c9e', + 130000: '8ab3b1862cacf7f1c4f6617f7280ba31547dadd62bec9c37ecc8d074c90be2b0', + 131000: '868d6520e526b8594c44a05eba67ec1de8e01b547b34b62ebc5dd09d656e9072', + 132000: 'ce09471822965da31db3188cd156538c44337da2952486a34dd2ab372ea490a8', + 133000: 'b0c34cd700630c9281c8657dd24baf609f72f897442538d77f4ac355199731c0', + 134000: 'a76e5dc895406a3951d55b557272915810420bdc3ac076863c7efdef3b115890', + 135000: '2b70fd7021d40f6df15f3f72bc8e034a8a0bcd8542847b5b7a2088253351b3a4', + 136000: '6161ee84f6fe100a02de0675bc9500dc18ccaec74e7da4b92135f0ccee6fb663', + 137000: 'ae5770a7158089ca9adab8bdd07f1259e6f204ed377af27fb6a772fe1c031864', + 138000: 'f3b8de21370968a90cedfcb1cb6803ec9a6b7740a94d8e4c60a80407fd55a12e', + 139000: 'f08f6431d7367b71b25b321809ebf48f797bf7fcc943ba366420fd3f3dc00e5b', + 140000: 'e76345dbb8c4c5ed1ea37249f892f64098557700259fb360401a559c45909041', + 141000: '776b333f5b221f6b443d6011bc8d0753eed5ce2446a6d3ee99a5800c7159fa92', + 142000: '9453693cb846f27ba2ac39194e5b70cedfaf435b73b5f7f288dd81343ff605ea', + 143000: '08aea64cc4eb0170d2704cbd3a4ee871d4f3e53a8c1395288c029eaddd1aa097', + 144000: '801d60c225301481c761114d5152780d5002b77aed18845c32c9e3804a5db254', + 145000: '78cc80d54933b84248aee32a18144d03237f948de7889c97e2112d17ac998009', + 146000: '1e954e8f2fe59da3feef0514f6b271f52961467782b2530501c294a83181f176', + 147000: 'c0ac42fe9c0e1d2b0b7e6e796c553eafbd5d7bbce7df84903dabe16dc56ba863', + 148000: '75b5441fc1785dcba69b9409f24224ba3207d624bb0e7d54af24118d08c788d7', + 149000: '20e53048515cc34507cf2d1dd160eabafaa97793d81626e9fce38d540142dbc4', + 150000: '5ff7a08461fc3fbacf8c6261dbd34a790e271bbc39908c23d5417f0f7a71b71c', + 151000: '5c14cab7670821966d5aa61f09b32d93c865764b6e258764a3c33a82a3133fe7', + 152000: 'e595968e040b54ed6572862b4089e005238399014d99bf4c361cc5922dbfa6c0', + 153000: '1292bac3fab11de09f46a8327511b7c37366d0f2a96a027f4bf2f16147a39af3', + 154000: '3db31267722b500dd9b26f5cfc4f06f26c5c343d181464499d3bb4aaf13c3c20', + 155000: 'b92962af73b42335125f28d7d3fb97e3aa6ccf47c3f4229eb7fa04b67fdb933e', + 156000: 'fdbcf12415284d8cc0adbf16996b9d37a943fd1a924c5dee5f47dc8add23f278', + 157000: '199781ca8e5929708f834d6f38ce3e2fe196ab519f9915f14401462273431e19', + 158000: '969017fd15cf6a73683de287c5e19fbd69ebb5aaf7c13339297f7177a716de3d', + 159000: '77c7995cf97218655c195fdac9010c599859a46f760d84750189114d2d2d1d9d', + 160000: '00ff84f0d845b4f1099d970cc0c3e2dd1c2584c611449f2318a3f27327246d51', + 161000: '08481dc5f61e776ae8be12236d595549abfa0be28b187d80dad573594d94c11f', + 162000: '71497ab26b05453f3c7057f8bf57fcc8ba30920a6032ab0ae3abdd29e0677582', + 163000: '08713e0b750233a3843241d24573c4800f32c894000b7815860048ef3e7a06db', + 164000: '01fd806f1879285ad5234f38789074b0ea3a0b2707bc6b49aa9bd51ecd26ddcb', + 165000: 'a990225a2f02a77c1c61d518d46dde1e1cff3f4df0ed24eb463b97f84c7a2616', + 166000: '7772a601a6e765ee340eb666cd645503c9988430ba4f558961246e9e69349b25', + 167000: '389c7a8979078b57d54936497680c7908b9f989e46631358f1c31a3094621557', + 168000: '2e1df97fe3cba7add5f05734a119464ff7c22bf92701ed85642da856ec653aff', + 169000: 'fe0adfe455f65cf90e1632ef39bf9ca5856020d995cb71ce8a4998a40eed5998', + 170000: '87487f255873e9f6411565553fa9ccb9518e7baf71ade67492536278f1ba0feb', + 171000: '37cf239bd630b7c891019adcefead4baf19a86b40e21f3dfe4a76a78f2985103', + 172000: 'e2f4087b868558af51dc8f40d77c300d76eacab17e5fb8bbf3b1f99c02d995a7', + 173000: '273f32717f740438e93063d259dd4f0f160077c34c937a7d6a9b1f9fbb34d87d', + 174000: '807a861813ca690a6fc0715f354f36bf491c34588b5ab778c0f4e692ec3fc152', + 175000: 'dd74cf3d686c59820885bab134c81b7079f2edca86a25944fe0aa36a96941550', + 176000: 'aba8163b91a905aab5ad6e7599b919fba650c446938dc7b2352759203bea8957', + 177000: '260bd0d053038a54565f9b25c3894cfa875c9426b9d6e9915ec1df03bcf90ac5', + 178000: '7a527f71743c4dff25f13b918ad4a6d91fb0bf9c873a7282e51cb4edf545edbb', + 179000: '945397818b80995ac4e23c785b6d5ed2eb01338a2d4ca6f0cf40986e87659d86', + 180000: 'ba1b5263f5418844796a19db06a53c095ec0428696d8ee953790d6c86de0795a', + 181000: '2b673859af43554accd8599bca53c2ca94500ecd65df7672a1e586595bdec7ff', + 182000: '08bae9f096e348c8c6b42ac2762298ec3faeba896524968478fff1d40017b5b0', + 183000: '211577ae9a4f93fa5dc3e33764c31048ee306b5e5ffa5081fabdf3f30e49a977', + 184000: 'a4301eead637043a8751a30c13935af971a731e03e90317c5a16c1677e50d837', + 185000: '1f4aa362281293e8328b6c5b32a9948ef03dcea81c6ba13f94531f0c5e42627a', + 186000: '0c6b15f2164a1b55bdf0eee2b0f5ab7cb9d2dfea6fcc82686483aa7a3659f6a4', + 187000: '5bfa261cad0e4d271814de1f4752352f35da10f24b8016ac0065172e1296052a', + 188000: '94c2490bdafe70d65951e4034e5b9122fa6b376a5ad8c9e6276289c4a5e059e0', + 189000: 'a0119c5d57523b7ccaf1e5c6671eae9f8562bbf2477f7dd8c6847c6da41707f6', + 190000: '4d82802c2464053c391374eb6124f8839faa9b343bae3f34e00f87fad9f45f9b', + 191000: 'd341f5b4d20894be520eeb533d56c5737ad02a7ab0c24b2463516ff04e3b8b0d', + 192000: 'ddf1a72ba512786d39f7c313f710bbd452040f1eb8a0c0ee7a4b9192199fcc3c', + 193000: '514e1b40e5e15dc72e75cabb8d490c5e76b968e6dcdb3acf518b47e3a2020407', + 194000: 'd0bac7ca0636b662d6be169bb5af807c891523a44fceda68f08c8934d6e9e254', + 195000: '00c37e81f0dfe63a8fdefd0ba91442019899d50f72b43c4de8017e767777a5af', + 196000: '581a9d093458c0157464c42fa2e2d9a1f61f7edf1ccc5dd77a4fdfdd9a4b37b8', + 197000: 'd7555fbf7671d2475e38f62418f6130ddfe85b8fd07fe8a9815849316165d401', + 198000: 'ea388884ad5abb06ccb1d0d2202dfef5c4ce4de0f3040df089017d01f14b9530', + 199000: '6a80cbb22620b7a5f99f4ea36380b7bcd22e87feaf4186bfba0b9aabcbcaab70', + 200000: '7a7231293d242887c6dfa517a78b60d40b41456b6f2fe833c0217074664a61d7', + 201000: '0d6bde8cdd0370964b83ee766891b3cda7b9eec098dfaa5ef05842b29bbd043c', + 202000: '223fc091bddfbaedec92099f873fc1a6bc50556f3421f472a63977b3351b95c1', + 203000: 'f87fb207fd397f8fb5c5c21f84f272cf33d840e20ccaf9de810af784d8459142', + 204000: '6273b130d819d84492bc50fa443ca401df748e58fa84a3708a1797f116f60f1f', + 205000: '498edfaaa9d5f0501aea5482ffdd9208b86af46fcea5a1cbc82d51d45eb6a256', + 206000: 'c069138610493a70742918b5c624cbf9275bf55ee24649e32244238433883ea0', + 207000: '5110744882ba11b98df95f11e100bc398a5aab4c8258133b518b5abb58c7fe64', + 208000: 'e5690d4e0561ef015d6e2997df3b8b95163cd884d999d8c9ac212bd7e1f06ab0', + 209000: '17a0a3bbf2fecbfa5c7667524cb2a452f49b8dd089af132f9d2c530ca0b677b1', + 210000: '3a9489f3aca89576152949a67df6462d25bf3a78976451b5cf8bd8385a10e6ee', + 211000: '45196e756c7c282c4a9e7133adb07f103d0f74ee90919cc8ad89ca3e5d38c6db', + 212000: '590f21c83694c35028b9c9d632a082307b8bb27ec87e2c0c4e2881a7be9c6637', + 213000: '46e56c45305c51e4d22927e449a1855064504cfb58cfceb0f293d8a4f1dca7cb', + 214000: '50b3f1bb82c1b213ed2cfd2c1144b8e67e5fa731dafabfefe063f8658c5536d7', + 215000: 'f53709db4ca6c12b80c64501eeb8f176211116b0fbc3a650082cf830792b206a', + 216000: '6818db3f6582e71021a84fffe751fcb47db2f734d59547be47a373575b5c7d8e', + 217000: '01f6f8a0699ac533ee7921f63334af07b04369d5e580c1c609c6fee64c8c6b8d', + 218000: '81e9ff93cdf5686fc42663a056c74fb0f560c29a5d23f21ef92364735a865392', + 219000: '80b32bb6bc3c54a19d3b604f56084b3bda13afd4af4130f791505e44fdad6c98', + 220000: 'c427f24ab14449b1430c5302385f3a77f90f19715d940e8dfb7c054828270163', + 221000: '4303a0e1c6f8489589290370a6843d97eb02a01be8c227d905cd0e30034feae6', + 222000: 'ccfdc1768afc18cccdf2b5d90d40158d86b110d878955b6570ad9e6e40063008', + 223000: '6baefb7061239c02472c4783de6d32936f7fa1da9fa6d2448b4d92f40b89e79d', + 224000: 'bdc334f5d1a3bd1d19cf589e3b1ab0e8a324d44ea1c70297ff2de6a66c99399f', + 225000: '82d29612f96c55dbe83655db68afe39146aff1418af929f05033fcbe612ebcb0', + 226000: '179b4144f4d644f546f001273c1e5382a8de50f84ec06a2c83b02808a9167d28', + 227000: 'a350e96895efe979223475151305eb8d1739121b566964033b9967031ee5581c', + 228000: 'd208c66a9ccb1afde8d1540d944bfb43522015f9d2df50851e66ce1b84b06c0d', + 229000: '95cfabe5d18dd8d8e4a57b7bf7052ab09b7138ba0b3c9d2efdb32352bfcca149', + 230000: '3e399d08c33b8fddeb687deb276afd3ce101d19f5a71b232532951f05d46d8f6', + 231000: '91c0c17d5df1c553bd7dd6db8feb58a4436d598ef4acb1ed9a06da65f3cca83f', + 232000: 'eb3928a522dddde176cd4c68fa95f1e953d74f3759af4b030f7e2be4c466c45f', + 233000: '3ac946df5a319d31716f50c752a5769471c0f3ba4b1002613996981c8f32fa3b', + 234000: '4c9cdbce1f2a4324368130c154e2d24e885877c13a126a847059ccc289fc922c', + 235000: '8701d98b254fd0b588f022ea140e936d496ca8ccc0890ce133523432f4c09e6a', + 236000: '0d2d45adcdf950541c25ff5b2b87dff5faad479094b1f68804a62c3151e6d598', + 237000: 'b8924ead720d0e7a304c45c209ef3fdb90cde437d138bfed2f1e8357fb3d951d', + 238000: '4c9c62a3b3f6bbfb08edc23d63f9b35ed94bb38393fd1761f9b9b810bb72f68d', + 239000: 'd01494b4999c2c4def2564eeb1207fd1f3d35a62fba2e61b386bfab498cb9b6f', + 240000: '3021d8f0edbc0cd2ab87bad5feffac54924b51eac9f6fae8ebe289f474f11a8c', + 241000: 'd2ecf0170093d3a2dfc876f673e565e3e4aa76d23896af51c96c004cb14ce67a', + 242000: '2b5ef67326178702a1a35fe4a848d22b791114b59121e41a25ddc1bfac82c6bd', + 243000: 'b2d546f127252c3fa5f7ca45c61268dbaef66027484a1bcdadd281fabf97d34f', + 244000: '58f9b398b871dd62952899d31661c118ca4f0d1c71dd165523b889d62b154393', + 245000: '504760f0e46e2de0a2c8b4dd3e93c19d0f3e0e1b36916f02cbff166655816206', + 246000: '15ffb23b844e6ac648a7fd099be8e631cb51b3a2496c67ac2dbedc4f6818d7a7', + 247000: 'e3e56e8575552202cf35fc883bfcfd92cacf8013a6536729ec19e53bdb0db128', + 248000: '22f75d137c7fe42a80ee04a386fdea8cad56e7ce8601234b6ec676498edf9e34', + 249000: 'f84709269c393cae8106ddb1fa3a48cf3226e97bcc108717e483493159921aa1', + 250000: 'f3aee545330f18ec49f6f880d7f9fd31ed6633279881dd13177b0a687c51b000', + 251000: 'c2e6a4cb3c6e88f523db2ac24fc7fff3b8a27d13ea7b5d614806ed96247025e4', + 252000: 'efba2c1f91a13ee8d7a8658e70893ccd3014a7019c225e356c3ee67955b58756', + 253000: '9097ecbbc1279bfa2c92822a6485714ca7ba35cd3dad9ccaff5a8d24645eae92', + 254000: '00a41b720ec0d7a9fea3229b16ad8d065cd6bb1d14587eb3404de33e45db9454', + 255000: '7523636406176d391928c638e15e959a9d0255aa71dc9f3bea8995cae02d1d51', + 256000: '8d3f7c5ded24e2488e25d6030d78d351354734b8b058caf6143430e96ebd5d0c', + 257000: '48fbbc3737a718163a501e52964eb58aaf4163aab7ceaa4f75f928b25c376ec2', + 258000: 'e1052e7ebf83246770a740c0c6ce8f011fc7007448777c533d67fec810fe5e64', + 259000: '043f7d98b500fafc248549c1f5a9727fb0cec4e3e5f13071eeba459fb9179d13', + 260000: 'ed2f3960b077651ac554b277dd271e12e6cd05942fb54b9464f18432adbe5ca2', + 261000: '234f17bf2054ad4d3efeae165c3359a14e08d2e53506c839cbed6ba3d2a48ab5', + 262000: '4342d662c4dab581bf1c123e057e4413bbf74b7a2a331e80f7099201539be54c', + 263000: 'a7b1380371fbfe64bfb94bed5ae256f7db59a4661e461221cae99dcd54c72a8a', + 264000: '4f2a606cdc5f346e0855d962cbef7c2e4993b2b07b86ce658a796b1baf6179d8', + 265000: 'cc09cde8c101eb17b5457bf976525940904cb839e70ef3335780eb0261976826', + 266000: '3c5f24533723afb124b2bc8c2f8d373eb5640dc4ea4dc76b841e81706dd6b29d', + 267000: '467c5ca63621be011a07129d0c3ea785e2a0853cc3c0d1e9cace122831d42507', + 268000: '07df13a794f6c0fc77142fbd0afa23121ee2a2a23bc9abca35c164edb5f6ad1c', + 269000: 'e2be277e83a997a111fb45948e9a7afdd7925feb1708a84c62968cf6a42da1c1', + 270000: '463425dddeeac2f27ee717364073229c333a7ba357888623c3b15a1db52542f4', + 271000: 'd62e0bd8584fae7e4893c18252ae8d1c245e2e4c8e2f2d73703c681c6259f19f', + 272000: '85cabf5254b6b6292c105e68d44f78d6effeec582467caf9d30a5deeb43b6a6a', + 273000: 'f5509cee847a757c84adea8e40f3c7c0cc6a4da33a8cd3019cf9ca850d0bc93b', + 274000: '2cfdb3477de2fe1c81c7133094094c79e933cfe6034c158de07ffd0c4c08b564', + 275000: '1c1b8e447e95ec3c306fc68528631682bd85e1c876d86223fea28b7ff383c16a', + 276000: 'b37e5e4f6d29590c9a58773a4d4384cf7c51d04fede4a602ba8095e14bf65993', + 277000: 'b92d6ffdecdca014c946c2fe8c6c404ccf60d8401e7631918da8885bd360daca', + 278000: 'c1f82e49c8f66aa580a216c8e08262080ff9fc4f4df1c482ce8b2695b90b7240', + 279000: '41c48f5965e8939d1df69127798b46af17646ea75e545a7b47049596d9bfb3a2', + 280000: '48da55fb8ce259432417ffacba5e064df4df2a8293cdd484ee0ad8ce4b0bc38f', + 281000: '0e892fd450a8ca018567e1a6e23bcdc5b4b872686f816a415991f59b3abd76f3', + 282000: 'b90fb78edf920d0bfb4bce1f39327c6777b511fd05d715729d7c4cdbf449d59e', + 283000: '80965fd3817b9509ddbe6a919aa73adb11af64cda5d8352ad2bd9b7824e95c42', + 284000: '43447043aea93b162d6f615e8cb7d992d02c176a41b0518256f7e0d8fb3162b1', + 285000: '33bc92c77ba88251822698a7bd02860ef9c12420e4c27f4020370acaab0a2f37', + 286000: 'eb664123736754c19a47e2afbbc8645c598d7b70c3d42132c334cfeea8fd6bf3', + 287000: '6af8f2fb3ea1f0fd0d6432599c3a681be7d384cdc553903bcaac8de2d472725f', + 288000: 'b5fcfef1b5f6373bfc9322164160fe98fc7fb4f3117bf91c8fe61324db62723a', + 289000: 'f6f1c6f8a2a2905b2ca18845b8bc771bad6e9cc32fd2c8e20a565cbd7781bac9', + 290000: 'cded625fb7b923d7471ee28276b6983bb4fb4f6281ded9a8637b6974710feab5', + 291000: '49af69a4855a36d00cfe2408f7d090fb5001913aa06615eca5af78ecb92b1fb9', + 292000: '2dfe91f7f1870539e6b51ce58f9cd6415ea8fd50dad376d74941a9ff1bf0f8f4', + 293000: 'b84f14b784cf6eca78d13f4b0db4714be5bcfa59b73974b7106a0206a8993767', + 294000: '07e95fc3c141dc64ae192cd233f97ac16b1adc9dfbbd685475bf693930b6f604', + 295000: 'c39cca37abe0b8839f8e797287385c73031496c632a064a89b9dc0730a3fffd2', + 296000: 'e69afa519ac761328358f5b3ac8b4395d5aa4b955bf1b4c5f3bc0af49cbc3756', + 297000: '20c93e2d1d9bce1c861eb16febe50e534acc3e2d142cd2d0767bfe076545d1f8', + 298000: '75970b654ac884cbd1328b68b59963bfa5babacca4c7d48368a1d938feb8af6c', + 299000: '05448c7f1230870e814858ac9a7858fc72a045dd9e0f1f14b8862d2fdd524b07', + 300000: 'e254223f85c3ba65a0295a5538387c08ae94e6070f5c989d27349978fef077c7', + 301000: '4a8bd2357b4ce05422d51e052018fb9f75ce3c07c53b5d1623ac19afff388e63', + 302000: '18f4a960b810034f5ea02458bbcb9bf3d38d681555be4e40d132f867444306c8', + 303000: '667cd927ba1e7ab162c17a711bb0ba65f838e543f46b093c86c14e39988da156', + 304000: '55280e92a3c3b9ab9e1032490aff10c1a2fe9e96cf4f88e49b7a756c0491701e', + 305000: '8af5b503f1819b23da7b3663314a15b10e39e072377d1b08518a373c85515c1a', + 306000: '439796b2b4add7dd0fbe14bb3d04f3917e92de2afa07da2a384b389ad9228888', + 307000: '07fd4cbacddb504e23880dc454fb70659386afa21dab3e862cbe7688218a43fb', + 308000: '2734a4205be5f2c8ce5850bee8d4ad0829c208911bf4ce833349a43af2db8fe1', + 309000: '261b84349406a913a4cb700fd218112920910c1c57922e289114c7b903fe2d9f', + 310000: 'bc652e239edd3476df7bd1e0563595145fd4f2b37dc2ac3bc28ec089a7550812', + 311000: 'fd7c1b32015a2418efc6e1a73470ea3205de05662c767b0d4198fcaf804c3b5a', + 312000: '4f017d27b81da7eaf62760cb7fe6613a6ec28fdc18859b6e727b216240a86315', + 313000: '2d09525e2364e7cd7fe45191fee11f1ce16a2fbd16b56e41f90a3c64ce2f944e', + 314000: 'c05770490a47280c5b318709c264b2ab951a5bf8c4575cbf1b1e3dbf35323f45', + 315000: 'ac5da2f5abcee0f92f29479c6355ddafd1f3b4b3cca6659117c17c9fc10fc743', + 316000: '4496d1a0e4c22a2521595bd4fae60abdb9404f79135311606a9e6713be373271', + 317000: '40ba7c87490677af9e8b033a19ee5ffe51ba192f5912f491738bb6485cfb76d4', + 318000: 'f42045295ce30bd4ba9fd9c845bc1228949b4152517185d3dd0598d8bae38481', + 319000: 'f488695e46b53e4734f4eb9fced93f067f037af7099d0f9510856a898a0c438d', + 320000: '83e42370911c70bfa2ad89a82ecd02bbc9cb13d24c2a900a0030771898ed618b', + 321000: '74eabfbb64eb89488b59393ed529b4ee97e0f88a7fc14f798f22715bcee3fd19', + 322000: '54f42438cf96fe8efa0180a47350002b34ccfe82123a85a9a6dc6f8b0894f0a2', + 323000: '5509357b42476732435a1d5a6a850e27f62cb3ca81219a94f08dded06dfcbbf8', + 324000: '1cd477361dc4c72b4f157469b6f3616e060f5d058b44ba864225210432b35540', + 325000: '060ae9c541eee90a80a11fe300583e47c0ad4d12093d0651310cc6c87ecf2b75', + 326000: '13ce59868c947ecf2d6aa986501c9f8411a59b4e0b8b6113f4f45dff4074fe59', + 327000: '16a1532ff983f5cdab6c2d465d58f0369665a926c9d877212d95f9387e8732ca', + 328000: 'c9608a16fc43e7e94f04b3309c067b0a4f68a66705009dabb39898f41fdc862e', + 329000: '029b02acbe0fc8610894f0b89d6bd274362195a443fb9e5e4d4485be7786d461', + 330000: '01181b9529da87eb39510619285c08b05cfa2ca2c7a2fb3ecadba474c1db3aa5', + 331000: '2371752c35da119184e53dcb593a193fb1c81d6edcff921599328a6142a57e3d', + 332000: 'fc025a8f319725faa9903eca86935116a2aad8fec6193055d5d5f0f431ba84b2', + 333000: 'b602651afd9988075887a3f48455d977357032185b7d482262efff9cc97a45a6', + 334000: 'a9a2bfb6b0f8db298b5ba732f4a23242fa89e1c5038916ccfba3751f7dd1b4a0', + 335000: 'eeb3e35b6ea94119bf058e6b02a2ec2a706089bc1c360ca28d1b4e2ce1b547ac', + 336000: 'e834f7ac6bd8e5b46ecebda4997ae73bccf353fa024fbaba7ad4d1d70d08d14b', + 337000: '229a7017b6d0ffa08374e297b07f12620bcdbaaf5977dd39f3f207ecb03c7adc', + 338000: '83d86500e8b2ef46a0f1505e29f7401b123ea5ed5feacb49864ab1de284eb574', + 339000: 'b08e9dcef3e38d631414b4d3a8d8b3dfa8a07d9e5829c515e7db2fec64c60827', + 340000: 'aa36ce6494df6905f4516530ddca77d88341c0202bf052347a86e090c03ab0ad', + 341000: '527b50125d5479117ca8bfe84fd5507b2312360908d4cafe8588d40c4ef5622d', + 342000: 'd1397e8b0f847b6d46c409659e9e10f3182ad9bdbca563cc2598af37cdc080bf', + 343000: '896576aef0e0dbdb2df8291f2832716029f69718ac79f355425d475a1b2b7b4a', + 344000: 'cf4b3c75279b0f7d71cbba934228709ff03e3254fdd83a47fde66e83ff393eb3', + 345000: 'e68c760edf370e758e815d9363ada456fa2568b06abf8626f28a32cc62944770', + 346000: 'ab19fe52e780e45a716ed97e93084bb3f7916e0a4479f131c7f5f7a2e7d69d9c', + 347000: 'bc77d338a10fd71daa6c68fa9df2fdccf44b6d9b1d7c6456bb281bf8547b622b', + 348000: 'f84469ebb26540cb3de0ae03f1f47452f9a9bc6b7195c3b5260d48623507ed86', + 349000: '5bafc3e6dbe4bf88257c4381cb4f2fea0a4cde1f272e1f1006600b507cb0f670', + 350000: 'e53ff96761b9c98bed635457acd1856a2ffe65b84e339cea94ad30f13a31e5e9', + 351000: '5e0eb50d32e90e5ca73639313e5d3a850a113fbe60ac494ca6cdb799e9604d7e', + 352000: '1a1a04aaef110493bf8fbdb8d1784a7706c6e303773ad0bad6511dd3c1da6c54', + 353000: 'd72f3b615e244584f131f2f8e7154534aa9d2d9c4a710bb4de44de556b5d1232', + 354000: 'e81b823c64229c555a50e90a4c3c4654b6673cbff4c62c6e897d09aac00d95f8', + 355000: '11eed14cac2024588b389ddcd54315923ffff03d6c4c5f32657ba551fd308516', + 356000: '465f28c63e564296f75a4a22c3ad16714cda29d7a2797e51e512eb1ed5f6a49e', + 357000: '2b544c79c6f74ae1c05c268669e57ad41d6807f1d0ce1a5024b7b08646df6861', + 358000: '8a76eccf113819aad8d7c17a459a614274bfdaeacfb4659d945f13dbf3907ccc', + 359000: 'e63743f458f1479593d73952bfb882c72e5889331add576a716761eff2cf13dd', + 360000: '0ab0bef439debf08dc3e1df9010c4e119d6862de6620de5f725cac0013c06301', + 361000: '50f649f0c8bc0d77ffddd53fd5b896705a8858d3684a3ec876cb320ad26d2c29', + 362000: 'e058c43c366c9c307ca3ce5a732f0ab2b4e09ff291a067366af25a82fb3cddf1', + 363000: 'f99d7f0ec93ec7fd74273bb6f8bc4ad8ec0fcda9f6406001668dcf62a088a867', + 364000: 'e5a69ea8763650542b1f90d6217af00da137a9bb72cd0dbd5379570d1fa696f0', + 365000: '5d0eb3063474c6bbf9d9581898fcf8c4cd097fe5c6e3875d7cd701a5a0302507', + 366000: '14da2978f4d4eeff5eb0a6bb663e514095acfe2210039df8dda9b76f522f74ba', + 367000: '4ad3ebe23fe9b0f407c0765adb69ee973de5b84db29c97167c7a358fe8696bfa', + 368000: '22db57ccf8c8193ae9fc08abe11f1ea8e869bfd6ab2c0b918820acdaafc19136', + 369000: '530052d61878d20282a69e0bace41857f7edea6d3e9e92fd0a7781c8b60d208e', + 370000: 'cd057d0c889c78e06d0eb71287c46deea4340b8c7a4c470dd69020e1a9029c43', + 371000: '41f32ab47863f0c44b8755ecf9b201aaa92999051ccbf8a5acdee0e12dee4110', + 372000: '37de29bd97d75a41dc3eb52bed9c09baf069c23120a393888e15e61ac7f81ea2', + 373000: 'd6b26bb791fdf6d79f59d5c37ef2af0a35149737029157724224d614477527f4', + 374000: 'de18c6a921838385950b10ed9429579e95a0bdc574f448f457ee486a59a04260', + 375000: '1f86edf4ae47f409f8de6a16c17c2da063937478a0ccc800633e4d33b531709f', + 376000: 'a0511050ef086052fe5671d3333c255507b03edbd365cd7ce7cecddb4000f391', + 377000: '7b8f0060f51fd170296b9e000bb2970097f77ebf0b5872a9ac01be5aa03d7ffa', + 378000: 'fc5669d194d822c55e141ddc9aa229059e4663169456dc927d43b44a2c8d3c0d', + 379000: '1c0bf45f052e0123d1a6806fc4a93ac8261ed26772f324c1b3e7c3fa44d7f442', + 380000: '73b69dcfa0a916a4c495a14d4d43e270b123ba37d267e3eb0d65890336fd3e32', + 381000: 'e89e3d548193df667c637f774c76f508ec79b7bf1f2ac5d9adf7468006e34454', + 382000: '6b835973c2efcd564fbd58d96226cb984d5c32dbf63d5b6044a0a11f77c2ccc7', + 383000: '50b7011656c5903941303c7a19e4ea0fc6bc10456ba85638b0919658556e47cb', + 384000: '74c509b238c1418ae527d370e8fe922f7639c88ef8d78fc3a45894deeb2252a2', + 385000: '6f9f6147b06d8fea8c440cdebb3869b7bface7cb2355d66639060056adc9bcc2', + 386000: '5b59e68fd62751b8be184a5621f1c63f1e9cbd22051487999816df392dca7837', + 387000: '86d8b9628b83ac63637c86c364001067ee1c3050f395758fd3f0bcab20cef703', + 388000: '5e570f7b69738285848bd8b7f2fbb82d05e99a72cd4a0faf20a9e3491d2d041c', + 389000: '78d31bda70842388c1cfb2b83e9e95ded37a0304aaed3292e2687a96c07be9b4', + 390000: '316da5e2ce79bbc4f4500ea6417271f0ae268f85f86f81223d0af38d1e0e0a67', + 391000: '45cebf8a25c25aaaac06e915fb3fcc99a84be61a266c2bd8997bd1fe00a7321c', + 392000: 'c5ff6e24f524bfb1f629245345052c0a0df33ab881ee9d11dcaf64915b2acad3', + 393000: 'a54b5d4824bae06f91566b73c99a07b812095d7c133d1042ded80211c0402b3f', + 394000: '0ae76082f8347ddf838ce0ad486f5decf6223e8b8cfc16dfe178b15830807c60', + 395000: 'dc436c97ee6ed427687e6d48e609d271b125495afdebbcf611c0fd5102ec900e', + 396000: 'a87871c13ca3119f309a4ed98e4224d4eff01387ea67e86148007ac0d759f2b9', + 397000: '58c143b302132ad4b38718ebe1c5fa81c1177a30045f1e4a52dfa749e84dde1d', + 398000: '33089400401a190c9375abb63c6ef489123f7369e5a35766b59e69edaf4e3b93', + 399000: '656305fae087b86758bc7a01e13051b1aaf4a5c53d9b71e56614217a82b20eea', + 400000: '7ea274642eac11d02aef589428b181e1c8aed913e248b357917604b00b23c752', + 401000: '1969a9f64e316e0b4fc864fd7162379821510a944652b8dea747e425c6222807', + 402000: '3700e1703e7803a4577de7e364a34c3be94d3f933e05c5dfbf6ff68349b6fa31', + 403000: '8b622599bf4b2d82bc11c897f35536036e6798ac101a6256e14477624147efe2', + 404000: '5f6f6dcea393ddccdec69b30881ab08e01e1a7e6a50d681883c759b742c12d6e', + 405000: 'c3fb0507a57da69042e246dc8ecac101f33cf2b1fb53fe2f507ce28a0d82e4b5', + 406000: '4f034663bb7b84439fa256f7b4ee2a5cf93e0048c1eb476aa2152453ee7be544', + 407000: 'ff99ff38df6f928242571fed5b03fe0c3cb481b510d3c943e47e26f26440cbda', + 408000: '89865af5e99e64cef61dd931061bbca7fe355218668f487f06474e46ebf892d8', + 409000: '1b1d050c0b98d55d9ecb0d1a5b03e75caa352b4e0ce6a57136551000e150c480', + 410000: 'fe52790595db35c7685e5261de78d139c830af0a5ae73a7e33da21be9ffcc9e6', + 411000: '1afe55e66e6e16abff282c9d6bcfbc6a3a6e40db3c60323cd44e3b3a5d4ba924', + 412000: 'b59a4057d73e8037f3e0e9b92310def7900c19917892ebe7689d5f8d77912388', + 413000: '603f3ca16a6a795d5aa807eac5d6ee7f80a1e2596cd91cf41cf90ee309b2871a', + 414000: '10e255807a43bf5b95f9af648f1724de1e953a5540819822be5de184ead24ae0', + 415000: '9faf6b1c33006e3da3ae2c7599c7fdeb5078a8427edd6666290e32aea67f5a23', + 416000: '973b77e4d491ce0c163ca64f5419b05822eebc2f5d85810f51370f56e0d812b6', + 417000: 'bf99710d77fe84ae3ef5a371887cacc2ca1fbc5549b04ef9f61cb9d911b06a9f', + 418000: 'f032acfc2d629fecfa180067b4531f9d72924c5f204cfe9053c6556a334373e8', + 419000: 'bc9f085612af6a21411fcf51259387d97e6ae1169e4c6f92766a168302a98bac', + 420000: '9136bd1fa7e8850c52e3cda39a96f754cdc605a69ea49b7d6d959d0cce81efba', + 421000: 'ec2a78bad1b4b7325d574122ad8c3072d27fe6854a6965e6917a28e0c868a229', + 422000: '536f4cc5d6ab082d32ccdba1caccb5d4f3145b60ee8538b1e18c6231dcfa23df', + 423000: 'ac7ad1f97da0e55661bb25d53c27ea72f705d54050f020aad9fd5856a27117cd', + 424000: 'f840eed1c0fdfba43965bfc175e4d92a5ddd0fd6022dfef988d70a5e7f363116', + 425000: 'c1a8f9b79a34b76c9ae58e005501dbb1893cfbda8bb25b0d54854657b3ff1a54', + 426000: 'e8895d8f797026a2f19ade2b548bd2cb655e2d756dcc0a50850fd37148d24ae5', + 427000: '0ca56cf4fd882583854fca00e339534346d53178c6679f8eb1e7a5dbb7b9e7b0', + 428000: 'd3ce901843f952df4b63e755a0f58924ae11324aad47afece9a93502dcae7c13', + 429000: '774cf2d1ba62f20f5e1ce50692f65f0ce35c7456d3988cfbb2e1b753e0ab0f51', + 430000: 'a07fc310f6201c30f1fa70a082348942c65743d51e3c19492596f6aab4bc9bb5', + 431000: '7bc7f405bdfc91dd01877ccfd6142c973d81dab12fa188f468abe6c9a30fe0d8', + 432000: '7fe40e145078c1952f87547156e2147144449971f108164d27bcfb93f1c840d7', + 433000: 'dbe52d2772d6c0461e53357b71a79096140039ef643876ec2d27e1bdfee7e81f', + 434000: '94b379e842d4979bbec171bb08881227ac4e9477186ab63f7eee9b4261b92365', + 435000: 'bc9cd265de563cfe80090ce12085539bc67d406c23e4d41751f191d6b6c63168', + 436000: '3ff03098d2c20b5cc080363a84d8295510db937272801454d429c4d17a1dbe10', + 437000: '5b8ebd40170e217eb66990af3574b80f4e66bedd092e44836aa5101ded50c13d', + 438000: 'f725813181d50c0b20552635ecc578685ab0218619d53ed23493c46fb060ecec', + 439000: 'cc7cb296ae79850ba2f6f7532c445189452831957f62cb823bc83aa6b51daee9', + 440000: '757b816392227c9d1a14f7350cab6e77a46e465e7853b1291abdaebcee2bba94', + 441000: '834515861eb72aa472843a6924fa9d6350630ed56e35181d9fe66e3a46b6952a', + 442000: 'cac31ac937ebb6f0cf4f3c33f19d66a05bdc11105a98ef12c32b494e19e772a0', + 443000: '552b03680b5495c7bd6ec58db0b6767e6bde5c5961a02b7b2130ea6116bddbad', + 444000: '2bea6ef419d9fe81de8bf5cfe21b2603f202e2b9a959d2cfbdcbccbb7a7406c8', + 445000: 'a4106f2115b356eca92e5e3f45b8ae0e13b82af94bae391942200d1c59d34d17', + 446000: 'ba49924afd8a40260e546e5e4633e945877ac9d5754c277b77bb154c0c9d0e40', + 447000: '83ea56135c2b052df0fb09c68cb7a085d876854f34338abaf3cf51760c83c3eb', + 448000: '806d77789673fa09d24c188ee024e71425df7baf623fda073076f3a9b87ddbce', + 449000: 'c41d60b93fca1d2adbbfcfa99475a8eda152630d815eb1f6c58cbd15a0c2eb65', + 450000: '732d315feeb485fcc5055916df17827d80e7f831c43df81ea79123882c4fb356', + 451000: '202156d29e57c51941cdb7256918128589f5b8dbae5762bbbeacff6eefbbfc71', + 452000: '0194e8482252f2492e1e2696fa5761f50875d0188310a0cf1d967edffb1b79f5', + 453000: '9e63570821d8b5cd7de0c5f9fda67300f531d87cd8f778fdc6bee9519b77e01e', + 454000: '632353fe048e83ebef942d64d550721d29b3fc8083a247b2443f2aa2b706c6fb', + 455000: '65863feebdbd60d16330000cbae496667a67dd5fa8b8e3b6dd3732843f6222a5', + 456000: 'bbc554cb3c0281d6853ff2f6c25b1251eb108c09e5a6c4a69a56e8f79c9206e4', + 457000: 'fe6fdabb84ad1ffa807fd55db29dc95dafeb0e9a5eeb5ce75be8a4d16f9f116c', + 458000: '949bd40d861e4b297352722b6f01efe9574711fe64b10c2e8dd769ad99cf9655', + 459000: '9c139c9ccb41fbb3ecee878cf47bb37e28c76816c257f99dd7b23ec1f3eb3ab8', + 460000: '61c3126d504fb38ea21948d2837368441389155e9675c7f1629eefcd03198f8a', + 461000: '28ff145fe2abc7cacfb1613d40b45e13580c643ad68c44db81d15125324f4f33', + 462000: 'e6b532924d07f1ff3976a53397327ccb232d3b2ac6ab090874cfa6ae47b8b315', + 463000: '15ca651db9b159978ba6b2adfae1669e3042f4311ed683c72838bf337d71274d', + 464000: '6903539d4f38f728e075441efa96b8013295ac166db50ddd4103fb2bc9937f0f', + 465000: '6b6c2640411d900c13f5b53a9a993d10fbbd7f03f6fd477e120ba06a7cd67778', + 466000: 'a05d282fcb9c091a0a96c2ddb3728c1e98eb767fd5311bcc08c9c10b5a0eb24f', + 467000: 'afb81fdfd562f2cd2d53434e6458c38462f85b65958ecade001bcf2b49f9b28a', + 468000: '97374586191b3a30654483722f7c6486ef9d9f71aaca44b72146c11d4f4b80c7', + 469000: '90a829ff58be9ad30024ba1851515cd96d567be1d99154cb966067316d1902af', + 470000: 'e9fbf2d2291e1b5af5dded3c223611d9f2e31fa6e1d8c075741aa155538bbddc', + 471000: '3b4513a6a6c05ae9d2ab880a2d1aed3b189e0fca9cfa2bbc7c31ebcbffaed9a4', + 472000: '32a57820226f672e695823620fccd42a6fe624dd2d88f7e7c76e0e9128883b27', + 473000: '15cb7e920799939eb01d06b1b713cba3ed31ef92ca037ee98eb8101b6331e5a6', + 474000: 'cc23e76e68dd726f31e592a4ea5d6aa3cacd827d068317a0a476f16ce468d9f9', + 475000: '9e7e33a8ee53a4bd94a26b35bb43054b42a238a85a859ed9e4a33db978a128ff', + 476000: '23c85e4d8edc7939fe0e5a47fa250f0921b366711c34910c394af00a4efd90c6', + 477000: 'fd9d489fba4175c1fb3b8a0f9bb947287e64da4bee9f8a7fe73411632c820368', + 478000: 'e287b69696013e70b276843e83af6cb5cd076000cb280e43de29cad7f706dec9', + 479000: 'e86c7a06d96887e404c052b0fcedf10d40423b02d6b9fa7ccb9827551e5d089f', + 480000: '74e7ad7d78dc585a5cd1054c0288aefcec0e031cd40f0ebaa0fefc7a325f212a', + 481000: 'a01f98758cd3e561d02bef30c054ef70405ebdce9e670a7d6165efe3a9030886', + 482000: 'cc9e22f04e4e2e4f52b8382281092ba50ec99722c9172a0e1c9ad654d26bd4d9', + 483000: 'c92e278d54111ab2f3906eafab87fdb932307e60836df209856a97cb9e1ae3a8', + 484000: '18fc89d07631aa01891aa44231dae6cfa43fd4733c193468b033532a4574eb80', + 485000: 'ab546bb694363e07469dec48b559479d34d724a43865fa8f3a178ddc7b83e8a2', + 486000: '59926fddc89abcfb55775a58e5e6e453e388f530f720dc61001f1ac59b59cd43', + 487000: 'b41bcbd91984a3dfe72c87bb7c2ef32ba3eb3cf4318f5d37654459143dceca1b', + 488000: '94e859b8f01ccbe9fc5bea5c274f252e23599cd444d5eb0962ece97110b86eda', + 489000: 'fd99b951714162e55d8944e8c3ce9b098c4b74ea24017e4b204f95359202df08', + 490000: '80b84d18f4bc6d624d4190db94920976926bbe48f2e3e0f4c08106b9f99ac5df', + 491000: 'b0c5b14b01225b1ed8272dbe9b2def41756cda0a3fe6a1a8e75b0795b72e6a7b', + 492000: '572dfe57507384278def462488416248a3039c1a76a8c4aa23d1ec706de1e3f6', + 493000: '75f22a8d4d202914c0f3bffaf95e47eeb82eb00dcd2913af5dd67acf6f8ed8f0', + 494000: '626e26524f9427582d61f340ee9181d8f238ad8acf96c1862724fab62ede8c8b', + 495000: '69a11346ba65c79b5fd566768a8b2880a5a29e2552ee8e343bc88bcf43e9ca54', + 496000: '314c88db22753d317c7b65f872d54d3ff672d04b3781293d59a1ea902811336e', + 497000: '7bfcf2614b54ac3808894694e8b15940d6740760a2252ba4b47b2926bdd4bf51', + 498000: 'cc02f90d2fd4e0f378d2ae3bf1011026ca3d7f6642dab316f5fbc2f57aa840d0', + 499000: '841182d8519cd7b2d8fd01ddd087de892ecfafaacf9d6aa029ce1ed71ee0d538', + 500000: 'e2742854018ecbfef71d626ab797b7c6137e84ff1e27a05b5da84b2fc7c43145', + 501000: '07334852d0b542f897164e4d74e699678c850af4e46111d495701a312be21105', + 502000: '6150771ac25f4185eb94ef6aa15e7f409b6beb411358ba7d982755655d431492', + 503000: '156d3165eb11d021c46ae5968aeac20f65eb05e68d841bdc98dde156d42fb500', + 504000: 'e9d474e59bf527d84f031ba41cf471f49e5bf0472ae1b3f48f036813a0b195e6', + 505000: '261cda40eed5ec5b2a44edcd995b24887a45a417fec3f68507a4d600ef2abc33', + 506000: 'ffbeca24a8f2b7cd6d64207911e153a2f0bb70d34bb4f073c33197e3a2276558', + 507000: '14e2664bed708d57478f6fb5a4228e4c26843bd6533056fb38dbf1e7595028a9', + 508000: '2750e25c3dde233447266f7b35f78d8bff18bbde4ae094d06e38fbf6cd2c6902', + 509000: '5fc210a72d142dd413bc3b76d5ac772564a5917aef13890e7cee190c61b059d0', + 510000: '7184fa5628bfe59481bb3542bb4212b535c3ede3c24425f0bf7b1b453c238c6a', + 511000: '2b4bec04d73f0e524cf12d04bbf7b3859b489275edd047f41b139ee6220074de', + 512000: 'ebf81767f4d26c98bce62a5f5d1a289da5bba770e88f346e2fbb1d676c155e09', + 513000: '8bd8a99c65a258ebec809ca76e2cc91b0b4fec5dfe0970736d2fcd706aa660ff', + 514000: '2dd7cba17366c483819ea090d3196950ee35a4bd9f28acd94575525c851234d5', + 515000: '33ccf5f5230b9a79cd89eace91cd1c2858f6d040a11494ffb8d51ee934753d57', + 516000: '30eafb000d199c8d9d1a55c5b09a382a3b10bcbca837e584db6647699ce6a3c0', + 517000: 'e7ed622512e2483fb8537579ce2d2519053f9a6c8d19c42d665007632d5f74d3', + 518000: '5180d11f98a25b8698b969c3c9a4b736522397859e7dc00665658f93c6cb8be4', + 519000: '17ca81b3a3c942c80384c4d2c176b5274c52569b90eef686e077f53a884130c7', + 520000: 'fe1f552778cd5250ba6556712217b86fd6f1d574143f8ad6238e1d6b67481f97', + 521000: 'd7ac10bcb8662ab9940c4212dbf0c4b9cf9750e3c04a905efdffbce63f065de7', + 522000: 'fa7625300aa890ccda58268eaf2daef14539c195f93311099754f955f367413f', + 523000: 'a55ef67a0e75bea12003609359443a28b42b9037290872d6b1672433cf5c0d34', + 524000: '08d1d556393bbddf6eda11df852a792515ab9c1d519bd91c48711a0de8faffcc', + 525000: '76b8691b16f7dafcf980ec3e7c69c2f3caee32057ac8d61b3d3aef0fd394642c', + 526000: '6d3ecbd7132a725e330f0a566d8d602a9cd96326a36387c1c1fe52f7f6ad975a', + 527000: '71d08ce7aec6ff50fa1ddc37f6b1574db6e71004e8e111f04a9b715103f68e01', + 528000: 'e9b103c11db50a121c9e3a1fb4cbbd4fb124b07a67167c5a461a539dac3f932c', + 529000: 'd2f784907a019c1eb5830a74799562cd6d21570c4591bb038f379897f3296d75', + 530000: '996bca5a7dfc4a8c7f9f8dfb38b1881fb97893a950f4c8f57462506782ee8ca7', + 531000: 'c84dea2e7fb0c9c884f8dce525b8b65bb7deb33ab7fdbcf72a5382e9bac2ade5', + 532000: '3e200d76a92dd1f6fcc420d44ebe446829fa5e0162177acd75a13c98f1477cd1', + 533000: '37fa7cd81c658ada13d04dcf0e674017c6b7430bb0cd5a080b1b924396377b52', + 534000: '9483c344293c376d2bd247f2846a5ff5038b27af48ea5badcde101fcd0113dfd', + 535000: 'c9d4f9982955c07960da1e4f92803ba135f4584416e0767a3567e4d2a21e62dc', + 536000: 'bc9abe4e4be06aa8f61b599d64923ba88d36886b638f413b12bf87ae852e4010', + 537000: 'df8307b0097b4882740c18432ab2d7e64486cb7bb70b503923bd630ea277e9e3', + 538000: '6e2accc5e65a85e0aad098dbc7021bba3cac37c70f6a0ff3135f86b90954bd2c', + 539000: 'b7d9f50727cb512dbaa1ba073d227805e7adc70f9aee46b9721a65ca3d34a8fb', + 540000: '19c695a6b8bd256fddfd00c53e6401829ae59ebdc1e71ac234f82ffb2991ed0f', + 541000: 'cc07c3740f96ec8929c7c8714892a19815fe64f4bcdc4d1bf12cd0a9e1e82125', + 542000: '23ffec3127ec3e093ec8221fa760510e63fcab31f9f0fb2b48727649e4308980', + 543000: '6c73b242bb05af125ce4ea324d921ece54c9510980128605aec23cc465dbeb00', + 544000: '51f215e5f11ae31853c4caaae1d01aac2dbafd00302b70e7198e213d485d5bfe', + 545000: '2c36150f62c86d050e4c64d83aa0071ef4af0bf8e1e17623c740ea4ee7f3275e', + 546000: '322e6d77f529412c4b7a1c7778253900f1976f238f96b771845b56aa080f4d23', + 547000: '1d3413530079389231bdba0a0a7de7c85a4ca991569b2f26a1e48ad2f34a995f', + 548000: '3576ec0d3ed469c5498229af531fdda290777fa1ab7cd83c49450f730c5ac0d6', + 549000: 'd2a5ed3d920f273bf891ac40ece534a56f8774c8809ed7d8482ce4b78bdf016e', + 550000: 'e1f75701ff829339e03c019eef2da1ee4239f38879173a833f81295200e96d29', + 551000: 'fe8339fb53a33e3b5c4a39cdd64d869f41b4fc88c73fa018eb5245d2a7a01e94', + 552000: '8c2180cc834d5c14c9801772caa65e82b44cde33da7aac586c81b2bfe77412dc', + 553000: 'db9b4ea2cbdb21fd48a69da3bac2dbe8cea2a16d45f9252776633661e28fd1d2', + 554000: 'd09c32f70dd94d737205922618b789204a39668b59fd3ef405ba6d4ba3bd8aea', + 555000: '6e8efd0190d866972b469cc827330f84f515d6466d8748fc33520e8e0505bf3e', + 556000: '8ce7cc8aa6ad30971014b332c64ce87303098e64391a9be03d1c621df851c00c', + 557000: '9959211d82372edae69cfac47cc5c937fecbb6de9e235a5a3954c653f3b1fda6', + 558000: '4f1d620babaa28a03d02c2ba8b7d1526ae47d45bac1220a1a794cf4a489536be', + 559000: '3a0f1617c6056a27b1da33f0e4ca80b5b6648341551c19622f7f4b4f016ce8ce', + 560000: '54877adf522c9abaec851a358ec6fba9957227e0e306d3a425167685380cf6ff', + 561000: '55db8f8ad8869221ee891244767a7698d48ec61264729cae19008779dadba422', + 562000: '4ac9c41b29253ed03c45222412c53ddde7b9da63441d06ea91b448d89dc220bb', + 563000: 'cce11479fe1fcda6af734a03be8ea044df7e2726df67927f5bc76c33728133a2', + 564000: 'e2e5c52e028ab1be7f5d182a217e16694ffd370930cdafa9443b627e8da256c5', + 565000: '81488a98ee93b0cf5704cb1f46383456bf5238fdae7bcde4d0826276b5a48235', + 566000: 'c9afdc804f7b0df600c51b9d596d6107a0cb71a4f8ed3ea764585ffc5f70644d', + 567000: 'efd3ac4baf1c6e1fb2f8bac40f1416f189c9eb1135330ae4c25f296f012907fa', + 568000: '225c3db3b83b78fbf9459c7e1e53ae35b01752608d329eefa99247d8a2b9b83e', + 569000: 'ac98f3fb9084261c19a63fc8f10baeb5b03aaf9ca432d7f763a3a6706f81eefc', + 570000: '4ee2eae15c9019466bb50d103947bdf60a5397f8fff56154ef782fd1b8579dd9', + 571000: 'a89f29bfc76988560ccd59a61f8a323fde7c4020abafd577f88d1b0b4c289260', + 572000: 'eb8380e096c79806443535f2f3b007378e43d2cf3ff56f3b7e6a1b2639ff3da2', + 573000: '3182ec1555aae0df8d76dd5f37bebac6936e0e0c5609263263eef2e6e87863ef', + 574000: 'fc45bbadf22417d49e7df89f02df358b85a4cc07adb4010d3bc0334f397b01d6', + 575000: '6faadf2b0c3a98952bb047a58b83466a223dd323a6e32d68be5e8d064482fa84', + 576000: '68465ee1d64b65ca71bbeeb611a8b4657d3ce765857d049536047e2ea0490044', + 577000: 'a019fabcf35e7d4fe15e4e3a499dea2bedc61e4c9cea2803731995def15a9794', + 578000: 'cc2fb6111fa1a338eeade31574b77a1012ec5948f88e7e5276f7ba4443df3db0', + 579000: 'c5e7c605c03e56f221397f55bcd164dff21856f7fa48e745e61f2d05d7de5f54', + 580000: '110fe7150988c930f9c324e00e108050b8be5ca2eaf200f54477073c72feef2a', + 581000: '0c3adb09dca9a9924fcfe30f16bc70734ec62fd4a306ce65a38b74c5d817ee40', + 582000: 'fc47433896c47765651d8cb31c88cbf80d0887bf071eaab9d2963af9f8bdfb56', + 583000: 'c7c518f8634bc7c5cc536c9b77fe119cd6cdce4fafb25d4c0aba53c5bd970e35', + 584000: '43b2357dd053608b30488aca98f5d9c151a68730fb8d008f1bd1488bfab0c391', + 585000: 'cdfdb4ebd1c5fc3689b4f013254c832da7bfb12f3440fd327c6b065afd093dce', + 586000: '48979b814cb6964b20ad8b322c4936c7b241c24fe6d610181c43d1cd003e5d90', + 587000: 'c262a418368aab465facb8f1a82a392eb74e3fd56760779033a97a3f930630ff', + 588000: 'b20dc4e0c17a223408a51b958b6b481ddeadad513e046c86c7418a8e96e35992', + 589000: '0411c5ce49578218dc8e225c11276ed9a9802aac47d8f5837c4eadb3bd72e1dd', + 590000: 'be8fe8b058722f4919af2834c076e2bfb602654a0343b57eb568d0ed4833fb39', + 591000: 'affcbbd2a9fa43a64332412d649c2c0bd4b85e90e5e67576f09ee0c207a66cdc', + 592000: '14a2d414220d240286ec6eddab0b0540ff4d1794b39f7913c00ab005e2ca2a3d', + 593000: 'a1e2744404626b5e8e101402f50eb751c282ce692d58493ac6fa2efa43254526', + 594000: '679da754e8f4cfd017ed5c8111361c0f9624104a87a50d077dd7bb5bb97c7ddb', + 595000: '0c31eeb25d718f96c68de13704a0904c3a524b5936690ddab8b4b38fbabad5f5', + 596000: 'f930b6516bb01d3365bd0e28aed56a580b4efbb45b0abed5ba941f7a56b545de', + 597000: '56287fa7bcb53be966ba75c15b92ddf94a7bb51f4ef51037e56ccca936df4d41', + 598000: '1a6303e68038921fe23644dd10f5cbad4af3648f7cbe8432860f439c84fb226b', + 599000: 'add270bb4fcb8dc9c254c9130fa037dcedb8ca922c2e35ed403661a896addf5e', + 600000: '786ea387bc7e1665655f1f683d56c67baeb9bc482bbe4401acd167793d34a508', + 601000: 'a4c5f398f28cfcdc6e0b6fd467582ff5abfcb4f003cc59ae7703e9467cdd9ee4', + 602000: '21b19a5a23344325581e7daf0a398b37cac5b4c335237bb8e6c14e270bdc1567', + 603000: '000eef5b4300cfd804554e199ca6ffcfe73ec8446e82ad32abbc51f45594a368', + 604000: 'fbf77437dfba9caa4d11b03a3673f2274dbd3d92e704bc5e8ed3f90d99d2c988', + 605000: '0bdba251f44baa12e2b33b1ba89eb8a5d31ecaf6193b836e6356845149ea3f33', + 606000: '2dcadcae1cdc68ab6404606937aa9981400e12ae77474462764b8077d391d2d4', + 607000: 'eff9dfe7fb3e31536dc12c78603f023f8f1f59b2b44a59359e8bfe50b2e22299', + 608000: '50fdd1223efaaa261f6fe592d7b9e1e32b5a50f78dfee7bc8af19250f8a82776', + 609000: '5d88ff14c7c36602251061379179ed87c5f7d60d746d8d10fe678a0303ce6596', + 610000: '5e56249989c623bd7f818ac1e64c5d1b1f3afec3678559d446e356ee5bb3394a', + 611000: '5a9b15a38dd35b9ef0a028b12ab69b0993eb4bc5577d50c486fdfd032f243a14', + 612000: '258213d8b7141418f22e303ec43005bae0508a3597fa99c4d59d656d7d768af2', + 613000: '5ee8b46ebdec5576550540f7fe01c5a91c3ba7bfab26db605ade4e620430cc21', + 614000: 'c41a592e2042d201bf028bbfef484d264a391db9dc9ef302708e7a39f723cf5c', + 615000: '70fa6591c0b7bb63665f040b87750668be47058942c79c863bc00232f536d725', + 616000: 'f4932c1c8fc84e3f4b97336424e27eeb09a65d7347097e983e1ab698a38dcc0a', + 617000: 'd82bd9b58adf54a50bfd471c66144f2969ab443648fb7bab8299408afc021544', + 618000: '72f7dfda9bd67b900106e632cf50204fe27cecdeea51d9b0cfde9ba6da8dcd5c', + 619000: 'ac434b8d723ba498bee878cf589c7d61b74c7a00ea5b7abe70149ba878522332', + 620000: '519930116d8a8f3b01ba443bbe4fe5346cebc783c632cdd5f62d305f61b6f446', + 621000: 'fed977a0174f8cdc3e979e7435ffde2f7dbac2d4e1c63593117cfe81f320bfec', + 622000: '1a6daef00425e6ad136d7bda36cad99baa762f6d788b92f630ed1af33995211e', + 623000: 'b9f1e2836bd2acae3a9d21516052007005f3f67ae1fc133c1c4c336abe0dce51', + 624000: '5d4a19f7e7f99b5c39bc844b0a90ebcbd97d88f817f8f2edafca78ae499c3167', + 625000: 'c1afa80aebb856746da25503eb786f247ab098a94ddc58fc96ca46caee7019c3', + 626000: '222c2ec50130a658ece226c420c76a8f7a15640033fd89ccaa29bd7c639cb94b', + 627000: 'de9830e9cc7537c42133153cccf891dcd422203827b1377df748ec27dbfd04a9', + 628000: '35d44ad84a3d9d548bbc8c91ca734979759e3a03e37153b650f794a0aed1b742', + 629000: 'c119f3acbd761598673b48eaf0d223e1729b8c8aaef9a715f1bc3dde561a4568', + 630000: 'ebfda8a47983ced56d40e95ce3be793305899cb92359ee2f9a83f7f6a59a4fe7', + 631000: '1479c2e6217b97e98f44bbb315ca99d1ab02aad14f52d60f7e6670f5c6d0fbe4', + 632000: '6ed7f9c90ea95825b1aae91644a68e36b6452e61f05a2821e94399823ad669e3', + 633000: 'e00eb0bffcf784f9f8fef21b92b9cd98e44db5d790b6eab309d0b11adf910820', + 634000: '3c71674dab2c2644d910acff4336fb02c2dd8281e30e5c97b9378d50f3d6eac0', + 635000: 'ee3fbedd791f31de2f820cf37cf5a7bcf1fddff354c1ab5736384f41846894f5', + 636000: '551b7598f5576c577b48ac7530e56422a5fd31a3860b37994f8489308830e9f6', + 637000: '531b1a767734f1750077ec9d1ab57fd4430e0154deca2fdb9dc5cfcbe980726c', + 638000: '3ae3a009b3822211b3bbe9745f006eaf3aeae365e81890205e0223925461af81', + 639000: '2aa1de8c824fcc1a8f34269010405a0a0fb921fa0f8b998010e4076fd722948c', + 640000: 'badc624e2ffabc438b7b2c9894e2134e36c2e80af700cebdf0980169c1cfe49a', + 641000: '0b550e8f9b66ac3a3d0e5a911819643600dec121959fe2b5128cc6c68503e655', + 642000: 'bfccbe9a0a127d4cf606adb5eb42f3f0ee2ebc780b719d0ec612e590aa7ec47f', + 643000: '4d44b13390410caf86d145103471469d0b10450166f5b96b8886a440aab5b1cf', + 644000: '2670950e311d8230e8d3447e97652cb5592248b59968bedb6a98a98c12b8d81d', + 645000: '6703a25f54614ff79c954438ab9fae5baff32c81fa47306b337f0f87bb11f949', + 646000: 'c93876f02c788041b400aa42149c176e624eef01bc66b49adf8fcbe255dd1833', + 647000: '276521b16f6a98f5399a9754621fbd7999b6b94e3cff17cd88d4a1176f431353', + 648000: '65b6238c589baec511c240cc87cb6ce758ad6cf119f465838a976a853c7f5be6', + 649000: 'c94f6f7fa7a8afdbecc7c291eab2a3cf218ff1af5117ed1f3f0e60214ec7041d', + 650000: 'b823008cdb0176fa0522c26b35962b4325fd526217f7eb19464cffcb3c24b4d9', + 651000: '77024b0bfd3cc7c8d52eff6db4dcec041f8675e6f3c413b7d581bedb1b53bb11', + 652000: '22a677eef4cfcee12756492eab7e184bda054481d4fc19cc46bc7147c097d118', + 653000: '66779d92b9fc28fddc46ed14b79b26b3ad7672f0dbc7cf4462d5a49769a89470', + 654000: '097eb77aede84c35822b3700673efd0129de247cb8ba5ef6b8e6dd7e85bb908b', + 655000: '83b5b4d4aeef3782c82b9a2fd53937e4523ea15867780e4264688e860fd99593', + 656000: '99646218016eda7976b45fbb6e891fc4bc130bfb3b5e6dcac8b11e0a4b2f59bd', + 657000: '9ad1995af3f7f6839a11bf5fcd856f471b200911e1e5a647390a7aa26abc6825', + 658000: 'fd5a4462737448dcfa9d81802d16f4621ea0b177f538144d836829a536cdd451', + 659000: '7f6baf86cafbdc820437d63260fcc434694dcf0ab8000307f134fc9c50f437b6', + 660000: '54a5f2c5a1f1534f23f54c5ac6e3f794ebb62f298dc9ed1aba4112e10cc778cb', + 661000: '91b46c2d247a43ea916adad7edcc37e18d149fe3ab970eaf39cd060a9c856ce2', + 662000: 'b49e0df59bfc9dcd32cd1a4b903a12accda70054fbd5f4bc480de0367f254c21', + 663000: '29d2d3e06f744ec2225659199bd43e446308fe1e1ff16c26e0e89caa468450f1', + 664000: '50b0665a8c7a9fd1ffb165fbb0148115f83fb7a5f6e9b8ad16c9dff9175d11c8', + 665000: '35754b82653110c6cb0f9a0e7011ed1ff8b360301e4280e6aa2f6faa5950a369', + 666000: 'fbc64efa658b00bbb70f5db8265e99465a07627d663c9a716ec8b42bed82338e', + 667000: 'a046d6957f24ec9f66c73612d3eb483c65e4216fc2761260d9818486cbebe56e', + 668000: '5d78ef63ea8ce0f66032368effcd56fa443b0f5c488ef8c243d3622113087fbe', + 669000: '0b1b551500e2c8617150a18deec1e3b9594dd98fc9b97e20dacf2f059f966692', + 670000: '5fa21e8edbdbb2ccccc2e3aca8e15b0965cd307ea54b6621760e654b53488d33', + 671000: '90affcb451a786aa42ff93dd1f39961a15cc8c332613f8dc591d9f9c34d359e6', + 672000: '1a8996b27fb79d43627caf0166f5c8ec72e1e0d09e6c0e17a0d9418d37719afc', + 673000: '491ec5299fa141a82b438ef5d2429eb5560a0f04412878165c86f9fbb94198af', + 674000: 'dd680d4c1101a98b6465ac85a63ef12f3ebb1eb88277d37aa341a5b387e2f1b6', + 675000: 'af8acc98b0fd9ff347acb01801f246c63c6698aff0a0cd69b0f91e17da3742df', + 676000: 'af24a5759888314e5ef719ee23b9f948b3962ac0b949231a540d45eedaa614f8', + 677000: '7ec568d4e5ceff99c0e9cefa6407784aa0246845420812bcfd8a5d9c03cb01a8', + 678000: '845df5d7e1d6f22242ffc4c976937da67ce0880a94c5bf189195d5728f208976', + 679000: '41f895ce11bd91e09ea8b16102c8f2192cdd2c754e2cb04d33edea688fa7c3b8', + 680000: 'cbf1e3a313abbe05d90ff1de6d861cf58da50b7f8f7ba43a593a3b6192d65f8a', + 681000: '1ac184be2b5edfdcc0a73a6a93d3f59a5197bd5bba2ddb1ce737251526998bc9', + 682000: '0e793523ba0a679dce1e9cf1d1189c5311e27ae8d88c35bce36ddb3c16dc34c4', + 683000: '08e07172a6b6c3d70c4379b206ed796ea8d916d314d0b6f02539becbb90077d5', + 684000: '0b67663fc37940b4d40ba88b8b7610776e35f070e4995c5ab09ccd4b86dc1143', + 685000: '8ae92af68bcc012326889b5ef89b899aa38f65dfd3d9f7dd5b29dba0c5fbebbd', + 686000: '5223ea0cffd3c59fa6597208425ab55b3a76366fbd27d22508cd208ebf2a2eec', + 687000: 'e160e55af7fd110905364f980543fca62a123c84c81e9fd38a0aeeebc30a501a', + 688000: 'fcf3b5e0afae8d665cd6f63dacd2a861aecac20f8b73a682f79cad9523c5c4f3', + 689000: '772ffab484f07ad91ed45cf6f569eff7440f4243e16e07698e4b7bd4a109e6e3', + 690000: '5b2a4fc69617ccf1787ee40a0f6d7e0af783bcd856c2c8e0ab747a91a7c68d19', + 691000: 'e4ba352a759e2425d508a4d5b58595e6ff5eb912d8bfece5a0bc646f61e77084', + 692000: 'fca2c5b6a721db85278ab55b5e2e39a445e5b1ee5ba69a17044623a6b945d0b2', + 693000: '99f4365eee70f86499ec26c373922d389cbc5e2a198e96af5d5823ad241748a1', + 694000: '0e19e76398e65cb01517dcd4ee702c9c04c0fd53cf9468ee539094aa6248e1c8', + 695000: '5addd92022007d81ced43595458e2eee2903227063af8e9edd75cccbe559930d', + 696000: 'ba25dfe3467cc9999eb8593d955032475b777cefc8006851f692266ebb83d140', + 697000: 'e10c1c734038198a99bf970ae89e3a56b2058612e35eac8141d48e147d2d47e0', + 698000: '472c4de8e57737ca48c3bdbf3c35c37f24a179f5dfa48af89efda3c3d33c131f', + 699000: 'b359f2e61a3ca3cf8b613045617e38b06767ccff72129f13971faf26a4d08234', + 700000: 'd0555d53358978ae0abcb09f911c1b3e7b5a282b4ff6d455ca9ad04299666dc5', + 701000: '5c763b8b4329809553fb58ab279ebbc6639695c45760f0c181168d41da95e6d7', + 702000: '8a6abdca484fbdac5fd6fb279a435377d964d43b62393b0d5b0cd3baaab1d2c3', + 703000: '50cf159af75b28c7a95ca990e480ebcd534c4601b03d7c0a979f26f4e33cecd7', + 704000: '0daf7f9c7eca5c6d91de9ebadb3ef1287c290700bab0aa9f6a2ac4bf42a8a98c', + 705000: 'a359e7584112eadd7fad5d961f1dee180481852d7706701c2211a98a009c128d', + 706000: '139ab9432b0c7f62739f818b114d57a59cbf20584230f1198c35f1da62c2ed62', + 707000: '43e1f4c83d26d419a35adc4dc5bd85c6253ec0faf7e70196bc5d5fac18c51746', + 708000: '1a05dba8d5d2ef4984c79cc939148ad71535043640fd5644141cdbc512623514', + 709000: 'fe7a8bd03f742de5227e5d0c8dcfd0b5cc13582703b74b1898c506fc6ffdab04', + 710000: '26217b4ecadd39e9a14ffb6cbcaab6ee7bf0954efcbbbec4de9bddb461819084', + 711000: '94aee5cc00e862e4952e573908449f4a48b630285f2f2fb20765d01dfcb68ac6', + 712000: '21fb256c0f5133634b94f6522cb8d2b0a9b982f1672ba561033be689410e7860', + 713000: '8d7d7ee4cb0598fecd85c8004b59074bedc3219f2376966e01a7faa92377cdcf', + 714000: '56a033531fc8dbb7ffe07d7365cce20281e522c3ff9fe7a19d0188af351a5799', + 715000: '4a6671846cccdc26d6fd1d77db50772315f932c24dc706004e11de68ad1ac387', + 716000: '4222447b25305bc063b668c0e010b16f9c0802fff9371e9d096ba2174926f073', + 717000: 'c231fcf5238a34cd15ac735d27b3dd0ee025714e48c408a798f0ba74be0aef77', + 718000: '0f96aaa30d20fc572d9db26871143a657b706c35a9668d31e3b64280a049787a', + 719000: '8557c088c4fc4745674c795d4a58aeb3df0dc91bb5ff93cc6268c4a88c64db5a', + 720000: 'd699e06ee7835c9d687853187125f27145ec91daf394fdee37218b8c34fee9f4', + 721000: 'c675e891a36425627e20de36f1fbdd5baba7114661f1c45f81f66a3fc55da902', + 722000: '0c6fedfb6d6c1a77254904fdd2400dedce7d45bdc2271beb77e2087a0ba30d1a', + 723000: 'a442c320886beccb3d7ea13276dbef8e98e1a47686cba2cdbeb6a2d2883af928', + 724000: '1592e2d6ac3be7535b44db6cc99080d00b19c6663f52eef3c28eae3dac27ba49', + 725000: '45b2a800f17b8571172a2658577dbe95b91ae88e611a91ac0c92609b3600f693', + 726000: 'ed6257a6567665747aa354e93ab7d3e6539d6dd41fced8a2f62cf848e2b30ce0', + 727000: '4b1a577c6c2358b0344bb1befd9b8e5572b787ec2bdc0bdcd4a150f26b2e2ab7', + 728000: '448765fbdf6261c376120ff9401db8a8841fcbed466365f982d3bc53775b93ca', + 729000: '99c2acea0af193d2e10498acd1c6d162d2a804a69157af46817b5ece5ea86491', + 730000: '94cec967e44f850f512d4240cb8a52ffaf953d0364b0a1dd7604b4a01406e669', + 731000: '6e63f5019439bc7e27a17a189baad0da8f5724883af3ca35efa0d4e5aaa75b97', + 732000: '53e1b373805f3236c7725415e872d5635b8679894c4fb630c62b6b75b4ec9d9c', +} diff --git a/lbry/wallet/header.py b/lbry/wallet/header.py index 01694e735..b6de019f1 100644 --- a/lbry/wallet/header.py +++ b/lbry/wallet/header.py @@ -1,16 +1,19 @@ +import base64 import os import struct import asyncio import hashlib import logging +import zlib from io import BytesIO from contextlib import asynccontextmanager -from typing import Optional, Iterator, Tuple +from typing import Optional, Iterator, Tuple, Callable from binascii import hexlify, unhexlify from lbry.crypto.hash import sha512, double_sha256, ripemd160 from lbry.wallet.util import ArithUint256 +from .checkpoints import HASHES log = logging.getLogger(__name__) @@ -43,6 +46,7 @@ class Headers: self.io = BytesIO() self.path = path self._size: Optional[int] = None + self.chunk_getter: Optional[Callable] = None async def open(self): if self.path != ':memory:': @@ -109,17 +113,45 @@ class Headers: async def get(self, height) -> dict: if isinstance(height, slice): raise NotImplementedError("Slicing of header chain has not been implemented yet.") - if not 0 <= height <= self.height: - raise IndexError(f"{height} is out of bounds, current height: {self.height}") return self.deserialize(height, await self.get_raw_header(height)) def estimated_timestamp(self, height): return self.first_block_timestamp + (height * self.timestamp_average_offset) async def get_raw_header(self, height) -> bytes: + await self.ensure_chunk_at(height) self.io.seek(height * self.header_size, os.SEEK_SET) return self.io.read(self.header_size) + async def chunk_hash(self, start, count): + self.io.seek(start * self.header_size, os.SEEK_SET) + return self.hash_header(self.io.read(count * self.header_size)).decode() + + async def ensure_tip(self): + await self.ensure_chunk_at(max(HASHES.keys())) + + async def ensure_chunk_at(self, height): + if await self.has_header(height): + log.info("has header %s", height) + return + log.info("on-demand fetching height %s", height) + start = (height // 1000) * 1000 + headers = await self.chunk_getter(start) # pylint: disable=not-callable + chunk = ( + zlib.decompress(base64.b64decode(headers['base64']), wbits=-15, bufsize=600_000) + ) + chunk_hash = self.hash_header(chunk).decode() + if HASHES[start] == chunk_hash: + return self._write(start, chunk) + raise Exception( + f"Checkpoint mismatch at height {start}. Expected {HASHES[start]}, but got {chunk_hash} instead." + ) + + async def has_header(self, height): + empty = '56944c5d3f98413ef45cf54545538103cc9f298e0575820ad3591376e2e0f65d' + all_zeroes = '789d737d4f448e554b318c94063bbfa63e9ccda6e208f5648ca76ee68896557b' + return await self.chunk_hash(height, 1) not in (empty, all_zeroes) + @property def height(self) -> int: return len(self)-1 diff --git a/lbry/wallet/ledger.py b/lbry/wallet/ledger.py index 4d1207bee..6577b4e54 100644 --- a/lbry/wallet/ledger.py +++ b/lbry/wallet/ledger.py @@ -329,8 +329,8 @@ class Ledger(metaclass=LedgerRegistry): async def join_network(self, *_): log.info("Subscribing and updating accounts.") - async with self._header_processing_lock: - await self.update_headers() + #async with self._header_processing_lock: + # await self.update_headers() await self.subscribe_accounts() await self._update_tasks.done.wait() self._on_ready_controller.add(True) @@ -349,17 +349,17 @@ class Ledger(metaclass=LedgerRegistry): async def initial_headers_sync(self): target = self.network.remote_height + 1 current = len(self.headers) - get_chunk = partial(self.network.retriable_call, self.network.get_headers, count=4096, b64=True) - chunks = [asyncio.create_task(get_chunk(height)) for height in range(current, target, 4096)] - total = 0 - async with self.headers.checkpointed_connector() as buffer: - for chunk in chunks: - headers = await chunk - total += buffer.write( - zlib.decompress(base64.b64decode(headers['base64']), wbits=-15, bufsize=600_000) - ) - self._download_height = current + total // self.headers.header_size + get_chunk = partial(self.network.retriable_call, self.network.get_headers, count=1000, b64=True) + self.headers.chunk_getter = get_chunk + await self.headers.ensure_tip() + + async def doit(): + for height in range(current, target, 1000): + await self.headers.ensure_chunk_at(height) + self._download_height = height log.info("Headers sync: %s / %s", self._download_height, target) + asyncio.ensure_future(doit()) + return async def update_headers(self, height=None, headers=None, subscription_update=False): rewound = 0 @@ -598,7 +598,7 @@ class Ledger(metaclass=LedgerRegistry): async def maybe_verify_transaction(self, tx, remote_height): tx.height = remote_height - if 0 < remote_height < len(self.headers): + if 0 < remote_height < self.network.remote_height: merkle = await self.network.retriable_call(self.network.get_merkle, tx.id, remote_height) merkle_root = self.get_root_of_merkle_tree(merkle['merkle'], merkle['pos'], tx.hash) header = await self.headers.get(remote_height) From b04a516063651537ef2344463a48e6ac8e53e25e Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 18 Mar 2020 14:17:01 -0300 Subject: [PATCH 05/10] better locking, stop corrupting headers, fix some tests --- lbry/wallet/header.py | 59 ++++++++++++++++++++----------- lbry/wallet/ledger.py | 17 ++++----- tests/unit/wallet/test_headers.py | 4 +++ tests/unit/wallet/test_ledger.py | 2 +- 4 files changed, 49 insertions(+), 33 deletions(-) diff --git a/lbry/wallet/header.py b/lbry/wallet/header.py index b6de019f1..c5af445aa 100644 --- a/lbry/wallet/header.py +++ b/lbry/wallet/header.py @@ -5,6 +5,7 @@ import asyncio import hashlib import logging import zlib +from concurrent.futures.thread import ThreadPoolExecutor from io import BytesIO from contextlib import asynccontextmanager @@ -47,6 +48,7 @@ class Headers: self.path = path self._size: Optional[int] = None self.chunk_getter: Optional[Callable] = None + self.executor = ThreadPoolExecutor(1) async def open(self): if self.path != ':memory:': @@ -54,8 +56,10 @@ class Headers: self.io = open(self.path, 'w+b') else: self.io = open(self.path, 'r+b') + self._size = self.io.seek(0, os.SEEK_END) // self.header_size async def close(self): + self.executor.shutdown() self.io.close() @staticmethod @@ -103,27 +107,34 @@ class Headers: return new_target def __len__(self) -> int: - if self._size is None: - self._size = self.io.seek(0, os.SEEK_END) // self.header_size return self._size def __bool__(self): return True async def get(self, height) -> dict: + if height < 0: + raise IndexError(f"Height cannot be negative!!") if isinstance(height, slice): raise NotImplementedError("Slicing of header chain has not been implemented yet.") - return self.deserialize(height, await self.get_raw_header(height)) + try: + return self.deserialize(height, await self.get_raw_header(height)) + except struct.error: + raise IndexError(f"failed to get {height}, at {len(self)}") def estimated_timestamp(self, height): return self.first_block_timestamp + (height * self.timestamp_average_offset) async def get_raw_header(self, height) -> bytes: - await self.ensure_chunk_at(height) - self.io.seek(height * self.header_size, os.SEEK_SET) - return self.io.read(self.header_size) + if self.chunk_getter: + await self.ensure_chunk_at(height) + return await asyncio.get_running_loop().run_in_executor(self.executor, self._read, height) - async def chunk_hash(self, start, count): + def _read(self, height, count=1): + self.io.seek(height * self.header_size, os.SEEK_SET) + return self.io.read(self.header_size * count) + + def chunk_hash(self, start, count): self.io.seek(start * self.header_size, os.SEEK_SET) return self.hash_header(self.io.read(count * self.header_size)).decode() @@ -141,16 +152,20 @@ class Headers: zlib.decompress(base64.b64decode(headers['base64']), wbits=-15, bufsize=600_000) ) chunk_hash = self.hash_header(chunk).decode() - if HASHES[start] == chunk_hash: - return self._write(start, chunk) + if HASHES.get(start) == chunk_hash: + return await asyncio.get_running_loop().run_in_executor(self.executor, self._write, start, chunk) + elif start not in HASHES: + return # todo: fixme raise Exception( f"Checkpoint mismatch at height {start}. Expected {HASHES[start]}, but got {chunk_hash} instead." ) async def has_header(self, height): - empty = '56944c5d3f98413ef45cf54545538103cc9f298e0575820ad3591376e2e0f65d' - all_zeroes = '789d737d4f448e554b318c94063bbfa63e9ccda6e208f5648ca76ee68896557b' - return await self.chunk_hash(height, 1) not in (empty, all_zeroes) + def _has_header(height): + empty = '56944c5d3f98413ef45cf54545538103cc9f298e0575820ad3591376e2e0f65d' + all_zeroes = '789d737d4f448e554b318c94063bbfa63e9ccda6e208f5648ca76ee68896557b' + return self.chunk_hash(height, 1) not in (empty, all_zeroes) + return await asyncio.get_running_loop().run_in_executor(self.executor, _has_header, height) @property def height(self) -> int: @@ -216,11 +231,11 @@ class Headers: def _write(self, height, verified_chunk): self.io.seek(height * self.header_size, os.SEEK_SET) written = self.io.write(verified_chunk) // self.header_size - self.io.truncate() + # self.io.truncate() # .seek()/.write()/.truncate() might also .flush() when needed # the goal here is mainly to ensure we're definitely flush()'ing self.io.flush() - self._size = self.io.tell() // self.header_size + self._size = self.io.seek(0, os.SEEK_END) // self.header_size return written async def validate_chunk(self, height, chunk): @@ -272,8 +287,9 @@ class Headers: previous_header_hash = fail = None batch_size = 36 for start_height in range(0, self.height, batch_size): - self.io.seek(self.header_size * start_height) - headers = self.io.read(self.header_size*batch_size) + headers = await asyncio.get_running_loop().run_in_executor( + self.executor, self._read, start_height, batch_size + ) if len(headers) % self.header_size != 0: headers = headers[:(len(headers) // self.header_size) * self.header_size] for header_hash, header in self._iterate_headers(start_height, headers): @@ -286,11 +302,12 @@ class Headers: fail = True if fail: log.warning("Header file corrupted at height %s, truncating it.", height - 1) - self.io.seek(max(0, (height - 1)) * self.header_size, os.SEEK_SET) - self.io.truncate() - self.io.flush() - self._size = None - return + def __truncate(at_height): + self.io.seek(max(0, (at_height - 1)) * self.header_size, os.SEEK_SET) + self.io.truncate() + self.io.flush() + self._size = self.io.seek(0, os.SEEK_END) // self.header_size + return await asyncio.get_running_loop().run_in_executor(self.executor, __truncate, height) previous_header_hash = header_hash @classmethod diff --git a/lbry/wallet/ledger.py b/lbry/wallet/ledger.py index 6577b4e54..0d1284dc2 100644 --- a/lbry/wallet/ledger.py +++ b/lbry/wallet/ledger.py @@ -316,9 +316,6 @@ class Ledger(metaclass=LedgerRegistry): first_connection = self.network.on_connected.first asyncio.ensure_future(self.network.start()) await first_connection - async with self._header_processing_lock: - await self._update_tasks.add(self.initial_headers_sync()) - await self._on_ready_controller.stream.first await asyncio.gather(*(a.maybe_migrate_certificates() for a in self.accounts)) await asyncio.gather(*(a.save_max_gap() for a in self.accounts)) if len(self.accounts) > 10: @@ -329,8 +326,10 @@ class Ledger(metaclass=LedgerRegistry): async def join_network(self, *_): log.info("Subscribing and updating accounts.") - #async with self._header_processing_lock: - # await self.update_headers() + self._update_tasks.add(self.initial_headers_sync()) + async with self._header_processing_lock: + await self.headers.ensure_tip() + await self.update_headers() await self.subscribe_accounts() await self._update_tasks.done.wait() self._on_ready_controller.add(True) @@ -348,16 +347,12 @@ class Ledger(metaclass=LedgerRegistry): async def initial_headers_sync(self): target = self.network.remote_height + 1 - current = len(self.headers) get_chunk = partial(self.network.retriable_call, self.network.get_headers, count=1000, b64=True) self.headers.chunk_getter = get_chunk - await self.headers.ensure_tip() async def doit(): - for height in range(current, target, 1000): + for height in reversed(range(0, target, 1000)): await self.headers.ensure_chunk_at(height) - self._download_height = height - log.info("Headers sync: %s / %s", self._download_height, target) asyncio.ensure_future(doit()) return @@ -598,7 +593,7 @@ class Ledger(metaclass=LedgerRegistry): async def maybe_verify_transaction(self, tx, remote_height): tx.height = remote_height - if 0 < remote_height < self.network.remote_height: + if 0 < remote_height < len(self.headers): merkle = await self.network.retriable_call(self.network.get_merkle, tx.id, remote_height) merkle_root = self.get_root_of_merkle_tree(merkle['merkle'], merkle['pos'], tx.hash) header = await self.headers.get(remote_height) diff --git a/tests/unit/wallet/test_headers.py b/tests/unit/wallet/test_headers.py index 8499a2b99..968766725 100644 --- a/tests/unit/wallet/test_headers.py +++ b/tests/unit/wallet/test_headers.py @@ -42,6 +42,7 @@ class TestHeaders(AsyncioTestCase): async def test_connect_from_genesis(self): headers = Headers(':memory:') + await headers.open() self.assertEqual(headers.height, -1) await headers.connect(0, HEADERS) self.assertEqual(headers.height, 19) @@ -49,6 +50,7 @@ class TestHeaders(AsyncioTestCase): async def test_connect_from_middle(self): h = Headers(':memory:') h.io.write(HEADERS[:block_bytes(10)]) + await h.open() self.assertEqual(h.height, 9) await h.connect(len(h), HEADERS[block_bytes(10):block_bytes(20)]) self.assertEqual(h.height, 19) @@ -140,6 +142,7 @@ class TestHeaders(AsyncioTestCase): async def test_checkpointed_writer(self): headers = Headers(':memory:') + await headers.open() getblocks = lambda start, end: HEADERS[block_bytes(start):block_bytes(end)] headers.checkpoint = 10, hexlify(sha256(getblocks(10, 11))) async with headers.checkpointed_connector() as buff: @@ -149,6 +152,7 @@ class TestHeaders(AsyncioTestCase): buff.write(getblocks(10, 19)) self.assertEqual(len(headers), 19) headers = Headers(':memory:') + await headers.open() async with headers.checkpointed_connector() as buff: buff.write(getblocks(0, 19)) self.assertEqual(len(headers), 19) diff --git a/tests/unit/wallet/test_ledger.py b/tests/unit/wallet/test_ledger.py index 13957d459..dc51ca240 100644 --- a/tests/unit/wallet/test_ledger.py +++ b/tests/unit/wallet/test_ledger.py @@ -67,7 +67,7 @@ class LedgerTestCase(AsyncioTestCase): serialized = self.make_header(**kwargs) self.ledger.headers.io.seek(0, os.SEEK_END) self.ledger.headers.io.write(serialized) - self.ledger.headers._size = None + self.ledger.headers._size = self.ledger.headers.io.seek(0, os.SEEK_END) // self.ledger.headers.header_size class TestSynchronization(LedgerTestCase): From 9fc7f9904bb2c66e4157aeb675c66615a0cf4321 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sat, 21 Mar 2020 02:01:21 -0300 Subject: [PATCH 06/10] fix tests, delete old code --- lbry/wallet/header.py | 55 +++++-------------- .../datanetwork/test_file_commands.py | 4 +- tests/unit/wallet/test_headers.py | 20 +------ 3 files changed, 19 insertions(+), 60 deletions(-) diff --git a/lbry/wallet/header.py b/lbry/wallet/header.py index c5af445aa..97956d54d 100644 --- a/lbry/wallet/header.py +++ b/lbry/wallet/header.py @@ -2,13 +2,11 @@ import base64 import os import struct import asyncio -import hashlib import logging import zlib from concurrent.futures.thread import ThreadPoolExecutor from io import BytesIO -from contextlib import asynccontextmanager from typing import Optional, Iterator, Tuple, Callable from binascii import hexlify, unhexlify @@ -36,7 +34,7 @@ class Headers: max_target = 0x0000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff genesis_hash = b'9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463' target_timespan = 150 - checkpoint = (600_000, b'100b33ca3d0b86a48f0d6d6f30458a130ecb89d5affefe4afccb134d5a40f4c2') + checkpoints = HASHES first_block_timestamp = 1466646588 # block 1, as 0 is off by a lot timestamp_average_offset = 160.6855883050695 # calculated at 733447 @@ -113,8 +111,6 @@ class Headers: return True async def get(self, height) -> dict: - if height < 0: - raise IndexError(f"Height cannot be negative!!") if isinstance(height, slice): raise NotImplementedError("Slicing of header chain has not been implemented yet.") try: @@ -128,6 +124,8 @@ class Headers: async def get_raw_header(self, height) -> bytes: if self.chunk_getter: await self.ensure_chunk_at(height) + if not 0 <= height <= self.height: + raise IndexError(f"{height} is out of bounds, current height: {self.height}") return await asyncio.get_running_loop().run_in_executor(self.executor, self._read, height) def _read(self, height, count=1): @@ -139,7 +137,8 @@ class Headers: return self.hash_header(self.io.read(count * self.header_size)).decode() async def ensure_tip(self): - await self.ensure_chunk_at(max(HASHES.keys())) + if self.checkpoints: + await self.ensure_chunk_at(max(self.checkpoints.keys())) async def ensure_chunk_at(self, height): if await self.has_header(height): @@ -152,12 +151,12 @@ class Headers: zlib.decompress(base64.b64decode(headers['base64']), wbits=-15, bufsize=600_000) ) chunk_hash = self.hash_header(chunk).decode() - if HASHES.get(start) == chunk_hash: + if self.checkpoints.get(start) == chunk_hash: return await asyncio.get_running_loop().run_in_executor(self.executor, self._write, start, chunk) - elif start not in HASHES: + elif start not in self.checkpoints: return # todo: fixme raise Exception( - f"Checkpoint mismatch at height {start}. Expected {HASHES[start]}, but got {chunk_hash} instead." + f"Checkpoint mismatch at height {start}. Expected {self.checkpoints[start]}, but got {chunk_hash} instead." ) async def has_header(self, height): @@ -186,33 +185,6 @@ class Headers: return b'0' * 64 return hexlify(double_sha256(header)[::-1]) - @asynccontextmanager - async def checkpointed_connector(self): - buf = BytesIO() - try: - yield buf - finally: - await asyncio.sleep(0) - final_height = len(self) + buf.tell() // self.header_size - verifiable_bytes = (self.checkpoint[0] - len(self)) * self.header_size if self.checkpoint else 0 - if verifiable_bytes > 0 and final_height >= self.checkpoint[0]: - buf.seek(0) - self.io.seek(0) - h = hashlib.sha256() - h.update(self.io.read()) - h.update(buf.read(verifiable_bytes)) - if h.hexdigest().encode() == self.checkpoint[1]: - buf.seek(0) - self._write(len(self), buf.read(verifiable_bytes)) - remaining = buf.read() - buf.seek(0) - buf.write(remaining) - buf.truncate() - else: - log.warning("Checkpoint mismatch, connecting headers through slow method.") - if buf.tell() > 0: - await self.connect(len(self), buf.getvalue()) - async def connect(self, start: int, headers: bytes) -> int: added = 0 bail = False @@ -223,7 +195,8 @@ class Headers: except InvalidHeader as e: bail = True chunk = chunk[:(height-e.height)*self.header_size] - added += self._write(height, chunk) if chunk else 0 + if chunk: + added += await asyncio.get_running_loop().run_in_executor(self.executor, self._write, height, chunk) if bail: break return added @@ -235,14 +208,15 @@ class Headers: # .seek()/.write()/.truncate() might also .flush() when needed # the goal here is mainly to ensure we're definitely flush()'ing self.io.flush() - self._size = self.io.seek(0, os.SEEK_END) // self.header_size + self._size = self.io.tell() // self.header_size return written async def validate_chunk(self, height, chunk): previous_hash, previous_header, previous_previous_header = None, None, None if height > 0: - previous_header = await self.get(height-1) - previous_hash = await self.hash(height-1) + raw = await self.get_raw_header(height-1) + previous_header = self.deserialize(height-1, raw) + previous_hash = self.hash_header(raw) if height > 1: previous_previous_header = await self.get(height-2) chunk_target = self.get_next_chunk_target(height // 2016 - 1) @@ -345,3 +319,4 @@ class UnvalidatedHeaders(Headers): validate_difficulty = False max_target = 0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff genesis_hash = b'6e3fcf1299d4ec5d79c3a4c91d624a4acf9e2e173d95a1a0504f677669687556' + checkpoints = {} diff --git a/tests/integration/datanetwork/test_file_commands.py b/tests/integration/datanetwork/test_file_commands.py index cfd14ef0a..24d9b1e71 100644 --- a/tests/integration/datanetwork/test_file_commands.py +++ b/tests/integration/datanetwork/test_file_commands.py @@ -125,14 +125,14 @@ class FileCommands(CommandTestCase): file_list = await self.file_list() self.assertEqual( file_list[0]['timestamp'], - None + self.ledger.headers.estimated_timestamp(file_list[0]['height']) ) self.assertEqual(file_list[0]['confirmations'], -1) await self.daemon.jsonrpc_resolve('foo') file_list = await self.file_list() self.assertEqual( file_list[0]['timestamp'], - self.ledger.headers[file_list[0]['height']]['timestamp'] + self.ledger.headers.estimated_timestamp(file_list[0]['height']) ) self.assertEqual(file_list[0]['confirmations'], 1) diff --git a/tests/unit/wallet/test_headers.py b/tests/unit/wallet/test_headers.py index 968766725..1d1589648 100644 --- a/tests/unit/wallet/test_headers.py +++ b/tests/unit/wallet/test_headers.py @@ -19,6 +19,7 @@ class TestHeaders(AsyncioTestCase): self.maxDiff = None h = Headers(':memory:') h.io.write(HEADERS) + await h.open() self.assertEqual(await h.get(0), { 'bits': 520159231, 'block_height': 0, @@ -140,23 +141,6 @@ class TestHeaders(AsyncioTestCase): await headers.connect(len(headers), HEADERS[block_bytes(8):]) self.assertEqual(19, headers.height) - async def test_checkpointed_writer(self): - headers = Headers(':memory:') - await headers.open() - getblocks = lambda start, end: HEADERS[block_bytes(start):block_bytes(end)] - headers.checkpoint = 10, hexlify(sha256(getblocks(10, 11))) - async with headers.checkpointed_connector() as buff: - buff.write(getblocks(0, 10)) - self.assertEqual(len(headers), 10) - async with headers.checkpointed_connector() as buff: - buff.write(getblocks(10, 19)) - self.assertEqual(len(headers), 19) - headers = Headers(':memory:') - await headers.open() - async with headers.checkpointed_connector() as buff: - buff.write(getblocks(0, 19)) - self.assertEqual(len(headers), 19) - async def test_concurrency(self): BLOCKS = 19 headers_temporary_file = tempfile.mktemp() @@ -168,7 +152,7 @@ class TestHeaders(AsyncioTestCase): await headers.connect(block_index, HEADERS[block_bytes(block_index):block_bytes(block_index + 1)]) async def reader(): for block_index in range(BLOCKS): - while len(headers) < block_index: + while len(headers) <= block_index: await asyncio.sleep(0.000001) assert (await headers.get(block_index))['block_height'] == block_index reader_task = asyncio.create_task(reader()) From 19c0a81c427acd7cf544fde5f91b35e6a31df704 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sat, 21 Mar 2020 04:32:03 -0300 Subject: [PATCH 07/10] fix bad usages of hash and some tests --- lbry/extras/daemon/components.py | 2 +- lbry/testcase.py | 1 + lbry/wallet/header.py | 10 +++++++--- lbry/wallet/ledger.py | 13 +++++++------ lbry/wallet/manager.py | 4 ++-- lbry/wallet/orchstr8/node.py | 1 + .../blockchain/test_blockchain_reorganization.py | 6 +++--- tests/integration/blockchain/test_network.py | 12 +++++------- .../integration/blockchain/test_resolve_command.py | 2 +- tests/integration/blockchain/test_sync.py | 1 + tests/unit/lbrynet_daemon/test_Daemon.py | 2 +- 11 files changed, 30 insertions(+), 24 deletions(-) diff --git a/lbry/extras/daemon/components.py b/lbry/extras/daemon/components.py index 7de4d9fe2..5271c1558 100644 --- a/lbry/extras/daemon/components.py +++ b/lbry/extras/daemon/components.py @@ -143,7 +143,7 @@ class WalletComponent(Component): progress = min(max(math.ceil(float(download_height) / float(target_height) * 100), 0), 100) else: progress = 100 - best_hash = self.wallet_manager.get_best_blockhash() + best_hash = await self.wallet_manager.get_best_blockhash() result.update({ 'headers_synchronization_progress': progress, 'blocks': max(local_height, 0), diff --git a/lbry/testcase.py b/lbry/testcase.py index 56c9cb4c6..667fab5e1 100644 --- a/lbry/testcase.py +++ b/lbry/testcase.py @@ -370,6 +370,7 @@ class CommandTestCase(IntegrationTestCase): ) self.extra_wallet_node_port += 1 await wallet_node.start(self.conductor.spv_node, seed=seed) + await wallet_node.ledger.on_ready.first self.extra_wallet_nodes.append(wallet_node) upload_dir = os.path.join(wallet_node.data_path, 'uploads') diff --git a/lbry/wallet/header.py b/lbry/wallet/header.py index 97956d54d..1375d5442 100644 --- a/lbry/wallet/header.py +++ b/lbry/wallet/header.py @@ -49,6 +49,8 @@ class Headers: self.executor = ThreadPoolExecutor(1) async def open(self): + if not self.executor: + self.executor = ThreadPoolExecutor(1) if self.path != ':memory:': if not os.path.exists(self.path): self.io = open(self.path, 'w+b') @@ -57,7 +59,9 @@ class Headers: self._size = self.io.seek(0, os.SEEK_END) // self.header_size async def close(self): - self.executor.shutdown() + if self.executor: + self.executor.shutdown() + self.executor = None self.io.close() @staticmethod @@ -142,7 +146,7 @@ class Headers: async def ensure_chunk_at(self, height): if await self.has_header(height): - log.info("has header %s", height) + log.debug("has header %s", height) return log.info("on-demand fetching height %s", height) start = (height // 1000) * 1000 @@ -208,7 +212,7 @@ class Headers: # .seek()/.write()/.truncate() might also .flush() when needed # the goal here is mainly to ensure we're definitely flush()'ing self.io.flush() - self._size = self.io.tell() // self.header_size + self._size = max(self._size or 0, self.io.tell() // self.header_size) return written async def validate_chunk(self, height, chunk): diff --git a/lbry/wallet/ledger.py b/lbry/wallet/ledger.py index 0d1284dc2..ad5801e61 100644 --- a/lbry/wallet/ledger.py +++ b/lbry/wallet/ledger.py @@ -316,6 +316,8 @@ class Ledger(metaclass=LedgerRegistry): first_connection = self.network.on_connected.first asyncio.ensure_future(self.network.start()) await first_connection + async with self._header_processing_lock: + await self._update_tasks.add(self.initial_headers_sync()) await asyncio.gather(*(a.maybe_migrate_certificates() for a in self.accounts)) await asyncio.gather(*(a.save_max_gap() for a in self.accounts)) if len(self.accounts) > 10: @@ -326,10 +328,8 @@ class Ledger(metaclass=LedgerRegistry): async def join_network(self, *_): log.info("Subscribing and updating accounts.") - self._update_tasks.add(self.initial_headers_sync()) async with self._header_processing_lock: - await self.headers.ensure_tip() - await self.update_headers() + await self._update_tasks.add(self.initial_headers_sync()) await self.subscribe_accounts() await self._update_tasks.done.wait() self._on_ready_controller.add(True) @@ -353,8 +353,9 @@ class Ledger(metaclass=LedgerRegistry): async def doit(): for height in reversed(range(0, target, 1000)): await self.headers.ensure_chunk_at(height) - asyncio.ensure_future(doit()) - return + await self.headers.ensure_tip() + self._update_tasks.add(doit()) + await self.update_headers() async def update_headers(self, height=None, headers=None, subscription_update=False): rewound = 0 @@ -894,7 +895,7 @@ class Ledger(metaclass=LedgerRegistry): headers = self.headers history = [] for tx in txs: # pylint: disable=too-many-nested-blocks - ts = headers.estimated_timestamp(tx.height)['timestamp'] + ts = headers.estimated_timestamp(tx.height) item = { 'txid': tx.id, 'timestamp': ts, diff --git a/lbry/wallet/manager.py b/lbry/wallet/manager.py index 6e8cc5db1..37b2ec992 100644 --- a/lbry/wallet/manager.py +++ b/lbry/wallet/manager.py @@ -248,10 +248,10 @@ class WalletManager: log.warning("Failed to migrate %s receiving addresses!", len(set(receiving_addresses).difference(set(migrated_receiving)))) - def get_best_blockhash(self): + async def get_best_blockhash(self): if len(self.ledger.headers) <= 0: return self.ledger.genesis_hash - return self.ledger.headers.hash(self.ledger.headers.height).decode() + return (await self.ledger.headers.hash(self.ledger.headers.height)).decode() def get_unused_address(self): return self.default_account.receiving.get_or_create_usable_address() diff --git a/lbry/wallet/orchstr8/node.py b/lbry/wallet/orchstr8/node.py index a94a7bc0a..411b57599 100644 --- a/lbry/wallet/orchstr8/node.py +++ b/lbry/wallet/orchstr8/node.py @@ -77,6 +77,7 @@ class Conductor: async def start_wallet(self): if not self.wallet_started: await self.wallet_node.start(self.spv_node) + await self.wallet_node.ledger.on_ready.first self.wallet_started = True async def stop_wallet(self): diff --git a/tests/integration/blockchain/test_blockchain_reorganization.py b/tests/integration/blockchain/test_blockchain_reorganization.py index 6aabb09fd..b271966df 100644 --- a/tests/integration/blockchain/test_blockchain_reorganization.py +++ b/tests/integration/blockchain/test_blockchain_reorganization.py @@ -8,7 +8,7 @@ class BlockchainReorganizationTests(IntegrationTestCase): async def assertBlockHash(self, height): self.assertEqual( - self.ledger.headers.hash(height).decode(), + (await self.ledger.headers.hash(height)).decode(), await self.blockchain.get_block_hash(height) ) @@ -16,7 +16,7 @@ class BlockchainReorganizationTests(IntegrationTestCase): # invalidate current block, move forward 2 self.assertEqual(self.ledger.headers.height, 200) await self.assertBlockHash(200) - await self.blockchain.invalidate_block(self.ledger.headers.hash(200).decode()) + await self.blockchain.invalidate_block((await self.ledger.headers.hash(200)).decode()) await self.blockchain.generate(2) await self.ledger.on_header.where(lambda e: e.height == 201) self.assertEqual(self.ledger.headers.height, 201) @@ -24,7 +24,7 @@ class BlockchainReorganizationTests(IntegrationTestCase): await self.assertBlockHash(201) # invalidate current block, move forward 3 - await self.blockchain.invalidate_block(self.ledger.headers.hash(200).decode()) + await self.blockchain.invalidate_block((await self.ledger.headers.hash(200)).decode()) await self.blockchain.generate(3) await self.ledger.on_header.where(lambda e: e.height == 202) self.assertEqual(self.ledger.headers.height, 202) diff --git a/tests/integration/blockchain/test_network.py b/tests/integration/blockchain/test_network.py index 95400e321..f2222d560 100644 --- a/tests/integration/blockchain/test_network.py +++ b/tests/integration/blockchain/test_network.py @@ -90,13 +90,11 @@ class ReconnectTests(IntegrationTestCase): while self.conductor.spv_node.server.session_mgr.notified_height < initial_height + 99: # off by 1 await asyncio.sleep(0.1) self.assertEqual(initial_height, self.ledger.local_height_including_downloaded_height) - # locks header processing so we make sure we are the only ones modifying it - async with self.ledger._header_processing_lock: - await self.ledger.headers.open() - await self.ledger.network.start() - await self.ledger.network.on_connected.first - await self.ledger.initial_headers_sync() - self.assertEqual(initial_height + 100, self.ledger.local_height_including_downloaded_height) + await self.ledger.headers.open() + await self.ledger.network.start() + await self.ledger.network.on_connected.first + await self.ledger.initial_headers_sync() + self.assertEqual(initial_height + 100, self.ledger.local_height_including_downloaded_height) async def test_connection_drop_still_receives_events_after_reconnected(self): address1 = await self.account.receiving.get_or_create_usable_address() diff --git a/tests/integration/blockchain/test_resolve_command.py b/tests/integration/blockchain/test_resolve_command.py index ea6874cfd..b6477199a 100644 --- a/tests/integration/blockchain/test_resolve_command.py +++ b/tests/integration/blockchain/test_resolve_command.py @@ -337,7 +337,7 @@ class ResolveAfterReorg(BaseResolveTestCase): blocks = self.ledger.headers.height - start self.blockchain.block_expected = start - 1 # go back to start - await self.blockchain.invalidate_block(self.ledger.headers.hash(start).decode()) + await self.blockchain.invalidate_block((await self.ledger.headers.hash(start)).decode()) # go to previous + 1 await self.generate(blocks + 2) diff --git a/tests/integration/blockchain/test_sync.py b/tests/integration/blockchain/test_sync.py index 7af2bd1aa..7a3ffd3a7 100644 --- a/tests/integration/blockchain/test_sync.py +++ b/tests/integration/blockchain/test_sync.py @@ -27,6 +27,7 @@ class SyncTests(IntegrationTestCase): wallet_node = WalletNode(WalletManager, RegTestLedger, port=self.api_port) await wallet_node.start(self.conductor.spv_node, seed) self.started_nodes.append(wallet_node) + await wallet_node.ledger.on_ready.first return wallet_node async def test_nodes_with_same_account_stay_in_sync(self): diff --git a/tests/unit/lbrynet_daemon/test_Daemon.py b/tests/unit/lbrynet_daemon/test_Daemon.py index 71f5325ed..6cbd57b04 100644 --- a/tests/unit/lbrynet_daemon/test_Daemon.py +++ b/tests/unit/lbrynet_daemon/test_Daemon.py @@ -92,7 +92,7 @@ class TestCostEst(unittest.TestCase): @unittest.SkipTest class TestJsonRpc(unittest.TestCase): def setUp(self): - def noop(): + async def noop(): return None test_utils.reset_time(self) From 3eebe301fe324e7221d75eee85c20f7ed81421de Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sun, 22 Mar 2020 23:45:14 -0300 Subject: [PATCH 08/10] move checkpoints out of folder into file --- lbry/wallet/{checkpoints/__init__.py => checkpoints.py} | 0 lbry/wallet/ledger.py | 2 -- 2 files changed, 2 deletions(-) rename lbry/wallet/{checkpoints/__init__.py => checkpoints.py} (100%) diff --git a/lbry/wallet/checkpoints/__init__.py b/lbry/wallet/checkpoints.py similarity index 100% rename from lbry/wallet/checkpoints/__init__.py rename to lbry/wallet/checkpoints.py diff --git a/lbry/wallet/ledger.py b/lbry/wallet/ledger.py index ad5801e61..83b297174 100644 --- a/lbry/wallet/ledger.py +++ b/lbry/wallet/ledger.py @@ -1,7 +1,5 @@ import os -import zlib import copy -import base64 import asyncio import logging from io import StringIO From 952fc01efd879ed878c9d61da8f84a12564a0a74 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 23 Mar 2020 00:05:36 -0300 Subject: [PATCH 09/10] add script that generates checkpoints --- lbry/wallet/checkpoints.py | 2 ++ lbry/wallet/ledger.py | 5 +++-- scripts/checkpoints.py | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 scripts/checkpoints.py diff --git a/lbry/wallet/checkpoints.py b/lbry/wallet/checkpoints.py index 3c103be4e..71b7770a3 100644 --- a/lbry/wallet/checkpoints.py +++ b/lbry/wallet/checkpoints.py @@ -732,4 +732,6 @@ HASHES = { 730000: '94cec967e44f850f512d4240cb8a52ffaf953d0364b0a1dd7604b4a01406e669', 731000: '6e63f5019439bc7e27a17a189baad0da8f5724883af3ca35efa0d4e5aaa75b97', 732000: '53e1b373805f3236c7725415e872d5635b8679894c4fb630c62b6b75b4ec9d9c', + 733000: '43e9ab6cf54fde5dcdc4c473af26b256435f4af4254d96fa728f2af9b078d630', + 734000: 'a3ef7f9257d591c7dcc0f82346cb162a768ee5fe1228353ec485e69be1bf585f', } diff --git a/lbry/wallet/ledger.py b/lbry/wallet/ledger.py index 83b297174..0e648789b 100644 --- a/lbry/wallet/ledger.py +++ b/lbry/wallet/ledger.py @@ -349,8 +349,9 @@ class Ledger(metaclass=LedgerRegistry): self.headers.chunk_getter = get_chunk async def doit(): - for height in reversed(range(0, target, 1000)): - await self.headers.ensure_chunk_at(height) + for height in reversed(range(0, target)): + async with self._header_processing_lock: + await self.headers.ensure_chunk_at(height) await self.headers.ensure_tip() self._update_tasks.add(doit()) await self.update_headers() diff --git a/scripts/checkpoints.py b/scripts/checkpoints.py new file mode 100644 index 000000000..07c877095 --- /dev/null +++ b/scripts/checkpoints.py @@ -0,0 +1,35 @@ +import asyncio +import os + +from lbry.extras.cli import ensure_directory_exists +from lbry.conf import Config +from lbry.wallet.header import Headers +import lbry.wallet.checkpoints + + +async def main(): + outpath = lbry.wallet.checkpoints.__file__ + ledger_path = os.path.join(Config().wallet_dir, 'lbc_mainnet') + ensure_directory_exists(ledger_path) + headers_path = os.path.join(ledger_path, 'headers') + headers = Headers(headers_path) + await headers.open() + print(f"Working on headers at {outpath}") + print("Verifying integrity, might take a while.") + await headers.repair() + target = ((headers.height - 100) // 1000) * 1000 + current_checkpoint_tip = max(lbry.wallet.checkpoints.HASHES.keys()) + if target <= current_checkpoint_tip: + print(f"We have nothing to add: Local: {target}, checkpoint: {current_checkpoint_tip}") + return + print(f"Headers file at {headers.height}, checkpointing up to {target}." + f"Current checkpoint at {current_checkpoint_tip}.") + with open(outpath, 'w') as outfile: + print('HASHES = {', file=outfile) + for height in range(0, target, 1000): + print(f" {height}: '{headers.chunk_hash(height, 1000)}',", file=outfile) + print('}', file=outfile) + + +if __name__ == "__main__": + asyncio.get_event_loop().run_until_complete(main()) From 342cb006257d5ff86913c82a09387b3cf71a16ec Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 23 Mar 2020 01:19:34 -0300 Subject: [PATCH 10/10] less concurrent repeated header checks --- lbry/wallet/header.py | 41 +++++++++++++++++++++++++------ lbry/wallet/ledger.py | 6 ++--- tests/unit/wallet/test_headers.py | 9 ++++--- 3 files changed, 42 insertions(+), 14 deletions(-) diff --git a/lbry/wallet/header.py b/lbry/wallet/header.py index 1375d5442..6af943e3b 100644 --- a/lbry/wallet/header.py +++ b/lbry/wallet/header.py @@ -47,6 +47,8 @@ class Headers: self._size: Optional[int] = None self.chunk_getter: Optional[Callable] = None self.executor = ThreadPoolExecutor(1) + self.known_missing_checkpointed_chunks = set() + self.check_chunk_lock = asyncio.Lock() async def open(self): if not self.executor: @@ -57,6 +59,8 @@ class Headers: else: self.io = open(self.path, 'r+b') self._size = self.io.seek(0, os.SEEK_END) // self.header_size + await self.ensure_checkpointed_size() + await self.get_all_missing_headers() async def close(self): if self.executor: @@ -140,14 +144,19 @@ class Headers: self.io.seek(start * self.header_size, os.SEEK_SET) return self.hash_header(self.io.read(count * self.header_size)).decode() - async def ensure_tip(self): - if self.checkpoints: - await self.ensure_chunk_at(max(self.checkpoints.keys())) + async def ensure_checkpointed_size(self): + max_checkpointed_height = max(self.checkpoints.keys() or [-1]) + if self.height < max_checkpointed_height: + self._write(max_checkpointed_height, bytes([0] * self.header_size * 1000)) async def ensure_chunk_at(self, height): - if await self.has_header(height): - log.debug("has header %s", height) - return + async with self.check_chunk_lock: + if await self.has_header(height): + log.debug("has header %s", height) + return + return await self.fetch_chunk(height) + + async def fetch_chunk(self, height): log.info("on-demand fetching height %s", height) start = (height // 1000) * 1000 headers = await self.chunk_getter(start) # pylint: disable=not-callable @@ -156,7 +165,10 @@ class Headers: ) chunk_hash = self.hash_header(chunk).decode() if self.checkpoints.get(start) == chunk_hash: - return await asyncio.get_running_loop().run_in_executor(self.executor, self._write, start, chunk) + await asyncio.get_running_loop().run_in_executor(self.executor, self._write, start, chunk) + if start in self.known_missing_checkpointed_chunks: + self.known_missing_checkpointed_chunks.remove(start) + return elif start not in self.checkpoints: return # todo: fixme raise Exception( @@ -164,12 +176,27 @@ class Headers: ) async def has_header(self, height): + normalized_height = (height // 1000) * 1000 + if normalized_height in self.checkpoints: + return normalized_height not in self.known_missing_checkpointed_chunks + def _has_header(height): empty = '56944c5d3f98413ef45cf54545538103cc9f298e0575820ad3591376e2e0f65d' all_zeroes = '789d737d4f448e554b318c94063bbfa63e9ccda6e208f5648ca76ee68896557b' return self.chunk_hash(height, 1) not in (empty, all_zeroes) return await asyncio.get_running_loop().run_in_executor(self.executor, _has_header, height) + async def get_all_missing_headers(self): + # Heavy operation done in one optimized shot + def _io_checkall(): + for chunk_height, expected_hash in reversed(list(self.checkpoints.items())): + if chunk_height in self.known_missing_checkpointed_chunks: + continue + if self.chunk_hash(chunk_height, 1000) != expected_hash: + self.known_missing_checkpointed_chunks.add(chunk_height) + return self.known_missing_checkpointed_chunks + return await asyncio.get_running_loop().run_in_executor(self.executor, _io_checkall) + @property def height(self) -> int: return len(self)-1 diff --git a/lbry/wallet/ledger.py b/lbry/wallet/ledger.py index 0e648789b..907aec8ba 100644 --- a/lbry/wallet/ledger.py +++ b/lbry/wallet/ledger.py @@ -344,15 +344,13 @@ class Ledger(metaclass=LedgerRegistry): return max(self.headers.height, self._download_height) async def initial_headers_sync(self): - target = self.network.remote_height + 1 get_chunk = partial(self.network.retriable_call, self.network.get_headers, count=1000, b64=True) self.headers.chunk_getter = get_chunk async def doit(): - for height in reversed(range(0, target)): - async with self._header_processing_lock: + async with self._header_processing_lock: + for height in reversed(sorted(self.headers.known_missing_checkpointed_chunks)): await self.headers.ensure_chunk_at(height) - await self.headers.ensure_tip() self._update_tasks.add(doit()) await self.update_headers() diff --git a/tests/unit/wallet/test_headers.py b/tests/unit/wallet/test_headers.py index 1d1589648..52dfc2117 100644 --- a/tests/unit/wallet/test_headers.py +++ b/tests/unit/wallet/test_headers.py @@ -1,12 +1,15 @@ import os import asyncio import tempfile -from binascii import hexlify, unhexlify +from binascii import unhexlify -from lbry.crypto.hash import sha256 from lbry.wallet.util import ArithUint256 from lbry.testcase import AsyncioTestCase -from lbry.wallet.ledger import Headers +from lbry.wallet.ledger import Headers as _Headers + + +class Headers(_Headers): + checkpoints = {} def block_bytes(blocks):