From 929448d64bcbe6c5e476757ec78456beaa85e56a Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Sat, 12 Nov 2022 16:48:01 -0500 Subject: [PATCH] collect db values for integrity check in chunks --- hub/db/revertable.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/hub/db/revertable.py b/hub/db/revertable.py index a3ba7d6..f2b2eed 100644 --- a/hub/db/revertable.py +++ b/hub/db/revertable.py @@ -135,9 +135,11 @@ class RevertableOpStack: existing = {} if self._enforce_integrity and unique_keys: unique_keys = list(unique_keys) - existing.update({ - k: v for k, v in zip(unique_keys, self._multi_get(unique_keys)) - }) + for idx in range(0, len(unique_keys), 10000): + batch = unique_keys[idx:idx+10000] + existing.update({ + k: v for k, v in zip(batch, self._multi_get(batch)) + }) for op in ops_to_apply: if op.key in self._items and len(self._items[op.key]) and self._items[op.key][-1] == op.invert():