Improve code

src/main/java/com/lbry/database/BasePrefixDB.java

@@ -45,8 +45,9 @@ public abstract class BasePrefixDB{
         this.operationStack = new RevertibleOperationStack((byte[] key) -> {
             try{
                 return Optional.of(this.get(key));
-            }catch(RocksDBException e){}
+            }catch(RocksDBException e){
-            return Optional.empty();
+                return Optional.empty();
+            }
         },(List<byte[]> keys) -> {
             List<Optional<byte[]>> optionalKeys = new ArrayList<>();
             for(byte[] key : keys){
@@ -78,8 +79,9 @@ public abstract class BasePrefixDB{
                 }else{
                     batch.delete(columnFamily,stagedChange.getKey());
                 }
-                this.database.write(writeOptions,batch);
             }
+            this.database.write(writeOptions,batch);
+            batch.close();
         }finally{
             writeOptions.close();
             this.operationStack.clear();
@@ -92,41 +94,40 @@ public abstract class BasePrefixDB{
         List<byte[]> deleteUndos = new ArrayList<>();
         if(height>this.maxUndoDepth){
             byte[] upperBound = ByteBuffer.allocate(1+8).order(ByteOrder.BIG_ENDIAN).put(Prefix.UNDO.getValue()).putLong(height-this.maxUndoDepth).array();
-            RocksIterator iterator = this.database.newIterator(new ReadOptions().setIterateUpperBound(new Slice(upperBound)));
+            ReadOptions readOptions = new ReadOptions().setIterateUpperBound(new Slice(upperBound));
+            RocksIterator iterator = this.database.newIterator(readOptions);
             iterator.seek(ByteBuffer.allocate(1+8).order(ByteOrder.BIG_ENDIAN).put(Prefix.UNDO.getValue()).array());
             while(iterator.isValid()){
                 deleteUndos.add(iterator.key());
                 iterator.next();
             }
+            readOptions.close();
         }
+        WriteOptions writeOptions = new WriteOptions().setSync(true);
         try{
             ColumnFamilyHandle undoColumnFamily = this.getColumnFamilyByPrefix(Prefix.UNDO);
-            WriteOptions writeOptions = new WriteOptions().setSync(true);
+            WriteBatch batch = new WriteBatch();
-            try{
+            for(RevertibleOperation stagedChange : this.operationStack.iterate()){
-                WriteBatch batch = new WriteBatch();
+                ColumnFamilyHandle columnFamily = this.getColumnFamilyByPrefix(Prefix.getByValue(stagedChange.getKey()[0]));
-                for(RevertibleOperation stagedChange : this.operationStack.iterate()){
+                if(!stagedChange.isDelete()){
-                    ColumnFamilyHandle columnFamily = this.getColumnFamilyByPrefix(Prefix.getByValue(stagedChange.getKey()[0]));
+                    batch.put(columnFamily,stagedChange.getKey(),stagedChange.getValue());
-                    if(!stagedChange.isDelete()){
+                }else{
-                        batch.put(columnFamily,stagedChange.getKey(),stagedChange.getValue());
+                    batch.delete(columnFamily,stagedChange.getKey());
-                    }else{
+                }
-                        batch.delete(columnFamily,stagedChange.getKey());
-                    }
 
-                }
-                for(byte[] undoToDelete : deleteUndos){
-                    batch.delete(undoColumnFamily,undoToDelete);
-                }
-                UndoKey undoKey = new UndoKey();
-                undoKey.height = height;
-                undoKey.block_hash = blockHash;
-                byte[] undoKeyBytes = ((PrefixDB)this).undo.packKey(undoKey);
-                batch.put(undoColumnFamily,undoKeyBytes,undoOperations);
-                this.database.write(writeOptions,batch);
-            }finally{
-                writeOptions.close();
-                this.operationStack.clear();
             }
+            for(byte[] undoToDelete : deleteUndos){
+                batch.delete(undoColumnFamily,undoToDelete);
+            }
+            final long batchHeight = height;
+            batch.put(undoColumnFamily,((PrefixDB)this).undo.packKey(new UndoKey(){{
+                this.block_hash = blockHash;
+                this.height = batchHeight;
+            }}),undoOperations);
+            this.database.write(writeOptions,batch);
+            batch.close();
         }finally{
+            writeOptions.close();
             this.operationStack.clear();
         }
     }

src/main/java/com/lbry/database/revert/RevertibleOperationStack.java

@@ -1,5 +1,6 @@
 package com.lbry.database.revert;
 
+import com.lbry.database.util.ArrayHelper;
 import com.lbry.database.util.MapHelper;
 import com.lbry.database.util.Tuple2;
 
@@ -53,7 +54,6 @@ public class RevertibleOperationStack{
     }
 
     public void validateAndApplyStashedOperations(){
-//        System.err.println("STASH = "+this.stash);
         if(this.stash.isEmpty()){
             return;
         }
@@ -68,8 +68,7 @@ public class RevertibleOperationStack{
             if(operationArr!=null && operationArr.length>=1 && operation.invert().equals(operationArr[operationArr.length-1])){
                 this.items.replace(operationArr[0].getKey(),Arrays.copyOfRange(operationArr,0,operationArr.length-1));
                 continue;
-            }
+            }else if(operationArr!=null && operationArr.length>=1 && operation.equals(operationArr[operationArr.length-1])){
-            if(operationArr!=null && operationArr.length>=1 && operation.equals(operationArr[operationArr.length-1])){
                 continue;
             }else{
                 needAppend.add(operation);
@@ -78,63 +77,37 @@ public class RevertibleOperationStack{
         }
 
         Map<byte[],byte[]> existing = new HashMap<>();
-//        if(this.enforceIntegrity && !uniqueKeys.isEmpty()){
+        if(this.enforceIntegrity && !uniqueKeys.isEmpty()){
-//            List<byte[]> uniqueKeysList = new ArrayList<>(uniqueKeys);
+            List<byte[]> uniqueKeysList = new ArrayList<>(uniqueKeys);
-//            for(int idx=0;idx<uniqueKeys.size();idx+=10000){
+            for(int idx=0;idx<uniqueKeys.size();idx+=10000){
-//                List<byte[]> batch = uniqueKeysList.subList(idx,Math.min(uniqueKeysList.size(),idx+10000));
+                List<byte[]> batch = uniqueKeysList.subList(idx,Math.min(uniqueKeysList.size(),idx+10000));
-//                Iterator<Optional<byte[]>> iterator = this.multiGet.apply(batch).iterator();
+                Iterator<Optional<byte[]>> iterator = this.multiGet.apply(batch).iterator();
-//                for(byte[] k : batch){
+                for(byte[] k : batch){
-//                    byte[] v = iterator.next().get();
+                    Optional<byte[]> vOpt = iterator.next();
-//                    System.err.println(new RevertiblePut(k,v));
+                    vOpt.ifPresent(bytes -> existing.put(k, bytes));
-//                    existing.put(k,v);
+                }
-//                }
+
-//
+            }
-//            }
+        }
-//        }
 
         for(RevertibleOperation operation : needAppend){
-            RevertibleOperation[] operationArr = MapHelper.getValue(this.items,operation.key);
+            RevertibleOperation[] operationArr = MapHelper.getValue(this.items,operation.getKey());
-//            System.err.println("@ "+operation+ " vs "+Arrays.toString(operationArr));
             if(operationArr!=null && operationArr.length>=1 && operationArr[operationArr.length-1].equals(operation)){
+                this.items.put(MapHelper.getKey(this.items,operation.getKey()),ArrayHelper.pop(MapHelper.getValue(this.items,operation.getKey())));
                 RevertibleOperation[] operationArr2 = MapHelper.getValue(this.items,operation.getKey());
-                List<RevertibleOperation> operationList = Arrays.asList(operationArr2!=null?operationArr2:new RevertibleOperation[0]);
+                if(operationArr2==null || operationArr2.length==0){
-                if(!operationList.isEmpty()){
-                    operationList.remove(operationList.size()-1);
-                }
-                if(operationList.isEmpty()){
                     MapHelper.remove(this.items,operation.getKey());
-                    continue;
                 }
             }
             if(!this.enforceIntegrity){
-                RevertibleOperation[] operationArr2 = MapHelper.getValue(this.items,operation.getKey());
+                byte[] newKey = MapHelper.getKey(this.items,operation.getKey());
-                List<RevertibleOperation> operationList = Arrays.asList(operationArr2!=null?operationArr2:new RevertibleOperation[0]);
+                this.items.put(newKey,ArrayHelper.append(MapHelper.getValue(this.items,newKey),operation));
-                operationList.add(operation);
+                continue;
-                this.items.put(operationList.get(0).getKey(),operationList.toArray(new RevertibleOperation[0]));
             }
-
-            RevertibleOperation[] operationArrX = null;
-            for(Map.Entry<byte[],RevertibleOperation[]> e : this.items.entrySet()){
-                if(Arrays.equals(e.getKey(),operation.getKey())){
-                    operationArrX = e.getValue();
-                }
-            }
-
             byte[] storedValue = MapHelper.getValue(existing,operation.getKey());
-//            System.err.println(operation);
             boolean hasStoredValue = storedValue!=null;
-            RevertibleOperation deleteStoredOperation = hasStoredValue?new RevertibleDelete(operation.getKey(),storedValue):null;
+            RevertibleOperation deleteStoredOperation = !hasStoredValue?null:new RevertibleDelete(operation.getKey(),storedValue);
-            boolean deleteStoredOperationInOperationList = false;
+            boolean willDeleteExistingRecord = deleteStoredOperation!=null && (MapHelper.getValue(this.items,operation.getKey())!=null);
-            if(operationArr!=null){
-                for(RevertibleOperation o : operationArr){
-                    if(o.equals(deleteStoredOperation)){
-                        deleteStoredOperationInOperationList = true;
-                        break;
-                    }
-                }
-            }
-            boolean willDeleteExistingRecord = deleteStoredOperation!=null && deleteStoredOperationInOperationList;
-
             try{
                 if(operation.isDelete()){
                     if(hasStoredValue && !Arrays.equals(storedValue,operation.value) && !willDeleteExistingRecord){
@@ -167,10 +140,8 @@ public class RevertibleOperationStack{
                     throw e;
                 }
             }
-
+            byte[] newKey = MapHelper.getKey(this.items,operation.getKey());
-            RevertibleOperation[] newArr = new RevertibleOperation[operationArrX==null?1:operationArrX.length+1];
+            this.items.put(newKey,ArrayHelper.append(MapHelper.getValue(this.items,newKey),operation));
-            newArr[newArr.length-1] = operation;
-            this.items.put(newArr[0].getKey(),newArr);
         }
 
         this.stashedLastOperationForKey.clear();

src/main/java/com/lbry/database/util/ArrayHelper.java

@@ -1,9 +1,29 @@
 package com.lbry.database.util;
 
+import com.lbry.database.revert.RevertibleOperation;
+
+import java.lang.reflect.Array;
 import java.util.Arrays;
 
 public class ArrayHelper{
 
+    public static RevertibleOperation[] append(RevertibleOperation[] arr,RevertibleOperation val){
+        RevertibleOperation[] newArr = new RevertibleOperation[arr!=null?(arr.length+1):1];
+        if(arr!=null){
+            System.arraycopy(arr,0,newArr,0,arr.length);
+        }
+        newArr[newArr.length-1] = val;
+        return newArr;
+    }
+
+    public static RevertibleOperation[] pop(RevertibleOperation[] arr){
+        RevertibleOperation[] newArr = new RevertibleOperation[arr!=null?(arr.length==0?0:arr.length-1):0];
+        if(arr!=null){
+            System.arraycopy(arr,0,newArr,0,arr.length-1);
+        }
+        return newArr;
+    }
+
     public static byte[] fill(byte[] arr,byte val){
         if(arr!=null){
             Arrays.fill(arr,val);

src/test/java/com/lbry/database/tests/RevertablePrefixDBTest.java

@@ -1,19 +1,33 @@
 package com.lbry.database.tests;
 
 import com.lbry.database.PrefixDB;
-import com.lbry.database.keys.*;
+import com.lbry.database.keys.ActiveAmountKey;
+import com.lbry.database.keys.ClaimExpirationKey;
+import com.lbry.database.keys.ClaimTakeoverKey;
+import com.lbry.database.keys.KeyInterface;
+import com.lbry.database.keys.TxNumKey;
 import com.lbry.database.util.ArrayHelper;
-import com.lbry.database.values.*;
+import com.lbry.database.values.ActiveAmountValue;
+import com.lbry.database.values.ClaimExpirationValue;
+import com.lbry.database.values.ClaimTakeoverValue;
+import com.lbry.database.values.TxNumValue;
+import com.lbry.database.values.DBState;
 
 import java.io.File;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 import java.nio.file.Files;
-import java.util.*;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
 import java.util.function.BiFunction;
 
-import org.junit.jupiter.api.*;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
 import org.rocksdb.ReadOptions;
 import org.rocksdb.RocksDBException;
 import org.rocksdb.RocksIterator;
@@ -33,19 +47,21 @@ public class RevertablePrefixDBTest{
     }
 
     @AfterAll
-    public void tearDown(){}
+    public void tearDown(){
+        this.database.close();
+        this.tmpDir.deleteOnExit();
+    }
 
     
-    @Disabled
     @Test
     public void testRollback() throws RocksDBException{
         String name = "derp";
-        byte[] claim_hash1 = new byte[20];
+        byte[] claimHash1 = new byte[20];
-        Arrays.fill(claim_hash1, (byte) 0x00);
+        Arrays.fill(claimHash1, (byte) 0x00);
-        byte[] claim_hash2 = new byte[20];
+        byte[] claimHash2 = new byte[20];
-        Arrays.fill(claim_hash2, (byte) 0x01);
+        Arrays.fill(claimHash2, (byte) 0x01);
-        byte[] claim_hash3 = new byte[20];
+        byte[] claimHash3 = new byte[20];
-        Arrays.fill(claim_hash3, (byte) 0x02);
+        Arrays.fill(claimHash3, (byte) 0x02);
 
         int takeoverHeight = 10000000;
 
@@ -55,7 +71,7 @@ public class RevertablePrefixDBTest{
         this.database.claim_takeover.stashPut(new ClaimTakeoverKey(){{
             this.normalized_name = name;
         }},new ClaimTakeoverValue(){{
-            this.claim_hash = claim_hash1;
+            this.claim_hash = claimHash1;
             this.height = takeoverHeight;
         }});
         assertNull(this.database.claim_takeover.get(new ClaimTakeoverKey(){{
@@ -65,31 +81,27 @@ public class RevertablePrefixDBTest{
             this.normalized_name = name;
         }})).height);
 
-        /////////////////////
-
         this.database.commit(10000000,ArrayHelper.fill(new byte[32],(byte) 0x00));
         assertEquals(10000000,((ClaimTakeoverValue)this.database.claim_takeover.get(new ClaimTakeoverKey(){{
             this.normalized_name = name;
         }})).height);
 
-        /////////////////////
-
         this.database.claim_takeover.stashDelete(new ClaimTakeoverKey(){{
             this.normalized_name = name;
         }},new ClaimTakeoverValue(){{
-            this.claim_hash = claim_hash1;
+            this.claim_hash = claimHash1;
             this.height = takeoverHeight;
         }});
         this.database.claim_takeover.stashPut(new ClaimTakeoverKey(){{
             this.normalized_name = name;
         }},new ClaimTakeoverValue(){{
-            this.claim_hash = claim_hash2;
+            this.claim_hash = claimHash2;
             this.height = takeoverHeight + 1;
         }});
         this.database.claim_takeover.stashDelete(new ClaimTakeoverKey(){{
             this.normalized_name = name;
         }},new ClaimTakeoverValue(){{
-            this.claim_hash = claim_hash2;
+            this.claim_hash = claimHash2;
             this.height = takeoverHeight + 1;
         }});
         this.database.commit(10000001,ArrayHelper.fill(new byte[32],(byte) 0x01));
@@ -99,7 +111,7 @@ public class RevertablePrefixDBTest{
         this.database.claim_takeover.stashPut(new ClaimTakeoverKey(){{
             this.normalized_name = name;
         }},new ClaimTakeoverValue(){{
-            this.claim_hash = claim_hash3;
+            this.claim_hash = claimHash3;
             this.height = takeoverHeight + 2;
         }});
         this.database.commit(10000002,ArrayHelper.fill(new byte[32],(byte) 0x02));
@@ -107,18 +119,16 @@ public class RevertablePrefixDBTest{
             this.normalized_name = name;
         }})).height);
 
-        /////////////////////
-
         this.database.claim_takeover.stashDelete(new ClaimTakeoverKey(){{
             this.normalized_name = name;
         }},new ClaimTakeoverValue(){{
-            this.claim_hash = claim_hash3;
+            this.claim_hash = claimHash3;
             this.height = takeoverHeight + 2;
         }});
         this.database.claim_takeover.stashPut(new ClaimTakeoverKey(){{
             this.normalized_name = name;
         }},new ClaimTakeoverValue(){{
-            this.claim_hash = claim_hash2;
+            this.claim_hash = claimHash2;
             this.height = takeoverHeight + 3;
         }});
         this.database.commit(10000003,ArrayHelper.fill(new byte[32],(byte) 0x03));
@@ -126,8 +136,6 @@ public class RevertablePrefixDBTest{
             this.normalized_name = name;
         }})).height);
 
-        /////////////////////
-
         this.database.rollback(10000003,ArrayHelper.fill(new byte[32],(byte) 0x03));
         assertEquals(10000002,((ClaimTakeoverValue)this.database.claim_takeover.get(new ClaimTakeoverKey(){{
             this.normalized_name = name;
@@ -224,7 +232,7 @@ public class RevertablePrefixDBTest{
             this.genesis = new byte[]{'n','?',(byte) 0xcf,0x12,(byte) 0x99,(byte) 0xd4,(byte) 0xec,']','y',(byte) 0xc3,(byte) 0xa4,(byte) 0xc9,0x1d,'b','J','J',(byte) 0xcf,(byte) 0x9e,'.',0x17,'=',(byte) 0x95,(byte) 0xa1,(byte) 0xa0,'P','O','g','v','i','h','u','V'};
             this.height = 0;
             this.tx_count = 1;
-            this.tip = new byte[32]; //TODO
+            this.tip = new byte[]{'V','u','h','i','v','g','O','P',(byte) 0xa0,(byte) 0xa1,(byte) 0x95,'=',0x17,'.',(byte) 0x9e,(byte) 0xcf,'J','J','b',0x1d,(byte) 0xc9,(byte) 0xa4,(byte) 0xc3,'y',']',(byte) 0xec,(byte) 0xd4,(byte) 0x99,0x12,(byte) 0xcf,'?','n'};
             this.utxo_flush_count = 1;
             this.wall_time = 0;
             this.bit_fields = 1;
@@ -251,10 +259,11 @@ public class RevertablePrefixDBTest{
                 }
                 iterator.next();
             }
+            iterator.close();
             assertEquals(0,actualMap.size());
         }
-        //TODO (start=98 & stop=99) vs (prefix=98)
+        //TODO: (start=98 & stop=99) vs (prefix=98)
-        //TODO (start=99 & stop=100) vs (prefix=99)
+        //TODO: (start=99 & stop=100) vs (prefix=99)
         {
             Map<byte[],byte[]> expectedMap = new HashMap<>();
             expectedMap.put(this.database.claim_expiration.packKey(new ClaimExpirationKey(){{
@@ -283,6 +292,7 @@ public class RevertablePrefixDBTest{
                 assertArrayEquals(expectedEntry.getKey(),actualEntry.getKey());
                 assertArrayEquals(expectedEntry.getValue(),actualEntry.getValue());
             }
+            iterator.close();
         }
         {
             Map<byte[],byte[]> expectedMap = new HashMap<>();
@@ -304,12 +314,12 @@ public class RevertablePrefixDBTest{
             }}));
             Map<byte[],byte[]> actualMap = new HashMap<>();
             RocksIterator iterator = this.database.claim_expiration.iterate();
-            iterator.seekToLast();
+            iterator.seekToFirst();
             while(iterator.isValid()){
                 if(this.database.claim_expiration.unpackKey(iterator.key()).expiration==100){
                     actualMap.put(iterator.key(),iterator.value());
                 }
-                iterator.prev(); // ?
+                iterator.next();
             }
             assertEquals(expectedMap.size(),actualMap.size());
             Iterator<Map.Entry<byte[],byte[]>> expectedEntrySetIterator = expectedMap.entrySet().iterator();
@@ -320,6 +330,7 @@ public class RevertablePrefixDBTest{
                 assertArrayEquals(expectedEntry.getKey(),actualEntry.getKey());
                 assertArrayEquals(expectedEntry.getValue(),actualEntry.getValue());
             }
+            iterator.close();
         }
         //TODO (start=100 & stop=101) vs (prefix=100)
         {
@@ -350,6 +361,7 @@ public class RevertablePrefixDBTest{
                 assertArrayEquals(expectedEntry.getKey(),actualEntry.getKey());
                 assertArrayEquals(expectedEntry.getValue(),actualEntry.getValue());
             }
+            iterator.close();
         }
         {
             Map<byte[],byte[]> expectedMap = new HashMap<>();
@@ -379,6 +391,7 @@ public class RevertablePrefixDBTest{
                 assertArrayEquals(expectedEntry.getKey(),actualEntry.getKey());
                 assertArrayEquals(expectedEntry.getValue(),actualEntry.getValue());
             }
+            iterator.close();
         }
     }