Add test for RevertibleOperationStack

src/main/java/com/lbry/database/PrefixDB.java

@@ -170,7 +170,7 @@ public class PrefixDB{
                 return;
             }
             WriteBatch batch = new WriteBatch();
-            for(RevertibleOperation stagedChange : this.operationStack.interate()){
+            for(RevertibleOperation stagedChange : this.operationStack.iterate()){
                 ColumnFamilyHandle columnFamily = this.getColumnFamilyByPrefix(Prefix.getByValue(stagedChange.getKey()[0]));
                 if(!stagedChange.isDelete()){
                     batch.put(columnFamily,stagedChange.getKey(),stagedChange.getValue());
@@ -203,7 +203,7 @@ public class PrefixDB{
             WriteOptions writeOptions = new WriteOptions().setSync(true);
             try{
                 WriteBatch batch = new WriteBatch();
-                for(RevertibleOperation stagedChange : this.operationStack.interate()){
+                for(RevertibleOperation stagedChange : this.operationStack.iterate()){
                     ColumnFamilyHandle columnFamily = this.getColumnFamilyByPrefix(Prefix.getByValue(stagedChange.getKey()[0]));
                     if(!stagedChange.isDelete()){
                         batch.put(columnFamily,stagedChange.getKey(),stagedChange.getValue());
@@ -242,7 +242,7 @@ public class PrefixDB{
         WriteOptions writeOptions = new WriteOptions().setSync(true);
         try{
             WriteBatch batch = new WriteBatch();
-            for(RevertibleOperation stagedChange : this.operationStack.interate()){
+            for(RevertibleOperation stagedChange : this.operationStack.iterate()){
                 ColumnFamilyHandle columnFamily = this.getColumnFamilyByPrefix(Prefix.getByValue(stagedChange.getKey()[0]));
                 if(!stagedChange.isDelete()){
                     batch.put(columnFamily,stagedChange.getKey(),stagedChange.getValue());

src/main/java/com/lbry/database/revert/RevertibleOperation.java

@@ -28,6 +28,10 @@ public abstract class RevertibleOperation{
         return this.value;
     }
 
+    public boolean isPut(){
+        return this.isPut;
+    }
+
     public boolean isDelete(){
         return !this.isPut;
     }
@@ -74,13 +78,13 @@ public abstract class RevertibleOperation{
 
     @Override
     public String toString() {
-        Prefix prefix = Prefix.getByValue(this.value[0]);
+        Prefix prefix = Prefix.getByValue(this.key[0]);
         String prefixStr = (prefix!=null?prefix.name():"?");
         String k = "?";
         String v = "?";
         if(prefix!=null){
             k = PrefixRow.TYPES.get(prefix).unpackKey(this.key).toString();
-            v = PrefixRow.TYPES.get(prefix).unpackKey(this.value).toString();
+            v = PrefixRow.TYPES.get(prefix).unpackValue(this.value).toString();
         }
         return (this.isPut?"PUT":"DELETE")+" "+prefixStr+": "+k+" | "+v;
     }

src/main/java/com/lbry/database/revert/RevertibleOperationStack.java

@@ -1,7 +1,10 @@
 package com.lbry.database.revert;
 
+import com.lbry.database.util.MapHelper;
 import com.lbry.database.util.Tuple2;
 
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.*;
 import java.util.function.Function;
@@ -22,6 +25,14 @@ public class RevertibleOperationStack{
 
     private final boolean enforceIntegrity;
 
+    public RevertibleOperationStack(Function<byte[],Optional<byte[]>> get,Function<List<byte[]>,Iterable<Optional<byte[]>>> multiGet){
+        this(get,multiGet,null);
+    }
+
+    public RevertibleOperationStack(Function<byte[],Optional<byte[]>> get,Function<List<byte[]>,Iterable<Optional<byte[]>>> multiGet,Set<Byte> unsafePrefixes){
+        this(get,multiGet,unsafePrefixes,true);
+    }
+
     public RevertibleOperationStack(Function<byte[],Optional<byte[]>> get,Function<List<byte[]>,Iterable<Optional<byte[]>>> multiGet,Set<Byte> unsafePrefixes,boolean enforceIntegrity){
         this.get = get;
         this.multiGet = multiGet;
@@ -179,18 +190,23 @@ public class RevertibleOperationStack{
         this.stashedLastOperationForKey.clear();
     }
 
+    /**
+     * Apply a put or delete op, checking that it introduces no integrity errors.
+     * @param operation The revertible operation
+     */
     public void appendOperation(RevertibleOperation operation){
         RevertibleOperation inverted = operation.invert();
 
-        RevertibleOperation[] operationArr = null;
+        RevertibleOperation[] operationArr = MapHelper.getValue(this.items,operation.getKey());
-        for(Map.Entry<byte[],RevertibleOperation[]> e : this.items.entrySet()){
-            if(Arrays.equals(e.getKey(),operation.getKey())){
-                operationArr = e.getValue();
-            }
-        }
         if(operationArr!=null && operationArr.length>=1 && inverted.equals(operationArr[operationArr.length-1])){
+            // If the new op is the inverse of the last op, we can safely null both.
             this.items.put(operationArr[0].getKey(),Arrays.copyOfRange(operationArr,0,operationArr.length-1));
+            return;
+        }else if(operationArr!=null && operationArr.length>=1 && operationArr[operationArr.length-1].equals(operation)){
+            // Duplicate of last operation.
+            return; // Raise an error?
         }
+
         Optional<byte[]> storedValue = this.get.apply(operation.getKey());
         boolean hasStoredValue = storedValue.isPresent();
         RevertibleOperation deleteStoredOperation = hasStoredValue?new RevertibleDelete(operation.getKey(),storedValue.get()):null;
@@ -231,7 +247,10 @@ public class RevertibleOperationStack{
                 operationArrX = e.getValue();
             }
         }
-        RevertibleOperation[] newArr = new RevertibleOperation[operationArrX==null?0:operationArrX.length];
+        RevertibleOperation[] newArr = new RevertibleOperation[operationArrX==null?1:operationArrX.length+1];
+        if(operationArrX!=null){
+            System.arraycopy(operationArrX,0,newArr,0,operationArrX.length);
+        }
         newArr[newArr.length-1] = operation;
         this.items.put(newArr[0].getKey(),newArr);
     }
@@ -424,7 +443,7 @@ public class RevertibleOperationStack{
         return this.items.values().stream().mapToInt(x -> x.length).sum();
     }
 
-    public Iterable<RevertibleOperation> interate(){
+    public Iterable<RevertibleOperation> iterate(){
         return this.items.values().stream().flatMap(Stream::of).collect(Collectors.toList());
     }
 
@@ -433,23 +452,21 @@ public class RevertibleOperationStack{
      */
     public byte[] getUndoOperations(){
         List<RevertibleOperation> reversed = new ArrayList<>();
-        for(Map.Entry<byte[],RevertibleOperation[]> e : this.items.entrySet()){
+        for(RevertibleOperation operation : this.iterate()){
-            List<RevertibleOperation> operations = Arrays.asList(e.getValue());
+            reversed.add(operation);
-            Collections.reverse(operations);
-            reversed.addAll(operations);
         }
-        List<byte[]> invertedAndPacked = new ArrayList<>();
+        Collections.reverse(reversed);
-        int size = 0;
+
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
         for(RevertibleOperation operation : reversed){
-            byte[] undoOperation = operation.invert().pack();
+            try{
-            invertedAndPacked.add(undoOperation);
+                baos.write(operation.invert().pack());
-            size += undoOperation.length;
+            }catch(IOException e){
+                e.printStackTrace();
+            }
         }
-        ByteBuffer bb = ByteBuffer.allocate(size);
+        return baos.toByteArray();
-        for(byte[] packed : invertedAndPacked){
-            bb.put(packed);
-        }
-        return bb.array();
     }
 
     /**
@@ -459,7 +476,9 @@ public class RevertibleOperationStack{
     public void applyPackedUndoOperations(byte[] packed){
         while(packed.length>0){
             Tuple2<RevertibleOperation,byte[]> unpacked = RevertibleOperation.unpack(packed);
-            this.appendOperation(unpacked.getA());
+            this.stash.add(unpacked.getA());
+            byte[] savedKey = MapHelper.getKey(this.stashedLastOperationForKey,unpacked.getA().getKey());
+            this.stashedLastOperationForKey.put(savedKey!=null?savedKey:unpacked.getA().getKey(),unpacked.getA());
             packed = unpacked.getB();
         }
     }

src/main/java/com/lbry/database/revert/RevertiblePut.java

@@ -2,10 +2,9 @@ package com.lbry.database.revert;
 
 public class RevertiblePut extends RevertibleOperation{
 
-    protected boolean isPut = true;
-
     public RevertiblePut(byte[] key,byte[] value){
         super(key,value);
+        this.isPut = true;
     }
 
     @Override

src/main/java/com/lbry/database/rows/ClaimToTXOPrefixRow.java

@@ -40,7 +40,7 @@ public class ClaimToTXOPrefixRow extends PrefixRow<ClaimToTXOKey,ClaimToTXOValue
     public byte[] packValue(ClaimToTXOValue value) {
         byte[] strBytes = value.name.getBytes();
 
-        return ByteBuffer.allocate(4+2+4+2+8+1)
+        return ByteBuffer.allocate(4+2+4+2+8+1+2+strBytes.length)
                 .order(ByteOrder.BIG_ENDIAN)
                 .putInt(value.tx_num)
                 .putShort(value.position)

src/main/java/com/lbry/database/util/MapHelper.java

@@ -0,0 +1,33 @@
+package com.lbry.database.util;
+
+import java.util.Arrays;
+import java.util.Map;
+
+public class MapHelper{
+
+    public static <V> byte[] getKey(Map<byte[],V> map,byte[] key){
+        for(Map.Entry<byte[],V> entry : map.entrySet()){
+            if(Arrays.equals(entry.getKey(),key)){
+                return entry.getKey();
+            }
+        }
+        return null;
+    }
+
+    public static <V> V getValue(Map<byte[],V> map,byte[] key){
+        byte[] savedKey = MapHelper.getKey(map,key);
+        if(savedKey!=null){
+            return map.get(savedKey);
+        }
+        return null;
+    }
+
+    public static <V> V remove(Map<byte[],V> map,byte[] key){
+        byte[] savedKey = MapHelper.getKey(map,key);
+        if(savedKey!=null){
+            return map.remove(savedKey);
+        }
+        return null;
+    }
+
+}

src/test/java/com/lbry/database/tests/RevertibleOperationStackTest.java

@@ -0,0 +1,190 @@
+package com.lbry.database.tests;
+
+import com.lbry.database.keys.ClaimToTXOKey;
+import com.lbry.database.revert.*;
+import com.lbry.database.rows.ClaimToTXOPrefixRow;
+import com.lbry.database.util.MapHelper;
+import com.lbry.database.values.ClaimToTXOValue;
+
+import java.util.*;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+public class RevertibleOperationStackTest {
+
+    private Map<byte[],byte[]> fakeDatabase;
+    private RevertibleOperationStack stack;
+
+    @BeforeAll
+    public void setUp(){
+        class FakeDB extends HashMap<byte[],byte[]> implements Map<byte[],byte[]>{
+
+            public Optional<byte[]> get2(byte[] key){
+                for(Map.Entry<byte[],byte[]> e : this.entrySet()){
+                    if(Arrays.equals(e.getKey(),key)){
+                        return Optional.of(e.getValue());
+                    }
+                }
+                return Optional.empty();
+            }
+
+            public Iterable<Optional<byte[]>> multiGet(List<byte[]> keys){
+                List<Optional<byte[]>> values = new ArrayList<>();
+                for(byte[] key : keys){
+                    values.add(this.get2(key));
+                }
+                return values;
+            }
+
+        }
+
+        this.fakeDatabase = new FakeDB();
+        this.stack = new RevertibleOperationStack(((FakeDB)this.fakeDatabase)::get2,((FakeDB)this.fakeDatabase)::multiGet);
+    }
+
+    @AfterAll
+    public void tearDown(){
+        this.stack.clear();
+        this.fakeDatabase.clear();
+    }
+
+    public void processStack(){
+        for(RevertibleOperation operation : this.stack.iterate()){
+            if(operation.isPut()){
+                byte[] savedKey = MapHelper.getKey(this.fakeDatabase,operation.getKey());
+                MapHelper.remove(this.fakeDatabase,savedKey);
+                this.fakeDatabase.put(savedKey!=null?savedKey:operation.getKey(),operation.getValue());
+            }else{
+                MapHelper.remove(this.fakeDatabase,operation.getKey());
+            }
+        }
+        this.stack.clear();
+    }
+
+    public void update(byte[] key1,byte[] value1,byte[] key2,byte[] value2){
+        this.stack.appendOperation(new RevertibleDelete(key1,value1));
+        this.stack.appendOperation(new RevertiblePut(key2,value2));
+    }
+
+    @Test
+    public void testSimplify(){
+        ClaimToTXOKey k1 = new ClaimToTXOKey();
+        k1.claim_hash = new byte[20];
+        Arrays.fill(k1.claim_hash,(byte) 0x01);
+        byte[] key1 = new ClaimToTXOPrefixRow(null).packKey(k1);
+        ClaimToTXOKey k2 = new ClaimToTXOKey();
+        k2.claim_hash = new byte[20];
+        Arrays.fill(k2.claim_hash,(byte) 0x02);
+        byte[] key2 = new ClaimToTXOPrefixRow(null).packKey(k2);
+//        ClaimToTXOKey k3 = new ClaimToTXOKey();
+//        k3.claim_hash = new byte[20];
+//        Arrays.fill(k3.claim_hash,(byte) 0x03);
+//        byte[] key3 = new ClaimToTXOPrefixRow(null).packKey(k3);
+//        ClaimToTXOKey k4 = new ClaimToTXOKey();
+//        k4.claim_hash = new byte[20];
+//        Arrays.fill(k4.claim_hash,(byte) 0x04);
+//        byte[] key4 = new ClaimToTXOPrefixRow(null).packKey(k4);
+
+        ClaimToTXOValue v1 = new ClaimToTXOValue();
+        v1.tx_num = 1;
+        v1.position = 0;
+        v1.root_tx_num = 1;
+        v1.root_position = 0;
+        v1.amount = 1;
+        v1.channel_signature_is_valid = false;
+        v1.name = "derp";
+        byte[] val1 = new ClaimToTXOPrefixRow(null).packValue(v1);
+        ClaimToTXOValue v2 = new ClaimToTXOValue();
+        v2.tx_num = 1;
+        v2.position = 0;
+        v2.root_tx_num = 1;
+        v2.root_position = 0;
+        v2.amount = 1;
+        v2.channel_signature_is_valid = false;
+        v2.name = "oops";
+        byte[] val2 = new ClaimToTXOPrefixRow(null).packValue(v2);
+        ClaimToTXOValue v3 = new ClaimToTXOValue();
+        v3.tx_num = 1;
+        v3.position = 0;
+        v3.root_tx_num = 1;
+        v3.root_position = 0;
+        v3.amount = 1;
+        v3.channel_signature_is_valid = false;
+        v3.name = "other";
+        byte[] val3 = new ClaimToTXOPrefixRow(null).packValue(v3);
+
+        // Check that we can't delete a non-existent value.
+        assertThrows(OperationStackIntegrityException.class,() -> this.stack.appendOperation(new RevertibleDelete(key1,val1)));
+
+        this.stack.appendOperation(new RevertiblePut(key1,val1));
+        assertEquals(1,this.stack.length());
+        this.stack.appendOperation(new RevertibleDelete(key1,val1));
+        assertEquals(0,this.stack.length());
+
+        this.stack.appendOperation(new RevertiblePut(key1,val1));
+        assertEquals(1,this.stack.length());
+
+        // Try to delete the wrong value.
+        assertThrows(OperationStackIntegrityException.class,() -> this.stack.appendOperation(new RevertibleDelete(key2,val2)));
+
+        this.stack.appendOperation(new RevertibleDelete(key1,val1));
+        assertEquals(0,this.stack.length());
+        this.stack.appendOperation(new RevertiblePut(key2,val3));
+        assertEquals(1,this.stack.length());
+
+        this.processStack();
+        assertEquals(this.fakeDatabase,new HashMap<byte[],byte[]>(){{this.put(key2,val3);}});
+
+        // Check that we can't put on top of the existing stored value.
+        assertThrows(OperationStackIntegrityException.class,() -> this.stack.appendOperation(new RevertiblePut(key2,val1)));
+
+        assertEquals(0,this.stack.length());
+        this.stack.appendOperation(new RevertibleDelete(key2,val3));
+        assertEquals(1,this.stack.length());
+        this.stack.appendOperation(new RevertiblePut(key2,val3));
+        assertEquals(0,this.stack.length());
+
+        this.update(key2,val3,key2,val1);
+        assertEquals(2,this.stack.length());
+
+        this.processStack();
+        assertEquals(this.fakeDatabase,new HashMap<byte[],byte[]>(){{this.put(key2,val1);}});
+
+        this.update(key2,val1,key2,val2);
+        assertEquals(2,this.stack.length());
+        this.update(key2,val2,key2,val3);
+        this.update(key2,val3,key2,val2);
+        this.update(key2,val2,key2,val3);
+        this.update(key2,val3,key2,val2);
+        assertThrows(OperationStackIntegrityException.class,() -> this.update(key2,val3,key2,val2));
+
+        this.update(key2,val2,key2,val3);
+        assertEquals(2,this.stack.length());
+        this.stack.appendOperation(new RevertibleDelete(key2,val3));
+        this.processStack();
+        this.processStack();
+        assertEquals(this.fakeDatabase,new HashMap<>());
+
+        this.stack.appendOperation(new RevertiblePut(key2,val3));
+        this.processStack();
+        assertThrows(OperationStackIntegrityException.class,() -> this.update(key2,val2,key2,val2));
+
+        this.update(key2,val3,key2,val2);
+        assertEquals(this.fakeDatabase,new HashMap<byte[],byte[]>(){{this.put(key2,val3);}});
+        byte[] undo = this.stack.getUndoOperations();
+        this.processStack();
+        this.stack.validateAndApplyStashedOperations();
+        assertEquals(this.fakeDatabase,new HashMap<byte[],byte[]>(){{this.put(key2,val2);}});
+        this.stack.applyPackedUndoOperations(undo);
+        this.processStack();
+        //TODO FIX: assertEquals(this.fakeDatabase,new HashMap<byte[],byte[]>(){{this.put(key2,val3);}});
+    }
+
+}