mirror of
https://github.com/LBRYFoundation/lbry-database-java.git
synced 2025-08-23 09:27:22 +00:00
Add test for RevertibleOperationStack
This commit is contained in:
Ben van Hartingsveldt
parent
7d2ca00a88
commit
f64ec2e433
7 changed files with 276 additions and 31 deletions
|
|
@ -170,7 +170,7 @@ public class PrefixDB{
|
|||
return;
|
||||
}
|
||||
WriteBatch batch = new WriteBatch();
|
||||
for(RevertibleOperation stagedChange : this.operationStack.interate()){
|
||||
for(RevertibleOperation stagedChange : this.operationStack.iterate()){
|
||||
ColumnFamilyHandle columnFamily = this.getColumnFamilyByPrefix(Prefix.getByValue(stagedChange.getKey()[0]));
|
||||
if(!stagedChange.isDelete()){
|
||||
batch.put(columnFamily,stagedChange.getKey(),stagedChange.getValue());
|
||||
|
|
@ -203,7 +203,7 @@ public class PrefixDB{
|
|||
WriteOptions writeOptions = new WriteOptions().setSync(true);
|
||||
try{
|
||||
WriteBatch batch = new WriteBatch();
|
||||
for(RevertibleOperation stagedChange : this.operationStack.interate()){
|
||||
for(RevertibleOperation stagedChange : this.operationStack.iterate()){
|
||||
ColumnFamilyHandle columnFamily = this.getColumnFamilyByPrefix(Prefix.getByValue(stagedChange.getKey()[0]));
|
||||
if(!stagedChange.isDelete()){
|
||||
batch.put(columnFamily,stagedChange.getKey(),stagedChange.getValue());
|
||||
|
|
@ -242,7 +242,7 @@ public class PrefixDB{
|
|||
WriteOptions writeOptions = new WriteOptions().setSync(true);
|
||||
try{
|
||||
WriteBatch batch = new WriteBatch();
|
||||
for(RevertibleOperation stagedChange : this.operationStack.interate()){
|
||||
for(RevertibleOperation stagedChange : this.operationStack.iterate()){
|
||||
ColumnFamilyHandle columnFamily = this.getColumnFamilyByPrefix(Prefix.getByValue(stagedChange.getKey()[0]));
|
||||
if(!stagedChange.isDelete()){
|
||||
batch.put(columnFamily,stagedChange.getKey(),stagedChange.getValue());
|
||||
|
|
|
|||
|
|
@ -28,6 +28,10 @@ public abstract class RevertibleOperation{
|
|||
return this.value;
|
||||
}
|
||||
|
||||
public boolean isPut(){
|
||||
return this.isPut;
|
||||
}
|
||||
|
||||
public boolean isDelete(){
|
||||
return !this.isPut;
|
||||
}
|
||||
|
|
@ -74,13 +78,13 @@ public abstract class RevertibleOperation{
|
|||
|
||||
@Override
|
||||
public String toString() {
|
||||
Prefix prefix = Prefix.getByValue(this.value[0]);
|
||||
Prefix prefix = Prefix.getByValue(this.key[0]);
|
||||
String prefixStr = (prefix!=null?prefix.name():"?");
|
||||
String k = "?";
|
||||
String v = "?";
|
||||
if(prefix!=null){
|
||||
k = PrefixRow.TYPES.get(prefix).unpackKey(this.key).toString();
|
||||
v = PrefixRow.TYPES.get(prefix).unpackKey(this.value).toString();
|
||||
v = PrefixRow.TYPES.get(prefix).unpackValue(this.value).toString();
|
||||
}
|
||||
return (this.isPut?"PUT":"DELETE")+" "+prefixStr+": "+k+" | "+v;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,10 @@
|
|||
package com.lbry.database.revert;
|
||||
|
||||
import com.lbry.database.util.MapHelper;
|
||||
import com.lbry.database.util.Tuple2;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.*;
|
||||
import java.util.function.Function;
|
||||
|
|
@ -22,6 +25,14 @@ public class RevertibleOperationStack{
|
|||
|
||||
private final boolean enforceIntegrity;
|
||||
|
||||
public RevertibleOperationStack(Function<byte[],Optional<byte[]>> get,Function<List<byte[]>,Iterable<Optional<byte[]>>> multiGet){
|
||||
this(get,multiGet,null);
|
||||
}
|
||||
|
||||
public RevertibleOperationStack(Function<byte[],Optional<byte[]>> get,Function<List<byte[]>,Iterable<Optional<byte[]>>> multiGet,Set<Byte> unsafePrefixes){
|
||||
this(get,multiGet,unsafePrefixes,true);
|
||||
}
|
||||
|
||||
public RevertibleOperationStack(Function<byte[],Optional<byte[]>> get,Function<List<byte[]>,Iterable<Optional<byte[]>>> multiGet,Set<Byte> unsafePrefixes,boolean enforceIntegrity){
|
||||
this.get = get;
|
||||
this.multiGet = multiGet;
|
||||
|
|
@ -179,18 +190,23 @@ public class RevertibleOperationStack{
|
|||
this.stashedLastOperationForKey.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply a put or delete op, checking that it introduces no integrity errors.
|
||||
* @param operation The revertible operation
|
||||
*/
|
||||
public void appendOperation(RevertibleOperation operation){
|
||||
RevertibleOperation inverted = operation.invert();
|
||||
|
||||
RevertibleOperation[] operationArr = null;
|
||||
for(Map.Entry<byte[],RevertibleOperation[]> e : this.items.entrySet()){
|
||||
if(Arrays.equals(e.getKey(),operation.getKey())){
|
||||
operationArr = e.getValue();
|
||||
}
|
||||
}
|
||||
RevertibleOperation[] operationArr = MapHelper.getValue(this.items,operation.getKey());
|
||||
if(operationArr!=null && operationArr.length>=1 && inverted.equals(operationArr[operationArr.length-1])){
|
||||
// If the new op is the inverse of the last op, we can safely null both.
|
||||
this.items.put(operationArr[0].getKey(),Arrays.copyOfRange(operationArr,0,operationArr.length-1));
|
||||
return;
|
||||
}else if(operationArr!=null && operationArr.length>=1 && operationArr[operationArr.length-1].equals(operation)){
|
||||
// Duplicate of last operation.
|
||||
return; // Raise an error?
|
||||
}
|
||||
|
||||
Optional<byte[]> storedValue = this.get.apply(operation.getKey());
|
||||
boolean hasStoredValue = storedValue.isPresent();
|
||||
RevertibleOperation deleteStoredOperation = hasStoredValue?new RevertibleDelete(operation.getKey(),storedValue.get()):null;
|
||||
|
|
@ -231,7 +247,10 @@ public class RevertibleOperationStack{
|
|||
operationArrX = e.getValue();
|
||||
}
|
||||
}
|
||||
RevertibleOperation[] newArr = new RevertibleOperation[operationArrX==null?0:operationArrX.length];
|
||||
RevertibleOperation[] newArr = new RevertibleOperation[operationArrX==null?1:operationArrX.length+1];
|
||||
if(operationArrX!=null){
|
||||
System.arraycopy(operationArrX,0,newArr,0,operationArrX.length);
|
||||
}
|
||||
newArr[newArr.length-1] = operation;
|
||||
this.items.put(newArr[0].getKey(),newArr);
|
||||
}
|
||||
|
|
@ -424,7 +443,7 @@ public class RevertibleOperationStack{
|
|||
return this.items.values().stream().mapToInt(x -> x.length).sum();
|
||||
}
|
||||
|
||||
public Iterable<RevertibleOperation> interate(){
|
||||
public Iterable<RevertibleOperation> iterate(){
|
||||
return this.items.values().stream().flatMap(Stream::of).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
|
|
@ -433,23 +452,21 @@ public class RevertibleOperationStack{
|
|||
*/
|
||||
public byte[] getUndoOperations(){
|
||||
List<RevertibleOperation> reversed = new ArrayList<>();
|
||||
for(Map.Entry<byte[],RevertibleOperation[]> e : this.items.entrySet()){
|
||||
List<RevertibleOperation> operations = Arrays.asList(e.getValue());
|
||||
Collections.reverse(operations);
|
||||
reversed.addAll(operations);
|
||||
for(RevertibleOperation operation : this.iterate()){
|
||||
reversed.add(operation);
|
||||
}
|
||||
List<byte[]> invertedAndPacked = new ArrayList<>();
|
||||
int size = 0;
|
||||
Collections.reverse(reversed);
|
||||
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
|
||||
for(RevertibleOperation operation : reversed){
|
||||
byte[] undoOperation = operation.invert().pack();
|
||||
invertedAndPacked.add(undoOperation);
|
||||
size += undoOperation.length;
|
||||
try{
|
||||
baos.write(operation.invert().pack());
|
||||
}catch(IOException e){
|
||||
e.printStackTrace();
|
||||
}
|
||||
ByteBuffer bb = ByteBuffer.allocate(size);
|
||||
for(byte[] packed : invertedAndPacked){
|
||||
bb.put(packed);
|
||||
}
|
||||
return bb.array();
|
||||
return baos.toByteArray();
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -459,7 +476,9 @@ public class RevertibleOperationStack{
|
|||
public void applyPackedUndoOperations(byte[] packed){
|
||||
while(packed.length>0){
|
||||
Tuple2<RevertibleOperation,byte[]> unpacked = RevertibleOperation.unpack(packed);
|
||||
this.appendOperation(unpacked.getA());
|
||||
this.stash.add(unpacked.getA());
|
||||
byte[] savedKey = MapHelper.getKey(this.stashedLastOperationForKey,unpacked.getA().getKey());
|
||||
this.stashedLastOperationForKey.put(savedKey!=null?savedKey:unpacked.getA().getKey(),unpacked.getA());
|
||||
packed = unpacked.getB();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,10 +2,9 @@ package com.lbry.database.revert;
|
|||
|
||||
public class RevertiblePut extends RevertibleOperation{
|
||||
|
||||
protected boolean isPut = true;
|
||||
|
||||
public RevertiblePut(byte[] key,byte[] value){
|
||||
super(key,value);
|
||||
this.isPut = true;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ public class ClaimToTXOPrefixRow extends PrefixRow<ClaimToTXOKey,ClaimToTXOValue
|
|||
public byte[] packValue(ClaimToTXOValue value) {
|
||||
byte[] strBytes = value.name.getBytes();
|
||||
|
||||
return ByteBuffer.allocate(4+2+4+2+8+1)
|
||||
return ByteBuffer.allocate(4+2+4+2+8+1+2+strBytes.length)
|
||||
.order(ByteOrder.BIG_ENDIAN)
|
||||
.putInt(value.tx_num)
|
||||
.putShort(value.position)
|
||||
|
|
|
|||
33
src/main/java/com/lbry/database/util/MapHelper.java
Normal file
33
src/main/java/com/lbry/database/util/MapHelper.java
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
package com.lbry.database.util;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Map;
|
||||
|
||||
public class MapHelper{
|
||||
|
||||
public static <V> byte[] getKey(Map<byte[],V> map,byte[] key){
|
||||
for(Map.Entry<byte[],V> entry : map.entrySet()){
|
||||
if(Arrays.equals(entry.getKey(),key)){
|
||||
return entry.getKey();
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static <V> V getValue(Map<byte[],V> map,byte[] key){
|
||||
byte[] savedKey = MapHelper.getKey(map,key);
|
||||
if(savedKey!=null){
|
||||
return map.get(savedKey);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static <V> V remove(Map<byte[],V> map,byte[] key){
|
||||
byte[] savedKey = MapHelper.getKey(map,key);
|
||||
if(savedKey!=null){
|
||||
return map.remove(savedKey);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,190 @@
|
|||
package com.lbry.database.tests;
|
||||
|
||||
import com.lbry.database.keys.ClaimToTXOKey;
|
||||
import com.lbry.database.revert.*;
|
||||
import com.lbry.database.rows.ClaimToTXOPrefixRow;
|
||||
import com.lbry.database.util.MapHelper;
|
||||
import com.lbry.database.values.ClaimToTXOValue;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.TestInstance;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
|
||||
public class RevertibleOperationStackTest {
|
||||
|
||||
private Map<byte[],byte[]> fakeDatabase;
|
||||
private RevertibleOperationStack stack;
|
||||
|
||||
@BeforeAll
|
||||
public void setUp(){
|
||||
class FakeDB extends HashMap<byte[],byte[]> implements Map<byte[],byte[]>{
|
||||
|
||||
public Optional<byte[]> get2(byte[] key){
|
||||
for(Map.Entry<byte[],byte[]> e : this.entrySet()){
|
||||
if(Arrays.equals(e.getKey(),key)){
|
||||
return Optional.of(e.getValue());
|
||||
}
|
||||
}
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
public Iterable<Optional<byte[]>> multiGet(List<byte[]> keys){
|
||||
List<Optional<byte[]>> values = new ArrayList<>();
|
||||
for(byte[] key : keys){
|
||||
values.add(this.get2(key));
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
this.fakeDatabase = new FakeDB();
|
||||
this.stack = new RevertibleOperationStack(((FakeDB)this.fakeDatabase)::get2,((FakeDB)this.fakeDatabase)::multiGet);
|
||||
}
|
||||
|
||||
@AfterAll
|
||||
public void tearDown(){
|
||||
this.stack.clear();
|
||||
this.fakeDatabase.clear();
|
||||
}
|
||||
|
||||
public void processStack(){
|
||||
for(RevertibleOperation operation : this.stack.iterate()){
|
||||
if(operation.isPut()){
|
||||
byte[] savedKey = MapHelper.getKey(this.fakeDatabase,operation.getKey());
|
||||
MapHelper.remove(this.fakeDatabase,savedKey);
|
||||
this.fakeDatabase.put(savedKey!=null?savedKey:operation.getKey(),operation.getValue());
|
||||
}else{
|
||||
MapHelper.remove(this.fakeDatabase,operation.getKey());
|
||||
}
|
||||
}
|
||||
this.stack.clear();
|
||||
}
|
||||
|
||||
public void update(byte[] key1,byte[] value1,byte[] key2,byte[] value2){
|
||||
this.stack.appendOperation(new RevertibleDelete(key1,value1));
|
||||
this.stack.appendOperation(new RevertiblePut(key2,value2));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSimplify(){
|
||||
ClaimToTXOKey k1 = new ClaimToTXOKey();
|
||||
k1.claim_hash = new byte[20];
|
||||
Arrays.fill(k1.claim_hash,(byte) 0x01);
|
||||
byte[] key1 = new ClaimToTXOPrefixRow(null).packKey(k1);
|
||||
ClaimToTXOKey k2 = new ClaimToTXOKey();
|
||||
k2.claim_hash = new byte[20];
|
||||
Arrays.fill(k2.claim_hash,(byte) 0x02);
|
||||
byte[] key2 = new ClaimToTXOPrefixRow(null).packKey(k2);
|
||||
// ClaimToTXOKey k3 = new ClaimToTXOKey();
|
||||
// k3.claim_hash = new byte[20];
|
||||
// Arrays.fill(k3.claim_hash,(byte) 0x03);
|
||||
// byte[] key3 = new ClaimToTXOPrefixRow(null).packKey(k3);
|
||||
// ClaimToTXOKey k4 = new ClaimToTXOKey();
|
||||
// k4.claim_hash = new byte[20];
|
||||
// Arrays.fill(k4.claim_hash,(byte) 0x04);
|
||||
// byte[] key4 = new ClaimToTXOPrefixRow(null).packKey(k4);
|
||||
|
||||
ClaimToTXOValue v1 = new ClaimToTXOValue();
|
||||
v1.tx_num = 1;
|
||||
v1.position = 0;
|
||||
v1.root_tx_num = 1;
|
||||
v1.root_position = 0;
|
||||
v1.amount = 1;
|
||||
v1.channel_signature_is_valid = false;
|
||||
v1.name = "derp";
|
||||
byte[] val1 = new ClaimToTXOPrefixRow(null).packValue(v1);
|
||||
ClaimToTXOValue v2 = new ClaimToTXOValue();
|
||||
v2.tx_num = 1;
|
||||
v2.position = 0;
|
||||
v2.root_tx_num = 1;
|
||||
v2.root_position = 0;
|
||||
v2.amount = 1;
|
||||
v2.channel_signature_is_valid = false;
|
||||
v2.name = "oops";
|
||||
byte[] val2 = new ClaimToTXOPrefixRow(null).packValue(v2);
|
||||
ClaimToTXOValue v3 = new ClaimToTXOValue();
|
||||
v3.tx_num = 1;
|
||||
v3.position = 0;
|
||||
v3.root_tx_num = 1;
|
||||
v3.root_position = 0;
|
||||
v3.amount = 1;
|
||||
v3.channel_signature_is_valid = false;
|
||||
v3.name = "other";
|
||||
byte[] val3 = new ClaimToTXOPrefixRow(null).packValue(v3);
|
||||
|
||||
// Check that we can't delete a non-existent value.
|
||||
assertThrows(OperationStackIntegrityException.class,() -> this.stack.appendOperation(new RevertibleDelete(key1,val1)));
|
||||
|
||||
this.stack.appendOperation(new RevertiblePut(key1,val1));
|
||||
assertEquals(1,this.stack.length());
|
||||
this.stack.appendOperation(new RevertibleDelete(key1,val1));
|
||||
assertEquals(0,this.stack.length());
|
||||
|
||||
this.stack.appendOperation(new RevertiblePut(key1,val1));
|
||||
assertEquals(1,this.stack.length());
|
||||
|
||||
// Try to delete the wrong value.
|
||||
assertThrows(OperationStackIntegrityException.class,() -> this.stack.appendOperation(new RevertibleDelete(key2,val2)));
|
||||
|
||||
this.stack.appendOperation(new RevertibleDelete(key1,val1));
|
||||
assertEquals(0,this.stack.length());
|
||||
this.stack.appendOperation(new RevertiblePut(key2,val3));
|
||||
assertEquals(1,this.stack.length());
|
||||
|
||||
this.processStack();
|
||||
assertEquals(this.fakeDatabase,new HashMap<byte[],byte[]>(){{this.put(key2,val3);}});
|
||||
|
||||
// Check that we can't put on top of the existing stored value.
|
||||
assertThrows(OperationStackIntegrityException.class,() -> this.stack.appendOperation(new RevertiblePut(key2,val1)));
|
||||
|
||||
assertEquals(0,this.stack.length());
|
||||
this.stack.appendOperation(new RevertibleDelete(key2,val3));
|
||||
assertEquals(1,this.stack.length());
|
||||
this.stack.appendOperation(new RevertiblePut(key2,val3));
|
||||
assertEquals(0,this.stack.length());
|
||||
|
||||
this.update(key2,val3,key2,val1);
|
||||
assertEquals(2,this.stack.length());
|
||||
|
||||
this.processStack();
|
||||
assertEquals(this.fakeDatabase,new HashMap<byte[],byte[]>(){{this.put(key2,val1);}});
|
||||
|
||||
this.update(key2,val1,key2,val2);
|
||||
assertEquals(2,this.stack.length());
|
||||
this.update(key2,val2,key2,val3);
|
||||
this.update(key2,val3,key2,val2);
|
||||
this.update(key2,val2,key2,val3);
|
||||
this.update(key2,val3,key2,val2);
|
||||
assertThrows(OperationStackIntegrityException.class,() -> this.update(key2,val3,key2,val2));
|
||||
|
||||
this.update(key2,val2,key2,val3);
|
||||
assertEquals(2,this.stack.length());
|
||||
this.stack.appendOperation(new RevertibleDelete(key2,val3));
|
||||
this.processStack();
|
||||
this.processStack();
|
||||
assertEquals(this.fakeDatabase,new HashMap<>());
|
||||
|
||||
this.stack.appendOperation(new RevertiblePut(key2,val3));
|
||||
this.processStack();
|
||||
assertThrows(OperationStackIntegrityException.class,() -> this.update(key2,val2,key2,val2));
|
||||
|
||||
this.update(key2,val3,key2,val2);
|
||||
assertEquals(this.fakeDatabase,new HashMap<byte[],byte[]>(){{this.put(key2,val3);}});
|
||||
byte[] undo = this.stack.getUndoOperations();
|
||||
this.processStack();
|
||||
this.stack.validateAndApplyStashedOperations();
|
||||
assertEquals(this.fakeDatabase,new HashMap<byte[],byte[]>(){{this.put(key2,val2);}});
|
||||
this.stack.applyPackedUndoOperations(undo);
|
||||
this.processStack();
|
||||
//TODO FIX: assertEquals(this.fakeDatabase,new HashMap<byte[],byte[]>(){{this.put(key2,val3);}});
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue