Examples with VersioningKVData com.jd.blockchain.storage.service.utils.VersioningKVData used on opensource projects

Example 31 with VersioningKVData

use of com.jd.blockchain.storage.service.utils.VersioningKVData in project jdchain-core by blockchain-jd-com.

the class MerkleHashTrieTest method testMerkleTreeDiffNewPathType.

@Test
public void testMerkleTreeDiffNewPathType() {
    CryptoSetting cryptoSetting = createCryptoSetting();
    MemoryKVStorage storage = new MemoryKVStorage();
    List<String> newdataListString = new ArrayList<String>();
    List<String> dataListString = new ArrayList<String>();
    List<VersioningKVData<String, byte[]>> newdataList = new ArrayList<VersioningKVData<String, byte[]>>();
    int count = 1;
    int newAddCount = 5;
    List<VersioningKVData<String, byte[]>> dataList = generateSpecKeyDatas(dataListString, count);
    VersioningKVData<String, byte[]>[] datas = toArray(dataList);
    MerkleHashTrie merkleTree = newMerkleTree_with_committed(datas, cryptoSetting, storage);
    HashDigest rootHash0 = merkleTree.getRootHash();
    assertNotNull(rootHash0);
    assertEquals(count, merkleTree.getTotalKeys());
    assertEquals(count, merkleTree.getTotalRecords());
    // reload and add random key data item;
    MerkleHashTrie merkleTree_reload = new MerkleHashTrie(rootHash0, cryptoSetting, KEY_PREFIX, storage, false);
    assertEquals(count, merkleTree_reload.getTotalKeys());
    assertEquals(count, merkleTree_reload.getTotalRecords());
    assertEquals(rootHash0, merkleTree_reload.getRootHash());
    VersioningKVData<String, byte[]> data0 = new VersioningKVData<String, byte[]>("KEY-1741789838495252", 0L, BytesUtils.concat(BytesUtils.toBytes(0), BytesUtils.toBytes("VALUE")));
    newdataList.add(data0);
    VersioningKVData<String, byte[]> data1 = new VersioningKVData<String, byte[]>("KEY-2741789838505562", 0L, BytesUtils.concat(BytesUtils.toBytes(0), BytesUtils.toBytes("VALUE")));
    newdataList.add(data1);
    VersioningKVData<String, byte[]> data2 = new VersioningKVData<String, byte[]>("KEY-0745023937104559", 0L, BytesUtils.concat(BytesUtils.toBytes(0), BytesUtils.toBytes("VALUE")));
    newdataList.add(data2);
    VersioningKVData<String, byte[]> data3 = new VersioningKVData<String, byte[]>("KEY-7745261599950097", 0L, BytesUtils.concat(BytesUtils.toBytes(0), BytesUtils.toBytes("VALUE")));
    newdataList.add(data3);
    VersioningKVData<String, byte[]> data4 = new VersioningKVData<String, byte[]>("KEY-9745261599963367", 0L, BytesUtils.concat(BytesUtils.toBytes(0), BytesUtils.toBytes("VALUE")));
    newdataList.add(data4);
    for (int i = 0; i < newdataList.size(); i++) {
        merkleTree_reload.setData(newdataList.get(i).getKey(), newdataList.get(i).getVersion(), newdataList.get(i).getValue());
        newdataListString.add(newdataList.get(i).getKey());
    }
    merkleTree_reload.commit();
    HashDigest rootHash1 = merkleTree_reload.getRootHash();
    assertNotNull(rootHash1);
    assertNotEquals(rootHash0, rootHash1);
    assertEquals(count + newAddCount, merkleTree_reload.getTotalKeys());
    assertEquals(count + newAddCount, merkleTree_reload.getTotalRecords());
    SkippingIterator<KVEntry> diffIterator = merkleTree_reload.getKeyDiffIterator(merkleTree);
    // max boundary skip test
    assertEquals(newAddCount, diffIterator.getTotalCount());
    assertEquals(-1, diffIterator.getCursor());
    assertTrue(diffIterator.hasNext());
    long skipped = diffIterator.skip(newAddCount);
    assertEquals(newAddCount, skipped);
    assertFalse(diffIterator.hasNext());
    // re-interator and random skip test
    int skipNum = 4;
    diffIterator = merkleTree_reload.getKeyDiffIterator(merkleTree);
    assertEquals(newAddCount, diffIterator.getTotalCount());
    assertEquals(-1, diffIterator.getCursor());
    assertTrue(diffIterator.hasNext());
    long skipped1 = diffIterator.skip(skipNum);
    assertEquals(skipNum, skipped1);
    int diffNum = 0;
    // TODO: 无效的验证逻辑； by huanghaiquan at 2020-07-15;
    // while (diffIterator.hasNext()) {
    // MerkleData data = diffIterator.next();
    // assertNotNull(data);
    // assertFalse(dataList.contains(new String(data.getKey())));
    // assertTrue(newdataListString.contains(new String(data.getKey())));
    // diffNum++;
    // }
    // assertEquals(diffNum, diffIterator.getCount() - skipNum);
    // re-interator and next test
    diffIterator = merkleTree_reload.getKeyDiffIterator(merkleTree);
    int diffNum1 = 0;
    assertEquals(newAddCount, diffIterator.getTotalCount());
    while (diffIterator.hasNext()) {
        KVEntry data = diffIterator.next();
        assertNotNull(data);
        diffNum1++;
    }
    assertFalse(diffIterator.hasNext());
    assertEquals(newAddCount - 1, diffIterator.getCursor());
    assertEquals(newAddCount, diffIterator.getTotalCount());
    assertEquals(diffNum1, diffIterator.getTotalCount());
    // re-interator and test next key consistency
    diffIterator = merkleTree_reload.getKeyDiffIterator(merkleTree);
// TODO: 无效的验证逻辑； by huanghaiquan at 2020-07-15;
// while (diffIterator.hasNext()) {
// MerkleData data = diffIterator.next();
// assertNotNull(data);
// assertFalse(dataList.contains(new String(data.getKey())));
// assertTrue(newdataListString.contains(new String(data.getKey())));
// }
}

Example 32 with VersioningKVData

use of com.jd.blockchain.storage.service.utils.VersioningKVData in project jdchain-core by blockchain-jd-com.

the class MerkleHashTrieTest method testReloadTreeAddRandomNewDataNode.

/**
 * 对已存在的树进行重载，增加新的数据节点，通过重载树验证新节点是否添加成功，total keys 与total records
 * 是否符合预期，新添加的数据节点Key随机产生
 */
@Test
public void testReloadTreeAddRandomNewDataNode() {
    Random random = new Random();
    byte[] bytes = new byte[200];
    random.nextBytes(bytes);
    String newDataKey = Arrays.toString(bytes);
    CryptoSetting cryptoSetting = createCryptoSetting();
    MemoryKVStorage storage = new MemoryKVStorage();
    int count = 1024;
    List<VersioningKVData<String, byte[]>> dataList = generateDatas(count);
    VersioningKVData<String, byte[]>[] datas = toArray(dataList);
    MerkleHashTrie merkleTree = newMerkleTree_with_committed(datas, cryptoSetting, storage);
    HashDigest rootHash0 = merkleTree.getRootHash();
    assertNotNull(rootHash0);
    assertEquals(count, merkleTree.getTotalKeys());
    assertEquals(count, merkleTree.getTotalRecords());
    // reload and add one data item;
    MerkleHashTrie merkleTree_reload = new MerkleHashTrie(rootHash0, cryptoSetting, KEY_PREFIX, storage, false);
    assertEquals(count, merkleTree_reload.getTotalKeys());
    assertEquals(count, merkleTree_reload.getTotalRecords());
    assertEquals(rootHash0, merkleTree_reload.getRootHash());
    VersioningKVData<String, byte[]> data1025 = new VersioningKVData<String, byte[]>(newDataKey, 0, BytesUtils.toBytes("NEW-VALUE-1025-VERSION-0"));
    merkleTree_reload.setData(data1025.getKey(), data1025.getVersion(), data1025.getValue());
    merkleTree_reload.commit();
    HashDigest rootHash1 = merkleTree_reload.getRootHash();
    assertNotNull(rootHash1);
    assertNotEquals(rootHash0, rootHash1);
    MerkleTrieData data1025_reload_0 = merkleTree_reload.getData(data1025.getKey(), 0);
    assertNotNull(data1025_reload_0);
    assertNull(data1025_reload_0.getPreviousEntryHash());
    MerkleTrieData data0_reload_0 = merkleTree_reload.getData("KEY-0", 0);
    assertNotNull(data0_reload_0);
    assertNull(data0_reload_0.getPreviousEntryHash());
    System.out.println("mkl reload total keys = " + merkleTree_reload.getTotalKeys());
    assertEquals(count + 1, merkleTree_reload.getTotalKeys());
    MerkleHashTrie merkleTree_reload_1 = new MerkleHashTrie(rootHash1, cryptoSetting, KEY_PREFIX, storage, false);
    assertEquals(count + 1, merkleTree_reload_1.getTotalKeys());
    assertEquals(count + 1, merkleTree_reload_1.getTotalRecords());
    assertEquals(rootHash1, merkleTree_reload_1.getRootHash());
    HashDigest rootHash2 = merkleTree_reload_1.getRootHash();
    assertNotNull(rootHash2);
    assertNotEquals(rootHash0, rootHash2);
    MerkleTrieData data1025_reload_1 = merkleTree_reload_1.getData(data1025.getKey(), 0);
    assertNotNull(data1025_reload_1);
    assertNull(data1025_reload_1.getPreviousEntryHash());
    MerkleTrieData data0_reload_1 = merkleTree_reload_1.getData("KEY-0", 0);
    assertNotNull(data0_reload_1);
    assertNull(data0_reload_1.getPreviousEntryHash());
    System.out.println("mkl reload total keys = " + merkleTree_reload_1.getTotalKeys());
    assertEquals(count + 1, merkleTree_reload_1.getTotalKeys());
}

Example 33 with VersioningKVData

use of com.jd.blockchain.storage.service.utils.VersioningKVData in project jdchain-core by blockchain-jd-com.

the class MerkleHashTrieTest method testReloading.

/**
 * 测试树的加载读取；
 */
@Test
public void testReloading() {
    CryptoSetting cryptoSetting = createCryptoSetting();
    MemoryKVStorage storage = new MemoryKVStorage();
    // 数据集合长度为 100 时也能正常生成；
    int count = 1024;
    List<VersioningKVData<String, byte[]>> dataList = generateDatas(count);
    VersioningKVData<String, byte[]>[] datas = toArray(dataList);
    MerkleHashTrie merkleTree = newMerkleTree_with_committed(datas, cryptoSetting, storage);
    HashDigest rootHash0 = merkleTree.getRootHash();
    assertNotNull(rootHash0);
    assertEquals(count, merkleTree.getTotalKeys());
    assertEquals(count, merkleTree.getTotalRecords());
    // TODO: 暂时忽略默克尔证明的测试；
    // for (VersioningKVData<String, byte[]> data : datas) {
    // assertMerkleProof(data, merkleTree);
    // }
    // testMerkleProof1024(datas, merkleTree);
    MerkleHashTrie merkleTree_reload = new MerkleHashTrie(rootHash0, cryptoSetting, KEY_PREFIX, storage, false);
    assertEquals(count, merkleTree_reload.getTotalKeys());
    assertEquals(count, merkleTree_reload.getTotalRecords());
    // TODO: 暂时忽略默克尔证明的测试；
    // testMerkleProof1024(datas, merkleTree_reload);
    VersioningKVData<String, byte[]> data28 = new VersioningKVData<String, byte[]>("KEY-28", 1, BytesUtils.toBytes("NEW-VALUE-VERSION-1"));
    VersioningKVData<String, byte[]> data606 = new VersioningKVData<String, byte[]>("KEY-606", 1, BytesUtils.toBytes("NEW-VALUE-VERSION-1"));
    VersioningKVData<String, byte[]> data770 = new VersioningKVData<String, byte[]>("KEY-770", 1, BytesUtils.toBytes("NEW-VALUE-VERSION-1"));
    VersioningKVData<String, byte[]> data898 = new VersioningKVData<String, byte[]>("KEY-898", 1, BytesUtils.toBytes("NEW-VALUE-VERSION-1"));
    VersioningKVData<String, byte[]> data69 = new VersioningKVData<String, byte[]>("KEY-69", 1, BytesUtils.toBytes("NEW-VALUE-VERSION-1"));
    merkleTree_reload.setData(data28.getKey(), data28.getVersion(), data28.getValue());
    merkleTree_reload.setData(data606.getKey(), data606.getVersion(), data606.getValue());
    merkleTree_reload.setData(data770.getKey(), data770.getVersion(), data770.getValue());
    merkleTree_reload.setData(data898.getKey(), data898.getVersion(), data898.getValue());
    merkleTree_reload.setData(data69.getKey(), data69.getVersion(), data69.getValue());
    merkleTree_reload.commit();
    HashDigest rootHash1 = merkleTree_reload.getRootHash();
    assertNotNull(rootHash1);
    assertNotEquals(rootHash0, rootHash1);
    // TODO: 暂时忽略默克尔证明的测试；
    // MerkleProof proof = merkleTree_reload.getProof(data28.getKey(), 1);
    // assertNotNull(proof);
    // MerkleProofLevel[] hashPaths = proof.getProofLevels();
    // assertEquals(5, hashPaths.length);
    // proof = merkleTree_reload.getProof(data28.getKey(), 0);
    // assertNotNull(proof);
    // hashPaths = proof.getProofLevels();
    // assertEquals(6, hashPaths.length);
    MerkleTrieData data28_reload_0 = merkleTree_reload.getData(data28.getKey(), 0);
    assertNotNull(data28_reload_0);
    assertNull(data28_reload_0.getPreviousEntryHash());
    assertEquals(data28.getKey(), data28_reload_0.getKey().toUTF8String());
    assertEquals(datas[28].getVersion(), data28_reload_0.getVersion());
    assertArrayEquals(datas[28].getValue(), data28_reload_0.getValue().toBytes());
    MerkleTrieData data28_reload_1 = merkleTree_reload.getData(data28.getKey(), 1);
    assertNotNull(data28_reload_1);
    assertNotNull(data28_reload_1.getPreviousEntryHash());
    assertEquals(data28.getKey(), data28_reload_1.getKey().toUTF8String());
    assertEquals(data28.getVersion(), data28_reload_1.getVersion());
    assertArrayEquals(data28.getValue(), data28_reload_1.getValue().toBytes());
    // merkleTree_reload.print();
    // 测试不同根哈希加载的默克尔树能够检索的最新版本；
    MerkleHashTrie merkleTree_0 = new MerkleHashTrie(rootHash0, cryptoSetting, KEY_PREFIX, storage, false);
    MerkleHashTrie merkleTree_1 = new MerkleHashTrie(rootHash1, cryptoSetting, KEY_PREFIX, storage, false);
    MerkleTrieData data28_reload = merkleTree_0.getData(data28.getKey());
    assertEquals(0, data28_reload.getVersion());
    data28_reload = merkleTree_1.getData(data28.getKey());
    assertEquals(1, data28_reload.getVersion());
    // 测试在修改状态下检索默克尔证明的正确性；
    VersioningKVData<String, byte[]> data28_2 = new VersioningKVData<String, byte[]>("KEY-28", 2, BytesUtils.toBytes("NEW-VALUE-VERSION-2"));
// TODO: 暂时忽略默克尔证明的测试；
// MerkleProof proof28_1 = merkleTree_1.getProof("KEY-28", 1);
// MerkleProof proof606_1 = merkleTree_1.getProof("KEY-606", 1);
// assertNotNull(proof28_1);
// assertNotNull(proof606_1);
// 针对编号为 28 的数据加入一条新版本记录；
// merkleTree_1.setData(data28_2.getKey(), data28_2.getVersion(), data28_2.getValue());
// 对于修改中的数据项，查询未提交的最新版本数据的默克尔证明为 null，但是其已提交版本的证明不受影响；
// 此外，其它未修改的数据项的默克尔证明也不受影响；
// MerkleProof proof28_1_1 = merkleTree_1.getProof("KEY-28", 1);
// MerkleProof proof28_2 = merkleTree_1.getProof("KEY-28", 2);
// MerkleProof proof606_1_1 = merkleTree_1.getProof("KEY-606", 1);
// assertNotNull(proof28_1_1);
// assertNotNull(proof606_1_1);
// assertNull(proof28_2);
// assertEquals(proof28_1, proof28_1_1);
// assertEquals(proof606_1, proof606_1_1);
// 当提交修改之后，可以获取到修改数据项的最新版本的证明，同时其旧版本的证明也刷新了中间路径（加入了新版本数据节点）； 
// merkleTree_1.commit();
// TODO: 暂时忽略默克尔证明的测试；
// MerkleProof proof28_1_2 = merkleTree_1.getProof("KEY-28", 1);
// MerkleProof proof28_2_1 = merkleTree_1.getProof("KEY-28", 2);
// MerkleProof proof606_1_2 = merkleTree_1.getProof("KEY-606", 1);
// assertNotNull(proof28_1_2);
// assertNotNull(proof28_2_1);
// assertNotNull(proof606_1_2);
// // 由于默克尔树发生了修改，所有默克尔证明发生了改变；
// assertFalse(proof28_1.equals(proof28_1_2));
// assertFalse(proof606_1.equals(proof606_1_2));
// 同一个key的数据项的最新版本的默克尔证明路径节点中数据节点部分（倒数第2项），出现在其前一个版本的更新后的数据证明中倒数第3项；
// 验证默克尔证明的长度增长；
// MerkleProof proof28_5 = merkleTree_1.getProof("KEY-28", 0);
// assertNotNull(proof28_5);
// hashPaths = proof28_5.getProofLevels();
// assertEquals(7, hashPaths.length);
// TODO: 暂时忽略默克尔证明的测试；
// 重新加载默克尔树，默克尔证明是一致的；
// MerkleHashTrie merkleTree_1_1 = new MerkleHashTrie(rootHash1, cryptoSetting, KEY_PREFIX, storage, false);
// MerkleProof proof28_4 = merkleTree_1_1.getProof("KEY-28", 1);
// assertNotNull(proof28_4);
// assertEquals(proof28_1, proof28_4);
// merkleTree_1.print();
}

Example 34 with VersioningKVData

use of com.jd.blockchain.storage.service.utils.VersioningKVData in project jdchain-core by blockchain-jd-com.

the class MerkleHashTrieTest method testMerkleTreeKeysDiffSimple.

/**
 * 构造KEY固定的两棵树，验证是否能得到预期的差异数据
 */
@Test
public void testMerkleTreeKeysDiffSimple() {
    List<VersioningKVData<String, byte[]>> newdataList = new ArrayList<VersioningKVData<String, byte[]>>();
    List<String> newdataListString = new ArrayList<String>();
    CryptoSetting cryptoSetting = createCryptoSetting();
    MemoryKVStorage storage = new MemoryKVStorage();
    int count = 6;
    int newAddCount = 6;
    List<VersioningKVData<String, byte[]>> dataList = generateDatas(count);
    VersioningKVData<String, byte[]>[] datas = toArray(dataList);
    MerkleHashTrie merkleTree = newMerkleTree_with_committed(datas, cryptoSetting, storage);
    HashDigest rootHash0 = merkleTree.getRootHash();
    assertNotNull(rootHash0);
    assertEquals(count, merkleTree.getTotalKeys());
    assertEquals(count, merkleTree.getTotalRecords());
    // reload and add random key data item;
    MerkleHashTrie merkleTree_reload = new MerkleHashTrie(rootHash0, cryptoSetting, KEY_PREFIX, storage, false);
    assertEquals(count, merkleTree_reload.getTotalKeys());
    assertEquals(count, merkleTree_reload.getTotalRecords());
    assertEquals(rootHash0, merkleTree_reload.getRootHash());
    newdataListString = generateNewDatasString(count, newAddCount);
    for (int i = 0; i < newAddCount; i++) {
        VersioningKVData<String, byte[]> newdataKV = new VersioningKVData<String, byte[]>(newdataListString.get(i), 0, BytesUtils.toBytes("NEW-VALUE"));
        merkleTree_reload.setData(newdataKV.getKey(), newdataKV.getVersion(), newdataKV.getValue());
    }
    merkleTree_reload.commit();
    HashDigest rootHash1 = merkleTree_reload.getRootHash();
    assertNotNull(rootHash1);
    assertNotEquals(rootHash0, rootHash1);
    assertEquals(count + newAddCount, merkleTree_reload.getTotalKeys());
    assertEquals(count + newAddCount, merkleTree_reload.getTotalRecords());
    SkippingIterator<KVEntry> diffIterator;
    diffIterator = merkleTree_reload.getKeyDiffIterator(merkleTree);
    // max boundary skip test
    assertEquals(newAddCount, diffIterator.getTotalCount());
    assertEquals(-1, diffIterator.getCursor());
    assertTrue(diffIterator.hasNext());
    long skipped = diffIterator.skip(newAddCount);
    assertEquals(newAddCount, skipped);
    assertFalse(diffIterator.hasNext());
    // re-interator and random skip test
    int skipNum = 5;
    diffIterator = merkleTree_reload.getKeyDiffIterator(merkleTree);
    assertEquals(newAddCount, diffIterator.getTotalCount());
    assertEquals(-1, diffIterator.getCursor());
    assertTrue(diffIterator.hasNext());
    long skipped1 = diffIterator.skip(skipNum);
    assertEquals(5, skipped1);
    int diffNum = 0;
    // TODO: 无效的验证逻辑； by huanghaiquan at 2020-07-15;
    // while (diffIterator.hasNext()) {
    // MerkleData data = diffIterator.next();
    // assertNotNull(data);
    // assertFalse(dataList.contains(new String(data.getKey())));
    // assertTrue(newdataListString.contains(new String(data.getKey())));
    // diffNum++;
    // }
    // assertEquals(diffNum, diffIterator.getCount() - skipNum);
    // re-interator and next test
    diffIterator = merkleTree_reload.getKeyDiffIterator(merkleTree);
    int diffNum1 = 0;
    assertEquals(newAddCount, diffIterator.getTotalCount());
    while (diffIterator.hasNext()) {
        KVEntry data = diffIterator.next();
        assertNotNull(data);
        diffNum1++;
    }
    assertFalse(diffIterator.hasNext());
    assertEquals(newAddCount - 1, diffIterator.getCursor());
    assertEquals(newAddCount, diffIterator.getTotalCount());
    assertEquals(diffNum1, diffIterator.getTotalCount());
    // re-interator and test next key consistency
    diffIterator = merkleTree_reload.getKeyDiffIterator(merkleTree);
// while (diffIterator.hasNext()) {
// MerkleData data = diffIterator.next();
// assertNotNull(data);
// assertFalse(dataList.contains(new String(data.getKey())));
// assertTrue(newdataListString.contains(new String(data.getKey())));
// }
}

Example 35 with VersioningKVData

use of com.jd.blockchain.storage.service.utils.VersioningKVData in project jdchain-core by blockchain-jd-com.

the class MerkleHashTrieTest method testExtendPersistedPathNodes.

/**
 * 测试在已经持久化的默克尔树上追加新节点时，扩展已有路径节点的正确性；
 */
@Test
public void testExtendPersistedPathNodes() {
    int count = 10240;
    List<VersioningKVData<String, byte[]>> dataList = generateDatas(count);
    VersioningKVData<String, byte[]>[] datas = toArray(dataList);
    CryptoSetting cryptoSetting = createCryptoSetting();
    MemoryKVStorage storage = new MemoryKVStorage();
    Bytes prefix = Bytes.fromString(LedgerTestUtils.LEDGER_KEY_PREFIX);
    MerkleHashTrie merkleTree = new MerkleHashTrie(cryptoSetting, prefix, storage);
    int firstBatch = 500;
    for (int i = 0; i < firstBatch; i++) {
        merkleTree.setData(datas[i].getKey(), datas[i].getVersion(), datas[i].getValue());
    }
    // 先提交；
    merkleTree.commit();
    // 加载默克尔树到新实例；
    HashDigest rootHash = merkleTree.getRootHash();
    MerkleHashTrie merkleTree1 = new MerkleHashTrie(rootHash, cryptoSetting, prefix, storage, false);
    for (int i = firstBatch; i < datas.length; i++) {
        merkleTree1.setData(datas[i].getKey(), datas[i].getVersion(), datas[i].getValue());
    }
    merkleTree1.commit();
    // 重新加载；未正确扩展路径节点时，部分已持久化的叶子节点有可能丢失，在重新加载默克尔树并进行检索时将发现此错误；
    rootHash = merkleTree1.getRootHash();
    MerkleHashTrie merkleTree2 = new MerkleHashTrie(rootHash, cryptoSetting, prefix, storage, false);
    for (int i = 0; i < datas.length; i++) {
        KVEntry data = merkleTree2.getData(datas[i].getKey());
        assertNotNull(data);
    }
}