Examples with RandomPageReplacementPolicy herddb.core.RandomPageReplacementPolicy used on opensource projects

Example 26 with RandomPageReplacementPolicy

use of herddb.core.RandomPageReplacementPolicy in project herddb by diennea.

the class BlockRangeIndexStorageTest method testSimpleReload.

@Test
public void testSimpleReload() throws Exception {
    PageReplacementPolicy policy = new RandomPageReplacementPolicy(10);
    IndexDataStorage<Sized<Integer>, Sized<String>> storage = new MemoryIndexDataStorage<>();
    BlockRangeIndex<Sized<Integer>, Sized<String>> index = new BlockRangeIndex<>(400, policy, storage);
    index.boot(BlockRangeIndexMetadata.empty());
    index.put(Sized.valueOf(1), Sized.valueOf("a"));
    index.put(Sized.valueOf(2), Sized.valueOf("b"));
    index.put(Sized.valueOf(3), Sized.valueOf("c"));
    BlockRangeIndexMetadata<Sized<Integer>> metadata = index.checkpoint();
    assertEquals(index.getNumBlocks(), metadata.getBlocksMetadata().size());
    BlockRangeIndex<Sized<Integer>, Sized<String>> indexAfterBoot = new BlockRangeIndex<>(1024, new RandomPageReplacementPolicy(10), storage);
    indexAfterBoot.boot(metadata);
    assertEquals(Sized.valueOf("a"), index.search(Sized.valueOf(1)).get(0));
    assertEquals(Sized.valueOf("b"), index.search(Sized.valueOf(2)).get(0));
    assertEquals(Sized.valueOf("c"), index.search(Sized.valueOf(3)).get(0));
    assertEquals(2, indexAfterBoot.getNumBlocks());
}

Example 27 with RandomPageReplacementPolicy

use of herddb.core.RandomPageReplacementPolicy in project herddb by diennea.

the class BlockRangeIndexStorageTest method testReloadAfterBlockDeletion.

@Test
public void testReloadAfterBlockDeletion() throws Exception {
    PageReplacementPolicy policy = new RandomPageReplacementPolicy(10);
    IndexDataStorage<Sized<Integer>, Sized<Integer>> storage = new MemoryIndexDataStorage<>();
    BlockRangeIndex<Sized<Integer>, Sized<Integer>> index = new BlockRangeIndex<>(400, policy, storage);
    index.boot(BlockRangeIndexMetadata.empty());
    int i = 0;
    do {
        Sized<Integer> si = Sized.valueOf(i++);
        index.put(si, si);
    } while (index.getNumBlocks() < 4);
    /* First checkpoint after insertion */
    index.checkpoint();
    /* Now we empty middle blocks */
    /* Map without first key */
    ConcurrentNavigableMap<BlockStartKey<Sized<Integer>>, Block<Sized<Integer>, Sized<Integer>>> sub = index.getBlocks().tailMap(index.getBlocks().firstKey(), false);
    /* Second block */
    Block<Sized<Integer>, Sized<Integer>> second = sub.firstEntry().getValue();
    /* Map without second key */
    sub = sub.tailMap(sub.firstKey(), false);
    /* Third block */
    Block<Sized<Integer>, Sized<Integer>> third = sub.firstEntry().getValue();
    /* Copy to avoid concurrent modification */
    List<Entry<Sized<Integer>, Sized<Integer>>> toDelete = new ArrayList<>();
    second.values.forEach((k, vl) -> vl.forEach(v -> toDelete.add(new SimpleEntry<>(k, v))));
    third.values.forEach((k, vl) -> vl.forEach(v -> toDelete.add(new SimpleEntry<>(k, v))));
    /* Delete blocks 2 and 3 */
    toDelete.forEach(e -> index.delete(e.getKey(), e.getValue()));
    /* Checkpoint, should remove a block */
    BlockRangeIndexMetadata<Sized<Integer>> metadata = index.checkpoint();
    assertEquals(2, index.getNumBlocks());
    assertEquals(index.getNumBlocks(), metadata.getBlocksMetadata().size());
    /* Load a new index from data */
    BlockRangeIndex<Sized<Integer>, Sized<Integer>> indexAfterBoot = new BlockRangeIndex<>(1024, new RandomPageReplacementPolicy(10), storage);
    indexAfterBoot.boot(metadata);
    /* Check data equality between new and old index */
    index.getBlocks().forEach((f, b) -> b.values.forEach((k, vl) -> {
        List<Sized<Integer>> search = indexAfterBoot.search(k);
        Assert.assertEquals(vl, search);
    }));
}

Example 28 with RandomPageReplacementPolicy

use of herddb.core.RandomPageReplacementPolicy in project herddb by diennea.

the class BlockRangeIndexTest method lookupVeryFirstEntry.

@Test
public void lookupVeryFirstEntry() {
    BlockRangeIndex<Sized<Integer>, Sized<String>> index = new BlockRangeIndex<>(1024, new RandomPageReplacementPolicy(10));
    index.boot(BlockRangeIndexMetadata.empty());
    index.put(Sized.valueOf(1), Sized.valueOf("a"));
    index.put(Sized.valueOf(2), Sized.valueOf("b"));
    index.put(Sized.valueOf(3), Sized.valueOf("c"));
    index.put(Sized.valueOf(4), Sized.valueOf("d"));
    index.put(Sized.valueOf(5), Sized.valueOf("e"));
    index.put(Sized.valueOf(6), Sized.valueOf("f"));
    dumpIndex(index);
    List<Sized<String>> searchResult = index.search(Sized.valueOf(1));
    System.out.println("searchResult:" + searchResult);
    assertEquals(1, searchResult.size());
    List<Sized<String>> searchResult2 = index.search(Sized.valueOf(1), Sized.valueOf(4));
    System.out.println("searchResult:" + searchResult2);
    assertEquals(4, searchResult2.size());
    assertEquals(Sized.valueOf("a"), searchResult2.get(0));
    assertEquals(Sized.valueOf("b"), searchResult2.get(1));
    assertEquals(Sized.valueOf("c"), searchResult2.get(2));
    assertEquals(Sized.valueOf("d"), searchResult2.get(3));
}

Example 29 with RandomPageReplacementPolicy

use of herddb.core.RandomPageReplacementPolicy in project herddb by diennea.

the class BlockRangeIndexConcurrentTest method testConcurrentWrites.

@Test
public void testConcurrentWrites() throws Exception {
    int testSize = 10000;
    int parallelism = 10;
    BlockRangeIndex<Sized<Integer>, Sized<String>> index = new BlockRangeIndex<>(1024, new RandomPageReplacementPolicy(3));
    index.boot(BlockRangeIndexMetadata.empty());
    try {
        ExecutorService threadpool = Executors.newFixedThreadPool(parallelism);
        CountDownLatch l = new CountDownLatch(testSize);
        try {
            for (int i = 0; i < testSize; i++) {
                int _i = i;
                threadpool.submit(() -> {
                    try {
                        index.put(Sized.valueOf(_i), Sized.valueOf("a" + _i));
                        l.countDown();
                    } catch (RuntimeException e) {
                        e.printStackTrace();
                    }
                });
            }
        } finally {
            threadpool.shutdown();
        }
        assertTrue(l.await(10, TimeUnit.SECONDS));
        index.checkpoint();
        dumpIndex(index);
        verifyIndex(index);
        List<Sized<String>> result = index.search(Sized.valueOf(0), Sized.valueOf(testSize + 1));
        for (Sized<String> res : result) {
            System.out.println("res " + res.dummy);
        }
        for (int i = 0; i < testSize; i++) {
            assertTrue("cannot find " + i, index.containsKey(Sized.valueOf(i)));
        }
    } catch (Exception | AssertionError e) {
        deepDumpIndex(index);
        throw e;
    }
}

Example 30 with RandomPageReplacementPolicy

use of herddb.core.RandomPageReplacementPolicy in project herddb by diennea.

the class BlockRangeIndexTest method testSplitAndDelete.

/**
 * Verify that BRIN blocks next field is managed when pointed block get deleted
 *
 * @author diego.salvi
 */
@Test
public void testSplitAndDelete() throws IOException {
    /* Must be 1 to keep just one page in memory keeping to unload on every page load */
    final PageReplacementPolicy policy = new RandomPageReplacementPolicy(1);
    final MemoryIndexDataStorage<Sized<Integer>, Sized<String>> storage = new MemoryIndexDataStorage<>();
    final BlockRangeIndex<Sized<Integer>, Sized<String>> index = new BlockRangeIndex<>(400, policy, storage);
    index.boot(BlockRangeIndexMetadata.empty());
    /* Add values until block split */
    int elements;
    for (elements = 0; index.getNumBlocks() < 2; elements++) {
        /* All entries have the same key to create contiguous blocks with same keys */
        index.put(Sized.valueOf(1), Sized.valueOf("test_" + elements));
    }
    /* NumBlocks must be greater than 1 (split done) */
    Assert.assertTrue(index.getNumBlocks() > 1);
    /* Check every value existance */
    {
        List<Sized<String>> result = index.search(Sized.valueOf(1));
        Assert.assertEquals(elements, result.size());
    }
    /* Remove every last value until last block is empty */
    for (int i = elements - 1; i > -1; i--) {
        index.delete(Sized.valueOf(1), Sized.valueOf("test_" + i));
        /* Check if last block got emptied */
        if (index.getBlocks().lastEntry().getValue().getSize() == 0) {
            elements = i;
            break;
        }
    }
    /* Now checkpoint to remove empty block */
    BlockRangeIndexMetadata<Sized<Integer>> metadata = index.checkpoint();
    /* Deletes unreferenced pages from memory store */
    storage.getPages().retainAll(metadata.getBlocksMetadata().stream().map(m -> m.pageId).collect(Collectors.toList()));
    /* Delete remaining values (next should have been handled to avoid errors) */
    for (int i = 0; i < elements; i++) {
        index.delete(Sized.valueOf(1), Sized.valueOf("test_" + i));
    }
    /* Check every value non existance */
    {
        List<Sized<String>> result = index.search(Sized.valueOf(1));
        Assert.assertEquals(0, result.size());
    }
    index.clear();
}