Examples with BPlusTree org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree used on opensource projects

Example 1 with BPlusTree

use of org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree in project ignite by apache.

the class BPlusTreeSelfTest method doTestSizeForRandomPutRmvMultithreadedAsync.

/**
 * Verifies that concurrent running of {@link BPlusTree#put} + {@link BPlusTree#remove} sequence
 * and {@link BPlusTree#size} methods results in correct calculation of tree size.
 *
 * Since in the presence of concurrent modifications the size may differ from the actual one, the test maintains
 * sliding window of records in the tree, uses a barrier between concurrent runs to limit runaway delta in
 * the calculated size, and checks that the measured size lies within certain bounds.
 *
 * NB: This test has to be changed with the integration of IGNITE-3478.
 */
public void doTestSizeForRandomPutRmvMultithreadedAsync(final int rmvPutSlidingWindowSize) throws Exception {
    MAX_PER_PAGE = 5;
    final boolean DEBUG_PRINT = false;
    final TestTree tree = createTestTree(false);
    final AtomicLong curRmvKey = new AtomicLong(0);
    final AtomicLong curPutKey = new AtomicLong(rmvPutSlidingWindowSize);
    for (long i = curRmvKey.get(); i < curPutKey.get(); ++i) assertNull(tree.put(i));
    final int putRmvThreadCnt = Math.min(Runtime.getRuntime().availableProcessors(), rmvPutSlidingWindowSize);
    final int sizeThreadCnt = putRmvThreadCnt;
    final CyclicBarrier putRmvOpBarrier = new CyclicBarrier(putRmvThreadCnt + sizeThreadCnt, new Runnable() {

        @Override
        public void run() {
            if (DEBUG_PRINT) {
                try {
                    X.println("===BARRIER=== size=" + tree.size() + "; contents=[" + tree.findFirst() + ".." + tree.findLast() + "]" + "; rmvVal=" + curRmvKey.get() + "; putVal=" + curPutKey.get());
                    X.println(tree.printTree());
                } catch (IgniteCheckedException e) {
                // ignore
                }
            }
        }
    });
    final int loopCnt = 500;
    IgniteInternalFuture<?> putRmvFut = multithreadedAsync(new Callable<Object>() {

        @Override
        public Object call() throws Exception {
            for (int i = 0; i < loopCnt && !stop.get(); ++i) {
                int order;
                try {
                    order = putRmvOpBarrier.await();
                } catch (BrokenBarrierException e) {
                    break;
                }
                Long putVal = curPutKey.getAndIncrement();
                if (DEBUG_PRINT || (i & 0x3ff) == 0)
                    X.println(order + ": --> put(" + putVal + ")");
                assertNull(tree.put(putVal));
                Long rmvVal = curRmvKey.getAndIncrement();
                if (DEBUG_PRINT || (i & 0x3ff) == 0)
                    X.println(order + ": --> rmv(" + rmvVal + ")");
                assertEquals(rmvVal, tree.remove(rmvVal));
                assertNull(tree.findOne(rmvVal));
            }
            return null;
        }
    }, putRmvThreadCnt, "put-remove");
    IgniteInternalFuture<?> sizeFut = multithreadedAsync(new Callable<Object>() {

        @Override
        public Object call() throws Exception {
            final List<Long> treeContents = new ArrayList<>(rmvPutSlidingWindowSize * 2);
            final BPlusTree.TreeRowClosure<Long, Long> rowDumper = new BPlusTree.TreeRowClosure<Long, Long>() {

                @Override
                public boolean apply(BPlusTree<Long, Long> tree, BPlusIO<Long> io, long pageAddr, int idx) throws IgniteCheckedException {
                    treeContents.add(io.getLookupRow(tree, pageAddr, idx));
                    return true;
                }
            };
            for (long iter = 0; !stop.get(); ++iter) {
                int order = 0;
                try {
                    order = putRmvOpBarrier.await();
                } catch (BrokenBarrierException e) {
                    break;
                }
                long correctSize = curPutKey.get() - curRmvKey.get();
                treeContents.clear();
                long treeSize = tree.size(rowDumper);
                long minBound = correctSize - putRmvThreadCnt;
                long maxBound = correctSize + putRmvThreadCnt;
                if (DEBUG_PRINT || (iter & 0x3ff) == 0)
                    X.println(order + ": size=" + treeSize + "; bounds=[" + minBound + ".." + maxBound + "]; contents=" + treeContents);
                if (treeSize < minBound || treeSize > maxBound) {
                    fail("Tree size is not in bounds [" + minBound + ".." + maxBound + "]: " + treeSize + "; Tree contents: " + treeContents);
                }
            }
            return null;
        }
    }, sizeThreadCnt, "size");
    IgniteInternalFuture<?> lockPrintingFut = multithreadedAsync(new Callable<Void>() {

        @Override
        public Void call() throws Exception {
            while (!stop.get()) {
                Thread.sleep(5000);
                X.println(TestTree.printLocks());
            }
            return null;
        }
    }, 1, "printLocks");
    asyncRunFut = new GridCompoundFuture<>();
    asyncRunFut.add((IgniteInternalFuture) putRmvFut);
    asyncRunFut.add((IgniteInternalFuture) sizeFut);
    asyncRunFut.add((IgniteInternalFuture) lockPrintingFut);
    asyncRunFut.markInitialized();
    try {
        putRmvFut.get(getTestTimeout(), TimeUnit.MILLISECONDS);
    } finally {
        stop.set(true);
        putRmvOpBarrier.reset();
        asyncRunFut.get();
    }
    tree.validateTree();
    assertNoLocks();
}

Example 2 with BPlusTree

use of org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree in project ignite by apache.

the class BPlusTreeSelfTest method testPutSizeLivelock.

/**
 * The test forces {@link BPlusTree#size} method to run into a livelock: during single run
 * the method is picking up new pages which are concurrently added to the tree until the new pages are not added
 * anymore. Test verifies that despite livelock condition a size from a valid range is returned.
 *
 * NB: This test has to be changed with the integration of IGNITE-3478.
 *
 * @throws Exception if test failed
 */
public void testPutSizeLivelock() throws Exception {
    MAX_PER_PAGE = 5;
    CNT = 800;
    final int SLIDING_WINDOW_SIZE = 16;
    final boolean DEBUG_PRINT = false;
    final TestTree tree = createTestTree(false);
    final AtomicLong curRmvKey = new AtomicLong(0);
    final AtomicLong curPutKey = new AtomicLong(SLIDING_WINDOW_SIZE);
    for (long i = curRmvKey.get(); i < curPutKey.get(); ++i) assertNull(tree.put(i));
    final int hwThreads = Runtime.getRuntime().availableProcessors();
    final int putRmvThreadCnt = Math.max(1, hwThreads / 2);
    final int sizeThreadCnt = hwThreads - putRmvThreadCnt;
    final CyclicBarrier putRmvOpBarrier = new CyclicBarrier(putRmvThreadCnt, new Runnable() {

        @Override
        public void run() {
            if (DEBUG_PRINT) {
                try {
                    X.println("===BARRIER=== size=" + tree.size() + " [" + tree.findFirst() + ".." + tree.findLast() + "]");
                } catch (IgniteCheckedException e) {
                // ignore
                }
            }
        }
    });
    final int loopCnt = CNT / hwThreads;
    IgniteInternalFuture<?> putRmvFut = multithreadedAsync(new Callable<Object>() {

        @Override
        public Object call() throws Exception {
            for (int i = 0; i < loopCnt && !stop.get(); ++i) {
                int order;
                try {
                    order = putRmvOpBarrier.await();
                } catch (BrokenBarrierException e) {
                    // barrier reset() has been called: terminate
                    break;
                }
                Long putVal = curPutKey.getAndIncrement();
                if ((i & 0xff) == 0)
                    X.println(order + ": --> put(" + putVal + ")");
                assertNull(tree.put(putVal));
                Long rmvVal = curRmvKey.getAndIncrement();
                if ((i & 0xff) == 0)
                    X.println(order + ": --> rmv(" + rmvVal + ")");
                assertEquals(rmvVal, tree.remove(rmvVal));
                assertNull(tree.findOne(rmvVal));
            }
            return null;
        }
    }, putRmvThreadCnt, "put-remove");
    IgniteInternalFuture<?> sizeFut = multithreadedAsync(new Callable<Object>() {

        @Override
        public Object call() throws Exception {
            final List<Long> treeContents = new ArrayList<>(SLIDING_WINDOW_SIZE * 2);
            final BPlusTree.TreeRowClosure<Long, Long> rowDumper = new BPlusTree.TreeRowClosure<Long, Long>() {

                @Override
                public boolean apply(BPlusTree<Long, Long> tree, BPlusIO<Long> io, long pageAddr, int idx) throws IgniteCheckedException {
                    treeContents.add(io.getLookupRow(tree, pageAddr, idx));
                    final long endMs = System.currentTimeMillis() + 10;
                    final long endPutKey = curPutKey.get() + MAX_PER_PAGE;
                    while (System.currentTimeMillis() < endMs && curPutKey.get() < endPutKey) Thread.yield();
                    return true;
                }
            };
            while (!stop.get()) {
                treeContents.clear();
                long treeSize = tree.size(rowDumper);
                long curPutVal = curPutKey.get();
                X.println(" ======> size=" + treeSize + "; last-put-value=" + curPutVal);
                if (treeSize < SLIDING_WINDOW_SIZE || treeSize > curPutVal)
                    fail("Tree size is not in bounds [" + SLIDING_WINDOW_SIZE + ".." + curPutVal + "]:" + treeSize + "; contents=" + treeContents);
            }
            return null;
        }
    }, sizeThreadCnt, "size");
    asyncRunFut = new GridCompoundFuture<>();
    asyncRunFut.add((IgniteInternalFuture) putRmvFut);
    asyncRunFut.add((IgniteInternalFuture) sizeFut);
    asyncRunFut.markInitialized();
    try {
        putRmvFut.get(getTestTimeout(), TimeUnit.MILLISECONDS);
    } finally {
        stop.set(true);
        putRmvOpBarrier.reset();
        asyncRunFut.get();
    }
    tree.validateTree();
    assertNoLocks();
}

Example 3 with BPlusTree

use of org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree in project ignite by apache.

the class BPlusTreeSelfTest method testSizeForPutRmvSequential.

/**
 * Verifies that {@link BPlusTree#size} and {@link BPlusTree#size} methods behave correctly
 * on single-threaded addition and removal of elements in random order.
 *
 * @throws IgniteCheckedException If failed.
 */
public void testSizeForPutRmvSequential() throws IgniteCheckedException {
    MAX_PER_PAGE = 5;
    boolean DEBUG_PRINT = false;
    int itemCnt = (int) Math.pow(MAX_PER_PAGE, 5) + rnd.nextInt(MAX_PER_PAGE * MAX_PER_PAGE);
    Long[] items = new Long[itemCnt];
    for (int i = 0; i < itemCnt; ++i) items[i] = (long) i;
    TestTree testTree = createTestTree(true);
    TreeMap<Long, Long> goldenMap = new TreeMap<>();
    assertEquals(0, testTree.size());
    assertEquals(0, goldenMap.size());
    final Predicate<Long> rowMatcher = new Predicate<Long>() {

        @Override
        public boolean apply(Long row) {
            return row % 7 == 0;
        }
    };
    final BPlusTree.TreeRowClosure<Long, Long> rowClosure = new BPlusTree.TreeRowClosure<Long, Long>() {

        @Override
        public boolean apply(BPlusTree<Long, Long> tree, BPlusIO<Long> io, long pageAddr, int idx) throws IgniteCheckedException {
            return rowMatcher.apply(io.getLookupRow(tree, pageAddr, idx));
        }
    };
    int correctMatchingRows = 0;
    Collections.shuffle(Arrays.asList(items), rnd);
    for (Long row : items) {
        if (DEBUG_PRINT) {
            X.println(" --> put(" + row + ")");
            X.print(testTree.printTree());
        }
        assertEquals(goldenMap.put(row, row), testTree.put(row));
        assertEquals(row, testTree.findOne(row));
        if (rowMatcher.apply(row))
            ++correctMatchingRows;
        assertEquals(correctMatchingRows, testTree.size(rowClosure));
        long correctSize = goldenMap.size();
        assertEquals(correctSize, testTree.size());
        assertEquals(correctSize, size(testTree.find(null, null)));
        assertNoLocks();
    }
    Collections.shuffle(Arrays.asList(items), rnd);
    for (Long row : items) {
        if (DEBUG_PRINT) {
            X.println(" --> rmv(" + row + ")");
            X.print(testTree.printTree());
        }
        assertEquals(row, goldenMap.remove(row));
        assertEquals(row, testTree.remove(row));
        assertNull(testTree.findOne(row));
        if (rowMatcher.apply(row))
            --correctMatchingRows;
        assertEquals(correctMatchingRows, testTree.size(rowClosure));
        long correctSize = goldenMap.size();
        assertEquals(correctSize, testTree.size());
        assertEquals(correctSize, size(testTree.find(null, null)));
        assertNoLocks();
    }
}