Examples with BinaryDequeReader org.voltdb.utils.BinaryDeque.BinaryDequeReader used on opensource projects

Example 6 with BinaryDequeReader

use of org.voltdb.utils.BinaryDeque.BinaryDequeReader in project voltdb by VoltDB.

the class TestPersistentBinaryDeque method testReaderIsEmpty.

@Test
public void testReaderIsEmpty() throws Exception {
    System.out.println("Running testReaderIsEmpty");
    BinaryDequeReader reader = m_pbd.openForRead(CURSOR_ID);
    assertTrue(reader.isEmpty());
    assertNull(reader.poll(PersistentBinaryDeque.UNSAFE_CONTAINER_FACTORY));
    assertTrue(reader.isEmpty());
    m_pbd.offer(defaultContainer());
    assertFalse(reader.isEmpty());
    BBContainer retval = reader.poll(PersistentBinaryDeque.UNSAFE_CONTAINER_FACTORY);
    retval.discard();
    assertTrue(reader.isEmpty());
    // more than one segment
    for (int i = 0; i < 50; i++) {
        m_pbd.offer(defaultContainer());
    }
    assertFalse(reader.isEmpty());
    for (int i = 0; i < 50; i++) {
        retval = reader.poll(PersistentBinaryDeque.UNSAFE_CONTAINER_FACTORY);
        retval.discard();
        if (i < 49) {
            assertFalse(reader.isEmpty());
        } else {
            assertTrue(reader.isEmpty());
        }
    }
}

Example 7 with BinaryDequeReader

use of org.voltdb.utils.BinaryDeque.BinaryDequeReader in project voltdb by VoltDB.

the class TestPersistentBinaryDeque method testOfferCloseReopenOfferLeaveData.

@Test
public void testOfferCloseReopenOfferLeaveData() throws Exception {
    System.out.println("Running testOfferCloseHoleReopenOffer");
    //Make it create two full segments
    for (int ii = 0; ii < 96; ii++) {
        m_pbd.offer(DBBPool.wrapBB(getFilledBuffer(ii)));
    }
    File[] files = TEST_DIR.listFiles();
    assertEquals(3, files.length);
    m_pbd.sync();
    m_pbd.close();
    m_pbd = null;
    System.gc();
    System.runFinalization();
    m_pbd = new PersistentBinaryDeque(TEST_NONCE, TEST_DIR, logger);
    BinaryDequeReader reader = m_pbd.openForRead(CURSOR_ID);
    int cnt = reader.getNumObjects();
    assertEquals(cnt, 96);
    for (int ii = 96; ii < 192; ii++) {
        m_pbd.offer(DBBPool.wrapBB(getFilledBuffer(ii)));
    }
    m_pbd.sync();
    cnt = reader.getNumObjects();
    assertEquals(cnt, 192);
    //Now poll half of it and make sure the data is correct
    for (int ii = 0; ii < 96; ii++) {
        ByteBuffer defaultBuffer = getFilledBuffer(ii);
        BBContainer retval = reader.poll(PersistentBinaryDeque.UNSAFE_CONTAINER_FACTORY);
        assertTrue(defaultBuffer.equals(retval.b()));
        retval.discard();
        defaultBuffer.clear();
    }
    m_pbd.sync();
    m_pbd.close();
    m_pbd = null;
    System.gc();
    System.runFinalization();
    //Expect just the current write segment
    TreeSet<String> names = getSortedDirectoryListing();
    assertEquals(3, names.size());
    assertTrue(names.first().equals("pbd_nonce.3.pbd"));
    //Reload
    m_pbd = new PersistentBinaryDeque(TEST_NONCE, TEST_DIR, logger);
    reader = m_pbd.openForRead(CURSOR_ID);
    cnt = reader.getNumObjects();
    assertEquals(cnt, 96);
    //Expect just the current write segment hole should be deleted.
    names = getSortedDirectoryListing();
    assertEquals(4, names.size());
    assertTrue(names.first().equals("pbd_nonce.3.pbd"));
    for (int ii = 96; ii < 192; ii++) {
        m_pbd.offer(DBBPool.wrapBB(getFilledBuffer(ii)));
    }
    m_pbd.sync();
    cnt = reader.getNumObjects();
    assertEquals(cnt, 192);
    //Now poll half of it and make sure the data is correct
    for (int ii = 96; ii < 192; ii++) {
        ByteBuffer defaultBuffer = getFilledBuffer(ii);
        BBContainer retval = reader.poll(PersistentBinaryDeque.UNSAFE_CONTAINER_FACTORY);
        assertTrue(defaultBuffer.equals(retval.b()));
        retval.discard();
        defaultBuffer.clear();
    }
    //Expect just the current write segment
    names = getSortedDirectoryListing();
    assertEquals(3, names.size());
    assertTrue(names.first().equals("pbd_nonce.6.pbd"));
    //Poll and leave one behind.
    for (int ii = 96; ii < 191; ii++) {
        ByteBuffer defaultBuffer = getFilledBuffer(ii);
        BBContainer retval = reader.poll(PersistentBinaryDeque.UNSAFE_CONTAINER_FACTORY);
        assertTrue(defaultBuffer.equals(retval.b()));
        retval.discard();
        defaultBuffer.clear();
    }
    //Expect just the current write segment
    names = getSortedDirectoryListing();
    assertEquals(1, names.size());
    assertTrue(names.first().equals("pbd_nonce.8.pbd"));
    //Push to get more segments at head
    BBContainer[] objs = new BBContainer[] { DBBPool.wrapBB(ByteBuffer.allocateDirect(1024 * 1024 * 32)), DBBPool.wrapBB(ByteBuffer.allocateDirect(1024 * 1024 * 32)), DBBPool.wrapBB(ByteBuffer.allocateDirect(1024 * 1024 * 32)) };
    m_pbd.push(objs);
    names = getSortedDirectoryListing();
    assertEquals(4, names.size());
    assertTrue(names.first().equals("pbd_nonce.5.pbd"));
    BBContainer retval = reader.poll(PersistentBinaryDeque.UNSAFE_CONTAINER_FACTORY);
    retval.discard();
    names = getSortedDirectoryListing();
    assertEquals(3, names.size());
    assertTrue(names.first().equals("pbd_nonce.6.pbd"));
}

Example 8 with BinaryDequeReader

use of org.voltdb.utils.BinaryDeque.BinaryDequeReader in project voltdb by VoltDB.

the class TestPersistentBinaryDeque method testOfferCloseReopenOfferSmall.

@Test
public void testOfferCloseReopenOfferSmall() throws Exception {
    System.out.println("Running testOfferCloseReopenOfferSmall");
    final String SMALL_TEST_NONCE = "asmall_pbd_nonce";
    PersistentBinaryDeque small_pbd = new PersistentBinaryDeque(SMALL_TEST_NONCE, TEST_DIR, logger);
    //Keep in 1 segment.
    for (int ii = 0; ii < 10; ii++) {
        small_pbd.offer(DBBPool.wrapBB(getFilledSmallBuffer(ii)));
    }
    File[] files = TEST_DIR.listFiles();
    //We have the default pbd and new one.
    assertEquals(2, files.length);
    small_pbd.sync();
    small_pbd.close();
    System.gc();
    System.runFinalization();
    small_pbd = new PersistentBinaryDeque(SMALL_TEST_NONCE, TEST_DIR, logger);
    BinaryDequeReader reader = small_pbd.openForRead(CURSOR_ID);
    int cnt = reader.getNumObjects();
    assertEquals(cnt, 10);
    for (int ii = 10; ii < 20; ii++) {
        small_pbd.offer(DBBPool.wrapBB(getFilledSmallBuffer(ii)));
    }
    small_pbd.sync();
    cnt = reader.getNumObjects();
    assertEquals(cnt, 20);
    small_pbd.sync();
    small_pbd.close();
    small_pbd = null;
    System.gc();
    System.runFinalization();
    TreeSet<String> names = getSortedDirectoryListing();
    assertEquals(3, names.size());
    assertTrue(names.first().equals("asmall_pbd_nonce.0.pbd"));
    small_pbd = new PersistentBinaryDeque(SMALL_TEST_NONCE, TEST_DIR, logger);
    reader = small_pbd.openForRead(CURSOR_ID);
    //Now poll all of it and make sure the data is correct dont poll everything out.
    for (int ii = 0; ii < 10; ii++) {
        ByteBuffer defaultBuffer = getFilledSmallBuffer(ii);
        BBContainer retval = reader.poll(PersistentBinaryDeque.UNSAFE_CONTAINER_FACTORY);
        try {
            assertTrue(defaultBuffer.equals(retval.b()));
        } finally {
            retval.discard();
        }
        defaultBuffer.clear();
    }
    small_pbd.sync();
    small_pbd.close();
    small_pbd = null;
    System.gc();
    System.runFinalization();
}

Example 9 with BinaryDequeReader

use of org.voltdb.utils.BinaryDeque.BinaryDequeReader in project voltdb by VoltDB.

the class TestPersistentBinaryDeque method testTruncator.

@Test
public void testTruncator() throws Exception {
    System.out.println("Running testTruncator");
    BinaryDequeReader reader = m_pbd.openForRead(CURSOR_ID);
    assertNull(reader.poll(PersistentBinaryDeque.UNSAFE_CONTAINER_FACTORY));
    for (int ii = 0; ii < 160; ii++) {
        m_pbd.offer(DBBPool.wrapBB(getFilledBuffer(ii)));
    }
    m_pbd.close();
    m_pbd = new PersistentBinaryDeque(TEST_NONCE, TEST_DIR, logger);
    TreeSet<String> listing = getSortedDirectoryListing();
    assertEquals(listing.size(), 5);
    m_pbd.parseAndTruncate(new BinaryDequeTruncator() {

        private long m_objectsParsed = 0;

        @Override
        public TruncatorResponse parse(BBContainer bbc) {
            ByteBuffer b = bbc.b();
            assertEquals(b.getLong(0), m_objectsParsed);
            assertEquals(b.remaining(), 1024 * 1024 * 2);
            if (b.getLong(0) == 45) {
                b.limit(b.remaining() / 2);
                return new PersistentBinaryDeque.ByteBufferTruncatorResponse(b.slice());
            }
            while (b.remaining() > 15) {
                assertEquals(b.getLong(), m_objectsParsed);
                b.getLong();
            }
            m_objectsParsed++;
            return null;
        }
    });
    reader = m_pbd.openForRead(CURSOR_ID);
    assertEquals(95420416, reader.sizeInBytes());
    listing = getSortedDirectoryListing();
    assertEquals(listing.size(), 2);
    for (int ii = 46; ii < 96; ii++) {
        m_pbd.offer(DBBPool.wrapBB(getFilledBuffer(ii)));
    }
    long actualSizeInBytes = 0;
    long reportedSizeInBytes = reader.sizeInBytes();
    long blocksFound = 0;
    BBContainer cont = null;
    while ((cont = reader.poll(PersistentBinaryDeque.UNSAFE_CONTAINER_FACTORY)) != null) {
        try {
            ByteBuffer buffer = cont.b();
            if (blocksFound == 45) {
                assertEquals(buffer.remaining(), 1024 * 1024);
            } else {
                assertEquals(buffer.remaining(), 1024 * 1024 * 2);
            }
            actualSizeInBytes += buffer.remaining();
            while (buffer.remaining() > 15) {
                assertEquals(buffer.getLong(), blocksFound);
                buffer.getLong();
            }
        } finally {
            blocksFound++;
            cont.discard();
        }
    }
    assertEquals(actualSizeInBytes, reportedSizeInBytes);
    assertEquals(blocksFound, 96);
}

Example 10 with BinaryDequeReader

use of org.voltdb.utils.BinaryDeque.BinaryDequeReader in project voltdb by VoltDB.

the class TestPersistentBinaryDeque method testIsEmptyWhileClosed.

@Test
public void testIsEmptyWhileClosed() throws Exception {
    System.out.println("Running testIsEmptyWhileClosed");
    BinaryDequeReader reader = m_pbd.openForRead(CURSOR_ID);
    m_pbd.close();
    try {
        reader.isEmpty();
    } catch (IOException e) {
        return;
    }
    fail();
}