Examples with BinaryDequeReader org.voltdb.utils.BinaryDeque.BinaryDequeReader used on opensource projects

Example 11 with BinaryDequeReader

use of org.voltdb.utils.BinaryDeque.BinaryDequeReader in project voltdb by VoltDB.

the class TestPersistentBinaryDeque method testDeleteOnNonEmptyNextSegment.

@Test
public void testDeleteOnNonEmptyNextSegment() throws Exception {
    System.out.println("Running testOfferPollOfferMoreThanPoll");
    BinaryDequeReader reader = m_pbd.openForRead(CURSOR_ID);
    assertNull(reader.poll(PersistentBinaryDeque.UNSAFE_CONTAINER_FACTORY));
    // Number of buffers it takes to fill a segment
    final int total = 47;
    //Make sure a single file with the appropriate data is created
    for (int i = 0; i < total; i++) {
        m_pbd.offer(defaultContainer());
    }
    assertEquals(1, TEST_DIR.listFiles().length);
    // Read read all the buffers from the segment (isEmpty() returns true)
    for (int i = 0; i < total; i++) {
        reader.poll(PersistentBinaryDeque.UNSAFE_CONTAINER_FACTORY).discard();
    }
    assert (reader.isEmpty());
    File[] files = TEST_DIR.listFiles();
    assertEquals(1, files.length);
    assert (files[0].getName().equals("pbd_nonce.0.pbd"));
    m_pbd.offer(defaultContainer());
    files = TEST_DIR.listFiles();
    // Make sure a new segment was created and the old segment was deleted
    assertEquals(1, files.length);
    assert (files[0].getName().equals("pbd_nonce.1.pbd"));
}

Example 12 with BinaryDequeReader

use of org.voltdb.utils.BinaryDeque.BinaryDequeReader in project voltdb by VoltDB.

the class TestPersistentBinaryDeque method testOfferThenPushThenPoll.

@Test
public void testOfferThenPushThenPoll() throws Exception {
    System.out.println("Running testOfferThenPushThenPoll");
    BinaryDequeReader reader = m_pbd.openForRead(CURSOR_ID);
    assertTrue(reader.isEmpty());
    //Make it create two full segments
    for (int ii = 0; ii < 96; ii++) {
        m_pbd.offer(defaultContainer());
        assertFalse(reader.isEmpty());
    }
    File[] files = TEST_DIR.listFiles();
    assertEquals(3, files.length);
    //Now create two buffers with different data to push at the front
    final ByteBuffer buffer1 = getFilledBuffer(16);
    final ByteBuffer buffer2 = getFilledBuffer(32);
    BBContainer[] pushContainers = new BBContainer[2];
    pushContainers[0] = DBBPool.dummyWrapBB(buffer1);
    pushContainers[1] = DBBPool.dummyWrapBB(buffer2);
    m_pbd.push(pushContainers);
    //Expect this to create a single new file
    TreeSet<String> names = getSortedDirectoryListing();
    assertEquals(4, names.size());
    assertTrue(names.first().equals("pbd_nonce.-1.pbd"));
    assertTrue(names.contains("pbd_nonce.0.pbd"));
    assertTrue(names.contains("pbd_nonce.1.pbd"));
    assertTrue(names.last().equals("pbd_nonce.2.pbd"));
    //Poll the two at the front and check that the contents are what is expected
    BBContainer retval1 = reader.poll(PersistentBinaryDeque.UNSAFE_CONTAINER_FACTORY);
    try {
        buffer1.clear();
        System.err.println(Long.toHexString(buffer1.getLong(0)) + " " + Long.toHexString(retval1.b().getLong(0)));
        assertEquals(retval1.b(), buffer1);
        BBContainer retval2 = reader.poll(PersistentBinaryDeque.UNSAFE_CONTAINER_FACTORY);
        try {
            buffer2.clear();
            assertEquals(retval2.b(), buffer2);
            //Expect the file for the two polled objects to still be there
            //until the discard
            names = getSortedDirectoryListing();
            assertEquals(4, names.size());
        } finally {
            retval2.discard();
        }
    } finally {
        retval1.discard();
    }
    names = getSortedDirectoryListing();
    assertEquals(3, names.size());
    assertTrue(names.first().equals("pbd_nonce.0.pbd"));
    ByteBuffer defaultBuffer = defaultBuffer();
    //Now poll the rest and make sure the data is correct
    for (int ii = 0; ii < 96; ii++) {
        defaultBuffer.clear();
        BBContainer retval = reader.poll(PersistentBinaryDeque.UNSAFE_CONTAINER_FACTORY);
        assertTrue(defaultBuffer.equals(retval.b()));
        retval.discard();
    }
    assertTrue(reader.isEmpty());
    //Expect just the current write segment
    names = getSortedDirectoryListing();
    assertEquals(1, names.size());
    assertTrue(names.first().equals("pbd_nonce.2.pbd"));
}

Example 13 with BinaryDequeReader

use of org.voltdb.utils.BinaryDeque.BinaryDequeReader in project voltdb by VoltDB.

the class TestPersistentBinaryDeque method testTruncatorWithFullTruncateReturn.

@Test
public void testTruncatorWithFullTruncateReturn() throws Exception {
    System.out.println("Running testTruncatorWithFullTruncateReturn");
    BinaryDequeReader reader = m_pbd.openForRead(CURSOR_ID);
    assertNull(reader.poll(PersistentBinaryDeque.UNSAFE_CONTAINER_FACTORY));
    for (int ii = 0; ii < 150; ii++) {
        m_pbd.offer(DBBPool.wrapBB(getFilledBuffer(ii)));
    }
    m_pbd.close();
    m_pbd = new PersistentBinaryDeque(TEST_NONCE, TEST_DIR, logger);
    TreeSet<String> listing = getSortedDirectoryListing();
    assertEquals(listing.size(), 5);
    m_pbd.parseAndTruncate(new BinaryDequeTruncator() {

        private long m_objectsParsed = 0;

        @Override
        public TruncatorResponse parse(BBContainer bbc) {
            ByteBuffer b = bbc.b();
            if (b.getLong(0) != m_objectsParsed) {
                System.out.println("asd");
            }
            assertEquals(b.getLong(0), m_objectsParsed);
            assertEquals(b.remaining(), 1024 * 1024 * 2);
            if (b.getLong(0) == 45) {
                b.limit(b.remaining() / 2);
                return PersistentBinaryDeque.fullTruncateResponse();
            }
            while (b.remaining() > 15) {
                assertEquals(b.getLong(), m_objectsParsed);
                b.getLong();
            }
            m_objectsParsed++;
            return null;
        }
    });
    listing = getSortedDirectoryListing();
    assertEquals(listing.size(), 2);
    for (int ii = 46; ii < 96; ii++) {
        m_pbd.offer(DBBPool.wrapBB(getFilledBuffer(ii)));
    }
    reader = m_pbd.openForRead(CURSOR_ID);
    long actualSizeInBytes = 0;
    long reportedSizeInBytes = reader.sizeInBytes();
    long blocksFound = 0;
    BBContainer cont = null;
    while ((cont = reader.poll(PersistentBinaryDeque.UNSAFE_CONTAINER_FACTORY)) != null) {
        try {
            ByteBuffer buffer = cont.b();
            if (blocksFound == 45) {
                //white lie, so we expect the right block contents
                blocksFound++;
            }
            assertEquals(buffer.remaining(), 1024 * 1024 * 2);
            actualSizeInBytes += buffer.remaining();
            while (buffer.remaining() > 15) {
                assertEquals(buffer.getLong(), blocksFound);
                buffer.getLong();
            }
        } finally {
            blocksFound++;
            cont.discard();
        }
    }
    assertEquals(actualSizeInBytes, reportedSizeInBytes);
    assertEquals(blocksFound, 96);
}

Example 14 with BinaryDequeReader

use of org.voltdb.utils.BinaryDeque.BinaryDequeReader in project voltdb by VoltDB.

the class TestPersistentBinaryDeque method testCloseOldSegments.

@Test
public void testCloseOldSegments() throws Exception {
    System.out.println("Running testCloseOldSegments");
    BinaryDequeReader reader = m_pbd.openForRead(CURSOR_ID);
    assertNull(reader.poll(PersistentBinaryDeque.UNSAFE_CONTAINER_FACTORY));
    final int total = 100;
    //Make sure several files with the appropriate data is created
    for (int i = 0; i < total; i++) {
        m_pbd.offer(defaultContainer());
    }
    File[] files = TEST_DIR.listFiles();
    assertEquals(3, files.length);
    Set<String> actualFiles = Sets.newHashSet();
    for (File f : files) {
        actualFiles.add(f.getName());
    }
    Set<String> expectedFiles = Sets.newHashSet();
    for (int i = 0; i < 3; i++) {
        expectedFiles.add("pbd_nonce." + i + ".pbd");
    }
    Assert.assertEquals(expectedFiles, actualFiles);
    //Now make sure the current write file is stolen and a new write file created
    for (int i = 0; i < total; i++) {
        BBContainer retval = reader.poll(PersistentBinaryDeque.UNSAFE_CONTAINER_FACTORY);
        retval.discard();
    }
}

Example 15 with BinaryDequeReader

use of org.voltdb.utils.BinaryDeque.BinaryDequeReader in project voltdb by VoltDB.

the class TestPersistentBinaryDeque method testOfferCloseReopenOffer.

@Test
public void testOfferCloseReopenOffer() throws Exception {
    System.out.println("Running testOfferCloseThenReopen");
    //Make it create two full segments
    for (int ii = 0; ii < 96; ii++) {
        m_pbd.offer(DBBPool.wrapBB(getFilledBuffer(ii)));
    }
    File[] files = TEST_DIR.listFiles();
    assertEquals(3, files.length);
    m_pbd.sync();
    m_pbd.close();
    m_pbd = new PersistentBinaryDeque(TEST_NONCE, TEST_DIR, logger);
    BinaryDequeReader reader = m_pbd.openForRead(CURSOR_ID);
    int cnt = reader.getNumObjects();
    assertEquals(cnt, 96);
    for (int ii = 96; ii < 192; ii++) {
        m_pbd.offer(DBBPool.wrapBB(getFilledBuffer(ii)));
    }
    m_pbd.sync();
    cnt = reader.getNumObjects();
    assertEquals(cnt, 192);
    //Now poll all of it and make sure the data is correct
    for (int ii = 0; ii < 192; ii++) {
        ByteBuffer defaultBuffer = getFilledBuffer(ii);
        BBContainer retval = reader.poll(PersistentBinaryDeque.UNSAFE_CONTAINER_FACTORY);
        try {
            assertTrue(defaultBuffer.equals(retval.b()));
        } finally {
            retval.discard();
        }
        defaultBuffer.clear();
    }
    //Expect just the current write segment
    TreeSet<String> names = getSortedDirectoryListing();
    assertEquals(1, names.size());
    assertTrue(names.first().equals("pbd_nonce.5.pbd"));
}