Examples with DbusEventIterator com.linkedin.databus.core.DbusEventBuffer.DbusEventIterator used on opensource projects

Example 16 with DbusEventIterator

use of com.linkedin.databus.core.DbusEventBuffer.DbusEventIterator in project databus by linkedin.

the class ReadEventsTestParams method testCopyIterator.

@Test
public void testCopyIterator() throws Exception {
    DbusEventBuffer dbuf = new DbusEventBuffer(getConfig(10000000, DbusEventBuffer.Config.DEFAULT_INDIVIDUAL_BUFFER_SIZE, 100000, 1000000, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    int numEntries = 50000;
    HashMap<Long, KeyValue> testDataMap = new HashMap<Long, KeyValue>(20000);
    dbuf.start(0);
    for (long i = 1; i < numEntries; ++i) {
        //LOG.info("Iteration:"+i);
        DbusEventKey key = new DbusEventKey(RngUtils.randomLong());
        String value = RngUtils.randomString(20);
        dbuf.startEvents();
        assertTrue(dbuf.appendEvent(key, pPartitionId, lPartitionId, timeStamp, srcId, schemaId, value.getBytes(Charset.defaultCharset()), false));
        testDataMap.put(i, new KeyValue(key, value));
        dbuf.endEvents(i);
    }
    final long minDbusEventBufferScn = dbuf.getMinScn();
    long expectedScn = minDbusEventBufferScn;
    DbusEventIterator eventIterator = dbuf.acquireIterator("eventIterator");
    DbusEventInternalWritable e = null;
    // searching for min scn
    int state = 0;
    DbusEventIterator copyIterator = eventIterator.copy(null, "copyIterator");
    while (eventIterator.hasNext()) {
        e = eventIterator.next();
        if (state == 0 && (e.sequence() >= minDbusEventBufferScn)) {
            // found min scn
            state = 1;
        }
        if (state == 1) {
            assertEquals(expectedScn, e.sequence());
            e = eventIterator.next();
            assertTrue(e.isEndOfPeriodMarker());
            assertEquals(expectedScn, e.sequence());
            expectedScn = (expectedScn + 1) % numEntries;
        }
    }
    dbuf.releaseIterator(eventIterator);
    state = 0;
    expectedScn = minDbusEventBufferScn;
    while (copyIterator.hasNext()) {
        e = copyIterator.next();
        if (state == 0 && (e.sequence() >= minDbusEventBufferScn)) {
            // found min scn
            state = 1;
        }
        if (state == 1) {
            assertEquals(expectedScn, e.sequence());
            e = copyIterator.next();
            assertTrue(e.isEndOfPeriodMarker());
            assertEquals(expectedScn, e.sequence());
            expectedScn = (expectedScn + 1) % numEntries;
        }
    }
    dbuf.releaseIterator(copyIterator);
}

Example 17 with DbusEventIterator

use of com.linkedin.databus.core.DbusEventBuffer.DbusEventIterator in project databus by linkedin.

the class ReadEventsTestParams method testResetBuffer.

/**
   * Add events to a buffer and verify that they are in there in the correct order.
   * Reset the buffer and ensure that there are no events and minScn is back to -1.
   * Add more events to the buffer and ensure that the only entries present are the
   * new ones added.
   */
@Test
public void testResetBuffer() throws Exception {
    int numEntries = 10;
    HashMap<Long, KeyValue> testDataMap = new HashMap<Long, KeyValue>(50);
    DbusEventBuffer dbuf = new DbusEventBuffer(getConfig(10000000, DbusEventBuffer.Config.DEFAULT_INDIVIDUAL_BUFFER_SIZE, 100000, 1000000, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    int valueLength = 20;
    // Fill 10 entries in buffer starting with 0, and verify they are placed in order.
    dbuf.start(0);
    for (long i = 1; i < numEntries; ++i) {
        // LOG.info("Iteration:"+i);
        DbusEventKey key = new DbusEventKey(RngUtils.randomLong());
        String value = RngUtils.randomString(valueLength);
        dbuf.startEvents();
        long ts = timeStamp + (i * 1000 * 1000);
        testDataMap.put(i, new KeyValue(key, value));
        assertTrue(dbuf.appendEvent(key, pPartitionId, lPartitionId, ts, srcId, schemaId, value.getBytes(Charset.defaultCharset()), false));
        dbuf.endEvents(i);
    }
    long minDbusEventBufferScn = dbuf.getMinScn();
    long expectedScn = minDbusEventBufferScn;
    DbusEventIterator eventIterator = dbuf.acquireIterator("eventIterator");
    DbusEventInternalWritable e = null;
    // searching for min scn
    int state = 0;
    long entryNum = 1;
    while (eventIterator.hasNext()) {
        e = eventIterator.next();
        if (state == 0 && (e.sequence() >= minDbusEventBufferScn)) {
            // found min scn
            state = 1;
        }
        if (state == 1) {
            assertEquals(expectedScn, e.sequence());
            long ts = e.timestampInNanos();
            assertEquals(ts, timeStamp + entryNum * 1000 * 1000);
            byte[] eventBytes = new byte[e.valueLength()];
            e.value().get(eventBytes);
            assertEquals(eventBytes, testDataMap.get(entryNum).value.getBytes(Charset.defaultCharset()));
            entryNum++;
            e = eventIterator.next();
            assertTrue(e.isEndOfPeriodMarker());
            assertEquals(expectedScn, e.sequence());
            expectedScn++;
        }
    }
    dbuf.releaseIterator(eventIterator);
    Assert.assertEquals(entryNum, numEntries);
    // Reset the buffer with prevScn = 3, and verify
    final long prevScn = 3;
    dbuf.reset(prevScn);
    assertTrue(dbuf.empty());
    assertEquals(-1, dbuf.getMinScn());
    assertEquals(prevScn, dbuf.getPrevScn());
    testDataMap.clear();
    long currentScn = prevScn;
    // Add more events and make sure that the new events are in the buffer.
    //dbuf.start(0);
    valueLength = 25;
    numEntries = 5;
    // Now add entries to and make sure that they appear.
    for (long i = 1; i < numEntries; ++i) {
        // LOG.info("Iteration:"+i);
        DbusEventKey key = new DbusEventKey(RngUtils.randomLong());
        String value = RngUtils.randomString(valueLength);
        dbuf.startEvents();
        long ts = timeStamp + (i * 1200 * 1000);
        assertTrue(dbuf.appendEvent(key, pPartitionId, lPartitionId, ts, srcId, schemaId, value.getBytes(Charset.defaultCharset()), false));
        testDataMap.put(i, new KeyValue(key, value));
        dbuf.endEvents(currentScn + i);
    }
    minDbusEventBufferScn = dbuf.getMinScn();
    assertEquals(minDbusEventBufferScn, prevScn + 1);
    expectedScn = minDbusEventBufferScn;
    eventIterator = dbuf.acquireIterator("eventIterator2");
    // searching for min scn
    state = 0;
    entryNum = 1;
    while (eventIterator.hasNext()) {
        e = eventIterator.next();
        if (state == 0) {
            if (e.sequence() >= minDbusEventBufferScn) {
                // found min scn
                state = 1;
            }
        }
        if (state == 1) {
            assertEquals(expectedScn, e.sequence());
            long ts = e.timestampInNanos();
            assertEquals(ts, timeStamp + entryNum * 1200 * 1000);
            byte[] eventBytes = new byte[e.valueLength()];
            e.value().get(eventBytes);
            assertEquals(eventBytes, testDataMap.get(entryNum).value.getBytes(Charset.defaultCharset()));
            entryNum++;
            e = eventIterator.next();
            assertTrue(e.isEndOfPeriodMarker());
            assertEquals(expectedScn, e.sequence());
            expectedScn++;
        }
    }
    dbuf.releaseIterator(eventIterator);
    Assert.assertEquals(entryNum, numEntries);
}

Example 18 with DbusEventIterator

use of com.linkedin.databus.core.DbusEventBuffer.DbusEventIterator in project databus by linkedin.

the class ReadEventsTestParams method testOpCode.

/**
   * Test if the UPSERT/DELETE opcodes are correctly set in the event buffer.
   * The test appends and upsert and a delete event and iterates though them to see if they have been
   * inserted with the correct opcode.
   */
public void testOpCode(String type) throws InvalidConfigException, UnsupportedKeyException {
    DbusEventBuffer dbuf = new DbusEventBuffer(getConfig(10000000, DbusEventBuffer.Config.DEFAULT_INDIVIDUAL_BUFFER_SIZE, 100000, 1000000, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    dbuf.start(0);
    //Write the upsert event
    dbuf.startEvents();
    String value = RngUtils.randomString(20);
    DbusOpcode opCode = DbusOpcode.UPSERT;
    DbusEventInfo dbusEventInfo = new DbusEventInfo(opCode, //scn
    1234, //pconfigId
    (short) 2600, //Partition id
    (short) 1, timeStamp, //logical source id
    (short) 2601, schemaId, value.getBytes(Charset.defaultCharset()), false, false);
    DbusEventKey key;
    long longKey = RngUtils.randomLong();
    String stringKey = RngUtils.randomString(10);
    if (type.equals("STRING")) {
        key = new DbusEventKey((Object) stringKey);
    } else if (type.equals("LONG")) {
        key = new DbusEventKey((Object) longKey);
    } else {
        throw new DatabusRuntimeException("Unknown key type provided for the test");
    }
    dbuf.appendEvent(key, dbusEventInfo, null);
    dbuf.endEvents(1234);
    //Write the delete event
    dbuf.startEvents();
    opCode = DbusOpcode.DELETE;
    dbusEventInfo = new DbusEventInfo(opCode, //scn
    1235, //pconfigId
    (short) 2600, //Partition id
    (short) 1, timeStamp, //logical source id
    (short) 2601, schemaId, value.getBytes(Charset.defaultCharset()), false, false);
    dbuf.appendEvent(key, dbusEventInfo, null);
    dbuf.endEvents(1235);
    //Iterate and verify if the both the upsert and delete events exist
    DbusEventIterator eventIterator = dbuf.acquireIterator("eventIterator");
    //Read EOP
    assert (eventIterator.hasNext());
    eventIterator.next();
    //Read upsert event
    assert (eventIterator.hasNext());
    DbusEventInternalWritable e = eventIterator.next();
    assertEquals(DbusOpcode.UPSERT, e.getOpcode());
    if (type.equals("STRING"))
        assertEquals(stringKey.getBytes(Charset.defaultCharset()), e.keyBytes());
    else if (type.equals("LONG"))
        assertEquals(longKey, e.key());
    assertEquals(1234, e.sequence());
    //Read EOP
    assert (eventIterator.hasNext());
    eventIterator.next();
    //Read delete event
    assert (eventIterator.hasNext());
    e = eventIterator.next();
    assertEquals(DbusOpcode.DELETE, e.getOpcode());
    if (type.equals("STRING"))
        assertEquals(stringKey.getBytes(Charset.defaultCharset()), e.keyBytes());
    else if (type.equals("LONG"))
        assertEquals(longKey, e.key());
    assertEquals(1235, e.sequence());
    //Read EOP
    assert (eventIterator.hasNext());
    eventIterator.next();
}