Examples with DbusEventIterator com.linkedin.databus.core.DbusEventBuffer.DbusEventIterator used on opensource projects

Example 11 with DbusEventIterator

use of com.linkedin.databus.core.DbusEventBuffer.DbusEventIterator in project databus by linkedin.

the class TestEventLogWriter method testEventLogReaderSameBuffer.

@Test
public void testEventLogReaderSameBuffer() throws InterruptedException, InvalidConfigException {
    DbusEventBuffer dbuf = new DbusEventBuffer(getConfig(10000000, DbusEventBuffer.Config.DEFAULT_INDIVIDUAL_BUFFER_SIZE, 100000, 1000000, AllocationPolicy.MMAPPED_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE));
    String sessionDir = _writeDir + File.separator + "sameBufferTest";
    //EventLogWriter.LOG.setLevel(Level.DEBUG);
    long lastWindowScn = performWrites(dbuf, _writeDir, sessionDir, Encoding.BINARY, 1, 50000);
    DbusEventBuffer readDbuf = new DbusEventBuffer(getConfig(10000000, DbusEventBuffer.Config.DEFAULT_INDIVIDUAL_BUFFER_SIZE, 100000, 10000, AllocationPolicy.MMAPPED_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE));
    EventLogReader.Config eventLogReaderConfig = new EventLogReader.Config();
    eventLogReaderConfig.setEventBuffer(readDbuf);
    eventLogReaderConfig.setEnabled(true);
    eventLogReaderConfig.setTopLevelLogDir(_writeDir.getAbsolutePath());
    eventLogReaderConfig.setReadSessionDir(sessionDir);
    EventLogReader logReader = new EventLogReader(eventLogReaderConfig.build());
    //EventLogReader.LOG.setLevel(Level.DEBUG);
    //DbusEventBuffer.LOG.setLevel(Level.DEBUG);
    Checkpoint cp = logReader.read();
    assertTrue(cp.getWindowOffset() == -1);
    assertTrue(cp.getWindowScn() == lastWindowScn);
    // Read and Write Buffers should be the same
    DbusEventIterator writeIter = dbuf.acquireIterator("testIterator");
    DbusEventIterator readIter = readDbuf.acquireIterator("testIterator");
    //LOG.setLevel(Level.DEBUG);
    checkIterEquals(writeIter, readIter);
    dbuf.releaseIterator(writeIter);
    readDbuf.releaseIterator(readIter);
}

Example 12 with DbusEventIterator

use of com.linkedin.databus.core.DbusEventBuffer.DbusEventIterator in project databus by linkedin.

the class TestEventLogWriter method testEventLogReaderSameBufferJSON.

@Test
public void testEventLogReaderSameBufferJSON() throws InterruptedException, InvalidConfigException {
    DbusEventBuffer dbuf = new DbusEventBuffer(getConfig(10000000, DbusEventBuffer.Config.DEFAULT_INDIVIDUAL_BUFFER_SIZE, 100000, 1000000, AllocationPolicy.MMAPPED_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE));
    String sessionDir = _writeDir + File.separator + "sameBufferTestJson";
    //EventLogWriter.LOG.setLevel(Level.DEBUG);
    long lastWindowScn = performWrites(dbuf, _writeDir, sessionDir, Encoding.JSON, 1, 50000);
    DbusEventBuffer readDbuf = new DbusEventBuffer(getConfig(10000000, DbusEventBuffer.Config.DEFAULT_INDIVIDUAL_BUFFER_SIZE, 100000, 10000, AllocationPolicy.MMAPPED_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE));
    EventLogReader.Config eventLogReaderConfig = new EventLogReader.Config();
    eventLogReaderConfig.setEventBuffer(readDbuf);
    eventLogReaderConfig.setEnabled(true);
    eventLogReaderConfig.setTopLevelLogDir(_writeDir.getAbsolutePath());
    eventLogReaderConfig.setReadSessionDir(sessionDir);
    EventLogReader logReader = new EventLogReader(eventLogReaderConfig.build());
    //EventLogReader.LOG.setLevel(Level.DEBUG);
    //DbusEventBuffer.LOG.setLevel(Level.DEBUG);
    Checkpoint cp = logReader.read();
    assertTrue(cp.getWindowOffset() == -1);
    assertTrue(cp.getWindowScn() == lastWindowScn);
    // Read and Write Buffers should be the same
    DbusEventIterator writeIter = dbuf.acquireIterator("testIterator");
    DbusEventIterator readIter = readDbuf.acquireIterator("testIterator");
    //LOG.setLevel(Level.DEBUG);
    checkIterEquals(writeIter, readIter);
    dbuf.releaseIterator(writeIter);
    readDbuf.releaseIterator(readIter);
}

Example 13 with DbusEventIterator

use of com.linkedin.databus.core.DbusEventBuffer.DbusEventIterator in project databus by linkedin.

the class TestDbusEventBufferPersistence method testRestoringBufferWithEvents.

@Test
public /**
   * test:
   * 1. create buffer, push events , validate events are there
   * 2. save meta info
   * 3. create a new buffer - it should pick up all the events.Validate it
   * 4. create another buffer (without calling save metainfo again) - this one should be empty.
   *
   * @throws InvalidConfigException
   * @throws IOException
   */
void testRestoringBufferWithEvents() throws InvalidConfigException, IOException {
    int maxEventBufferSize = 1144;
    int maxIndividualBufferSize = 500;
    // must be even
    int numEvents = 10;
    DbusEventBuffer.StaticConfig conf = getConfig(maxEventBufferSize, maxIndividualBufferSize, 100, 500, AllocationPolicy.MMAPPED_MEMORY, _mmapDirStr, true);
    DbusEventBuffer dbusBuf = new DbusEventBuffer(conf);
    pushEventsToBuffer(dbusBuf, numEvents);
    // verify events are in the buffer
    DbusEventIterator it = dbusBuf.acquireIterator("allevents");
    // first event is "prev" event
    int count = -1;
    while (it.hasNext()) {
        DbusEvent e = it.next();
        Assert.assertTrue(e.isValid());
        count++;
    }
    dbusBuf.releaseIterator(it);
    Assert.assertEquals(count, numEvents);
    // save meta data
    dbusBuf.saveBufferMetaInfo(false);
    // create another similar buffer and see if has the events
    dbusBuf = new DbusEventBuffer(conf);
    dbusBuf.validateEventsInBuffer();
    it = dbusBuf.acquireIterator("alleventsNew");
    // first event is "prev" event
    count = -1;
    while (it.hasNext()) {
        DbusEvent e = it.next();
        Assert.assertTrue(e.isValid());
        count++;
    }
    Assert.assertEquals(count, numEvents);
    dbusBuf.releaseIterator(it);
    // meta file should be gone by now
    File metaFile = new File(_mmapDir, dbusBuf.metaFileName());
    Assert.assertFalse(metaFile.exists());
    // we can start a buffer and would be empty
    dbusBuf = new DbusEventBuffer(conf);
    it = dbusBuf.acquireIterator("alleventsNewEmpty");
    Assert.assertFalse(it.hasNext());
    dbusBuf.releaseIterator(it);
}

Example 14 with DbusEventIterator

use of com.linkedin.databus.core.DbusEventBuffer.DbusEventIterator in project databus by linkedin.

the class TestDbusEvent method testAppendToEventBuffer_one.

@Test
public void testAppendToEventBuffer_one() throws Exception {
    String valueStr = "testvalue";
    ByteBuffer serializationBuffer = ByteBuffer.allocate(1000).order(_eventV1Factory.getByteOrder());
    DbusEventInfo eventInfo = new DbusEventInfo(null, 2L, (short) 0, (short) 3, 4L, (short) 5, schemaId, valueStr.getBytes(Charset.defaultCharset()), false, true);
    // make this explicit
    eventInfo.setEventSerializationVersion(DbusEventFactory.DBUS_EVENT_V1);
    DbusEventFactory.serializeEvent(new DbusEventKey(1L), serializationBuffer, eventInfo);
    DbusEventInternalReadable event1 = _eventV1Factory.createReadOnlyDbusEventFromBuffer(serializationBuffer, 0);
    assertTrue("event crc correct", event1.isValid());
    //test JSON_PLAIN_VALUE
    ByteArrayOutputStream jsonOut = new ByteArrayOutputStream();
    WritableByteChannel jsonOutChannel = Channels.newChannel(jsonOut);
    event1.writeTo(jsonOutChannel, Encoding.JSON_PLAIN_VALUE);
    byte[] jsonBytes = jsonOut.toByteArray();
    String jsonString = new String(jsonBytes);
    ObjectMapper mapper = new ObjectMapper();
    Map<String, Object> jsonMap = mapper.readValue(jsonString, new TypeReference<Map<String, Object>>() {
    });
    assertEquals("key correct", 1L, ((Number) jsonMap.get("key")).longValue());
    assertEquals("sequence correct", 2L, ((Number) jsonMap.get("sequence")).longValue());
    assertEquals("partitionId correct", 3, ((Number) jsonMap.get("logicalPartitionId")).shortValue());
    assertEquals("timestamp correct", 4L, ((Number) jsonMap.get("timestampInNanos")).longValue());
    assertEquals("srcId correct", 5, ((Number) jsonMap.get("srcId")).longValue());
    assertEquals("schemaId correct", Base64.encodeBytes(schemaId), jsonMap.get("schemaId"));
    assertEquals("valueEnc correct", Encoding.JSON_PLAIN_VALUE.toString(), jsonMap.get("valueEnc"));
    assertEquals("value correct", valueStr, jsonMap.get("value"));
    DbusEventBuffer eventBuffer1 = new DbusEventBuffer(getConfig(100000, DbusEventBuffer.Config.DEFAULT_INDIVIDUAL_BUFFER_SIZE, 10000, 1000, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE));
    eventBuffer1.startEvents();
    assertEquals("json deserialization", 1, DbusEventSerializable.appendToEventBuffer(jsonString, eventBuffer1, null, false));
    eventBuffer1.endEvents(2);
    DbusEventIterator it1 = eventBuffer1.acquireIterator("it1");
    assertTrue("buffer has event", it1.hasNext());
    DbusEvent testEvent = it1.next();
    assertEquals("key correct", 1L, testEvent.key());
    assertEquals("sequence correct", 2L, testEvent.sequence());
    assertEquals("partitionId correct", 3, testEvent.logicalPartitionId());
    assertEquals("timestamp correct", 4L, testEvent.timestampInNanos());
    assertEquals("srcId correct", 5, testEvent.srcId());
    assertEquals("schemaId correct", new String(schemaId), new String(testEvent.schemaId()));
    assertEquals("value correct", valueStr, Utils.byteBufferToString(testEvent.value()));
    assertEquals("Get DbusEventKey", 1L, ((DbusEventInternalReadable) testEvent).getDbusEventKey().getLongKey().longValue());
    //test JSON
    jsonOut = new ByteArrayOutputStream();
    jsonOutChannel = Channels.newChannel(jsonOut);
    event1.writeTo(jsonOutChannel, Encoding.JSON);
    jsonBytes = jsonOut.toByteArray();
    jsonString = new String(jsonBytes);
    jsonMap = mapper.readValue(jsonString, new TypeReference<Map<String, Object>>() {
    });
    assertEquals("key correct", 1L, ((Number) jsonMap.get("key")).longValue());
    assertEquals("sequence correct", 2L, ((Number) jsonMap.get("sequence")).longValue());
    assertEquals("logicalPartitionId correct", 3, ((Number) jsonMap.get("logicalPartitionId")).shortValue());
    assertEquals("timestampInNanos correct", 4L, ((Number) jsonMap.get("timestampInNanos")).longValue());
    assertEquals("srcId correct", 5, ((Number) jsonMap.get("srcId")).longValue());
    assertEquals("schemaId correct", Base64.encodeBytes(schemaId), jsonMap.get("schemaId"));
    assertEquals("valueEnc correct", Encoding.JSON.toString(), jsonMap.get("valueEnc"));
    assertEquals("value correct", Base64.encodeBytes(valueStr.getBytes(Charset.defaultCharset())), jsonMap.get("value"));
    assertTrue("buffer has event", it1.hasNext());
    testEvent = it1.next();
    assertTrue("end of window", testEvent.isEndOfPeriodMarker());
    DbusEventBuffer eventBuffer2 = new DbusEventBuffer(getConfig(100000, DbusEventBuffer.Config.DEFAULT_INDIVIDUAL_BUFFER_SIZE, 10000, 1000, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE));
    eventBuffer2.startEvents();
    assertTrue("json deserialization", (DbusEventSerializable.appendToEventBuffer(jsonString, eventBuffer2, null, false) > 0));
    eventBuffer2.endEvents(2);
    DbusEventIterator it2 = eventBuffer2.acquireIterator("it2");
    assertTrue("buffer has event", it2.hasNext());
    testEvent = it2.next();
    assertEquals("key correct", 1L, testEvent.key());
    assertEquals("partitionId correct", 3, testEvent.logicalPartitionId());
    assertEquals("timestamp correct", 4L, testEvent.timestampInNanos());
    assertEquals("srcId correct", 5, testEvent.srcId());
    assertEquals("schemaId correct", new String(schemaId), new String(testEvent.schemaId()));
    assertEquals("value correct", valueStr, Utils.byteBufferToString(testEvent.value()));
    assertEquals("Get DbusEventKey", 1L, ((DbusEventInternalReadable) testEvent).getDbusEventKey().getLongKey().longValue());
    assertTrue("buffer has event", it2.hasNext());
    testEvent = it2.next();
    assertTrue("end of window", testEvent.isEndOfPeriodMarker());
}

Example 15 with DbusEventIterator

use of com.linkedin.databus.core.DbusEventBuffer.DbusEventIterator in project databus by linkedin.

the class ReadEventsTestParams method testIteration.

@Test
public void testIteration() throws IOException, InvalidConfigException {
    int numEntries = 50000;
    HashMap<Long, KeyValue> testDataMap = new HashMap<Long, KeyValue>(20000);
    DbusEventBuffer dbuf = new DbusEventBuffer(getConfig(10000000, DbusEventBuffer.Config.DEFAULT_INDIVIDUAL_BUFFER_SIZE, 100000, 1000000, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    dbuf.start(0);
    for (long i = 1; i < numEntries; ++i) {
        //LOG.info("Iteration:"+i);
        DbusEventKey key = new DbusEventKey(RngUtils.randomLong());
        String value = RngUtils.randomString(20);
        dbuf.startEvents();
        long ts = timeStamp + (i * 1000 * 1000);
        assertTrue(dbuf.appendEvent(key, pPartitionId, lPartitionId, ts, srcId, schemaId, value.getBytes(Charset.defaultCharset()), false));
        testDataMap.put(i, new KeyValue(key, value));
        dbuf.endEvents(i);
    }
    long minDbusEventBufferScn = dbuf.getMinScn();
    long expectedScn = minDbusEventBufferScn;
    DbusEventIterator eventIterator = dbuf.acquireIterator("eventIterator");
    DbusEventInternalWritable e = null;
    // searching for min scn
    int state = 0;
    while (eventIterator.hasNext()) {
        e = eventIterator.next();
        if (state == 0 && (e.sequence() >= minDbusEventBufferScn)) {
            // found min scn
            state = 1;
        }
        if (state == 1) {
            assertEquals(expectedScn, e.sequence());
            long ts = e.timestampInNanos();
            e = eventIterator.next();
            assertTrue(e.isEndOfPeriodMarker());
            assertEquals(expectedScn, e.sequence());
            assertEquals(ts, e.timestampInNanos());
            expectedScn = (expectedScn + 1) % numEntries;
        }
    }
    dbuf.releaseIterator(eventIterator);
}