Examples with DbusEventIterator com.linkedin.databus.core.DbusEventBuffer.DbusEventIterator used on opensource projects

Example 1 with DbusEventIterator

use of com.linkedin.databus.core.DbusEventBuffer.DbusEventIterator in project databus by linkedin.

the class TestEventLogWriter method testEventLogReaderSmallerBufferJSONPlusMoreWrites.

@Test
public void testEventLogReaderSmallerBufferJSONPlusMoreWrites() throws InterruptedException, InvalidConfigException {
    DbusEventBuffer dbuf = new DbusEventBuffer(getConfig(10000000, DbusEventBuffer.Config.DEFAULT_INDIVIDUAL_BUFFER_SIZE, 100000, 1000000, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE));
    String sessionDir = _writeDir + File.separator + "smallerBufferTestJSON";
    long lastWindowScn = performWrites(dbuf, _writeDir, sessionDir, Encoding.JSON, 1, 10000);
    DbusEventBuffer readDbuf = new DbusEventBuffer(getConfig(100000, DbusEventBuffer.Config.DEFAULT_INDIVIDUAL_BUFFER_SIZE, 100000, 10000, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE));
    EventLogReader.Config eventLogReaderConfig = new EventLogReader.Config();
    eventLogReaderConfig.setEventBuffer(readDbuf);
    eventLogReaderConfig.setEnabled(true);
    eventLogReaderConfig.setTopLevelLogDir(_writeDir.getAbsolutePath());
    eventLogReaderConfig.setReadSessionDir(sessionDir);
    EventLogReader logReader = new EventLogReader(eventLogReaderConfig.build());
    Checkpoint cp = logReader.read();
    assertTrue(cp.getWindowOffset() == -1);
    assertTrue(cp.getWindowScn() == lastWindowScn);
    // Read and Write Buffers should be the same
    DbusEventIterator writeIter = dbuf.acquireIterator("testIterator");
    DbusEventIterator readIter = readDbuf.acquireIterator("testIterator");
    // LOG.info("First event = " + readIter.next());
    // LOG.info("Second event = " + readIter.next());
    // LOG.info("Third event = "  + readIter.next());
    checkIterProperSubset(writeIter, readIter);
    dbuf.releaseIterator(writeIter);
    readDbuf.releaseIterator(readIter);
    dbuf.endEvents(lastWindowScn);
    lastWindowScn = performWrites(dbuf, _writeDir, sessionDir, Encoding.JSON, lastWindowScn, 200);
}

Example 2 with DbusEventIterator

use of com.linkedin.databus.core.DbusEventBuffer.DbusEventIterator in project databus by linkedin.

the class TestEventLogWriter method testEventLogReaderSmallerBuffer.

@Test
public void testEventLogReaderSmallerBuffer() throws InterruptedException, InvalidConfigException {
    DbusEventBuffer dbuf = new DbusEventBuffer(getConfig(10000000, DbusEventBuffer.Config.DEFAULT_INDIVIDUAL_BUFFER_SIZE, 100000, 1000000, AllocationPolicy.MMAPPED_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE));
    String sessionDir = _writeDir + File.separator + "smallerBufferTest";
    long lastWindowScn = performWrites(dbuf, _writeDir, sessionDir, Encoding.BINARY, 1, 50000);
    //    ScnIndex.LOG.setLevel(Level.DEBUG);
    DbusEventBuffer readDbuf = new DbusEventBuffer(getConfig(100000, DbusEventBuffer.Config.DEFAULT_INDIVIDUAL_BUFFER_SIZE, 100000, 10000, AllocationPolicy.MMAPPED_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE));
    EventLogReader.Config eventLogReaderConfig = new EventLogReader.Config();
    eventLogReaderConfig.setEventBuffer(readDbuf);
    eventLogReaderConfig.setEnabled(true);
    eventLogReaderConfig.setTopLevelLogDir(_writeDir.getAbsolutePath());
    eventLogReaderConfig.setReadSessionDir(sessionDir);
    EventLogReader logReader = new EventLogReader(eventLogReaderConfig.build());
    Checkpoint cp = logReader.read();
    assertTrue(cp.getWindowOffset() == -1);
    assertTrue(cp.getWindowScn() == lastWindowScn);
    // Read and Write Buffers should be the same
    DbusEventIterator writeIter = dbuf.acquireIterator("testIterator");
    DbusEventIterator readIter = readDbuf.acquireIterator("testIterator");
    // LOG.info("First event = " + readIter.next());
    // LOG.info("Second event = " + readIter.next());
    // LOG.info("Third event = "  + readIter.next());
    checkIterProperSubset(writeIter, readIter);
    dbuf.releaseIterator(writeIter);
    readDbuf.releaseIterator(readIter);
}

Example 3 with DbusEventIterator

use of com.linkedin.databus.core.DbusEventBuffer.DbusEventIterator in project databus by linkedin.

the class TestEventLogWriter method testEventLogReaderSmallerBufferJSON.

@Test
public void testEventLogReaderSmallerBufferJSON() throws InterruptedException, InvalidConfigException {
    DbusEventBuffer dbuf = new DbusEventBuffer(getConfig(10000000, DbusEventBuffer.Config.DEFAULT_INDIVIDUAL_BUFFER_SIZE, 100000, 1000000, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE));
    String sessionDir = _writeDir + File.separator + "smallerBufferTestJSON";
    long lastWindowScn = performWrites(dbuf, _writeDir, sessionDir, Encoding.JSON, 1, 50000);
    DbusEventBuffer readDbuf = new DbusEventBuffer(getConfig(100000, DbusEventBuffer.Config.DEFAULT_INDIVIDUAL_BUFFER_SIZE, 100000, 10000, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE));
    EventLogReader.Config eventLogReaderConfig = new EventLogReader.Config();
    eventLogReaderConfig.setEventBuffer(readDbuf);
    eventLogReaderConfig.setEnabled(true);
    eventLogReaderConfig.setTopLevelLogDir(_writeDir.getAbsolutePath());
    eventLogReaderConfig.setReadSessionDir(sessionDir);
    EventLogReader logReader = new EventLogReader(eventLogReaderConfig.build());
    Checkpoint cp = logReader.read();
    assertTrue(cp.getWindowOffset() == -1);
    assertTrue(cp.getWindowScn() == lastWindowScn);
    // Read and Write Buffers should be the same
    DbusEventIterator writeIter = dbuf.acquireIterator("testIterator");
    DbusEventIterator readIter = readDbuf.acquireIterator("testIterator");
    // LOG.info("First event = " + readIter.next());
    // LOG.info("Second event = " + readIter.next());
    // LOG.info("Third event = "  + readIter.next());
    checkIterProperSubset(writeIter, readIter);
    dbuf.releaseIterator(writeIter);
    readDbuf.releaseIterator(readIter);
}

Example 4 with DbusEventIterator

use of com.linkedin.databus.core.DbusEventBuffer.DbusEventIterator in project databus by linkedin.

the class TestGoldenGateEventProducer method testAddEventToBuffer.

@Test
public void testAddEventToBuffer() throws InvalidConfigException, UnsupportedKeyException, DatabusException {
    // No rate control
    long rate = 0;
    PhysicalSourceStaticConfig pssc = buildPssc(rate, 0L);
    long scn = 10;
    DbusEventBuffer mb = (DbusEventBuffer) createBufMult(pssc);
    GoldenGateEventProducer gg = new GoldenGateEventProducer(pssc, null, mb, null, null);
    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates = new ArrayList<TransactionState.PerSourceTransactionalUpdate>(10);
    int sourceId = 505;
    HashSet<DBUpdateImage> db = new HashSet<DBUpdateImage>();
    Object key = new String("name");
    Schema.Type keyType = Schema.Type.RECORD;
    ColumnsState.KeyPair kp = new ColumnsState.KeyPair(key, keyType);
    ArrayList<ColumnsState.KeyPair> keyPairs = new ArrayList<ColumnsState.KeyPair>(1);
    keyPairs.add(kp);
    Schema s = Schema.parse(avroSchema);
    GenericRecord gr = new GenericData.Record(s);
    gr.put("name", "phani");
    DBUpdateImage dbi = new DBUpdateImage(keyPairs, scn, gr, s, DbUpdateState.DBUpdateImage.OpType.INSERT, false);
    db.add(dbi);
    TransactionState.PerSourceTransactionalUpdate dbUpdate = new TransactionState.PerSourceTransactionalUpdate(sourceId, db);
    dbUpdates.add(dbUpdate);
    long timestamp = System.nanoTime();
    gg.addEventToBuffer(dbUpdates, new TransactionInfo(0, 0, timestamp, scn));
    Assert.assertEquals(gg.getRateControl().getNumSleeps(), 0);
    DbusEventIterator iter = mb.acquireIterator("test");
    int count = 0;
    long eventTs = 0;
    while (iter.hasNext()) {
        DbusEvent e = iter.next();
        if (count == 1) {
            // first event prev control event
            eventTs = e.timestampInNanos();
        }
        count++;
    }
    Assert.assertEquals("Event timestamp in Ns", timestamp, eventTs);
    Assert.assertEquals("Got events ", 3, count);
    return;
}

Example 5 with DbusEventIterator

use of com.linkedin.databus.core.DbusEventBuffer.DbusEventIterator in project databus by linkedin.

the class DbusEventBufferConsumer method run.

@Override
public void run() {
    reset();
    try {
        int totalEvents = 0;
        long allTotalEvents = 0;
        DbusEventIterator iDbusEvent = _buffer.acquireIterator("Test_DbusEventBufferConsumer");
        do {
            if (!iDbusEvent.hasNext()) {
                if (!_invalidEvent) {
                    if (_deletionInterval > 0) {
                        iDbusEvent.remove();
                    }
                    try {
                        iDbusEvent.await();
                    } catch (InterruptedException e) {
                        // TODO Auto-generated catch block
                        e.printStackTrace();
                        return;
                    }
                } else {
                    //try and consume as many events as possible
                    if (allTotalEvents >= eventsReadTillInvalidEvent()) {
                        LOG.info("total events read until invalid event=" + allTotalEvents + "; terminating");
                        stop();
                    }
                }
            }
            while (iDbusEvent.hasNext()) {
                DbusEventInternalReadable e = iDbusEvent.next();
                ++allTotalEvents;
                if (!e.isCheckpointMessage() && !e.isControlMessage() && !e.isEndOfPeriodMarker()) {
                    //needs to be idempotent; so - ensure that duplicates are dropped;
                    if (!_seenKeys.contains(e.key())) {
                        //deep copy
                        _out.add(e.createCopy());
                        _seenKeys.add(e.key());
                        ++totalEvents;
                    }
                }
                if ((_deletionInterval > 0) && allTotalEvents % _deletionInterval == 0) {
                    iDbusEvent.remove();
                }
            }
        } while (totalEvents < _maxEvents && !_stop);
        iDbusEvent.remove();
    } catch (RuntimeException e) {
        _exceptionThrown = e;
        LOG.error("consumer exception:" + e.getMessage(), e);
    } catch (Error e) {
        _exceptionThrown = e;
        LOG.error("consumer error:" + e.getMessage(), e);
    }
}