Examples with ReadableByteChannel java.nio.channels.ReadableByteChannel used on opensource projects

Example 91 with ReadableByteChannel

use of java.nio.channels.ReadableByteChannel in project databus by linkedin.

the class ReadEventsTestParams method testMissingEOWUnhappyPath.

/**
 * Testcase sends events with appropriate EOW events in between
 * valid packet, no EOW for prior packet, valid packet with a higher scn.
 * @throws Exception
 */
@Test
public void testMissingEOWUnhappyPath() throws Exception {
    final DbusEventBuffer dbusBuf = new DbusEventBuffer(getConfig(1000, 1000, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE, AssertLevel.NONE));
    dbusBuf.setDropOldEvents(true);
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(3, 1, 100, 10, events);
    // conversion of readable events to writable is for TESTING ONLY:
    DbusEventInternalWritable writableEvent = DbusEventCorrupter.makeWritable(events.get(1));
    // No EOW for event 0.
    writableEvent.setSequence(events.get(0).sequence() + 50);
    writableEvent.applyCrc();
    writableEvent = DbusEventCorrupter.makeWritable(events.get(2));
    // No EOW for event 1.
    writableEvent.setSequence(events.get(0).sequence() + 100);
    writableEvent.applyCrc();
    // Increment the SCN and reuse
    for (int i = 0; i < 3; ++i) {
        DbusEvent e = events.get(i);
        assertTrue("invalid event #" + i, e.isValid());
        LOG.info("DbusEvent " + i + " " + e);
    }
    // Set up the ReadChannel with 2 events
    ByteArrayOutputStream oStream = new ByteArrayOutputStream();
    WritableByteChannel oChannel = Channels.newChannel(oStream);
    for (int i = 0; i < 3; ++i) {
        ((DbusEventInternalReadable) events.get(i)).writeTo(oChannel, Encoding.BINARY);
    }
    byte[] writeBytes = oStream.toByteArray();
    ByteArrayInputStream iStream = new ByteArrayInputStream(writeBytes);
    final ReadableByteChannel rChannel = Channels.newChannel(iStream);
    try {
        dbusBuf.readEvents(rChannel);
        Assert.fail();
    } catch (InvalidEventException ie) {
        LOG.info("Exception trace is " + ie);
        return;
    }
}

Example 92 with ReadableByteChannel

use of java.nio.channels.ReadableByteChannel in project databus by linkedin.

the class ReadEventsTestParams method testBufferOverFlow.

/*
   * TestCase to recreate the BufferOverFlowException issue tracked in DDS-793
   */
@Test
public void testBufferOverFlow() throws Exception {
    // DbusEventBuffer.LOG.setLevel(Level.DEBUG);
    final Logger log = Logger.getLogger("TestDbusEventBuffer.testBufferOverflow");
    // log.setLevel(Level.INFO);
    log.info("starting");
    DbusEventBuffer dbusBuf = new DbusEventBuffer(getConfig(1000, 1000, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE, AssertLevel.ALL));
    final DbusEventBuffer dbusBuf2 = new DbusEventBuffer(getConfig(2000, 2000, 100, 1000, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE, AssertLevel.NONE));
    BufferPositionParser parser = dbusBuf.getBufferPositionParser();
    final BufferPositionParser parser2 = dbusBuf2.getBufferPositionParser();
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(12, 12, 100, 10, events);
    log.info("generate sample events to the EventBuffer");
    DbusEventAppender appender = new DbusEventAppender(events, dbusBuf, null);
    appender.run();
    log.info("dbusBuf : Head:" + parser.toString(dbusBuf.getHead()) + ",Tail:" + parser.toString(dbusBuf.getTail()));
    ByteBuffer[] buf = dbusBuf.getBuffer();
    byte[] b = new byte[(int) dbusBuf.getTail()];
    buf[0].position(0);
    buf[0].get(b);
    log.info("copy data to the destination buffer: 1");
    ReadableByteChannel rChannel = Channels.newChannel(new ByteArrayInputStream(b));
    dbusBuf2.readEvents(rChannel);
    log.info("dbusBuf2 : Head:" + parser2.toString(dbusBuf2.getHead()) + ",Tail:" + parser2.toString(dbusBuf2.getTail()));
    rChannel.close();
    log.info("copy data to the destination buffer: 2");
    rChannel = Channels.newChannel(new ByteArrayInputStream(b));
    dbusBuf2.readEvents(rChannel);
    log.info("dbusBuf2 : Head:" + parser2.toString(dbusBuf2.getHead()) + ",Tail:" + parser2.toString(dbusBuf2.getTail()));
    log.info("Buffer Size is :" + dbusBuf2.getBuffer().length);
    rChannel.close();
    log.info("process data in destination buffer: 1");
    DbusEventBuffer.DbusEventIterator itr = dbusBuf2.acquireIterator("dummy1");
    for (int i = 0; i < 15; i++) {
        itr.next();
        itr.remove();
    }
    itr.close();
    log.info("dbusBuf2 : Head:" + parser2.toString(dbusBuf2.getHead()) + ",Tail:" + parser2.toString(dbusBuf2.getTail()));
    log.info("copy data to the destination buffer: 3");
    rChannel = Channels.newChannel(new ByteArrayInputStream(b));
    dbusBuf2.readEvents(rChannel);
    ByteBuffer[] buf2 = dbusBuf2.getBuffer();
    log.info("dbusBuf2 : Head:" + parser2.toString(dbusBuf2.getHead()) + ",Tail:" + parser2.toString(dbusBuf2.getTail()));
    log.info("dbusBuf2 : Buffer :" + buf2[0]);
    rChannel.close();
    log.info("process data in destination buffer: 2");
    itr = dbusBuf2.acquireIterator("dummy2");
    for (int i = 0; i < 15; i++) {
        itr.next();
        itr.remove();
    }
    itr.close();
    log.info("dbusBuf2 : Head:" + parser2.toString(dbusBuf2.getHead()) + ",Tail:" + parser2.toString(dbusBuf2.getTail()));
    log.info("dbusBuf2 : Buffer :" + buf2[0]);
    log.info("generate more sample events to the EventBuffer");
    dbusBuf = new DbusEventBuffer(getConfig(2000, 2000, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE, AssertLevel.ALL));
    events = new Vector<DbusEvent>();
    generator.generateEvents(8, 9, 150, 52, events);
    log.info("Events Size is :" + events.get(0).size());
    appender = new DbusEventAppender(events, dbusBuf, null);
    appender.run();
    final AtomicBoolean stopReader = new AtomicBoolean(false);
    Runnable reader = new Runnable() {

        @Override
        public void run() {
            try {
                Thread.sleep(5 * 1000);
            } catch (InterruptedException ie) {
            }
            DbusEventBuffer.DbusEventIterator itr = dbusBuf2.acquireIterator("dummy3");
            log.info("Reader iterator:" + itr);
            while (!stopReader.get() || itr.hasNext()) {
                while (itr.hasNext()) {
                    itr.next();
                    itr.remove();
                }
                itr.await(100, TimeUnit.MILLISECONDS);
            }
            itr.close();
            log.info("Reader Thread: dbusBuf2 : Head:" + parser2.toString(dbusBuf2.getHead()) + ",Tail:" + parser2.toString(dbusBuf2.getTail()));
            ByteBuffer[] buf = dbusBuf2.getBuffer();
            log.info("Reader Tread : dbusBuf2 : Buffer :" + buf[0]);
            log.info("Reader iterator:" + itr);
        }
    };
    log.info("generate sample events to the EventBuffer");
    Thread t = new Thread(reader, "BufferOverflowReader");
    b = new byte[(int) dbusBuf.getTail()];
    buf = dbusBuf.getBuffer();
    buf[0].position(0);
    buf[0].get(b);
    log.info("copy data to the destination buffer: 4");
    log.info("Size is :" + b.length);
    rChannel = Channels.newChannel(new ByteArrayInputStream(b));
    // <=== Overflow happened at this point
    dbusBuf2.readEvents(rChannel);
    rChannel.close();
    log.info("dbusBuf2 : Head:" + parser2.toString(dbusBuf2.getHead()) + ",Tail:" + parser2.toString(dbusBuf2.getTail()));
    log.info("dbusBuf2 : Buffer :" + buf2[0]);
    log.info("test if the readEvents can allow reader to proceed while it is blocked");
    rChannel = Channels.newChannel(new ByteArrayInputStream(b));
    log.info("start reader thread");
    t.start();
    log.info("copy data to the destination buffer: 5");
    dbusBuf2.readEvents(rChannel);
    rChannel.close();
    log.info("data copied to the destination buffer: 5");
    stopReader.set(true);
    t.join(20000);
    log.info("check if dbusBuf2 is empty");
    Assert.assertTrue(!t.isAlive());
    if (!dbusBuf2.empty()) {
        log.error("dbusBuf2 not empty: " + dbusBuf2);
    }
    Assert.assertTrue(dbusBuf2.toString(), dbusBuf2.empty());
    log.info("dbusBuf2 : Head:" + parser2.toString(dbusBuf2.getHead()) + ",Tail:" + parser2.toString(dbusBuf2.getTail()));
    log.info("dbusBuf2 : Buffer :" + buf2[0]);
    log.info("done");
}

Example 93 with ReadableByteChannel

use of java.nio.channels.ReadableByteChannel in project databus by linkedin.

the class ReadEventsTestParams method testBigReadEventBuffer.

@Test
public /*
   * ReadBuffer Size is bigger than the overall EVB size.
   * A large read happens to EVB which is bigger than its size.
   */
void testBigReadEventBuffer() throws Exception {
    final Logger log = Logger.getLogger("TestDbusEventBuffer.testBigReadEventBuffer");
    // log.setLevel(Level.INFO);
    log.info("starting");
    final DbusEventBuffer dbusBuf = new DbusEventBuffer(getConfig(4000, 4000, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE, AssertLevel.ALL));
    final DbusEventBuffer dbusBuf2 = new DbusEventBuffer(getConfig(1000, 1000, 100, 3000, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE, AssertLevel.NONE));
    // dbusBuf2.getLog().setLevel(Level.DEBUG);
    BufferPositionParser parser = dbusBuf.getBufferPositionParser();
    final BufferPositionParser parser2 = dbusBuf2.getBufferPositionParser();
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(24, 24, 100, 10, events);
    log.info("Num Events :" + events.size());
    // Add events to the EventBuffer
    DbusEventAppender appender = new DbusEventAppender(events, dbusBuf, null);
    appender.run();
    final AtomicBoolean stopReader = new AtomicBoolean(false);
    log.info("dbusBuf : Head:" + parser.toString(dbusBuf.getHead()) + ",Tail:" + parser.toString(dbusBuf.getTail()));
    class EvbReader implements Runnable {

        private int _count = 0;

        public EvbReader() {
            _count = 0;
        }

        public int getCount() {
            return _count;
        }

        @Override
        public void run() {
            try {
                Thread.sleep(5 * 1000);
            } catch (InterruptedException ie) {
            }
            DbusEventBuffer.DbusEventIterator itr = dbusBuf2.acquireIterator("dummy");
            log.info("Reader iterator:" + itr);
            while (!stopReader.get() || itr.hasNext()) {
                while (itr.hasNext()) {
                    itr.next();
                    itr.remove();
                    _count++;
                }
                itr.await(100, TimeUnit.MILLISECONDS);
            }
            log.info("Reader Thread: dbusBuf2 : Head:" + parser2.toString(dbusBuf2.getHead()) + ",Tail:" + parser2.toString(dbusBuf2.getTail()));
            ByteBuffer[] buf = dbusBuf2.getBuffer();
            log.info("Reader Thread : dbusBuf2 : Buffer :" + buf[0]);
            log.info("Count is :" + _count);
            log.info("Reader iterator:" + itr);
        }
    }
    ;
    EvbReader reader = new EvbReader();
    Thread t = new Thread(reader, "BigReadEventReader");
    ByteBuffer[] buf = dbusBuf.getBuffer();
    byte[] b = new byte[(int) dbusBuf.getTail()];
    buf[0].position(0);
    buf[0].get(b);
    ReadableByteChannel rChannel = Channels.newChannel(new ByteArrayInputStream(b));
    t.start();
    dbusBuf2.readEvents(rChannel);
    stopReader.set(true);
    t.join(20000);
    Assert.assertTrue(!t.isAlive());
    DbusEventBuffer.DbusEventIterator itr2 = dbusBuf2.acquireIterator("dummy");
    int count = 0;
    while (itr2.hasNext()) {
        itr2.next();
        itr2.remove();
        count++;
    }
    log.info("Total Count :" + (count + reader.getCount()));
    log.info("Head :" + dbusBuf2.getHead() + ", Tail :" + dbusBuf2.getTail());
    assertEquals("Total Count", 26, (count + reader.getCount()));
    assertEquals("Head == Tail", dbusBuf2.getHead(), dbusBuf2.getTail());
    assertEquals("Head Check:", 2890, dbusBuf2.getHead());
    log.info("done");
}

Example 94 with ReadableByteChannel

use of java.nio.channels.ReadableByteChannel in project databus by linkedin.

the class ReadEventsTestParams method readDataAtDestination.

public void readDataAtDestination() throws InterruptedException, IOException {
    final ReadableByteChannel readChannel1 = Channels.newChannel(new ByteArrayInputStream(_srcByteStr.toByteArray()));
    final AtomicInteger eventsRead1 = new AtomicInteger(-1);
    final AtomicBoolean hasError1 = new AtomicBoolean(false);
    _log.info("Reading events on client side");
    // run the read in a separate thread in case it hangs
    Thread readThread1 = new Thread(new Runnable() {

        @Override
        public void run() {
            try {
                eventsRead1.set(_destBuf.readEvents(readChannel1, _destBufStats));
            } catch (InvalidEventException e) {
                _log.error("readEvents error: " + e.getMessage(), e);
                hasError1.set(true);
            }
        }
    }, "readEvents");
    readThread1.setDaemon(true);
    readThread1.start();
    final long timeout = _debuggingMode ? 100000000 : 1000;
    readThread1.join(timeout);
    readChannel1.close();
    // smoke tests
    Assert.assertTrue(!readThread1.isAlive());
    Assert.assertEquals(_expectDestReadError, hasError1.get());
    if (!_expectDestReadError)
        Assert.assertEquals(_numStreamedEvents, eventsRead1.get());
}

Example 95 with ReadableByteChannel

use of java.nio.channels.ReadableByteChannel in project databus by linkedin.

the class RelayPullThread method writeEventToRelayDispatcher.

private void writeEventToRelayDispatcher(ConnectionState curState, DbusEvent event, String message) throws InterruptedException, InvalidEventException {
    boolean success = false;
    // Create a infinite backoff timer that waits for maximum of 1 sec
    // for writing the control message to evb
    BackoffTimerStaticConfig timerConfig = new BackoffTimerStaticConfig(1, 1000, 1, 1, -1);
    BackoffTimer timer = new BackoffTimer("EVB More Space Timer", timerConfig);
    timer.reset();
    byte[] eventBytes = new byte[event.size()];
    _log.info("Event size: " + eventBytes.length);
    _log.info("Event:" + event.toString());
    event.getRawBytes().get(eventBytes);
    UnifiedClientStats unifiedClientStats = _sourcesConn.getUnifiedClientStats();
    while ((!success) && (timer.getRemainingRetriesNum() > 0)) {
        ByteArrayInputStream cpIs = new ByteArrayInputStream(eventBytes);
        ReadableByteChannel cpRbc = Channels.newChannel(cpIs);
        sendHeartbeat(unifiedClientStats);
        int ecnt = curState.getDataEventsBuffer().readEvents(cpRbc);
        if (ecnt <= 0) {
            _log.error("Not enough spece in the event buffer to add a control message :" + message);
            boolean interrupted = !timer.backoffAndSleep();
            if (interrupted)
                throw new InterruptedException("Got interrupted while waiting to write control Message to EVB : " + message);
        } else {
            _log.info("Sent a control message :" + message);
            success = true;
        }
    }
}