Examples with WritableByteChannel java.nio.channels.WritableByteChannel used on opensource projects

Example 71 with WritableByteChannel

use of java.nio.channels.WritableByteChannel in project hbase by apache.

the class ByteBufferOutputStream method writeTo.

/**
  * Writes the complete contents of this byte buffer output stream to
  * the specified output stream argument.
  *
  * @param      out   the output stream to which to write the data.
  * @exception  IOException  if an I/O error occurs.
  */
public void writeTo(OutputStream out) throws IOException {
    WritableByteChannel channel = Channels.newChannel(out);
    ByteBuffer bb = curBuf.duplicate();
    bb.flip();
    channel.write(bb);
}

Example 72 with WritableByteChannel

use of java.nio.channels.WritableByteChannel in project databus by linkedin.

the class ReadEventsTestParams method testNoMissingEOWHappyPathWithMultiEventWindow.

/**
   * Testcase send the following sequence:
   * two valid packets, EOW, two valid packets, EOW
   */
@Test
public void testNoMissingEOWHappyPathWithMultiEventWindow() throws Exception {
    final DbusEventBuffer dbusBuf = new DbusEventBuffer(getConfig(1000, 1000, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE, AssertLevel.NONE));
    dbusBuf.setDropOldEvents(true);
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(6, 1, 100, 10, events);
    // conversion of readable events to writable is for TESTING ONLY:
    DbusEventInternalWritable writableEvent = DbusEventCorrupter.makeWritable(events.get(1));
    writableEvent.setSequence(events.get(0).sequence());
    writableEvent.applyCrc();
    writableEvent = DbusEventCorrupter.makeWritable(events.get(2));
    writableEvent.setSequence(events.get(0).sequence());
    writableEvent.setSrcId((short) -2);
    writableEvent.applyCrc();
    writableEvent = DbusEventCorrupter.makeWritable(events.get(3));
    writableEvent.setSequence(events.get(0).sequence() + 100);
    writableEvent.applyCrc();
    writableEvent = DbusEventCorrupter.makeWritable(events.get(4));
    writableEvent.setSequence(events.get(3).sequence());
    writableEvent.applyCrc();
    writableEvent = DbusEventCorrupter.makeWritable(events.get(5));
    writableEvent.setSequence(events.get(0).sequence() + 100);
    writableEvent.setSrcId((short) -2);
    writableEvent.applyCrc();
    // Increment the SCN and reuse
    for (int i = 0; i < 6; ++i) {
        DbusEvent e = events.get(i);
        assertTrue("invalid event #" + i, e.isValid());
    }
    // Set up the ReadChannel with 2 events
    ByteArrayOutputStream oStream = new ByteArrayOutputStream();
    WritableByteChannel oChannel = Channels.newChannel(oStream);
    for (int i = 0; i < 6; ++i) {
        ((DbusEventInternalReadable) events.get(i)).writeTo(oChannel, Encoding.BINARY);
    }
    byte[] writeBytes = oStream.toByteArray();
    ByteArrayInputStream iStream = new ByteArrayInputStream(writeBytes);
    final ReadableByteChannel rChannel = Channels.newChannel(iStream);
    try {
        dbusBuf.readEvents(rChannel);
    // Should NOT throw invalid event exception
    } catch (InvalidEventException ie) {
        LOG.error("Exception trace is " + ie.getMessage(), ie);
        Assert.fail();
        return;
    }
}

Example 73 with WritableByteChannel

use of java.nio.channels.WritableByteChannel in project databus by linkedin.

the class ReadEventsTestParams method testNoMissingEOWHappyPath.

/**
   * Testcase sends events with the following sequence :
   * valid packet, EOW, valid packet
   */
@Test
public void testNoMissingEOWHappyPath() throws Exception {
    final DbusEventBuffer dbusBuf = new DbusEventBuffer(getConfig(1000, 1000, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE, AssertLevel.NONE));
    dbusBuf.setDropOldEvents(true);
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(3, 1, 100, 10, events);
    // don't try this at home, kids!
    DbusEventInternalWritable writableEvent = DbusEventCorrupter.makeWritable(events.get(1));
    writableEvent.setSequence(events.get(0).sequence());
    writableEvent.setSrcId((short) -2);
    writableEvent.applyCrc();
    writableEvent = DbusEventCorrupter.makeWritable(events.get(2));
    writableEvent.setSequence(events.get(0).sequence() + 100);
    writableEvent.applyCrc();
    // Increment the SCN and reuse
    for (int i = 0; i < 3; ++i) {
        DbusEvent e = events.get(i);
        assertTrue("invalid event #" + i, e.isValid());
        LOG.info("DbusEvent " + i + " " + e);
    }
    // set up the ReadChannel with 2 events
    ByteArrayOutputStream oStream = new ByteArrayOutputStream();
    WritableByteChannel oChannel = Channels.newChannel(oStream);
    for (int i = 0; i < 3; ++i) {
        ((DbusEventInternalReadable) events.get(i)).writeTo(oChannel, Encoding.BINARY);
    }
    byte[] writeBytes = oStream.toByteArray();
    ByteArrayInputStream iStream = new ByteArrayInputStream(writeBytes);
    final ReadableByteChannel rChannel = Channels.newChannel(iStream);
    try {
        dbusBuf.readEvents(rChannel);
    // Should NOT throw invalid event exception
    } catch (InvalidEventException ie) {
        LOG.error("Exception trace is " + ie);
        Assert.fail();
        return;
    }
}

Example 74 with WritableByteChannel

use of java.nio.channels.WritableByteChannel in project databus by linkedin.

the class ReadEventsTestParams method testNoMissingEOWHappyPathWithMultiEventWindowWithMultiEOWs.

/**
   * Testcase send the following sequence:
   * two valid packets, EOW, EOW, one valid packet, EOW
   * @throws Exception
   */
@Test
public void testNoMissingEOWHappyPathWithMultiEventWindowWithMultiEOWs() throws Exception {
    final DbusEventBuffer dbusBuf = new DbusEventBuffer(getConfig(1000, 1000, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE, AssertLevel.NONE));
    dbusBuf.setDropOldEvents(true);
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(6, 1, 100, 10, events);
    // conversion of readable events to writable is for TESTING ONLY:
    DbusEventInternalWritable writableEvent = DbusEventCorrupter.makeWritable(events.get(1));
    writableEvent.setSequence(events.get(0).sequence());
    writableEvent.applyCrc();
    writableEvent = DbusEventCorrupter.makeWritable(events.get(2));
    writableEvent.setSequence(events.get(0).sequence());
    writableEvent.setSrcId((short) -2);
    writableEvent.applyCrc();
    writableEvent = DbusEventCorrupter.makeWritable(events.get(3));
    writableEvent.setSequence(events.get(0).sequence() + 50);
    writableEvent.setSrcId((short) -2);
    writableEvent.applyCrc();
    writableEvent = DbusEventCorrupter.makeWritable(events.get(4));
    writableEvent.setSequence(events.get(0).sequence() + 100);
    writableEvent.applyCrc();
    writableEvent = DbusEventCorrupter.makeWritable(events.get(5));
    writableEvent.setSequence(events.get(0).sequence() + 100);
    writableEvent.setSrcId((short) -2);
    writableEvent.applyCrc();
    // Increment the SCN and reuse
    for (int i = 0; i < 6; ++i) {
        DbusEvent e = events.get(i);
        assertTrue("invalid event #" + i, e.isValid());
    }
    // Set up the ReadChannel with 2 events
    ByteArrayOutputStream oStream = new ByteArrayOutputStream();
    WritableByteChannel oChannel = Channels.newChannel(oStream);
    for (int i = 0; i < 6; ++i) {
        ((DbusEventInternalReadable) events.get(i)).writeTo(oChannel, Encoding.BINARY);
    }
    byte[] writeBytes = oStream.toByteArray();
    ByteArrayInputStream iStream = new ByteArrayInputStream(writeBytes);
    final ReadableByteChannel rChannel = Channels.newChannel(iStream);
    try {
        dbusBuf.readEvents(rChannel);
    // Should NOT throw invalid event exception
    } catch (InvalidEventException ie) {
        LOG.error("Exception trace is " + ie.getMessage(), ie);
        Assert.fail();
        return;
    }
}

Example 75 with WritableByteChannel

use of java.nio.channels.WritableByteChannel in project databus by linkedin.

the class TestDbusEventBufferMult method testMultiPPartionStreamFromLatest.

@Test
public void testMultiPPartionStreamFromLatest() throws Exception {
    createBufMult();
    PhysicalPartition[] p = { _pConfigs[0].getPhysicalPartition(), _pConfigs[1].getPhysicalPartition(), _pConfigs[2].getPhysicalPartition() };
    //generate a bunch of windows for 3 partitions
    int windowsNum = 10;
    for (int i = 1; i <= windowsNum; ++i) {
        DbusEventBufferAppendable buf = _eventBufferMult.getDbusEventBufferAppendable(p[0]);
        buf.startEvents();
        byte[] schema = "abcdefghijklmnop".getBytes(Charset.defaultCharset());
        assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null));
        buf.endEvents(100 * i, null);
        buf = _eventBufferMult.getDbusEventBufferAppendable(p[1]);
        buf.startEvents();
        assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null));
        assertTrue(buf.appendEvent(new DbusEventKey(2), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null));
        buf.endEvents(100 * i + 1, null);
        buf = _eventBufferMult.getDbusEventBufferAppendable(p[2]);
        buf.startEvents();
        assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null));
        assertTrue(buf.appendEvent(new DbusEventKey(2), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null));
        assertTrue(buf.appendEvent(new DbusEventKey(3), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null));
        buf.endEvents(100 * i + 2, null);
    }
    String[] pnames = new String[p.length];
    int count = 0;
    for (PhysicalPartition ip : p) {
        pnames[count++] = ip.toSimpleString();
    }
    StatsCollectors<DbusEventsStatisticsCollector> statsColl = createStats(pnames);
    PhysicalPartitionKey[] pkeys = { new PhysicalPartitionKey(p[0]), new PhysicalPartitionKey(p[1]), new PhysicalPartitionKey(p[2]) };
    CheckpointMult cpMult = new CheckpointMult();
    for (int i = 0; i < 3; ++i) {
        Checkpoint cp = new Checkpoint();
        cp.setFlexible();
        cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
        cpMult.addCheckpoint(p[i], cp);
    }
    DbusEventBufferBatchReadable reader = _eventBufferMult.getDbusEventBufferBatchReadable(cpMult, Arrays.asList(pkeys), statsColl);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    WritableByteChannel writeChannel = Channels.newChannel(baos);
    // Set streamFromLatestScn == true
    reader.streamEvents(true, 1000000, writeChannel, Encoding.BINARY, new AllowAllDbusFilter());
    writeChannel.close();
    baos.close();
    //make sure we got the physical partition names right
    List<String> ppartNames = statsColl.getStatsCollectorKeys();
    assertEquals(ppartNames.size(), 3);
    HashSet<String> expectedPPartNames = new HashSet<String>(Arrays.asList(p[0].toSimpleString(), p[1].toSimpleString(), p[2].toSimpleString()));
    for (String ppartName : ppartNames) {
        assertTrue(expectedPPartNames.contains(ppartName));
    }
    //verify event counts per partition
    DbusEventsTotalStats[] ppartStats = { statsColl.getStatsCollector(p[0].toSimpleString()).getTotalStats(), statsColl.getStatsCollector(p[1].toSimpleString()).getTotalStats(), statsColl.getStatsCollector(p[2].toSimpleString()).getTotalStats() };
    // Only the last window is returned in each of the partitions
    assertEquals(ppartStats[0].getNumDataEvents(), 1);
    assertEquals(ppartStats[1].getNumDataEvents(), 2);
    assertEquals(ppartStats[2].getNumDataEvents(), 3);
    assertEquals(ppartStats[0].getNumSysEvents(), 1);
    assertEquals(ppartStats[1].getNumSysEvents(), 1);
    assertEquals(ppartStats[2].getNumSysEvents(), 1);
    assertEquals(statsColl.getStatsCollector().getTotalStats().getNumDataEvents(), (1 + 2 + 3));
    assertEquals(statsColl.getStatsCollector().getTotalStats().getNumSysEvents(), (1 + 1 + 1));
    assertEquals(statsColl.getStatsCollector().getTotalStats().getMaxTimeLag(), Math.max(ppartStats[0].getTimeLag(), Math.max(ppartStats[1].getTimeLag(), ppartStats[2].getTimeLag())));
    assertEquals(statsColl.getStatsCollector().getTotalStats().getMinTimeLag(), Math.min(ppartStats[0].getTimeLag(), Math.min(ppartStats[1].getTimeLag(), ppartStats[2].getTimeLag())));
}