Examples with WritableByteChannel java.nio.channels.WritableByteChannel used on opensource projects

Example 81 with WritableByteChannel

use of java.nio.channels.WritableByteChannel in project databus by linkedin.

the class TestDbusEvent method testAppendToEventBuffer_one.

@Test
public void testAppendToEventBuffer_one() throws Exception {
    String valueStr = "testvalue";
    ByteBuffer serializationBuffer = ByteBuffer.allocate(1000).order(_eventV1Factory.getByteOrder());
    DbusEventInfo eventInfo = new DbusEventInfo(null, 2L, (short) 0, (short) 3, 4L, (short) 5, schemaId, valueStr.getBytes(Charset.defaultCharset()), false, true);
    // make this explicit
    eventInfo.setEventSerializationVersion(DbusEventFactory.DBUS_EVENT_V1);
    DbusEventFactory.serializeEvent(new DbusEventKey(1L), serializationBuffer, eventInfo);
    DbusEventInternalReadable event1 = _eventV1Factory.createReadOnlyDbusEventFromBuffer(serializationBuffer, 0);
    assertTrue("event crc correct", event1.isValid());
    //test JSON_PLAIN_VALUE
    ByteArrayOutputStream jsonOut = new ByteArrayOutputStream();
    WritableByteChannel jsonOutChannel = Channels.newChannel(jsonOut);
    event1.writeTo(jsonOutChannel, Encoding.JSON_PLAIN_VALUE);
    byte[] jsonBytes = jsonOut.toByteArray();
    String jsonString = new String(jsonBytes);
    ObjectMapper mapper = new ObjectMapper();
    Map<String, Object> jsonMap = mapper.readValue(jsonString, new TypeReference<Map<String, Object>>() {
    });
    assertEquals("key correct", 1L, ((Number) jsonMap.get("key")).longValue());
    assertEquals("sequence correct", 2L, ((Number) jsonMap.get("sequence")).longValue());
    assertEquals("partitionId correct", 3, ((Number) jsonMap.get("logicalPartitionId")).shortValue());
    assertEquals("timestamp correct", 4L, ((Number) jsonMap.get("timestampInNanos")).longValue());
    assertEquals("srcId correct", 5, ((Number) jsonMap.get("srcId")).longValue());
    assertEquals("schemaId correct", Base64.encodeBytes(schemaId), jsonMap.get("schemaId"));
    assertEquals("valueEnc correct", Encoding.JSON_PLAIN_VALUE.toString(), jsonMap.get("valueEnc"));
    assertEquals("value correct", valueStr, jsonMap.get("value"));
    DbusEventBuffer eventBuffer1 = new DbusEventBuffer(getConfig(100000, DbusEventBuffer.Config.DEFAULT_INDIVIDUAL_BUFFER_SIZE, 10000, 1000, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE));
    eventBuffer1.startEvents();
    assertEquals("json deserialization", 1, DbusEventSerializable.appendToEventBuffer(jsonString, eventBuffer1, null, false));
    eventBuffer1.endEvents(2);
    DbusEventIterator it1 = eventBuffer1.acquireIterator("it1");
    assertTrue("buffer has event", it1.hasNext());
    DbusEvent testEvent = it1.next();
    assertEquals("key correct", 1L, testEvent.key());
    assertEquals("sequence correct", 2L, testEvent.sequence());
    assertEquals("partitionId correct", 3, testEvent.logicalPartitionId());
    assertEquals("timestamp correct", 4L, testEvent.timestampInNanos());
    assertEquals("srcId correct", 5, testEvent.srcId());
    assertEquals("schemaId correct", new String(schemaId), new String(testEvent.schemaId()));
    assertEquals("value correct", valueStr, Utils.byteBufferToString(testEvent.value()));
    assertEquals("Get DbusEventKey", 1L, ((DbusEventInternalReadable) testEvent).getDbusEventKey().getLongKey().longValue());
    //test JSON
    jsonOut = new ByteArrayOutputStream();
    jsonOutChannel = Channels.newChannel(jsonOut);
    event1.writeTo(jsonOutChannel, Encoding.JSON);
    jsonBytes = jsonOut.toByteArray();
    jsonString = new String(jsonBytes);
    jsonMap = mapper.readValue(jsonString, new TypeReference<Map<String, Object>>() {
    });
    assertEquals("key correct", 1L, ((Number) jsonMap.get("key")).longValue());
    assertEquals("sequence correct", 2L, ((Number) jsonMap.get("sequence")).longValue());
    assertEquals("logicalPartitionId correct", 3, ((Number) jsonMap.get("logicalPartitionId")).shortValue());
    assertEquals("timestampInNanos correct", 4L, ((Number) jsonMap.get("timestampInNanos")).longValue());
    assertEquals("srcId correct", 5, ((Number) jsonMap.get("srcId")).longValue());
    assertEquals("schemaId correct", Base64.encodeBytes(schemaId), jsonMap.get("schemaId"));
    assertEquals("valueEnc correct", Encoding.JSON.toString(), jsonMap.get("valueEnc"));
    assertEquals("value correct", Base64.encodeBytes(valueStr.getBytes(Charset.defaultCharset())), jsonMap.get("value"));
    assertTrue("buffer has event", it1.hasNext());
    testEvent = it1.next();
    assertTrue("end of window", testEvent.isEndOfPeriodMarker());
    DbusEventBuffer eventBuffer2 = new DbusEventBuffer(getConfig(100000, DbusEventBuffer.Config.DEFAULT_INDIVIDUAL_BUFFER_SIZE, 10000, 1000, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE));
    eventBuffer2.startEvents();
    assertTrue("json deserialization", (DbusEventSerializable.appendToEventBuffer(jsonString, eventBuffer2, null, false) > 0));
    eventBuffer2.endEvents(2);
    DbusEventIterator it2 = eventBuffer2.acquireIterator("it2");
    assertTrue("buffer has event", it2.hasNext());
    testEvent = it2.next();
    assertEquals("key correct", 1L, testEvent.key());
    assertEquals("partitionId correct", 3, testEvent.logicalPartitionId());
    assertEquals("timestamp correct", 4L, testEvent.timestampInNanos());
    assertEquals("srcId correct", 5, testEvent.srcId());
    assertEquals("schemaId correct", new String(schemaId), new String(testEvent.schemaId()));
    assertEquals("value correct", valueStr, Utils.byteBufferToString(testEvent.value()));
    assertEquals("Get DbusEventKey", 1L, ((DbusEventInternalReadable) testEvent).getDbusEventKey().getLongKey().longValue());
    assertTrue("buffer has event", it2.hasNext());
    testEvent = it2.next();
    assertTrue("end of window", testEvent.isEndOfPeriodMarker());
}

Example 82 with WritableByteChannel

use of java.nio.channels.WritableByteChannel in project databus by linkedin.

the class ReadEventsTestParams method testReadEventsAssertSpanError.

/**
   * Test to reproduce DDSDBUS-388.
   */
@Test
public void testReadEventsAssertSpanError() throws Exception {
    final Logger log = Logger.getLogger("TestDbusEventBuffer.testReadEventsAssertSpanError");
    //log.setLevel(Level.INFO);
    log.info("start");
    DbusEventBuffer dbusBuf = new DbusEventBuffer(getConfig(10000, 10000, 320, 5000, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    //dbusBuf.getLog().setLevel(Level.DEBUG);
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    //event size:  10 + 61 = 71
    //window size:  5 * 71 + 61 = 416
    //layout: [61 + 23 * 416 + 5 * 71 = 9984 ] <WrapAround> [ 61 + 22 * 416 + 2 * 71 + 61 = 9416 ]
    //      [ H=Gen:0,Ofs:0,    T=Gen:0,Ofs:9984, L=10000 ]
    //      [ H=Gen:0,Ofs:9629, T=Gen:1,Ofs:9416, L=9984 ]
    generator.generateEvents(232, 5, 100, 10, events);
    // Add events to the EventBuffer
    DbusEventAppender appender = new DbusEventAppender(events, dbusBuf, null);
    appender.run();
    log.info("Dbus Event Buffer is :" + dbusBuf);
    log.info("SCNIndex is :" + dbusBuf.getScnIndex());
    final BufferPosition headPos = new BufferPosition(dbusBuf.getHead(), dbusBuf.getBufferPositionParser(), dbusBuf.getBuffer());
    final BufferPosition tailPos = new BufferPosition(dbusBuf.getTail(), dbusBuf.getBufferPositionParser(), dbusBuf.getBuffer());
    Assert.assertEquals(0, headPos.bufferGenId());
    Assert.assertEquals(0, headPos.bufferIndex());
    Assert.assertEquals(9629, headPos.bufferOffset());
    Assert.assertEquals(1, tailPos.bufferGenId());
    Assert.assertEquals(0, tailPos.bufferIndex());
    Assert.assertEquals(9416, tailPos.bufferOffset());
    Assert.assertEquals(9984, dbusBuf.getBuffer()[0].limit());
    dbusBuf.getScnIndex().assertHeadPosition(dbusBuf.getHead());
    final BufferPosition lastWinStart = new BufferPosition(61 + 22 * 416, dbusBuf.getBufferPositionParser(), dbusBuf.getBuffer());
    dbusBuf.getScnIndex().assertLastWrittenPos(lastWinStart);
    long lastScn = events.get(events.size() - 1).sequence();
    generator = new DbusEventGenerator(lastScn - 1);
    events.clear();
    //event size: 10 + 61 = 71
    generator.generateEvents(2, 2, 75, 10, events);
    //intended layout: append 2 * 71  = 142 bytes
    //      [ H=Gen:0,Ofs:9629, T=Gen:1,Ofs:9558, L=9984 ]
    lastScn = events.get(events.size() - 1).sequence();
    generator = new DbusEventGenerator(lastScn - 1);
    //event size: 500 + 61 = 561
    //partial window size: 561  = 622
    generator.generateEvents(1, 1, 800, 500, events);
    //intended layout: append 561 bytes;
    //      [ H=Gen:1,Ofs:893, T=Gen:2,Ofs:561, L=9558 ]
    // head moves to first window after 561: 61 + 2 * 416 = 893 > 561
    //Set up the ReadChannel with new events
    ByteArrayOutputStream oStream = new ByteArrayOutputStream();
    WritableByteChannel oChannel = Channels.newChannel(oStream);
    for (DbusEvent e : events) {
        assertTrue("invalid event", e.isValid());
        ((DbusEventInternalReadable) e).writeTo(oChannel, Encoding.BINARY);
        log.debug("event size is: " + e.size());
    }
    byte[] writeBytes = oStream.toByteArray();
    ByteArrayInputStream iStream = new ByteArrayInputStream(writeBytes);
    final ReadableByteChannel rChannel = Channels.newChannel(iStream);
    dbusBuf.readEvents(rChannel);
    log.info("Dbus Event Buffer is :" + dbusBuf);
    log.info("SCNIndex is :" + dbusBuf.getScnIndex());
    //expected layout:
    //      [ H=Gen:1,Ofs:893, T=Gen:2,Ofs:561, L=9558 ]
    headPos.setPosition(dbusBuf.getHead());
    Assert.assertEquals(1, headPos.bufferGenId());
    Assert.assertEquals(0, headPos.bufferIndex());
    Assert.assertEquals(893, headPos.bufferOffset());
    tailPos.setPosition(dbusBuf.getTail());
    Assert.assertEquals(2, tailPos.bufferGenId());
    Assert.assertEquals(0, tailPos.bufferIndex());
    Assert.assertEquals(561, tailPos.bufferOffset());
    Assert.assertEquals(9558, dbusBuf.getBuffer()[0].limit());
    dbusBuf.getBufferPositionParser().assertSpan(dbusBuf.getHead(), dbusBuf.getTail(), true);
    dbusBuf.getScnIndex().assertHeadPosition(dbusBuf.getHead());
    //the second write did not add EOP so the SCN index tail should not have changed
    dbusBuf.getScnIndex().assertLastWrittenPos(lastWinStart);
}

Example 83 with WritableByteChannel

use of java.nio.channels.WritableByteChannel in project databus by linkedin.

the class ReadEventsTestParams method testNoMissingEOWHappyPathWithOneEvent.

/**
   * Testcase send the following sequence
   * 1 valid packets, EOW
   * @throws Exception
   */
@Test
public void testNoMissingEOWHappyPathWithOneEvent() throws Exception {
    final DbusEventBuffer dbusBuf = new DbusEventBuffer(getConfig(1000, 1000, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE, AssertLevel.NONE));
    dbusBuf.setDropOldEvents(true);
    dbusBuf.start(0);
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(2, 1, 100, 10, events);
    // conversion of readable events to writable is for TESTING ONLY:
    DbusEventInternalWritable writableEvent = DbusEventCorrupter.makeWritable(events.get(1));
    writableEvent.setSequence(events.get(0).sequence());
    writableEvent.setSrcId((short) -2);
    writableEvent.applyCrc();
    // Increment the SCN and reuse
    for (int i = 0; i < 2; ++i) {
        DbusEvent e = events.get(i);
        assertTrue("invalid event #" + i, e.isValid());
    }
    // Set up the ReadChannel with 2 events
    ByteArrayOutputStream oStream = new ByteArrayOutputStream();
    WritableByteChannel oChannel = Channels.newChannel(oStream);
    for (int i = 0; i < 2; ++i) {
        ((DbusEventInternalReadable) events.get(i)).writeTo(oChannel, Encoding.BINARY);
    }
    byte[] writeBytes = oStream.toByteArray();
    ByteArrayInputStream iStream = new ByteArrayInputStream(writeBytes);
    final ReadableByteChannel rChannel = Channels.newChannel(iStream);
    try {
        dbusBuf.readEvents(rChannel);
    // Should NOT throw invalid event exception
    } catch (InvalidEventException ie) {
        LOG.error("Exception trace is " + ie);
        Assert.fail();
        return;
    }
}

Example 84 with WritableByteChannel

use of java.nio.channels.WritableByteChannel in project databus by linkedin.

the class ReadEventsTestParams method streamEvents.

public void streamEvents() throws ScnNotFoundException, OffsetNotFoundException {
    _srcByteStr = new ByteArrayOutputStream();
    final WritableByteChannel srcChannel = Channels.newChannel(_srcByteStr);
    Checkpoint cp = new Checkpoint();
    cp.setFlexible();
    _srcBufStats.reset();
    StreamEventsArgs args = new StreamEventsArgs(_srcBufferSize).setStatsCollector(_srcBufStats);
    _numStreamedEvents = _srcBuf.streamEvents(cp, srcChannel, args).getNumEventsStreamed();
    Assert.assertEquals(_numSrcEvents, _srcBufStats.getTotalStats().getNumDataEvents());
}

Example 85 with WritableByteChannel

use of java.nio.channels.WritableByteChannel in project databus by linkedin.

the class ReadEventsTestParams method testMissingEOWUnhappyPathWithMultiEventWindow.

/**
   * Testcase sends events with appropriate EOW events in between:
   * 2 valid packets, no EOW for prior packet, 2 valid packets with a higher scn.
   * @throws Exception
   */
@Test
public void testMissingEOWUnhappyPathWithMultiEventWindow() throws Exception {
    final DbusEventBuffer dbusBuf = new DbusEventBuffer(getConfig(1000, 1000, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE, AssertLevel.NONE));
    dbusBuf.setDropOldEvents(true);
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(6, 1, 100, 10, events);
    // conversion of readable events to writable is for TESTING ONLY:
    DbusEventInternalWritable writableEvent = DbusEventCorrupter.makeWritable(events.get(1));
    writableEvent.setSequence(events.get(0).sequence());
    writableEvent.applyCrc();
    writableEvent = DbusEventCorrupter.makeWritable(events.get(2));
    writableEvent.setSequence(events.get(0).sequence());
    writableEvent.applyCrc();
    writableEvent = DbusEventCorrupter.makeWritable(events.get(3));
    // NO EOW for event 0 sequence.
    writableEvent.setSequence(events.get(0).sequence() + 100);
    writableEvent.applyCrc();
    writableEvent = DbusEventCorrupter.makeWritable(events.get(4));
    writableEvent.setSequence(events.get(3).sequence());
    writableEvent.applyCrc();
    writableEvent = DbusEventCorrupter.makeWritable(events.get(5));
    writableEvent.setSrcId((short) -2);
    writableEvent.applyCrc();
    // Increment the SCN and reuse
    for (int i = 0; i < 6; ++i) {
        DbusEvent e = events.get(i);
        assertTrue("invalid event #" + i, e.isValid());
    }
    // Set up the ReadChannel with 6 events
    ByteArrayOutputStream oStream = new ByteArrayOutputStream();
    WritableByteChannel oChannel = Channels.newChannel(oStream);
    for (int i = 0; i < 6; ++i) {
        ((DbusEventInternalReadable) events.get(i)).writeTo(oChannel, Encoding.BINARY);
    }
    byte[] writeBytes = oStream.toByteArray();
    ByteArrayInputStream iStream = new ByteArrayInputStream(writeBytes);
    final ReadableByteChannel rChannel = Channels.newChannel(iStream);
    try {
        dbusBuf.readEvents(rChannel);
        Assert.fail();
    } catch (InvalidEventException ie) {
        LOG.info("Exception trace is " + ie);
        return;
    }
}