Examples with WritableByteChannel java.nio.channels.WritableByteChannel used on opensource projects

Example 76 with WritableByteChannel

use of java.nio.channels.WritableByteChannel in project databus by linkedin.

the class TestDbusEventBufferMult method testSinglePPartionStreamFromLatest.

@Test
public void testSinglePPartionStreamFromLatest() throws Exception {
    createBufMult();
    PhysicalPartition[] p = { _pConfigs[0].getPhysicalPartition() };
    //generate a bunch of windows for 3 partitions
    int windowsNum = 10;
    for (int i = 1; i <= windowsNum; ++i) {
        DbusEventBufferAppendable buf = _eventBufferMult.getDbusEventBufferAppendable(p[0]);
        buf.startEvents();
        byte[] schema = "abcdefghijklmnop".getBytes(Charset.defaultCharset());
        assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null));
        buf.endEvents(100 * i, null);
    }
    String[] pnames = new String[p.length];
    int count = 0;
    for (PhysicalPartition ip : p) {
        pnames[count++] = ip.toSimpleString();
    }
    StatsCollectors<DbusEventsStatisticsCollector> statsColl = createStats(pnames);
    PhysicalPartitionKey[] pkeys = { new PhysicalPartitionKey(p[0]) };
    CheckpointMult cpMult = new CheckpointMult();
    Checkpoint cp = new Checkpoint();
    cp.setFlexible();
    cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
    cpMult.addCheckpoint(p[0], cp);
    DbusEventBufferBatchReadable reader = _eventBufferMult.getDbusEventBufferBatchReadable(cpMult, Arrays.asList(pkeys), statsColl);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    WritableByteChannel writeChannel = Channels.newChannel(baos);
    // Set streamFromLatestScn == true
    reader.streamEvents(true, 1000000, writeChannel, Encoding.BINARY, new AllowAllDbusFilter());
    writeChannel.close();
    baos.close();
    //make sure we got the physical partition names right
    List<String> ppartNames = statsColl.getStatsCollectorKeys();
    assertEquals(ppartNames.size(), 1);
    HashSet<String> expectedPPartNames = new HashSet<String>(Arrays.asList(p[0].toSimpleString()));
    for (String ppartName : ppartNames) {
        assertTrue(expectedPPartNames.contains(ppartName));
    }
    //verify event counts per partition
    DbusEventsTotalStats[] ppartStats = { statsColl.getStatsCollector(p[0].toSimpleString()).getTotalStats() };
    // Only the last window is returned in each of the partitions
    assertEquals(ppartStats[0].getNumDataEvents(), 1);
    assertEquals(ppartStats[0].getNumSysEvents(), 1);
    assertEquals(statsColl.getStatsCollector().getTotalStats().getNumDataEvents(), (1));
    assertEquals(statsColl.getStatsCollector().getTotalStats().getNumSysEvents(), (1));
    assertEquals(statsColl.getStatsCollector().getTotalStats().getMaxTimeLag(), ppartStats[0].getTimeLag());
    assertEquals(statsColl.getStatsCollector().getTotalStats().getMinTimeLag(), ppartStats[0].getTimeLag());
}

Example 77 with WritableByteChannel

use of java.nio.channels.WritableByteChannel in project databus by linkedin.

the class TestDbusEventBufferMult method batchReading.

private void batchReading(Set<Integer> srcIds, final int expectEvents, final int batchFetchSize) throws IOException, ScnNotFoundException, DatabusException, OffsetNotFoundException {
    // for debug generate this line
    String inputSrcIds = Arrays.toString(srcIds.toArray());
    LOG.info("Reading events from " + inputSrcIds);
    // create checkpoints
    CheckpointMult cpMult = new CheckpointMult();
    for (PhysicalSourceStaticConfig pConf : _pConfigs) {
        PhysicalPartition pPart = pConf.getPhysicalPartition();
        Checkpoint cp;
        if (cpMult.getCheckpoint(pPart) == null) {
            // needs a new checkpoint
            cp = new Checkpoint();
            cp.setFlexible();
            cpMult.addCheckpoint(pPart, cp);
        }
    }
    DbusEventBufferBatchReadable read = _eventBufferMult.getDbusEventBufferBatchReadable(srcIds, cpMult, null);
    int totalRead = 0;
    int numEventsRead = Integer.MAX_VALUE;
    int numNonControlEventsRead = 0;
    int maxIterNum = 100, iterCount = 0;
    String prevEvent = "";
    while (numEventsRead > 0) {
        if (iterCount++ > maxIterNum) {
            fail("Checkpoint doesn't work - it is a never-ending loop");
        }
        ByteArrayOutputStream jsonOut = new ByteArrayOutputStream();
        WritableByteChannel jsonOutChannel = Channels.newChannel(jsonOut);
        numEventsRead = read.streamEvents(false, batchFetchSize, jsonOutChannel, Encoding.JSON_PLAIN_VALUE, new SourceDbusFilter(srcIds)).getNumEventsStreamed();
        totalRead += numEventsRead;
        LOG.info("read for " + inputSrcIds + ": " + numEventsRead + " events");
        byte[] jsonBytes = jsonOut.toByteArray();
        if (jsonBytes.length == 0)
            // nothing more to read
            break;
        String jsonString = new String(jsonBytes);
        String[] jsonStrings = jsonString.split("\n");
        assertEquals(jsonStrings.length, numEventsRead);
        ObjectMapper mapper = new ObjectMapper();
        for (int i = 0; i < jsonStrings.length; i++) {
            // verify what was written
            String evtStr = jsonStrings[i];
            if (evtStr.equals(prevEvent)) {
                // offered buffer is small. This check gets around the issue.
                continue;
            }
            prevEvent = evtStr;
            Map<String, Object> jsonMap = mapper.readValue(evtStr, new TypeReference<Map<String, Object>>() {
            });
            //assertEquals(jsonMap.size(), 10);
            Integer srcId = (Integer) jsonMap.get("srcId");
            if (!DbusEventUtils.isControlSrcId(srcId)) {
                // not a control message
                numNonControlEventsRead++;
                Integer physicalPartitionId = (Integer) jsonMap.get("physicalPartitionId");
                Integer logicalPartitionId = (Integer) jsonMap.get("logicalPartitionId");
                PhysicalPartition pPartition = _eventBufferMult.getPhysicalPartition(srcId, new LogicalPartition(logicalPartitionId.shortValue()));
                LOG.info("EVENT: " + jsonMap.toString());
                assertTrue(srcIds.contains(srcId), "src id " + srcId + " doesn't match to " + inputSrcIds);
                assertEquals(physicalPartitionId, pPartition.getId(), "physical partition id didn't match");
            } else {
                LOG.info("Control event: " + jsonMap.toString());
            }
        }
    }
    assertTrue(totalRead > numNonControlEventsRead);
    assertEquals(numNonControlEventsRead, expectEvents);
}

Example 78 with WritableByteChannel

use of java.nio.channels.WritableByteChannel in project databus by linkedin.

the class TestDbusEventBufferMult method testMultiPPartionStreamStats.

@Test
public void testMultiPPartionStreamStats() throws Exception {
    createBufMult();
    PhysicalPartition[] p = { _pConfigs[0].getPhysicalPartition(), _pConfigs[1].getPhysicalPartition(), _pConfigs[2].getPhysicalPartition() };
    //generate a bunch of windows for 3 partitions
    int windowsNum = 10;
    for (int i = 1; i <= windowsNum; ++i) {
        DbusEventBufferAppendable buf = _eventBufferMult.getDbusEventBufferAppendable(p[0]);
        buf.startEvents();
        byte[] schema = "abcdefghijklmnop".getBytes(Charset.defaultCharset());
        assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null));
        buf.endEvents(100 * i, null);
        buf = _eventBufferMult.getDbusEventBufferAppendable(p[1]);
        buf.startEvents();
        assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null));
        assertTrue(buf.appendEvent(new DbusEventKey(2), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null));
        buf.endEvents(100 * i + 1, null);
        buf = _eventBufferMult.getDbusEventBufferAppendable(p[2]);
        buf.startEvents();
        assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null));
        assertTrue(buf.appendEvent(new DbusEventKey(2), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null));
        assertTrue(buf.appendEvent(new DbusEventKey(3), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null));
        buf.endEvents(100 * i + 2, null);
    }
    String[] pnames = new String[p.length];
    int count = 0;
    for (PhysicalPartition ip : p) {
        pnames[count++] = ip.toSimpleString();
    }
    StatsCollectors<DbusEventsStatisticsCollector> statsColl = createStats(pnames);
    PhysicalPartitionKey[] pkeys = { new PhysicalPartitionKey(p[0]), new PhysicalPartitionKey(p[1]), new PhysicalPartitionKey(p[2]) };
    CheckpointMult cpMult = new CheckpointMult();
    for (int i = 0; i < 3; ++i) {
        Checkpoint cp = new Checkpoint();
        cp.setFlexible();
        cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
        cpMult.addCheckpoint(p[i], cp);
    }
    DbusEventBufferBatchReadable reader = _eventBufferMult.getDbusEventBufferBatchReadable(cpMult, Arrays.asList(pkeys), statsColl);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    WritableByteChannel writeChannel = Channels.newChannel(baos);
    reader.streamEvents(false, 1000000, writeChannel, Encoding.BINARY, new AllowAllDbusFilter());
    writeChannel.close();
    baos.close();
    //make sure we got the physical partition names right
    List<String> ppartNames = statsColl.getStatsCollectorKeys();
    assertEquals(ppartNames.size(), 3);
    HashSet<String> expectedPPartNames = new HashSet<String>(Arrays.asList(p[0].toSimpleString(), p[1].toSimpleString(), p[2].toSimpleString()));
    for (String ppartName : ppartNames) {
        assertTrue(expectedPPartNames.contains(ppartName));
    }
    //verify event counts per partition
    DbusEventsTotalStats[] ppartStats = { statsColl.getStatsCollector(p[0].toSimpleString()).getTotalStats(), statsColl.getStatsCollector(p[1].toSimpleString()).getTotalStats(), statsColl.getStatsCollector(p[2].toSimpleString()).getTotalStats() };
    assertEquals(ppartStats[0].getNumDataEvents(), windowsNum);
    assertEquals(ppartStats[1].getNumDataEvents(), windowsNum * 2);
    assertEquals(ppartStats[2].getNumDataEvents(), windowsNum * 3);
    assertEquals(ppartStats[0].getNumSysEvents(), windowsNum);
    assertEquals(ppartStats[1].getNumSysEvents(), windowsNum);
    assertEquals(ppartStats[2].getNumSysEvents(), windowsNum);
    assertEquals(statsColl.getStatsCollector().getTotalStats().getNumDataEvents(), windowsNum * (1 + 2 + 3));
    assertEquals(statsColl.getStatsCollector().getTotalStats().getNumSysEvents(), windowsNum * 3);
    assertEquals(statsColl.getStatsCollector().getTotalStats().getMaxTimeLag(), Math.max(ppartStats[0].getTimeLag(), Math.max(ppartStats[1].getTimeLag(), ppartStats[2].getTimeLag())));
    assertEquals(statsColl.getStatsCollector().getTotalStats().getMinTimeLag(), Math.min(ppartStats[0].getTimeLag(), Math.min(ppartStats[1].getTimeLag(), ppartStats[2].getTimeLag())));
}

Example 79 with WritableByteChannel

use of java.nio.channels.WritableByteChannel in project databus by linkedin.

the class TestDbusEventBufferPersistence method pushEventsToBuffer.

private void pushEventsToBuffer(DbusEventBuffer dbusBuf, int numEvents) {
    dbusBuf.start(1);
    dbusBuf.startEvents();
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(numEvents, 1, 100, 10, events);
    // set end of windows
    for (int i = 0; i < numEvents - 1; ++i) {
        long scn = events.get(i).sequence();
        ++i;
        DbusEventInternalWritable writableEvent;
        try {
            writableEvent = DbusEventCorrupter.makeWritable(events.get(i));
        } catch (InvalidEventException ie) {
            LOG.error("Exception trace is " + ie);
            Assert.fail();
            return;
        }
        writableEvent.setSrcId((short) -2);
        writableEvent.setSequence(scn);
        writableEvent.applyCrc();
        assertTrue("invalid event #" + i, writableEvent.isValid(true));
    }
    // set up the ReadChannel with 2 events
    ByteArrayOutputStream oStream = new ByteArrayOutputStream();
    WritableByteChannel oChannel = Channels.newChannel(oStream);
    for (int i = 0; i < numEvents; ++i) {
        ((DbusEventInternalReadable) events.get(i)).writeTo(oChannel, Encoding.BINARY);
    }
    byte[] writeBytes = oStream.toByteArray();
    ByteArrayInputStream iStream = new ByteArrayInputStream(writeBytes);
    ReadableByteChannel rChannel = Channels.newChannel(iStream);
    try {
        dbusBuf.readEvents(rChannel);
    } catch (InvalidEventException ie) {
        LOG.error("Exception trace is " + ie);
        Assert.fail();
        return;
    }
}

Example 80 with WritableByteChannel

use of java.nio.channels.WritableByteChannel in project databus by linkedin.

the class TestDbusEventBufferStreamEvents method testStreamLargeScn.

@Test
public /** Tests streamEvents with a large scn with concurrent writes  */
void testStreamLargeScn() throws Exception {
    final Logger log = Logger.getLogger("TestDbusEventBuffer.testStreamLargeScn");
    //log.setLevel(Level.INFO);
    log.info("starting");
    final DbusEventBuffer dbusBuf = new DbusEventBuffer(TestDbusEventBuffer.getConfig(120000, 500000, 1024, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    final Checkpoint cp = Checkpoint.createOnlineConsumptionCheckpoint(1000);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final WritableByteChannel ochannel = Channels.newChannel(baos);
    final AtomicBoolean hasError = new AtomicBoolean(false);
    final AtomicInteger num = new AtomicInteger(0);
    final AtomicBoolean genComplete = new AtomicBoolean(false);
    UncaughtExceptionTrackingThread streamThread = new UncaughtExceptionTrackingThread(new Runnable() {

        @Override
        public void run() {
            try {
                while (!genComplete.get() && 0 >= num.get()) {
                    StreamEventsArgs args = new StreamEventsArgs(10000);
                    num.set(dbusBuf.streamEvents(cp, ochannel, args).getNumEventsStreamed());
                }
            } catch (ScnNotFoundException e) {
                hasError.set(true);
            } catch (OffsetNotFoundException e) {
                hasError.set(true);
            }
        }
    }, "testGetLargeScn.streamThread");
    streamThread.setDaemon(true);
    streamThread.start();
    log.info("append initial events");
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(70000, 1, 120, 39, events);
    // Add events to the EventBuffer. Now the buffer is full
    DbusEventAppender appender = new DbusEventAppender(events, dbusBuf, null);
    // running in the same thread
    appender.run();
    genComplete.set(true);
    streamThread.join(10000);
    Assert.assertFalse(streamThread.isAlive());
    Assert.assertNull(streamThread.getLastException());
    Assert.assertFalse(hasError.get());
    Assert.assertTrue(num.get() > 0);
}