Search in sources :

Example 86 with ReadableByteChannel

use of java.nio.channels.ReadableByteChannel in project Lazy by l123456789jy.

the class NioFileUtiles method writeToFile.

public static void writeToFile(byte[] data, File target) throws IOException {
    FileOutputStream fo = null;
    ReadableByteChannel src = null;
    FileChannel out = null;
    try {
        src = Channels.newChannel(new ByteArrayInputStream(data));
        fo = new FileOutputStream(target);
        out = fo.getChannel();
        out.transferFrom(src, 0, data.length);
    } finally {
        if (fo != null) {
            fo.close();
        }
        if (src != null) {
            src.close();
        }
        if (out != null) {
            out.close();
        }
    }
}
Also used : ReadableByteChannel(java.nio.channels.ReadableByteChannel) ByteArrayInputStream(java.io.ByteArrayInputStream) FileChannel(java.nio.channels.FileChannel) FileOutputStream(java.io.FileOutputStream)

Example 87 with ReadableByteChannel

use of java.nio.channels.ReadableByteChannel in project databus by linkedin.

the class TestDatabusRelayEvents method testV2Events.

/**
 * Stuffs an event buffer with both a v1 and a v2 event, then reads the buffer two ways:
 * first accepting only v1 events (verifying conversion of the v2 event to v1); then accepting
 * both v1 and v2 events.
 *
 * Note that the version of the _EOP_ events must match the version of the event factory,
 * regardless of the versions of any preceding "real" events.  (This matches DbusEventBuffer
 * behavior; see the serializeLongKeyEndOfPeriodMarker() call in endEvents() for details.)
 */
@Test
public void testV2Events() throws KeyTypeNotImplementedException, InvalidEventException, IOException, DatabusException {
    final Logger log = Logger.getLogger("TestDatabusRelayEvents.testV2Events");
    log.setLevel(Level.DEBUG);
    String[] srcs = { "com.linkedin.events.example.fake.FakeSchema" };
    String pSourceName = DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]);
    short srcId = 2;
    short pId = 1;
    int relayPort = Utils.getAvailablePort(11993);
    // create relay
    final DatabusRelayMain relay1 = createRelay(relayPort, pId, srcs);
    DatabusRelayTestUtil.RelayRunner r1 = null;
    ClientRunner cr = null;
    try {
        // EventProducer[] producers = relay1.getProducers();
        r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
        log.info("Relay created");
        DbusEventBufferMult bufMult = relay1.getEventBuffer();
        PhysicalPartition pPartition = new PhysicalPartition((int) pId, pSourceName);
        DbusEventBuffer buf = (DbusEventBuffer) bufMult.getDbusEventBufferAppendable(pPartition);
        log.info("create some events");
        long windowScn = 100L;
        ByteBuffer serializationBuffer = addEvent(windowScn, srcId, relay1.getSchemaRegistryService().fetchSchemaIdForSourceNameAndVersion(srcs[0], 2).getByteArray(), pId, DbusEventFactory.DBUS_EVENT_V2);
        ReadableByteChannel channel = Channels.newChannel(new ByteBufferInputStream(serializationBuffer));
        int readEvents = buf.readEvents(channel);
        log.info("successfully read in " + readEvents + " events ");
        channel.close();
        windowScn = 101L;
        serializationBuffer = addEvent(windowScn, srcId, relay1.getSchemaRegistryService().fetchSchemaIdForSourceNameAndVersion(srcs[0], 2).getByteArray(), pId, DbusEventFactory.DBUS_EVENT_V1);
        channel = Channels.newChannel(new ByteBufferInputStream(serializationBuffer));
        readEvents = buf.readEvents(channel);
        log.info("successfully read in " + readEvents + " events ");
        channel.close();
        log.info("starting relay on port " + relayPort);
        r1.start();
        // TestUtil.sleep(10*1000);
        // wait until relay comes up
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return relay1.isRunningStatus();
            }
        }, "Relay hasn't come up completely ", 30000, LOG);
        log.info("now create client");
        String srcSubscriptionString = TestUtil.join(srcs, ",");
        String serverName = "localhost:" + relayPort;
        final EventsCountingConsumer countingConsumer = new EventsCountingConsumer();
        int id = (RngUtils.randomPositiveInt() % 10000) + 1;
        DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", id, serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true, DatabusClientNettyThreadPools.createNettyThreadPools(id), 0, DbusEventFactory.DBUS_EVENT_V1, 0);
        cr = new ClientRunner(clientConn);
        log.info("starting client");
        cr.start();
        // wait till client gets the event
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                int events = countingConsumer.getNumDataEvents();
                LOG.info("client got " + events + " events");
                return events == 2;
            }
        }, "Consumer didn't get 2 events ", 64 * 1024, LOG);
        // asserts
        Assert.assertEquals(countingConsumer.getNumDataEvents(), 2);
        Assert.assertEquals(countingConsumer.getNumWindows(), 2);
        Assert.assertEquals(countingConsumer.getNumDataEvents(DbusEventFactory.DBUS_EVENT_V1), 2);
        log.info("shutdown first client");
        clientConn.stop();
        cr.shutdown();
        TestUtil.sleep(1000);
        cr = null;
        log.info("start another client who understands V2");
        final EventsCountingConsumer countingConsumer1 = new EventsCountingConsumer();
        clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", id, serverName, srcSubscriptionString, countingConsumer1, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true, DatabusClientNettyThreadPools.createNettyThreadPools(id), 0, DbusEventFactory.DBUS_EVENT_V2, 0);
        cr = new ClientRunner(clientConn);
        cr.start();
        log.info("wait till client gets the event");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                int events = countingConsumer1.getNumDataEvents();
                LOG.debug("client got " + events + " events");
                return events == 2;
            }
        }, "Consumer didn't get 2 events ", 64 * 1024, LOG);
        // asserts
        Assert.assertEquals(countingConsumer1.getNumDataEvents(), 2);
        Assert.assertEquals(countingConsumer1.getNumWindows(), 2);
        Assert.assertEquals(countingConsumer1.getNumDataEvents(DbusEventFactory.DBUS_EVENT_V1), 1);
        Assert.assertEquals(countingConsumer1.getNumDataEvents(DbusEventFactory.DBUS_EVENT_V2), 1);
    } finally {
        cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1 }, cr);
    }
}
Also used : ConditionCheck(com.linkedin.databus2.test.ConditionCheck) ReadableByteChannel(java.nio.channels.ReadableByteChannel) ClientRunner(com.linkedin.databus2.relay.TestDatabusRelayMain.ClientRunner) ByteBufferInputStream(org.apache.zookeeper.server.ByteBufferInputStream) Logger(org.apache.log4j.Logger) ByteBuffer(java.nio.ByteBuffer) DbusEventBuffer(com.linkedin.databus.core.DbusEventBuffer) DatabusSourcesConnection(com.linkedin.databus.client.DatabusSourcesConnection) DatabusRelayTestUtil(com.linkedin.databus2.relay.util.test.DatabusRelayTestUtil) DbusEventBufferMult(com.linkedin.databus.core.DbusEventBufferMult) PhysicalPartition(com.linkedin.databus.core.data_model.PhysicalPartition) Test(org.testng.annotations.Test)

Example 88 with ReadableByteChannel

use of java.nio.channels.ReadableByteChannel in project databus by linkedin.

the class MockBootstrapConnection method testBootstrapPendingEvent.

// Make sure that we suspend on error when we get the x-dbus-pending-size header with a size that is
// larger than our dbusevent size.
@Test
public void testBootstrapPendingEvent() throws Exception {
    List<String> sources = Arrays.asList("source1");
    Properties clientProps = new Properties();
    clientProps.setProperty("client.container.httpPort", "0");
    clientProps.setProperty("client.container.jmx.rmiEnabled", "false");
    clientProps.setProperty("client.runtime.bootstrap.enabled", "true");
    clientProps.setProperty("client.runtime.bootstrap.service(1).name", "bs1");
    clientProps.setProperty("client.runtime.bootstrap.service(1).host", "localhost");
    clientProps.setProperty("client.runtime.bootstrap.service(1).port", "10001");
    clientProps.setProperty("client.runtime.bootstrap.service(1).sources", "source1");
    clientProps.setProperty("client.runtime.relay(1).name", "relay1");
    clientProps.setProperty("client.runtime.relay(1).port", "10001");
    clientProps.setProperty("client.runtime.relay(1).sources", "source1");
    clientProps.setProperty("client.connectionDefaults.eventBuffer.maxSize", "100000");
    clientProps.setProperty("client.connectionDefaults.pullerRetries.maxRetryNum", "3");
    DatabusHttpClientImpl.Config clientConfBuilder = new DatabusHttpClientImpl.Config();
    ConfigLoader<DatabusHttpClientImpl.StaticConfig> configLoader = new ConfigLoader<DatabusHttpClientImpl.StaticConfig>("client.", clientConfBuilder);
    configLoader.loadConfig(clientProps);
    DatabusHttpClientImpl.StaticConfig clientConf = clientConfBuilder.build();
    DatabusSourcesConnection.StaticConfig srcConnConf = clientConf.getConnectionDefaults();
    DatabusHttpClientImpl client = new DatabusHttpClientImpl(clientConf);
    client.registerDatabusBootstrapListener(new LoggingConsumer(), null, "source1");
    Assert.assertNotNull(client, "client instantiation failed");
    DatabusHttpClientImpl.RuntimeConfig clientRtConf = clientConf.getRuntime().build();
    // we keep the index of the next server we expect to see
    AtomicInteger serverIdx = new AtomicInteger(-1);
    List<IdNamePair> sourcesResponse = new ArrayList<IdNamePair>();
    sourcesResponse.add(new IdNamePair(1L, "source1"));
    Map<Long, List<RegisterResponseEntry>> registerResponse = new HashMap<Long, List<RegisterResponseEntry>>();
    List<RegisterResponseEntry> regResponse = new ArrayList<RegisterResponseEntry>();
    regResponse.add(new RegisterResponseEntry(1L, (short) 1, SCHEMA$.toString()));
    registerResponse.put(1L, regResponse);
    ChunkedBodyReadableByteChannel channel = EasyMock.createMock(ChunkedBodyReadableByteChannel.class);
    // getting the pending-event-size header is called twice, once for checking and once for logging.
    EasyMock.expect(channel.getMetadata(DatabusHttpHeaders.DATABUS_PENDING_EVENT_SIZE)).andReturn("1000000").times(2);
    EasyMock.expect(channel.getMetadata("x-dbus-error-cause")).andReturn(null).times(2);
    EasyMock.expect(channel.getMetadata("x-dbus-error")).andReturn(null).times(2);
    EasyMock.replay(channel);
    DbusEventBuffer dbusBuffer = EasyMock.createMock(DbusEventBuffer.class);
    dbusBuffer.endEvents(false, -1, false, false, null);
    EasyMock.expectLastCall().anyTimes();
    EasyMock.expect(dbusBuffer.injectEvent(EasyMock.<DbusEventInternalReadable>notNull())).andReturn(true).anyTimes();
    EasyMock.expect(dbusBuffer.getEventSerializationVersion()).andReturn(DbusEventFactory.DBUS_EVENT_V1).anyTimes();
    EasyMock.expect(dbusBuffer.getMaxReadBufferCapacity()).andReturn(600).times(2);
    EasyMock.expect(dbusBuffer.getBufferFreeReadSpace()).andReturn(600000).times(2);
    EasyMock.expect(dbusBuffer.readEvents(EasyMock.<ReadableByteChannel>notNull())).andReturn(1).times(1);
    EasyMock.expect(dbusBuffer.readEvents(EasyMock.<ReadableByteChannel>notNull(), EasyMock.<List<InternalDatabusEventsListener>>notNull(), EasyMock.<DbusEventsStatisticsCollector>isNull())).andReturn(0).times(1);
    EasyMock.replay(dbusBuffer);
    ConnectionStateFactory connStateFactory = new ConnectionStateFactory(sources);
    // This guy succeeds on /sources but fails on /register
    MockBootstrapConnection mockSuccessConn = new MockBootstrapConnection(10, 10, channel, serverIdx, false);
    DatabusBootstrapConnectionFactory mockConnFactory = org.easymock.EasyMock.createMock("mockRelayFactory", DatabusBootstrapConnectionFactory.class);
    // each server should be tried MAX_RETRIES time until all retries are exhausted
    EasyMock.expect(mockConnFactory.createConnection(EasyMock.<ServerInfo>notNull(), EasyMock.<ActorMessageQueue>notNull(), EasyMock.<RemoteExceptionHandler>notNull())).andReturn(mockSuccessConn).anyTimes();
    List<DatabusSubscription> sourcesSubList = DatabusSubscription.createSubscriptionList(sources);
    DatabusSourcesConnection sourcesConn2 = EasyMock.createMock(DatabusSourcesConnection.class);
    EasyMock.expect(sourcesConn2.getSourcesNames()).andReturn(Arrays.asList("source1")).anyTimes();
    EasyMock.expect(sourcesConn2.getSubscriptions()).andReturn(sourcesSubList).anyTimes();
    EasyMock.expect(sourcesConn2.getConnectionConfig()).andReturn(srcConnConf).anyTimes();
    EasyMock.expect(sourcesConn2.getConnectionStatus()).andReturn(new DatabusComponentStatus("dummy")).anyTimes();
    EasyMock.expect(sourcesConn2.getLocalRelayCallsStatsCollector()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getRelayCallsStatsCollector()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getUnifiedClientStats()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getBootstrapConnFactory()).andReturn(mockConnFactory).anyTimes();
    EasyMock.expect(sourcesConn2.loadPersistentCheckpoint()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getDataEventsBuffer()).andReturn(dbusBuffer).anyTimes();
    EasyMock.expect(sourcesConn2.isBootstrapEnabled()).andReturn(true).anyTimes();
    EasyMock.expect(sourcesConn2.getBootstrapRegistrations()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getBootstrapServices()).andReturn(null).anyTimes();
    EasyMock.expect(sourcesConn2.getBootstrapEventsStatsCollector()).andReturn(null).anyTimes();
    EasyMock.makeThreadSafe(mockConnFactory, true);
    EasyMock.makeThreadSafe(sourcesConn2, true);
    EasyMock.replay(mockConnFactory);
    EasyMock.replay(sourcesConn2);
    BootstrapPullThread bsPuller = new BootstrapPullThread("RelayPuller", sourcesConn2, dbusBuffer, connStateFactory, clientRtConf.getBootstrap().getServicesSet(), new ArrayList<DbusKeyCompositeFilterConfig>(), clientConf.getPullerBufferUtilizationPct(), ManagementFactory.getPlatformMBeanServer(), new DbusEventV2Factory(), null, null);
    mockSuccessConn.setCallback(bsPuller);
    bsPuller.getComponentStatus().start();
    Checkpoint cp = _ckptHandlerSource1.createInitialBootstrapCheckpoint(null, 0L);
    // TODO remove
    // cp.setSnapshotSource("source1");
    // cp.setCatchupSource("source1");
    // cp.setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);
    ConnectionState connState = bsPuller.getConnectionState();
    connState.switchToBootstrap(cp);
    testTransitionCase(bsPuller, StateId.BOOTSTRAP, StateId.REQUEST_START_SCN, cp);
    bsPuller.getMessageQueue().clear();
    testTransitionCase(bsPuller, StateId.REQUEST_START_SCN, StateId.START_SCN_RESPONSE_SUCCESS, null);
    bsPuller.getMessageQueue().clear();
    Map<Long, List<RegisterResponseEntry>> entries = new HashMap<Long, List<RegisterResponseEntry>>();
    entries.put(1L, new ArrayList<RegisterResponseEntry>());
    connState.setSourcesSchemas(entries);
    connState.setCurrentBSServerInfo(bsPuller.getCurentServer());
    testTransitionCase(bsPuller, StateId.START_SCN_RESPONSE_SUCCESS, StateId.REQUEST_STREAM, null);
    bsPuller.getMessageQueue().clear();
    connState.getSourcesNameMap().put("source1", new IdNamePair(1L, "source1"));
    connState.getSourceIdMap().put(1L, new IdNamePair(1L, "source1"));
    testTransitionCase(bsPuller, StateId.REQUEST_STREAM, StateId.STREAM_REQUEST_SUCCESS, null);
    bsPuller.getMessageQueue().clear();
    testTransitionCase(bsPuller, StateId.STREAM_REQUEST_SUCCESS, StateId.STREAM_REQUEST_SUCCESS, "SUSPEND_ON_ERROR", null);
    EasyMock.verify(channel);
    EasyMock.verify(sourcesConn2);
    EasyMock.verify(dbusBuffer);
    EasyMock.verify(channel);
    EasyMock.verify(mockConnFactory);
}
Also used : ReadableByteChannel(java.nio.channels.ReadableByteChannel) DatabusComponentStatus(com.linkedin.databus.core.DatabusComponentStatus) HashMap(java.util.HashMap) DbusKeyCompositeFilterConfig(com.linkedin.databus2.core.filter.DbusKeyCompositeFilterConfig) ServerInfo(com.linkedin.databus.client.pub.ServerInfo) DbusEventInternalReadable(com.linkedin.databus.core.DbusEventInternalReadable) ArrayList(java.util.ArrayList) DbusEventsStatisticsCollector(com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector) Properties(java.util.Properties) IdNamePair(com.linkedin.databus.core.util.IdNamePair) ArrayList(java.util.ArrayList) List(java.util.List) RemoteExceptionHandler(com.linkedin.databus.client.netty.RemoteExceptionHandler) DbusKeyCompositeFilterConfig(com.linkedin.databus2.core.filter.DbusKeyCompositeFilterConfig) ConfigLoader(com.linkedin.databus.core.util.ConfigLoader) AbstractActorMessageQueue(com.linkedin.databus.core.async.AbstractActorMessageQueue) ActorMessageQueue(com.linkedin.databus.core.async.ActorMessageQueue) DatabusSubscription(com.linkedin.databus.core.data_model.DatabusSubscription) DbusEventBuffer(com.linkedin.databus.core.DbusEventBuffer) Checkpoint(com.linkedin.databus.core.Checkpoint) LoggingConsumer(com.linkedin.databus.client.consumer.LoggingConsumer) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) RegisterResponseEntry(com.linkedin.databus2.core.container.request.RegisterResponseEntry) DbusEventV2Factory(com.linkedin.databus.core.DbusEventV2Factory) Test(org.testng.annotations.Test)

Example 89 with ReadableByteChannel

use of java.nio.channels.ReadableByteChannel in project databus by linkedin.

the class ReadEventsTestParams method testGetStreamedEventsWithRegression.

@Test
public void testGetStreamedEventsWithRegression() throws IOException, InvalidEventException, InvalidConfigException, OffsetNotFoundException {
    DbusEventBuffer dbuf = new DbusEventBuffer(getConfig(10000000, DbusEventBuffer.Config.DEFAULT_INDIVIDUAL_BUFFER_SIZE, 100000, 1000000, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    int numEntries = 50000;
    int eventWindowSize = 20;
    HashMap<Long, KeyValue> testDataMap = new HashMap<Long, KeyValue>(20000);
    dbuf.start(0);
    for (long i = 1; i < numEntries; i += 20) {
        // LOG.info("Iteration:"+i);
        DbusEventKey key = new DbusEventKey(RngUtils.randomLong());
        String value = RngUtils.randomString(20);
        dbuf.startEvents();
        for (int j = 0; j < eventWindowSize; ++j) {
            assertTrue(dbuf.appendEvent(key, pPartitionId, lPartitionId, timeStamp, srcId, schemaId, value.getBytes(Charset.defaultCharset()), false));
            testDataMap.put(i, new KeyValue(key, value));
        }
        dbuf.endEvents(i);
    }
    long minDbusEventBufferScn = dbuf.getMinScn();
    // TODO (medium) try out corner cases, more batches, etc.
    int batchFetchSize = 5000;
    Checkpoint cp = new Checkpoint();
    cp.setWindowScn(minDbusEventBufferScn);
    cp.setWindowOffset(0);
    cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
    WritableByteChannel writeChannel = null;
    File directory = new File(".");
    File writeFile = File.createTempFile("test", ".dbus", directory);
    try {
        writeChannel = Utils.openChannel(writeFile, true);
        StreamEventsArgs args = new StreamEventsArgs(batchFetchSize);
        dbuf.streamEvents(cp, writeChannel, args);
    } catch (ScnNotFoundException e) {
    }
    writeChannel.close();
    LOG.debug(writeFile.canRead());
    ReadableByteChannel readChannel = Utils.openChannel(writeFile, false);
    DbusEventBuffer checkDbusEventBuffer = new DbusEventBuffer(getConfig(50000, DbusEventBuffer.Config.DEFAULT_INDIVIDUAL_BUFFER_SIZE, 100000, 10000, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    int messageSize = 0;
    long lastWindowScn = 0;
    checkDbusEventBuffer.clear();
    checkDbusEventBuffer.readEvents(readChannel);
    LOG.debug("Reading events");
    DbusEventIterator eventIterator = checkDbusEventBuffer.acquireIterator("check");
    DbusEventInternalWritable e = null;
    while (eventIterator.hasNext()) {
        e = eventIterator.next();
        // LOG.info(e.scn()+"," + e.windowScn());
        messageSize += e.size();
        lastWindowScn = e.sequence();
    }
    assertTrue(messageSize <= batchFetchSize);
    LOG.debug("Reading events 2");
    // now we regress
    cp.setWindowScn(lastWindowScn - 5);
    cp.setWindowOffset(0);
    checkDbusEventBuffer.releaseIterator(eventIterator);
    LOG.debug("Reading events 3");
    writeFile.delete();
    writeFile = File.createTempFile("test", ".dbus", directory);
    try {
        writeChannel = Utils.openChannel(writeFile, true);
        StreamEventsArgs args = new StreamEventsArgs(batchFetchSize);
        dbuf.streamEvents(cp, writeChannel, args);
    } catch (ScnNotFoundException e1) {
        LOG.error("mainDbus threw ScnNotFound exception");
    }
    LOG.debug("mainDbus Read status a = " + dbuf.getReadStatus());
    assertEquals(0, dbuf.getReadStatus());
    LOG.debug("Reading events 4");
    LOG.debug(writeFile.canRead());
    readChannel = Utils.openChannel(writeFile, false);
    checkDbusEventBuffer.clear();
    messageSize = 0;
    lastWindowScn = 0;
    checkDbusEventBuffer.readEvents(readChannel);
    LOG.debug("Reading events 5");
    eventIterator = checkDbusEventBuffer.acquireIterator("eventIterator");
    LOG.debug("Reading events 6");
    while (eventIterator.hasNext()) {
        e = eventIterator.next();
        // LOG.info(e.scn()+"," + e.windowScn());
        // assertEquals(startScn+messageOffset + messageNum, e.scn());
        messageSize += e.size();
        lastWindowScn = e.sequence();
    // LOG.info("Reading events...");
    }
    assertTrue(messageSize <= batchFetchSize);
    LOG.debug("Reading events 7");
    checkDbusEventBuffer.releaseIterator(eventIterator);
    LOG.debug("mainDbus Read status = " + dbuf.getReadStatus());
    writeFile.delete();
}
Also used : ReadableByteChannel(java.nio.channels.ReadableByteChannel) HashMap(java.util.HashMap) WritableByteChannel(java.nio.channels.WritableByteChannel) File(java.io.File) DbusEventIterator(com.linkedin.databus.core.DbusEventBuffer.DbusEventIterator) Test(org.testng.annotations.Test)

Example 90 with ReadableByteChannel

use of java.nio.channels.ReadableByteChannel in project databus by linkedin.

the class ReadEventsTestParams method testReadEventOverlap.

@Test
public /*
   * This testcase is to recreate the bug where pull thread incorrectly writes to the head of
   * the iterator when in BLOCK_ON_WRITE mode. The error was because readEvents() incorrectly
   * relies on remaining() to give an accurate value.
   */
void testReadEventOverlap() throws Exception {
    /*
     * Recreate the head and tail position such that the eventBuffer is in the below state
     * --------------------------------------------------------------
     * ^      ^                                              ^      ^
     * |      |                                              |      |
     * 0      head                                           tail   capacity
     *
     * The space between tail and capacity is such that with no internal fragmentation, the free space
     * will be sufficient to store 2 events but with internal fragmentation, the free space will not be
     * enough. In this case, the readEvents should block until an event is removed by the other thread.
     */
    final DbusEventBuffer dbusBuf = new DbusEventBuffer(getConfig(1000, 1000, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE, AssertLevel.NONE));
    BufferPositionParser parser = dbusBuf.getBufferPositionParser();
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(11, 11, 100, 10, events);
    // Add events to the EventBuffer
    DbusEventAppender appender = new DbusEventAppender(events, dbusBuf, null);
    // running in the same thread
    appender.run();
    LOG.info("Head:" + dbusBuf.getHead() + ",Tail:" + dbusBuf.getTail());
    assertEquals("Head Check", 0, dbusBuf.getHead());
    assertEquals("Tail Check", 903, dbusBuf.getTail());
    // Remove the first event
    DbusEventIterator itr = dbusBuf.acquireIterator("dummy");
    assertTrue(itr.hasNext());
    DbusEvent event = itr.next();
    assertTrue(event.isValid());
    itr.remove();
    LOG.info("Head:" + parser.toString(dbusBuf.getHead()) + ",Tail:" + parser.toString(dbusBuf.getTail()));
    assertEquals("Head Check", 61, dbusBuf.getHead());
    assertEquals("Tail Check", 903, dbusBuf.getTail());
    for (DbusEvent e : events) {
        assertTrue("invalid event", e.isValid());
    }
    // set up the ReadChannel with 2 events
    ByteArrayOutputStream oStream = new ByteArrayOutputStream();
    WritableByteChannel oChannel = Channels.newChannel(oStream);
    for (int i = 0; i < 2; ++i) {
        ((DbusEventInternalReadable) events.get(i)).writeTo(oChannel, Encoding.BINARY);
    }
    byte[] writeBytes = oStream.toByteArray();
    ByteArrayInputStream iStream = new ByteArrayInputStream(writeBytes);
    final ReadableByteChannel rChannel = Channels.newChannel(iStream);
    // Create a Thread to call readEvents on the channel
    Runnable writer = new Runnable() {

        @Override
        public void run() {
            try {
                dbusBuf.readEvents(rChannel);
            } catch (InvalidEventException ie) {
                ie.printStackTrace();
                throw new RuntimeException(ie);
            }
        }
    };
    Thread writerThread = new Thread(writer);
    writerThread.start();
    // Check if the thread is alive (blocked) and head/tail is not overlapped
    trySleep(1000);
    assertTrue(writerThread.isAlive());
    LOG.info("Head:" + parser.toString(dbusBuf.getHead()) + ",Tail:" + parser.toString(dbusBuf.getTail()));
    assertEquals("Head Check", 61, dbusBuf.getHead());
    // GenId set here but tail is not yet overlapped
    assertEquals("Tail Check", 2048, dbusBuf.getTail());
    // Read the next event to unblock the writer
    event = itr.next();
    assertTrue(event.isValid());
    itr.remove();
    try {
        writerThread.join(1000);
    } catch (InterruptedException ie) {
        ie.printStackTrace();
    }
    LOG.info("Head:" + parser.toString(dbusBuf.getHead()) + ",Tail:" + parser.toString(dbusBuf.getTail()));
    assertFalse(writerThread.isAlive());
    assertEquals("Head Check", 132, dbusBuf.getHead());
    assertEquals("Tail Check", 2119, dbusBuf.getTail());
    while (itr.hasNext()) {
        assertTrue(itr.next().isValid(true));
        itr.remove();
    }
    LOG.info("Head:" + parser.toString(dbusBuf.getHead()) + ",Tail:" + parser.toString(dbusBuf.getTail()));
    assertEquals("Head Check", dbusBuf.getHead(), dbusBuf.getTail());
}
Also used : DbusEventAppender(com.linkedin.databus.core.test.DbusEventAppender) ReadableByteChannel(java.nio.channels.ReadableByteChannel) DbusEventGenerator(com.linkedin.databus.core.test.DbusEventGenerator) WritableByteChannel(java.nio.channels.WritableByteChannel) ByteArrayOutputStream(java.io.ByteArrayOutputStream) UncaughtExceptionTrackingThread(com.linkedin.databus.core.util.UncaughtExceptionTrackingThread) BufferPositionParser(com.linkedin.databus.core.util.BufferPositionParser) ByteArrayInputStream(java.io.ByteArrayInputStream) Vector(java.util.Vector) DbusEventIterator(com.linkedin.databus.core.DbusEventBuffer.DbusEventIterator) Test(org.testng.annotations.Test)

Aggregations

ReadableByteChannel (java.nio.channels.ReadableByteChannel)307 ByteBuffer (java.nio.ByteBuffer)111 IOException (java.io.IOException)84 FileOutputStream (java.io.FileOutputStream)62 WritableByteChannel (java.nio.channels.WritableByteChannel)62 Test (org.junit.Test)52 File (java.io.File)50 FileChannel (java.nio.channels.FileChannel)49 FileInputStream (java.io.FileInputStream)43 ByteArrayInputStream (java.io.ByteArrayInputStream)38 InputStream (java.io.InputStream)36 URL (java.net.URL)35 ByteArrayOutputStream (java.io.ByteArrayOutputStream)21 Path (java.nio.file.Path)18 Test (org.testng.annotations.Test)14 FileNotFoundException (java.io.FileNotFoundException)13 ArrayList (java.util.ArrayList)12 DbusEventGenerator (com.linkedin.databus.core.test.DbusEventGenerator)11 MalformedURLException (java.net.MalformedURLException)11 Vector (java.util.Vector)11