Examples with DbusEventInfo com.linkedin.databus.core.DbusEventInfo used on opensource projects

Example 11 with DbusEventInfo

use of com.linkedin.databus.core.DbusEventInfo in project databus by linkedin.

the class OpenReplicatorAvroEventFactory method createAndAppendEvent.

public int createAndAppendEvent(DbChangeEntry changeEntry, DbusEventBufferAppendable eventBuffer, boolean enableTracing, DbusEventsStatisticsCollector dbusEventsStatisticsCollector) throws EventCreationException, UnsupportedKeyException, DatabusException {
    Object keyObj = obtainKey(changeEntry);
    //Construct the Databus Event key, determine the key type and construct the key
    DbusEventKey eventKey = new DbusEventKey(keyObj);
    short lPartitionId = _partitionFunction.getPartition(eventKey);
    //Get the md5 for the schema
    SchemaId schemaId = SchemaId.createWithMd5(changeEntry.getSchema());
    byte[] payload = serializeEvent(changeEntry.getRecord());
    DbusEventInfo eventInfo = new DbusEventInfo(changeEntry.getOpCode(), changeEntry.getScn(), (short) _pSourceId, lPartitionId, changeEntry.getTimestampInNanos(), (short) _sourceId, schemaId.getByteArray(), payload, enableTracing, false);
    boolean success = eventBuffer.appendEvent(eventKey, eventInfo, dbusEventsStatisticsCollector);
    return success ? payload.length : -1;
}

Example 12 with DbusEventInfo

use of com.linkedin.databus.core.DbusEventInfo in project databus by linkedin.

the class DbusEventGenerator method generateEvents.

/**
   * Generate specified number of constant sized events.
   *
   * @param numEvents : Number of events desired
   * @param windowSize : Max window size (transaction size)
   * @param maxEventSize : maximum event size expected
   * @param payloadSize : payload size in bytes
   * @param useLastEventAsScn : if true, use count of last event of window as window scn; else use i/windowSize+1
   * @param eventVector : output container that is populated with the events
   * @return last window number generated
   */
public long generateEvents(int numEvents, int windowSize, int maxEventSize, int payloadSize, boolean useLastEventAsScn, Vector<DbusEvent> eventVector) {
    long lastScn = 0;
    try {
        long beginningOfTime = System.currentTimeMillis() / 1000;
        beginningOfTime *= 1000;
        short srcId = 1;
        for (int i = 0; i < numEvents; ++i) {
            if (_srcIdList != null && _srcIdList.size() > 0) {
                int srcIdIndex = RngUtils.randomPositiveInt() % _srcIdList.size();
                srcId = _srcIdList.get(srcIdIndex);
            } else {
                srcId = RngUtils.randomPositiveShort();
            }
            if (srcId == 0) {
                //0 srcId not allowed
                srcId = 1;
            }
            //assumption: serialized event fits in maxEventSize
            ByteBuffer buf = ByteBuffer.allocate(maxEventSize).order(_eventFactory.getByteOrder());
            DbusEventInfo eventInfo = new DbusEventInfo(DbusOpcode.UPSERT, // sequence number
            0L, // physical Partition
            (short) 0, RngUtils.randomPositiveShort(), (beginningOfTime - ((numEvents - i) * 1000)) * 1000 * 1000, //nanoseconds ; first event is numEvents seconds ago
            srcId, RngUtils.schemaMd5, RngUtils.randomString(payloadSize).getBytes(Charset.defaultCharset()), false, false);
            // make this explicit
            eventInfo.setEventSerializationVersion(DbusEventFactory.DBUS_EVENT_V1);
            DbusEventFactory.serializeEvent(new DbusEventKey(RngUtils.randomLong()), buf, eventInfo);
            lastScn = (useLastEventAsScn) ? _startScn + ((i / windowSize) + 1) * (long) windowSize : _startScn + (i / windowSize) + 1;
            eventVector.add((DbusEvent) DbusEventFactoryForTesting.createReadOnlyDbusEventFromBuffer(buf, 0, lastScn, DbusEventFactory.DBUS_EVENT_V1));
        }
    } catch (KeyTypeNotImplementedException e) {
    // ignore
    }
    return lastScn;
}

Example 13 with DbusEventInfo

use of com.linkedin.databus.core.DbusEventInfo in project databus by linkedin.

the class DummySuccessfulErrorCountingConsumer method testRelayFailoverPartialWindow1.

/**
   * Client switches from R1 to R2 on server close and from R2 to R3 on timeout while in the middle of the windows.
   *
   * <pre>
   * R1 EVB Boundaries --------------------------------------------------------------------------------
   *                   ^      ^        ^         ^         ^       ^       ^      ^     ^        ^
   *                   0      10       20        30        40      50      60     70    80       90
   *
   * R2 EVB Boundaries --------------------------------------------------------------------------------
   *                   ^               ^                   ^               ^            ^
   *                   0               20                  40              60           80
   *
   * R3 EVB Boundaries --------------------------------------------------------------------------------
   *                   ^                        ^                          ^                    ^
   *                   0                        30                         60                   90
   * </pre>
   *
   * Switch from R1 to R2 happens when windowScn = 10 and windowOffset = 8th event.
   * Switch from R2 to R3 happens when windowScn = 40 and windowOffset = 3rd event.
   *
   * @throws Exception
   */
@Test
public void testRelayFailoverPartialWindow1() throws Exception {
    final boolean debugOn = false;
    final Logger log = Logger.getLogger("TestDatabusHttpClient.testRelayFailoverPartialWindow1");
    log.setLevel(Level.INFO);
    final int eventsNum = 200;
    DbusEventInfo[] eventInfos = createSampleSchema1Events(eventsNum);
    final long timeoutMult = debugOn ? 100000 : 1;
    log.info("simulate relay buffers");
    DbusEventBuffer[] relayBuffer = new DbusEventBuffer[RELAY_PORT.length];
    List<List<Integer>> eventOfs = new ArrayList<List<Integer>>(3);
    List<List<DbusEventKey>> eventKeys = new ArrayList<List<DbusEventKey>>(3);
    for (int i = 0; i < RELAY_PORT.length; ++i) {
        relayBuffer[i] = new DbusEventBuffer(_bufCfg);
        relayBuffer[i].start(0);
        WriteEventsResult wrRes = writeEventsToBuffer(relayBuffer[i], eventInfos, (i + 1) * 10);
        List<Integer> ofs = wrRes.getOffsets();
        eventOfs.add(ofs);
        eventKeys.add(wrRes.getKeys());
    }
    List<DbusEventKey> key = eventKeys.get(0);
    for (int i = 1; i < RELAY_PORT.length; ++i) {
        assertEquals(" For Size index : " + i, key.size(), eventKeys.get(i).size());
        assertEquals(" For index : " + i, key, eventKeys.get(i));
        key = eventKeys.get(i);
    }
    int resp1EnfOfs = eventOfs.get(0).get(8);
    log.info("figure out an event offset inside a window:" + resp1EnfOfs);
    log.info("create client");
    _stdClientCfgBuilder.getContainer().setReadTimeoutMs(DEFAULT_READ_TIMEOUT_MS);
    final DatabusHttpClientImpl client = new DatabusHttpClientImpl(_stdClientCfgBuilder.build());
    final TestConsumer consumer = new TestConsumer();
    client.registerDatabusStreamListener(consumer, null, SOURCE1_NAME);
    client.start();
    try {
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return client._relayConnections.size() == 1;
            }
        }, "sources connection present", 100, log);
        final DatabusSourcesConnection clientConn = client._relayConnections.get(0);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != clientConn.getRelayPullThread().getLastOpenConnection();
            }
        }, "relay connection present", 100, log);
        final NettyHttpDatabusRelayConnection relayConn = (NettyHttpDatabusRelayConnection) clientConn.getRelayPullThread().getLastOpenConnection();
        final NettyHttpDatabusRelayConnectionInspector relayConnInsp = new NettyHttpDatabusRelayConnectionInspector(relayConn);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != relayConnInsp.getChannel() && relayConnInsp.getChannel().isConnected();
            }
        }, "client connected", 200, log);
        log.info("figure out the connection to the relay");
        Channel clientChannel = relayConnInsp.getChannel();
        InetSocketAddress relayAddr = (InetSocketAddress) clientChannel.getRemoteAddress();
        SocketAddress clientAddr = clientChannel.getLocalAddress();
        int relayPort = relayAddr.getPort();
        log.info("relay selected: " + relayPort);
        SimpleTestServerConnection relay = null;
        for (int i = 0; i < RELAY_PORT.length; ++i) {
            if (relayPort == RELAY_PORT[i])
                relay = _dummyServer[i];
        }
        assertTrue(null != relay);
        final SocketAddress testClientAddr = clientAddr;
        final SimpleTestServerConnection testRelay = relay;
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != testRelay.getChildChannel(testClientAddr);
            }
        }, "relay detects new connection", 1000, log);
        Channel serverChannel = relay.getChildChannel(clientAddr);
        assertTrue(null != serverChannel);
        ChannelPipeline serverPipeline = serverChannel.getPipeline();
        SimpleObjectCaptureHandler objCapture = (SimpleObjectCaptureHandler) serverPipeline.get("3");
        log.info("process the /sources request");
        NettyTestUtils.waitForHttpRequest(objCapture, SOURCES_REQUEST_REGEX, 1000);
        objCapture.clear();
        log.info("send back the /sources response");
        HttpResponse sourcesResp = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
        sourcesResp.setHeader(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
        sourcesResp.setHeader(HttpHeaders.Names.TRANSFER_ENCODING, HttpHeaders.Values.CHUNKED);
        HttpChunk body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(("[{\"id\":1,\"name\":\"" + SOURCE1_NAME + "\"}]").getBytes(Charset.defaultCharset())));
        NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
        log.info("make sure the client processes the response correctly");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                String idListString = clientConn.getRelayPullThread()._currentState.getSourcesIdListString();
                return "1".equals(idListString);
            }
        }, "client processes /sources response", 100, log);
        log.info("process the /register request");
        NettyTestUtils.waitForHttpRequest(objCapture, "/register.*", 1000);
        objCapture.clear();
        log.info("send back the /register response");
        RegisterResponseEntry entry = new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR);
        String responseStr = NettyTestUtils.generateRegisterResponse(entry);
        body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(responseStr.getBytes(Charset.defaultCharset())));
        NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
        log.info("make sure the client processes the response correctly");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                DispatcherState dispState = clientConn.getRelayDispatcher().getDispatcherState();
                return null != dispState.getSchemaMap() && 1 == dispState.getSchemaMap().size();
            }
        }, "client processes /register response", 100, log);
        log.info("process /stream call and return a partial window");
        NettyTestUtils.waitForHttpRequest(objCapture, "/stream.*", 1000);
        objCapture.clear();
        log.info("send back the /stream response");
        final DbusEventsStatisticsCollector stats = new DbusEventsStatisticsCollector(1, "test1", true, false, null);
        Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
        ChannelBuffer streamRes = NettyTestUtils.streamToChannelBuffer(relayBuffer[0], cp, resp1EnfOfs, stats);
        HttpResponse streamResp = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
        streamResp.setHeader(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
        streamResp.setHeader(HttpHeaders.Names.TRANSFER_ENCODING, HttpHeaders.Values.CHUNKED);
        NettyTestUtils.sendServerResponses(relay, clientAddr, streamResp, new DefaultHttpChunk(streamRes));
        log.info("make sure the client processes the /stream response correctly");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.debug("LastWritten SCN:" + clientConn.getDataEventsBuffer().lastWrittenScn());
                return clientConn.getDataEventsBuffer().lastWrittenScn() == 10;
            }
        }, "client receives /stream response", 1100, log);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.debug("events num=" + consumer.getEventNum());
                return stats.getTotalStats().getNumDataEvents() == consumer.getEventNum();
            }
        }, "client processes /stream response", 110000, log);
        assertEquals(-1, consumer.getRollbackScn());
        int rollbackNum = 0;
        assertEquals(stats.getTotalStats().getNumSysEvents() + 1 + rollbackNum, consumer.getWinNum());
        List<DbusEventKey> expKeys = eventKeys.get(0).subList(0, (int) stats.getTotalStats().getNumDataEvents());
        List<Long> expSeqs = new ArrayList<Long>();
        for (int i = 0; i < stats.getTotalStats().getNumDataEvents(); i++) expSeqs.add(10L);
        long numEvents = stats.getTotalStats().getNumDataEvents();
        assertEquals("Keys", expKeys, consumer.getKeys());
        assertEquals("Sequences", expSeqs, consumer.getSequences());
        log.info("now kill the relay and wait for a failover");
        serverChannel.close();
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != relayConnInsp.getChannel() && !relayConnInsp.getChannel().isConnected();
            }
        }, "client disconnected", 200, log);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return clientConn.getRelayPullThread().getLastOpenConnection() != relayConn;
            }
        }, "new netty connection", 200, log);
        log.info("/////////// FAKING CONNECTION TO NEW RELAY //////////////");
        final NettyHttpDatabusRelayConnection newRelayConn = (NettyHttpDatabusRelayConnection) clientConn.getRelayPullThread().getLastOpenConnection();
        final NettyHttpDatabusRelayConnectionInspector newRelayConnInsp = new NettyHttpDatabusRelayConnectionInspector(newRelayConn);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != newRelayConnInsp.getChannel() && newRelayConnInsp.getChannel().isConnected();
            }
        }, "client connected to new relay", 200, log);
        log.info("figure out the connection to the relay");
        clientChannel = newRelayConnInsp.getChannel();
        relayAddr = (InetSocketAddress) clientChannel.getRemoteAddress();
        clientAddr = clientChannel.getLocalAddress();
        relayPort = relayAddr.getPort();
        log.info("new relay selected: " + relayPort);
        relay = null;
        int relayIdx = 0;
        for (; relayIdx < RELAY_PORT.length; ++relayIdx) {
            if (relayPort == RELAY_PORT[relayIdx])
                relay = _dummyServer[relayIdx];
        }
        assertTrue(null != relay);
        serverChannel = relay.getChildChannel(clientAddr);
        assertTrue(null != serverChannel);
        serverPipeline = serverChannel.getPipeline();
        objCapture = (SimpleObjectCaptureHandler) serverPipeline.get("3");
        log.info("process the /sources request");
        NettyTestUtils.waitForHttpRequest(objCapture, SOURCES_REQUEST_REGEX, 1000);
        objCapture.clear();
        log.info("send back the /sources response");
        body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(("[{\"id\":1,\"name\":\"" + SOURCE1_NAME + "\"}]").getBytes(Charset.defaultCharset())));
        NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
        log.info("make sure the client processes the response correctly");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                String idListString = clientConn.getRelayPullThread()._currentState.getSourcesIdListString();
                return "1".equals(idListString);
            }
        }, "client processes /sources response", 100, log);
        log.info("process the /register request");
        NettyTestUtils.waitForHttpRequest(objCapture, "/register.*", 1000);
        objCapture.clear();
        log.info("send back the /register response");
        body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(responseStr.getBytes(Charset.defaultCharset())));
        NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
        log.info("make sure the client processes the /register response correctly");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                DispatcherState dispState = clientConn.getRelayDispatcher().getDispatcherState();
                return null != dispState.getSchemaMap() && 1 == dispState.getSchemaMap().size();
            }
        }, "client processes /register response", 100, log);
        log.info("process /stream call and return a partial window");
        Matcher streamMatcher = NettyTestUtils.waitForHttpRequest(objCapture, "/stream.*checkPoint=([^&]*)&.*", 1000);
        String cpString = streamMatcher.group(1);
        objCapture.clear();
        int respStartOfs = eventOfs.get(1).get(1);
        int respEndOfs = eventOfs.get(1).get(34);
        cp = new Checkpoint(cpString);
        //last window read was partial
        assertTrue(cp.getWindowOffset() > 0);
        streamRes = NettyTestUtils.streamToChannelBuffer(relayBuffer[1], cp, respEndOfs - respStartOfs, stats);
        NettyTestUtils.sendServerResponses(relay, clientAddr, streamResp, new DefaultHttpChunk(streamRes));
        log.info("make sure the client processes the response correctly");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.debug("lastWrittenScn=" + clientConn.getDataEventsBuffer().lastWrittenScn());
                return clientConn.getDataEventsBuffer().lastWrittenScn() == 40;
            }
        }, "client receives /stream response", 1100, log);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.debug("events num=" + consumer.getEventNum());
                return stats.getTotalStats().getNumDataEvents() == consumer.getEventNum();
            }
        }, "client processes /stream response", 11000, log);
        assertEquals(20, consumer.getRollbackScn());
        log.info("one more onStartDataEventSequence because of the rolback");
        ++rollbackNum;
        assertEquals(stats.getTotalStats().getNumSysEvents() + 1 + rollbackNum, consumer.getWinNum());
        assertEquals(clientConn.getRelayPullThread().getConnectionState().getDataEventsBuffer().isSCNRegress(), false);
        expKeys.addAll(eventKeys.get(1).subList(0, (int) (stats.getTotalStats().getNumDataEvents() - numEvents)));
        for (int i = 0; i < stats.getTotalStats().getNumDataEvents() - numEvents; i++) expSeqs.add((i / 20) * 20 + 20L);
        assertEquals("Keys", expKeys, consumer.getKeys());
        assertEquals("Sequences", expSeqs, consumer.getSequences());
        numEvents = stats.getTotalStats().getNumDataEvents();
        ///////////////////////////////////
        //simulate a timeout on the server; the client would have sent a /stream call and there
        //will be no response from the server, so eventually it should time out and switch servers
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.debug("Channel :" + newRelayConnInsp.getChannel());
                return (null != newRelayConnInsp.getChannel()) && (!newRelayConnInsp.getChannel().isConnected());
            }
        }, "waiting for a reconnect", (long) (DEFAULT_READ_TIMEOUT_MS * 1.5), log);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return clientConn.getRelayPullThread().getLastOpenConnection() != relayConn;
            }
        }, "new netty connection", 200, log);
        final NettyHttpDatabusRelayConnection new2RelayConn = (NettyHttpDatabusRelayConnection) clientConn.getRelayPullThread().getLastOpenConnection();
        final NettyHttpDatabusRelayConnectionInspector new2RelayConnInsp = new NettyHttpDatabusRelayConnectionInspector(new2RelayConn);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != new2RelayConnInsp.getChannel() && new2RelayConnInsp.getChannel().isConnected();
            }
        }, "client connected to third relay", 200, log);
        log.info("figure out the connection to the relay");
        clientChannel = new2RelayConnInsp.getChannel();
        relayAddr = (InetSocketAddress) clientChannel.getRemoteAddress();
        clientAddr = clientChannel.getLocalAddress();
        relayPort = relayAddr.getPort();
        log.info("third relay selected: " + relayPort);
        relay = null;
        relayIdx = 0;
        for (; relayIdx < RELAY_PORT.length; ++relayIdx) {
            if (relayPort == RELAY_PORT[relayIdx])
                relay = _dummyServer[relayIdx];
        }
        assertTrue(null != relay);
        serverChannel = relay.getChildChannel(clientAddr);
        assertTrue(null != serverChannel);
        serverPipeline = serverChannel.getPipeline();
        objCapture = (SimpleObjectCaptureHandler) serverPipeline.get("3");
        log.info("process the /sources request");
        NettyTestUtils.waitForHttpRequest(objCapture, SOURCES_REQUEST_REGEX, 1000);
        objCapture.clear();
        log.info("send back the /sources response");
        body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(("[{\"id\":1,\"name\":\"" + SOURCE1_NAME + "\"}]").getBytes(Charset.defaultCharset())));
        NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
        log.info("make sure the client processes the response correctly");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                String idListString = clientConn.getRelayPullThread()._currentState.getSourcesIdListString();
                return "1".equals(idListString);
            }
        }, "client processes /sources response", 100, log);
        log.info("process the /register request");
        NettyTestUtils.waitForHttpRequest(objCapture, "/register.*", 1000);
        objCapture.clear();
        log.info("SEND BACK THE /register RESPONSE");
        //clientConn.getRelayDispatcher().getLog().setLevel(Level.DEBUG);
        //RangeBasedReaderWriterLock.LOG.setLevel(Level.DEBUG);
        body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(responseStr.getBytes(Charset.defaultCharset())));
        NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
        log.info("make sure the client processes the response correctly");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                DispatcherState dispState = clientConn.getRelayDispatcher().getDispatcherState();
                return null != dispState.getSchemaMap() && 1 == dispState.getSchemaMap().size();
            }
        }, "client processes /register response", timeoutMult * 100, log);
        log.info("PROCESS the /stream CALL AND RETURN A PARTIAL WINDOW");
        streamMatcher = NettyTestUtils.waitForHttpRequest(objCapture, "/stream.*checkPoint=([^&]*)&.*", 1000);
        cpString = streamMatcher.group(1);
        objCapture.clear();
        respStartOfs = eventOfs.get(2).get(1);
        respEndOfs = eventOfs.get(2).get(84);
        log.debug("Checkpoint String is :" + cpString);
        cp = new Checkpoint(cpString);
        log.info("last window read was partial. So the client would have reset the windowOffset");
        assertTrue("Is WindowOffset Cleared", cp.getWindowOffset() == -1);
        assertEquals("WindowSCN == PrevSCN. Ckpt :" + cp, cp.getWindowScn(), cp.getPrevScn());
        streamRes = NettyTestUtils.streamToChannelBuffer(relayBuffer[2], cp, respEndOfs - respStartOfs, stats);
        NettyTestUtils.sendServerResponses(relay, clientAddr, streamResp, new DefaultHttpChunk(streamRes));
        log.debug("NumEvents already seen :" + numEvents);
        log.info("make sure the client processes the response correctly");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.debug("lastWrittenScn=" + clientConn.getDataEventsBuffer().lastWrittenScn() + ", NumEvents :" + stats.getTotalStats().getNumDataEvents());
                return clientConn.getDataEventsBuffer().lastWrittenScn() == 90;
            }
        }, "client receives /stream response, Sequences :" + consumer.getSequences(), timeoutMult * 1100, log);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.debug("events num=" + consumer.getEventNum());
                return stats.getTotalStats().getNumDataEvents() == consumer.getEventNum();
            }
        }, "client processes /stream response", timeoutMult * 1100, log);
        log.info("one more onStartDataEventSequence because of the rollback");
        assertEquals(30, consumer.getRollbackScn());
        ++rollbackNum;
        assertEquals(stats.getTotalStats().getNumSysEvents() + 1 + rollbackNum, consumer.getWinNum());
        expKeys.addAll(eventKeys.get(2).subList(0, (int) (stats.getTotalStats().getNumDataEvents() - numEvents)));
        for (int i = 0; i < stats.getTotalStats().getNumDataEvents() - numEvents; i++) expSeqs.add((i / 30) * 30 + 30L);
        assertEquals("Keys", expKeys, consumer.getKeys());
        assertEquals("Sequences", expSeqs, consumer.getSequences());
        numEvents = stats.getTotalStats().getNumDataEvents();
        assertEquals(clientConn.getRelayPullThread().getConnectionState().getDataEventsBuffer().isSCNRegress(), false);
    } finally {
        client.shutdown();
    }
}

Example 14 with DbusEventInfo

use of com.linkedin.databus.core.DbusEventInfo in project databus by linkedin.

the class DummySuccessfulErrorCountingConsumer method createSampleSchema1Events.

static DbusEventInfo[] createSampleSchema1Events(int eventsNum) throws IOException {
    Random rng = new Random();
    DbusEventInfo[] result = new DbusEventInfo[eventsNum];
    GenericDatumWriter<GenericRecord> writer = new GenericDatumWriter<GenericRecord>(SOURCE1_SCHEMA);
    for (int i = 0; i < eventsNum; ++i) {
        GenericRecord r = new GenericData.Record(SOURCE1_SCHEMA);
        String s = RngUtils.randomString(rng.nextInt(100));
        r.put("s", s);
        ByteArrayOutputStream baos = new ByteArrayOutputStream(s.length() + 100);
        BinaryEncoder out = new BinaryEncoder(baos);
        try {
            writer.write(r, out);
            out.flush();
            result[i] = new DbusEventInfo(DbusOpcode.UPSERT, 1, (short) 1, (short) 1, System.nanoTime(), (short) 1, SOURCE1_SCHEMAID, baos.toByteArray(), false, true);
            result[i].setEventSerializationVersion(_eventFactory.getVersion());
        } finally {
            baos.close();
        }
    }
    return result;
}

Example 15 with DbusEventInfo

use of com.linkedin.databus.core.DbusEventInfo in project databus by linkedin.

the class DummySuccessfulErrorCountingConsumer method testInStreamError.

/**
     * Tests the logic of the client to disconnect from a relay while still processing
     * data from the relay. We want to make sure that ChunkedBodyReadableByteChannel
     * and thus the StreamHttpResponseProcessor do net get blocked with unprocessed data.
     * One way to trigger such an error is to insert some invalid data inside the
     * response and then some more data ( > 4MB which is the current buffer size). */
@Test
public void testInStreamError() throws Exception {
    final Logger log = Logger.getLogger("TestDatabusHttpClient.testInStreamError");
    //log.setLevel(Level.DEBUG);
    final int eventsNum = 20;
    DbusEventInfo[] eventInfos = createSampleSchema1Events(eventsNum);
    //simulate relay buffers
    DbusEventBuffer relayBuffer = new DbusEventBuffer(_bufCfg);
    relayBuffer.start(0);
    writeEventsToBuffer(relayBuffer, eventInfos, 4);
    //prepare stream response
    Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
    final DbusEventsStatisticsCollector stats = new DbusEventsStatisticsCollector(1, "test1", true, false, null);
    ChannelBuffer streamResPrefix = NettyTestUtils.streamToChannelBuffer(relayBuffer, cp, 20000, stats);
    final StringBuilder alotOfDeadbeef = new StringBuilder();
    for (int i = 0; i < 1024 * 1024; ++i) {
        alotOfDeadbeef.append("DEADBEEF ");
    }
    ChannelBuffer streamResSuf = ChannelBuffers.wrappedBuffer(alotOfDeadbeef.toString().getBytes("UTF-8"));
    final ChannelBuffer streamRes = ChannelBuffers.wrappedBuffer(streamResPrefix, streamResSuf);
    //create client
    _stdClientCfgBuilder.getContainer().setReadTimeoutMs(DEFAULT_READ_TIMEOUT_MS);
    final DatabusHttpClientImpl client = new DatabusHttpClientImpl(_stdClientCfgBuilder.build());
    final TestConsumer consumer = new TestConsumer();
    client.registerDatabusStreamListener(consumer, null, SOURCE1_NAME);
    client.start();
    try {
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return client._relayConnections.size() == 1;
            }
        }, "sources connection present", 100, log);
        final DatabusSourcesConnection clientConn = client._relayConnections.get(0);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != clientConn.getRelayPullThread().getLastOpenConnection();
            }
        }, "relay connection present", 100, log);
        final NettyHttpDatabusRelayConnection relayConn = (NettyHttpDatabusRelayConnection) clientConn.getRelayPullThread().getLastOpenConnection();
        final NettyHttpDatabusRelayConnectionInspector relayConnInsp = new NettyHttpDatabusRelayConnectionInspector(relayConn);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != relayConnInsp.getChannel() && relayConnInsp.getChannel().isConnected();
            }
        }, "client connected", 200, log);
        //figure out the connection to the relay
        Channel clientChannel = relayConnInsp.getChannel();
        InetSocketAddress relayAddr = (InetSocketAddress) clientChannel.getRemoteAddress();
        SocketAddress clientAddr = clientChannel.getLocalAddress();
        int relayPort = relayAddr.getPort();
        log.info("relay selected: " + relayPort);
        SimpleTestServerConnection relay = null;
        for (int i = 0; i < RELAY_PORT.length; ++i) {
            if (relayPort == RELAY_PORT[i])
                relay = _dummyServer[i];
        }
        assertTrue(null != relay);
        final SocketAddress testClientAddr = clientAddr;
        final SimpleTestServerConnection testRelay = relay;
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != testRelay.getChildChannel(testClientAddr);
            }
        }, "relay detects new connection", 1000, log);
        Channel serverChannel = relay.getChildChannel(clientAddr);
        assertTrue(null != serverChannel);
        ChannelPipeline serverPipeline = serverChannel.getPipeline();
        SimpleObjectCaptureHandler objCapture = (SimpleObjectCaptureHandler) serverPipeline.get("3");
        //process the /sources request
        NettyTestUtils.waitForHttpRequest(objCapture, SOURCES_REQUEST_REGEX, 1000);
        objCapture.clear();
        //send back the /sources response
        HttpResponse sourcesResp = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
        sourcesResp.setHeader(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
        sourcesResp.setHeader(HttpHeaders.Names.TRANSFER_ENCODING, HttpHeaders.Values.CHUNKED);
        HttpChunk body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(("[{\"id\":1,\"name\":\"" + SOURCE1_NAME + "\"}]").getBytes(Charset.defaultCharset())));
        NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
        //make sure the client processes the response correctly
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                String idListString = clientConn.getRelayPullThread()._currentState.getSourcesIdListString();
                return "1".equals(idListString);
            }
        }, "client processes /sources response", 100, log);
        log.debug("process the /register request");
        NettyTestUtils.waitForHttpRequest(objCapture, "/register.*", 1000);
        objCapture.clear();
        log.debug("send back the /register response");
        RegisterResponseEntry entry = new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR);
        String responseStr = NettyTestUtils.generateRegisterResponse(entry);
        body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(responseStr.getBytes(Charset.defaultCharset())));
        NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
        log.debug("make sure the client processes the response /register correctly");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                DispatcherState dispState = clientConn.getRelayDispatcher().getDispatcherState();
                return null != dispState.getSchemaMap() && 1 == dispState.getSchemaMap().size();
            }
        }, "client processes /register response", 100, log);
        log.debug("process /stream call and return a response with garbled suffix");
        NettyTestUtils.waitForHttpRequest(objCapture, "/stream.*", 1000);
        objCapture.clear();
        final HttpResponse streamResp = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
        streamResp.setHeader(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
        streamResp.setHeader(HttpHeaders.Names.TRANSFER_ENCODING, HttpHeaders.Values.CHUNKED);
        //send the response asynchronously in case the client blocks
        final Thread streamRespThread = new Thread(new Runnable() {

            @Override
            public void run() {
                NettyTestUtils.sendServerResponses(testRelay, testClientAddr, streamResp, new DefaultHttpChunk(streamRes), 60000);
            }
        }, "send /stream resp");
        streamRespThread.setDaemon(true);
        streamRespThread.start();
        log.debug("make sure the client disconnects and recovers from the /stream response");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != relayConnInsp.getChannel() && !relayConnInsp.getChannel().isConnected();
            }
        }, "client disconnected", 30000, log);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return clientConn.getRelayPullThread().getLastOpenConnection() != relayConn;
            }
        }, "new netty connection", 30000, log);
        //make sure the relay send thread is dead
        streamRespThread.join(100);
        Assert.assertTrue(!streamRespThread.isAlive());
        log.debug("PHASE 2: make sure the client has fully recovered and able to connect to another relay");
        final NettyHttpDatabusRelayConnection newRelayConn = (NettyHttpDatabusRelayConnection) clientConn.getRelayPullThread().getLastOpenConnection();
        final NettyHttpDatabusRelayConnectionInspector newRelayConnInsp = new NettyHttpDatabusRelayConnectionInspector(newRelayConn);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return null != newRelayConnInsp.getChannel() && newRelayConnInsp.getChannel().isConnected();
            }
        }, "client connected to new relay", 200, log);
        //figure out the connection to the relay
        clientChannel = newRelayConnInsp.getChannel();
        relayAddr = (InetSocketAddress) clientChannel.getRemoteAddress();
        clientAddr = clientChannel.getLocalAddress();
        relayPort = relayAddr.getPort();
        log.info("new relay selected: " + relayPort);
        relay = null;
        int relayIdx = 0;
        for (; relayIdx < RELAY_PORT.length; ++relayIdx) {
            if (relayPort == RELAY_PORT[relayIdx])
                relay = _dummyServer[relayIdx];
        }
        assertTrue(null != relay);
        serverChannel = relay.getChildChannel(clientAddr);
        assertTrue(null != serverChannel);
        serverPipeline = serverChannel.getPipeline();
        objCapture = (SimpleObjectCaptureHandler) serverPipeline.get("3");
        //process the /sources request
        NettyTestUtils.waitForHttpRequest(objCapture, SOURCES_REQUEST_REGEX, 1000);
        objCapture.clear();
        //send back the /sources response
        body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(("[{\"id\":1,\"name\":\"" + SOURCE1_NAME + "\"}]").getBytes(Charset.defaultCharset())));
        NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
        //make sure the client processes the response correctly
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                String idListString = clientConn.getRelayPullThread()._currentState.getSourcesIdListString();
                return "1".equals(idListString);
            }
        }, "client processes /sources response", 100, log);
        //process the /register request
        NettyTestUtils.waitForHttpRequest(objCapture, "/register.*", 1000);
        objCapture.clear();
        //send back the /register response
        body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(responseStr.getBytes(Charset.defaultCharset())));
        NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);
        //make sure the client processes the response correctly
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                DispatcherState dispState = clientConn.getRelayDispatcher().getDispatcherState();
                return null != dispState.getSchemaMap() && 1 == dispState.getSchemaMap().size();
            }
        }, "client processes /register response", 100, log);
        //process /stream call and return a partial window
        Matcher streamMatcher = NettyTestUtils.waitForHttpRequest(objCapture, "/stream.*checkPoint=([^&]*)&.*", 1000);
        String cpString = streamMatcher.group(1);
        log.debug("/stream checkpoint: " + cpString);
        objCapture.clear();
        NettyTestUtils.sendServerResponses(relay, clientAddr, streamResp, new DefaultHttpChunk(streamResPrefix));
        //make sure the client processes the response correctly
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.debug("lastWrittenScn=" + clientConn.getDataEventsBuffer().lastWrittenScn());
                return clientConn.getDataEventsBuffer().lastWrittenScn() == 20;
            }
        }, "client receives /stream response", 1100, log);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.debug("events num=" + consumer.getEventNum());
                return stats.getTotalStats().getNumDataEvents() == consumer.getEventNum();
            }
        }, "client processes /stream response", 11000, log);
    } finally {
        client.shutdown();
    }
}