Examples with CheckpointMult com.linkedin.databus.core.CheckpointMult used on opensource projects

Example 1 with CheckpointMult

use of com.linkedin.databus.core.CheckpointMult in project databus by linkedin.

the class RelayPullThread method doRequestStream.

protected void doRequestStream(ConnectionState curState) {
    boolean debugEnabled = _log.isDebugEnabled();
    if (debugEnabled)
        _log.debug("Checking for free space in buffer");
    int freeBufferThreshold = (int) (_sourcesConn.getConnectionConfig().getFreeBufferThreshold() * 100.0 / _pullerBufferUtilizationPct);
    try {
        curState.getDataEventsBuffer().waitForFreeSpace(freeBufferThreshold);
    } catch (InterruptedException ie) {
        //loop
        enqueueMessage(curState);
        return;
    }
    Checkpoint cp = curState.getCheckpoint();
    if (debugEnabled)
        _log.debug("Checkpoint at RequestDataEvents: " + cp.toString());
    if (null == _relayFilter) {
        if (debugEnabled)
            _log.debug("Initializing relay filter config");
        _relayFilter = new DbusKeyCompositeFilter();
        Map<String, IdNamePair> srcNameIdMap = curState.getSourcesNameMap();
        for (DbusKeyCompositeFilterConfig conf : _relayFilterConfigs) {
            Map<String, KeyFilterConfigHolder> cMap = conf.getConfigMap();
            Map<Long, KeyFilterConfigHolder> fConfMap = new HashMap<Long, KeyFilterConfigHolder>();
            for (Entry<String, KeyFilterConfigHolder> e : cMap.entrySet()) {
                IdNamePair idName = srcNameIdMap.get(e.getKey());
                if (null != idName) {
                    fConfMap.put(idName.getId(), e.getValue());
                }
            }
            if (debugEnabled)
                _log.debug("FilterConfMap is :" + fConfMap);
            _relayFilter.merge(new DbusKeyCompositeFilter(fConfMap));
        }
        if (debugEnabled)
            _log.debug("Merged Filter (before deduping) is :" + _relayFilter);
        _relayFilter.dedupe();
        if (debugEnabled)
            _log.debug("Merged Filter (after deduping) is :" + _relayFilter);
    }
    _streamCallStartMs = System.currentTimeMillis();
    if (null != _relayCallsStats)
        _relayCallsStats.registerStreamRequest(cp, EMPTY_STREAM_LIST);
    int fetchSize = (int) ((curState.getDataEventsBuffer().getBufferFreeReadSpace() / 100.0) * _pullerBufferUtilizationPct);
    fetchSize = Math.max(freeBufferThreshold, fetchSize);
    CheckpointMult cpMult = new CheckpointMult();
    String args;
    if (curState.getRelayConnection().getProtocolVersion() >= 3) {
        // for version 3 and higher we pass subscriptions
        args = curState.getSubsListString();
        for (DatabusSubscription sub : curState.getSubscriptions()) {
            PhysicalPartition p = sub.getPhysicalPartition();
            cpMult.addCheckpoint(p, cp);
        }
    } else {
        args = curState.getSourcesIdListString();
        cpMult.addCheckpoint(PhysicalPartition.ANY_PHYSICAL_PARTITION, cp);
    }
    curState.switchToStreamRequestSent();
    sendHeartbeat(_sourcesConn.getUnifiedClientStats());
    curState.getRelayConnection().requestStream(args, _relayFilter, fetchSize, cpMult, _sourcesConn.getConnectionConfig().getKeyRange(), curState);
}

Example 2 with CheckpointMult

use of com.linkedin.databus.core.CheckpointMult in project databus by linkedin.

the class DummyRemoteExceptionHandler method runServerStreamReadTimeoutIteration.

private void runServerStreamReadTimeoutIteration(final Logger log, TestingConnectionCallback callback, DummyRemoteExceptionHandler remoteExceptionHandler, final NettyHttpDatabusRelayConnection conn) throws JsonGenerationException, JsonMappingException, IOException, ScnNotFoundException, OffsetNotFoundException {
    //connect to server and send /sources
    TestResponseProcessors.TestConnectionStateMessage msg = new TestResponseProcessors.TestConnectionStateMessage();
    conn.requestSources(msg);
    waitForServerConnection(conn, log);
    //introspect connection to server
    Channel channel = conn._channel;
    SocketAddress clientAddr = channel.getLocalAddress();
    Channel serverChannel = _dummyServer.getChildChannel(clientAddr);
    ChannelPipeline serverPipeline = serverChannel.getPipeline();
    SimpleObjectCaptureHandler objCapture = (SimpleObjectCaptureHandler) serverPipeline.get("3");
    //verify server gets the /source request
    HttpResponse sourcesResp = runHappyPathSources(log, callback, remoteExceptionHandler, clientAddr, objCapture);
    //send /register
    runHappyPathRegister(log, callback, remoteExceptionHandler, conn, msg, clientAddr, objCapture, sourcesResp);
    //send partial /stream
    callback.clearLastMsg();
    objCapture.clear();
    Checkpoint cp = new Checkpoint();
    cp.setFlexible();
    CheckpointMult cpm = new CheckpointMult();
    cpm.addCheckpoint(PhysicalPartition.ANY_PHYSICAL_PARTITION, cp);
    conn.requestStream("1", null, 1000, cpm, null, msg);
    //////// verify server gets the /stream request
    HttpRequest msgReq = captureRequest(objCapture);
    Assert.assertTrue(msgReq.getUri().startsWith("/stream"));
    //Trigger a read timeout
    TestUtil.sleep(DEFAULT_READ_TIMEOUT_MS + 100);
    waitForCallback(callback, TestResponseProcessors.TestConnectionStateMessage.State.STREAM_RESPONSE_ERROR, log);
    Assert.assertNull(remoteExceptionHandler.getLastException());
    Assert.assertEquals(1, callback.getAllMsgs().size());
    callback.clearLastMsg();
    objCapture.clear();
}

Example 3 with CheckpointMult

use of com.linkedin.databus.core.CheckpointMult in project databus by linkedin.

the class DummyRemoteExceptionHandler method runHappyPathStream.

private void runHappyPathStream(final Logger log, DbusEventBuffer buf, TestingConnectionCallback callback, DummyRemoteExceptionHandler remoteExceptionHandler, final NettyHttpDatabusRelayConnection conn, TestResponseProcessors.TestConnectionStateMessage msg, SocketAddress clientAddr, SimpleObjectCaptureHandler objCapture, HttpResponse sourcesResp) throws ScnNotFoundException, OffsetNotFoundException, IOException {
    HttpRequest msgReq;
    objCapture.clear();
    Checkpoint cp = new Checkpoint();
    cp.setFlexible();
    CheckpointMult cpm = new CheckpointMult();
    cpm.addCheckpoint(PhysicalPartition.ANY_PHYSICAL_PARTITION, cp);
    conn.requestStream("1", null, 1000, cpm, null, msg);
    //verify server gets the /stream request
    msgReq = captureRequest(objCapture);
    Assert.assertTrue(msgReq.getUri().startsWith("/stream"));
    // verify url construction for adding max event version
    String expectedVersion = "&" + DatabusHttpHeaders.MAX_EVENT_VERSION + "=" + MAX_EVENT_VERSION;
    Assert.assertTrue(msgReq.getUri().contains(expectedVersion));
    //send back some response
    ChannelBuffer tmpBuf = NettyTestUtils.streamToChannelBuffer(buf, cp, 10000, null);
    NettyTestUtils.sendServerResponses(_dummyServer, clientAddr, sourcesResp, new DefaultHttpChunk(tmpBuf));
    waitForCallback(callback, TestResponseProcessors.TestConnectionStateMessage.State.STREAM_RESPONSE_SUCCESS, log);
    Assert.assertNull(remoteExceptionHandler.getLastException());
    //wait for the readable byte channel to process the response and verify nothing has changed
    TestUtil.sleep(1000);
    waitForCallback(callback, TestResponseProcessors.TestConnectionStateMessage.State.STREAM_RESPONSE_SUCCESS, log);
    Assert.assertNull(remoteExceptionHandler.getLastException());
}

Example 4 with CheckpointMult

use of com.linkedin.databus.core.CheckpointMult in project databus by linkedin.

the class DummyRemoteExceptionHandler method runServerPartialStreamTimeoutIteration.

private void runServerPartialStreamTimeoutIteration(final Logger log, DbusEventBuffer buf, TestingConnectionCallback callback, DummyRemoteExceptionHandler remoteExceptionHandler, final NettyHttpDatabusRelayConnection conn) throws JsonGenerationException, JsonMappingException, IOException, ScnNotFoundException, OffsetNotFoundException {
    //connect to server and send /sources
    TestResponseProcessors.TestConnectionStateMessage msg = new TestResponseProcessors.TestConnectionStateMessage();
    conn.requestSources(msg);
    waitForServerConnection(conn, log);
    //introspect connection to server
    Channel channel = conn._channel;
    SocketAddress clientAddr = channel.getLocalAddress();
    Channel serverChannel = _dummyServer.getChildChannel(clientAddr);
    ChannelPipeline serverPipeline = serverChannel.getPipeline();
    SimpleObjectCaptureHandler objCapture = (SimpleObjectCaptureHandler) serverPipeline.get("3");
    //verify server gets the /source request
    HttpResponse sourcesResp = runHappyPathSources(log, callback, remoteExceptionHandler, clientAddr, objCapture);
    //send /register
    runHappyPathRegister(log, callback, remoteExceptionHandler, conn, msg, clientAddr, objCapture, sourcesResp);
    //send partial /stream
    callback.clearLastMsg();
    objCapture.clear();
    Checkpoint cp = new Checkpoint();
    cp.setFlexible();
    CheckpointMult cpm = new CheckpointMult();
    cpm.addCheckpoint(PhysicalPartition.ANY_PHYSICAL_PARTITION, cp);
    conn.requestStream("1", null, 1000, cpm, null, msg);
    //////// verify server gets the /stream request
    HttpRequest msgReq = captureRequest(objCapture);
    Assert.assertTrue(msgReq.getUri().startsWith("/stream"));
    ////// send back some partial response
    ChannelBuffer tmpBuf = NettyTestUtils.streamToChannelBuffer(buf, cp, 10000, null);
    _dummyServer.sendServerResponse(clientAddr, sourcesResp, 1000);
    _dummyServer.sendServerResponse(clientAddr, new DefaultHttpChunk(tmpBuf), 1000);
    //Trigger a read timeout
    TestUtil.sleep(DEFAULT_READ_TIMEOUT_MS + 100);
    waitForCallback(callback, TestResponseProcessors.TestConnectionStateMessage.State.STREAM_RESPONSE_SUCCESS, log);
    Assert.assertNull(remoteExceptionHandler.getLastException());
    Assert.assertEquals(1, callback.getAllMsgs().size());
    callback.clearLastMsg();
    objCapture.clear();
}

Example 5 with CheckpointMult

use of com.linkedin.databus.core.CheckpointMult in project databus by linkedin.

the class DummyRemoteExceptionHandler method runServerStreamReqDisconnectIteration.

private void runServerStreamReqDisconnectIteration(final Logger log, TestingConnectionCallback callback, DummyRemoteExceptionHandler remoteExceptionHandler, final NettyHttpDatabusRelayConnection conn) throws JsonGenerationException, JsonMappingException, IOException, ScnNotFoundException, OffsetNotFoundException {
    //connect to server and send /sources
    TestResponseProcessors.TestConnectionStateMessage msg = new TestResponseProcessors.TestConnectionStateMessage();
    conn.requestSources(msg);
    waitForServerConnection(conn, log);
    //introspect connection to server
    Channel channel = conn._channel;
    SocketAddress clientAddr = channel.getLocalAddress();
    Channel serverChannel = _dummyServer.getChildChannel(clientAddr);
    ChannelPipeline serverPipeline = serverChannel.getPipeline();
    SimpleObjectCaptureHandler objCapture = (SimpleObjectCaptureHandler) serverPipeline.get("3");
    //verify server gets the /source request
    HttpResponse sourcesResp = runHappyPathSources(log, callback, remoteExceptionHandler, clientAddr, objCapture);
    //send /register
    runHappyPathRegister(log, callback, remoteExceptionHandler, conn, msg, clientAddr, objCapture, sourcesResp);
    //send partial /stream
    callback.clearLastMsg();
    objCapture.clear();
    serverChannel.close();
    Checkpoint cp = new Checkpoint();
    cp.setFlexible();
    CheckpointMult cpm = new CheckpointMult();
    cpm.addCheckpoint(PhysicalPartition.ANY_PHYSICAL_PARTITION, cp);
    conn.requestStream("1", null, 1000, cpm, null, msg);
    waitForCallback(callback, TestResponseProcessors.TestConnectionStateMessage.State.STREAM_REQUEST_ERROR, log);
    Assert.assertNull(remoteExceptionHandler.getLastException());
    Assert.assertEquals(1, callback.getAllMsgs().size());
    callback.clearLastMsg();
    objCapture.clear();
}