Examples with Checkpoint com.linkedin.databus.core.Checkpoint used on opensource projects

Example 61 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class TestResponseProcessors method testStartSCNExceptionAfterStartCase1.

@Test
public void testStartSCNExceptionAfterStartCase1() throws Exception {
    TestAbstractQueue queue = new TestAbstractQueue();
    TestConnectionStateMessage stateMsg = new TestConnectionStateMessage();
    HttpResponse httpResponse = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
    TestRemoteExceptionHandler remoteExHandler = new TestRemoteExceptionHandler();
    Checkpoint cp = new Checkpoint();
    cp.setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);
    BootstrapStartScnHttpResponseProcessor processor = new BootstrapStartScnHttpResponseProcessor(null, queue, stateMsg, cp, remoteExHandler, null);
    ChannelBuffer buf = getScnResponse();
    HttpChunk httpChunk = new DefaultHttpChunk(buf);
    HttpChunkTrailer httpChunkTrailer = new DefaultHttpChunkTrailer();
    processor.startResponse(httpResponse);
    processor.channelException(new Exception("dummy exception"));
    processor.addChunk(httpChunk);
    processor.addTrailer(httpChunkTrailer);
    processor.finishResponse();
    Assert.assertEquals("Error Handled", true, processor._errorHandled);
    Assert.assertEquals("Processor Response State", AbstractHttpResponseProcessorDecorator.ResponseStatus.CHUNKS_FINISHED, processor._responseStatus);
    Assert.assertEquals("Actor Queue Size", 1, queue.getMessages().size());
    Assert.assertEquals("Expected ConnectionStateMessage", "TestConnectionStateMessage", queue.getMessages().get(0).getClass().getSimpleName());
    TestConnectionStateMessage gotMsg = (TestConnectionStateMessage) (queue.getMessages().get(0));
    Assert.assertEquals("Expected ConnectionStateMessage State", TestConnectionStateMessage.State.STARTSCN_RESPONSE_ERROR, gotMsg._state);
    Assert.assertEquals("No response", true, (null == stateMsg._cp));
}

Example 62 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class DbusEventAppender method run.

@Override
public void run() {
    //append events into buffer serially with varying  window sizes;
    long lastScn = -1;
    _count = 0;
    int dataEventCount = 0;
    int bootstrapCheckpoints = _bootstrapCheckpointPerWindow;
    int maxCount = (int) (_fraction * _events.size());
    for (DbusEvent ev : _events) {
        if (dataEventCount >= maxCount) {
            break;
        }
        long evScn = ev.sequence();
        if (lastScn != evScn) {
            //new window;
            if (lastScn == -1) {
                // Test DDSDBUS-1109 by skipping the start() call. The scn Index should be set for streamEvents() to work correctly
                if (_invokeStartOnBuffer) {
                    _buffer.start(evScn - 1);
                }
                _buffer.startEvents();
                if (_bootstrapCheckpoint != null) {
                    //add the initial checkpoint event to dispatcher's buffer to simulate bootstrap
                    addBootstrapCheckpointEventToBuffer(evScn - 1, dataEventCount, 1);
                }
            } else {
                ++_count;
                if (_callInternalListeners) {
                    if (0 == bootstrapCheckpoints) {
                        _buffer.endEvents(lastScn, _stats);
                    } else {
                        addBootstrapCheckpointEventToBuffer(lastScn, dataEventCount, 1);
                    }
                    --bootstrapCheckpoints;
                } else {
                    if (0 == bootstrapCheckpoints) {
                        _buffer.endEvents(true, lastScn, false, false, _stats);
                    } else {
                        //simulate bootstrap calls (snapshot)
                        addBootstrapCheckpointEventToBuffer(lastScn, dataEventCount, 1);
                    }
                    --bootstrapCheckpoints;
                }
                if (!_bufferReflector.validateBuffer()) {
                    throw new RuntimeException("Buffer validation 1 failed");
                }
                if (bootstrapCheckpoints < 0) {
                    _buffer.startEvents();
                    bootstrapCheckpoints = _bootstrapCheckpointPerWindow;
                }
            }
            if (!_bufferReflector.validateBuffer()) {
                throw new RuntimeException("Buffer validation 2 failed");
            }
            lastScn = evScn;
        }
        dataEventCount = addEventToBuffer(ev, dataEventCount);
        ++_count;
    }
    if ((lastScn != -1) && (maxCount == _events.size())) {
        ++_count;
        _buffer.endEvents(lastScn, _stats);
    }
    _dataEvents = dataEventCount;
}

Example 63 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class RelayEventProducer method getCheckpoint.

protected Checkpoint getCheckpoint(long sinceSCN, MaxSCNReaderWriter scnReaderWriter) {
    long scn = sinceSCN;
    if ((scn < 0) && (_scnReaderWriter != null)) {
        try {
            scn = _scnReaderWriter.getMaxScn();
        } catch (DatabusException e) {
            LOG.info("Cannot read persisted SCN " + e);
            scn = -1;
        }
    }
    // return no cp if unable to read from saved SCN
    if (scn <= 0) {
        return null;
    }
    Checkpoint cp = new Checkpoint();
    cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
    // always have greater than semantic
    cp.setWindowOffset(-1);
    cp.setWindowScn(scn);
    return cp;
}

Example 64 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class RelayEventProducer method start.

@Override
public synchronized void start(long sinceSCN) {
    if (_dbusConnection != null && !_dbusConnection.getConnectionStatus().isRunningStatus()) {
        LOG.info("In RelayEventProducer start:  running =" + _dbusConnection.getConnectionStatus().isRunningStatus());
        //translate relay saved scn to client checkpoint
        LOG.info("Requested sinceSCN = " + sinceSCN);
        Checkpoint cp = getCheckpoint(sinceSCN, _scnReaderWriter);
        //check if the relay chaining consumer has been initialized [it could when the leader passes its active buffer]
        if ((cp != null) && (_consumerEventBuffer.getStartSCN() < 0)) {
            //note that the restartScnOffset comes into the picture only iff the buffer is empty
            long savedScn = cp.getWindowScn();
            LOG.info("Checkpoint read = " + savedScn + " restartScnOffset=" + _restartScnOffset);
            long newScn = (savedScn >= _restartScnOffset) ? savedScn - _restartScnOffset : 0;
            cp.setWindowScn(newScn);
            LOG.info("Setting start scn of event buffer to " + cp.getWindowScn());
            _consumerEventBuffer.setStartSCN(cp.getWindowScn());
        }
        LOG.info("Eventbuffer start scn = " + _consumerEventBuffer.getStartSCN());
        //now set the checkpoint in the databus client fetcher
        _dbusConnection.getRelayPullThread().getConnectionState().setCheckpoint(cp);
        //start the connection
        _dbusConnection.start();
        _relayLogger.setDaemon(true);
        _relayLogger.start();
    } else {
        if (_dbusConnection == null) {
            LOG.error("Not started! Connection is null");
        } else {
            LOG.warn("dbusConnection status=" + _dbusConnection.getConnectionStatus().getStatus());
        }
    }
}

Example 65 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class TestRelayCommandsLocal method doTestOneDataStreamCommand.

private void doTestOneDataStreamCommand() throws Exception {
    //try to read it
    Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
    String streamRequest = "/stream?sources=100&size=100000&output=json&checkPoint=" + cp.toString();
    HttpRequest httpRequest = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, streamRequest);
    SimpleTestHttpClient httpClient = SimpleTestHttpClient.createLocal(TimeoutPolicy.ALL_TIMEOUTS);
    SimpleHttpResponseHandler respHandler = httpClient.sendRequest(_serverAddress, httpRequest);
    assertTrue("failed to get a response", respHandler.awaitResponseUninterruptedly(1, TimeUnit.SECONDS));
    HttpResponse respObj = respHandler.getResponse();
    assertNull("/stream returned unexpected error", respObj.getHeader(DatabusHttpHeaders.DATABUS_ERROR_CLASS_HEADER));
    if (LOG.isDebugEnabled()) {
        LOG.debug("/stream response:" + new String(respHandler.getReceivedBytes()));
    }
    ObjectMapper objMapper = new ObjectMapper();
    ByteArrayInputStream in = new ByteArrayInputStream(respHandler.getReceivedBytes());
    objMapper.readValue(in, new TypeReference<Map<String, String>>() {
    });
}