Examples with Checkpoint com.linkedin.databus.core.Checkpoint used on opensource projects

Example 6 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class TestBootstrap method testBootstrapService.

@Test
public void testBootstrapService() throws InstantiationException, IllegalAccessException, ClassNotFoundException, SQLException, IOException, BootstrapProcessingException, DatabusException, BootstrapDatabaseTooOldException, BootstrapDBException {
    final Logger log = Logger.getLogger("TestBootstrap.testBootstrapService");
    EventProcessor processorCallback = new EventProcessor();
    BootstrapConfig config = new BootstrapConfig();
    BootstrapReadOnlyConfig staticConfig = config.build();
    String[] sources = new String[4];
    sources[0] = "TestBootstrap.testBootstrapService.event";
    sources[1] = "TestBootstrap.testBootstrapService.event1";
    sources[2] = "TestBootstrap.testBootstrapService.event2";
    sources[3] = "TestBootstrap.testBootstrapService.event3";
    // Create the tables for all the sources before starting up the threads
    BootstrapConn _bootstrapConn = new BootstrapConn();
    final boolean autoCommit = true;
    _bootstrapConn.initBootstrapConn(autoCommit, staticConfig.getBootstrapDBUsername(), staticConfig.getBootstrapDBPassword(), staticConfig.getBootstrapDBHostname(), staticConfig.getBootstrapDBName());
    BootstrapDBMetaDataDAO dao = new BootstrapDBMetaDataDAO(_bootstrapConn, staticConfig.getBootstrapDBHostname(), staticConfig.getBootstrapDBUsername(), staticConfig.getBootstrapDBPassword(), staticConfig.getBootstrapDBName(), autoCommit);
    for (String source : sources) {
        SourceStatusInfo srcStatusInfo = dao.getSrcIdStatusFromDB(source, false);
        if (srcStatusInfo.getSrcId() >= 0) {
            dao.dropSourceInDB(srcStatusInfo.getSrcId());
        }
        dao.addNewSourceInDB(source, BootstrapProducerStatus.ACTIVE);
    }
    setBootstrapLoginfoTable(_bootstrapConn, 1, 1);
    DatabusBootstrapClient s = new DatabusBootstrapClient(sources);
    Checkpoint cp;
    while ((cp = s.getNextBatch(10, processorCallback)).getConsumptionMode() != DbusClientMode.ONLINE_CONSUMPTION) {
        log.debug(cp);
    }
}

Example 7 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class ClusterCheckpointPersistenceProvider method loadCheckpoint.

/**
 * Function called by Databus Client to load checkpoint;
 */
@Override
public Checkpoint loadCheckpoint(List<String> sourceNames) {
    if (_propertyStore != null) {
        String key = makeKey(sourceNames);
        Checkpoint cp = getCheckpoint(key);
        return cp;
    }
    return null;
}

Example 8 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class RelayEventProducer method getCheckpoint.

protected Checkpoint getCheckpoint(long sinceSCN, MaxSCNReaderWriter scnReaderWriter) {
    long scn = sinceSCN;
    if ((scn < 0) && (_scnReaderWriter != null)) {
        try {
            scn = _scnReaderWriter.getMaxScn();
        } catch (DatabusException e) {
            LOG.info("Cannot read persisted SCN " + e);
            scn = -1;
        }
    }
    // return no cp if unable to read from saved SCN
    if (scn <= 0) {
        return null;
    }
    Checkpoint cp = new Checkpoint();
    cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
    // always have greater than semantic
    cp.setWindowOffset(-1);
    cp.setWindowScn(scn);
    return cp;
}

Example 9 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class RelayEventProducer method start.

@Override
public synchronized void start(long sinceSCN) {
    if (_dbusConnection != null && !_dbusConnection.getConnectionStatus().isRunningStatus()) {
        LOG.info("In RelayEventProducer start:  running =" + _dbusConnection.getConnectionStatus().isRunningStatus());
        // translate relay saved scn to client checkpoint
        LOG.info("Requested sinceSCN = " + sinceSCN);
        Checkpoint cp = getCheckpoint(sinceSCN, _scnReaderWriter);
        // check if the relay chaining consumer has been initialized [it could when the leader passes its active buffer]
        if ((cp != null) && (_consumerEventBuffer.getStartSCN() < 0)) {
            // note that the restartScnOffset comes into the picture only iff the buffer is empty
            long savedScn = cp.getWindowScn();
            LOG.info("Checkpoint read = " + savedScn + " restartScnOffset=" + _restartScnOffset);
            long newScn = (savedScn >= _restartScnOffset) ? savedScn - _restartScnOffset : 0;
            cp.setWindowScn(newScn);
            LOG.info("Setting start scn of event buffer to " + cp.getWindowScn());
            _consumerEventBuffer.setStartSCN(cp.getWindowScn());
        }
        LOG.info("Eventbuffer start scn = " + _consumerEventBuffer.getStartSCN());
        // now set the checkpoint in the databus client fetcher
        _dbusConnection.getRelayPullThread().getConnectionState().setCheckpoint(cp);
        // start the connection
        _dbusConnection.start();
        _relayLogger.setDaemon(true);
        _relayLogger.start();
    } else {
        if (_dbusConnection == null) {
            LOG.error("Not started! Connection is null");
        } else {
            LOG.warn("dbusConnection status=" + _dbusConnection.getConnectionStatus().getStatus());
        }
    }
}

Example 10 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class TestRelayCommandsLocal method doTestOneDataStreamCommand.

private void doTestOneDataStreamCommand() throws Exception {
    // try to read it
    Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
    String streamRequest = "/stream?sources=100&size=100000&output=json&checkPoint=" + cp.toString();
    HttpRequest httpRequest = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, streamRequest);
    SimpleTestHttpClient httpClient = SimpleTestHttpClient.createLocal(TimeoutPolicy.ALL_TIMEOUTS);
    SimpleHttpResponseHandler respHandler = httpClient.sendRequest(_serverAddress, httpRequest);
    assertTrue("failed to get a response", respHandler.awaitResponseUninterruptedly(1, TimeUnit.SECONDS));
    HttpResponse respObj = respHandler.getResponse();
    assertNull("/stream returned unexpected error", respObj.getHeader(DatabusHttpHeaders.DATABUS_ERROR_CLASS_HEADER));
    if (LOG.isDebugEnabled()) {
        LOG.debug("/stream response:" + new String(respHandler.getReceivedBytes()));
    }
    ObjectMapper objMapper = new ObjectMapper();
    ByteArrayInputStream in = new ByteArrayInputStream(respHandler.getReceivedBytes());
    objMapper.readValue(in, new TypeReference<Map<String, String>>() {
    });
}