Examples with Checkpoint com.linkedin.databus.core.Checkpoint used on opensource projects

Example 1 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class BootstrapProcessor method streamSnapShotRows.

// Get specificed number of snapshot rows
public boolean streamSnapShotRows(Checkpoint currState, BootstrapEventCallback callBack) throws SQLException, BootstrapProcessingException, BootstrapDatabaseTooOldException, BootstrapDatabaseTooYoungException {
    assert (currState.getConsumptionMode() == DbusClientMode.BOOTSTRAP_SNAPSHOT);
    boolean phaseCompleted = false;
    long startSCN = currState.getBootstrapStartScn();
    long sinceSCN = currState.getBootstrapSinceScn();
    if (startSCN <= sinceSCN) {
        LOG.info("StartSCN is less than or equal to sinceSCN. Bypassing snapshot phase !! startSCN:" + startSCN + ",sinceSCN:" + sinceSCN);
        return true;
    }
    Connection conn = _dbDao.getBootstrapConn().getDBConn();
    BootstrapDBMetaDataDAO.SourceStatusInfo srcIdStatusPair = _dbDao.getSrcIdStatusFromDB(currState.getSnapshotSource(), true);
    if (!srcIdStatusPair.isValidSource())
        throw new BootstrapProcessingException("Bootstrap DB not servicing source :" + currState.getCatchupSource());
    PreparedStatement stmt = null;
    ResultSet rs = null;
    try {
        if (config.isEnableMinScnCheck()) {
            long minScn = _dbDao.getMinScnOfSnapshots(srcIdStatusPair.getSrcId());
            LOG.info("Min scn for tab tables is: " + minScn);
            if (minScn == BootstrapDBMetaDataDAO.DEFAULT_WINDOWSCN) {
                throw new BootstrapDatabaseTooYoungException("BootstrapDB has no minScn for these sources, but minScn check is enabled! minScn=" + minScn);
            }
            // sinceSCN should be greater than minScn, except when sinceSCN == minScn == 0.
            if ((sinceSCN <= minScn) && !(sinceSCN == 0 && minScn == 0)) {
                LOG.error("Bootstrap Snapshot doesn't have requested data . sinceScn too old! sinceScn is " + sinceSCN + " but minScn available is " + minScn);
                throw new BootstrapDatabaseTooYoungException("Min scn=" + minScn + " Since scn=" + sinceSCN);
            }
        } else {
            LOG.debug("Bypassing minScn check!");
        }
        String snapshotSQL = getSnapshotSQLString(_dbDao.getBootstrapConn().getSrcTableName(srcIdStatusPair.getSrcId()), currState.getSnapshotSource());
        stmt = conn.prepareStatement(snapshotSQL);
        long offset = currState.getSnapshotOffset();
        int i = 1;
        stmt.setLong(i++, offset);
        stmt.setLong(i++, currState.getBootstrapStartScn());
        stmt.setLong(i++, currState.getBootstrapSinceScn());
        stmt.setLong(i++, _maxSnapshotRowsPerFetch);
        LOG.info("SnapshotSQL string: " + snapshotSQL + ", " + offset + ", " + currState.getBootstrapStartScn() + ", " + currState.getBootstrapSinceScn() + ", " + _maxSnapshotRowsPerFetch);
        rs = new BootstrapDBTimedQuery(stmt, _queryTimeInSec).executeQuery();
        phaseCompleted = streamOutRows(currState, rs, callBack, _maxSnapshotRowsPerFetch);
    } catch (SQLException e) {
        DBHelper.close(rs, stmt, null);
        LOG.error("Exception occurred when getting snapshot rows" + e);
        throw e;
    } finally {
        if (stmt != null) {
            stmt.close();
            stmt = null;
        }
        mergeAndResetStats();
    }
    return phaseCompleted;
}

Example 2 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class BootstrapProcessor method streamCatchupRows.

// Get specified number of catchup rows
public boolean streamCatchupRows(Checkpoint currState, BootstrapEventCallback callBack) throws SQLException, BootstrapProcessingException, BootstrapDatabaseTooOldException {
    assert (currState.getConsumptionMode() == DbusClientMode.BOOTSTRAP_CATCHUP);
    boolean foundRows = false;
    BootstrapDBMetaDataDAO.SourceStatusInfo srcIdStatusPair = _dbDao.getSrcIdStatusFromDB(currState.getCatchupSource(), true);
    if (!srcIdStatusPair.isValidSource())
        throw new BootstrapProcessingException("Bootstrap DB not servicing source :" + currState.getCatchupSource());
    int curSrcId = srcIdStatusPair.getSrcId();
    int curLogId = _dbDao.getLogIdToCatchup(curSrcId, currState.getWindowScn());
    int targetLogId = _dbDao.getLogIdToCatchup(curSrcId, currState.getBootstrapTargetScn());
    boolean phaseCompleted = false;
    PreparedStatement stmt = null;
    try {
        ResultSet rs = null;
        while (!foundRows && curLogId <= targetLogId) {
            stmt = createCatchupStatement(curSrcId, curLogId, currState);
            rs = new BootstrapDBTimedQuery(stmt, _queryTimeInSec).executeQuery();
            foundRows = rs.isBeforeFirst();
            if (!foundRows) {
                // move to next log
                curLogId++;
                // reset rid to 0 for the next log
                currState.setCatchupOffset(0);
                LOG.info("Moving to next log table log_" + curSrcId + "_" + curLogId + " because current log table exhausted!");
            }
        }
        phaseCompleted = streamOutRows(currState, rs, callBack, _maxCatchupRowsPerFetch);
    } catch (SQLException e) {
        LOG.error("Exception occured during fetching catchup rows" + e);
        throw e;
    } finally {
        if (stmt != null) {
            stmt.close();
            stmt = null;
        }
        mergeAndResetStats();
    }
    return phaseCompleted;
}

Example 3 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class BootstrapRequestProcessorBase method process.

@Override
public final DatabusRequest process(DatabusRequest request) throws IOException, RequestProcessingException {
    initBootstrapServerInfo();
    String ckptStr = request.getParams().getProperty(CHECKPOINT_PARAM);
    Checkpoint ckpt = null;
    if (null != ckptStr) {
        ckpt = new Checkpoint(ckptStr);
        String bsServerInfo = ckpt.getBootstrapServerInfo();
        if ((null != bsServerInfo) && (null != _serverInfo)) {
            ServerInfo expServerInfo = null;
            try {
                expServerInfo = ServerInfo.buildServerInfoFromHostPort(bsServerInfo.trim(), DbusConstants.HOSTPORT_DELIMITER);
            } catch (Exception ex) {
                LOG.error("Unable to fetch ServerInfo from ckpt. Passed ServerInfo was :" + bsServerInfo.trim());
            }
            if ((null != expServerInfo) && (!_serverInfo.equals(expServerInfo))) {
                String msg = "Bootstrap Server Request should be served by different host : " + bsServerInfo + ", This instance is :" + _serverHostPort;
                LOG.error(msg);
                throw new RequestProcessingException(msg);
            }
        }
    }
    DatabusRequest req = doProcess(request);
    if (null != _serverHostPort) {
        req.getResponseContent().setMetadata(DbusConstants.SERVER_INFO_HOSTPORT_HEADER_PARAM, _serverHostPort);
    }
    return req;
}

Example 4 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class StartSCNRequestProcessor method doProcess.

@Override
protected DatabusRequest doProcess(DatabusRequest request) throws IOException, RequestProcessingException {
    BootstrapHttpStatsCollector bootstrapStatsCollector = _bootstrapServer.getBootstrapStatsCollector();
    long startTime = System.currentTimeMillis();
    String sources = request.getRequiredStringParam(SOURCES_PARAM);
    List<String> srcList = getSources(sources);
    Checkpoint ckpt = new Checkpoint(request.getRequiredStringParam(CHECKPOINT_PARAM));
    LOG.info("StartSCN requested for sources : (" + sources + "). CheckPoint is :" + ckpt);
    long sinceScn = ckpt.getBootstrapSinceScn();
    ObjectMapper mapper = new ObjectMapper();
    StringWriter out = new StringWriter(1024);
    long startSCN = -1;
    BootstrapSCNProcessor processor = null;
    try {
        processor = new BootstrapSCNProcessor(_config, _bootstrapServer.getInboundEventStatisticsCollector());
        List<SourceStatusInfo> srcStatusPairs = null;
        try {
            srcStatusPairs = processor.getSourceIdAndStatusFromName(srcList);
            startSCN = processor.getMinApplierWindowScn(sinceScn, srcStatusPairs);
            if (processor.shouldBypassSnapshot(sinceScn, startSCN, srcStatusPairs)) {
                LOG.info("Bootstrap Snapshot phase will be bypassed for startScn request :" + request);
                LOG.info("Original startSCN is:" + startSCN + ", Setting startSCN to the sinceSCN:" + sinceScn);
                startSCN = sinceScn;
            } else {
                if (startSCN == BootstrapDBMetaDataDAO.DEFAULT_WINDOWSCN) {
                    throw new RequestProcessingException("Bootstrap DB is being initialized! startSCN=" + startSCN);
                }
                if (_config.isEnableMinScnCheck()) {
                    // snapshot isn't bypassed. Check if snapshot is possible from sinceScn by checking minScn
                    long minScn = processor.getBootstrapMetaDataDAO().getMinScnOfSnapshots(srcStatusPairs);
                    LOG.info("Min scn for tab tables is: " + minScn);
                    if (minScn == BootstrapDBMetaDataDAO.DEFAULT_WINDOWSCN) {
                        throw new BootstrapDatabaseTooYoungException("BootstrapDB has no minScn for these sources, but minScn check is enabled! minScn=" + minScn);
                    }
                    // sinceSCN should be greater than minScn, unless sinceScn=minScn=0
                    if ((sinceScn <= minScn) && !(sinceScn == 0 && minScn == 0)) {
                        LOG.error("Bootstrap Snapshot doesn't have requested data . sinceScn too old! sinceScn is " + sinceScn + " but minScn available is " + minScn);
                        throw new BootstrapDatabaseTooYoungException("Min scn=" + minScn + " Since scn=" + sinceScn);
                    }
                } else {
                    LOG.debug("Bypassing minScn check! ");
                }
            }
        } catch (BootstrapDatabaseTooOldException tooOldException) {
            if (bootstrapStatsCollector != null) {
                bootstrapStatsCollector.registerErrStartSCN();
                bootstrapStatsCollector.registerErrDatabaseTooOld();
            }
            LOG.error("The bootstrap database is too old!", tooOldException);
            throw new RequestProcessingException(tooOldException);
        } catch (BootstrapDatabaseTooYoungException e) {
            if (bootstrapStatsCollector != null) {
                bootstrapStatsCollector.registerErrStartSCN();
                bootstrapStatsCollector.registerErrBootstrap();
            }
            LOG.error("The bootstrap database is too young!", e);
            throw new RequestProcessingException(e);
        } catch (SQLException e) {
            if (bootstrapStatsCollector != null) {
                bootstrapStatsCollector.registerErrStartSCN();
                bootstrapStatsCollector.registerErrSqlException();
            }
            LOG.error("Error encountered while fetching startSCN from database.", e);
            throw new RequestProcessingException(e);
        }
        mapper.writeValue(out, String.valueOf(startSCN));
        byte[] resultBytes = out.toString().getBytes(Charset.defaultCharset());
        request.getResponseContent().write(ByteBuffer.wrap(resultBytes));
        LOG.info("startSCN: " + startSCN + " with server Info :" + _serverHostPort);
    } catch (RequestProcessingException ex) {
        LOG.error("Got exception while calculating startSCN", ex);
        throw ex;
    } catch (Exception ex) {
        LOG.error("Got exception while calculating startSCN", ex);
        throw new RequestProcessingException(ex);
    } finally {
        if (null != processor)
            processor.shutdown();
    }
    if (bootstrapStatsCollector != null) {
        bootstrapStatsCollector.registerStartSCNReq(System.currentTimeMillis() - startTime);
    }
    return request;
}

Example 5 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class TestBootstrapProcessor method testIsPhaseCompletedFlag.

@Test
public void testIsPhaseCompletedFlag() throws Exception {
    BootstrapProcessor bp = new BootstrapProcessor();
    long processedRowCount = 1;
    // order of args : processedRowCount, isLimitExceeded, isDropped, isError
    BootstrapEventProcessResult result_ok = new BootstrapEventProcessResult(processedRowCount, false, false, false);
    BootstrapEventProcessResult result_err = new BootstrapEventProcessResult(processedRowCount, false, false, true);
    BootstrapEventProcessResult result_ile = new BootstrapEventProcessResult(processedRowCount, true, false, false);
    Checkpoint ckpt_ss = new Checkpoint("{\"consumption_mode\":\"BOOTSTRAP_SNAPSHOT\", \"bootstrap_since_scn\":0," + "\"bootstrap_start_scn\":1000,\"bootstrap_target_scn\":2000,\"bootstrap_catchup_source_index\":0," + "\"bootstrap_snapshot_source_index\":1}");
    ckpt_ss.assertCheckpoint();
    Checkpoint ckpt_cu = new Checkpoint("{\"consumption_mode\":\"BOOTSTRAP_CATCHUP\", \"bootstrap_since_scn\":0," + "\"bootstrap_start_scn\":1000,\"bootstrap_target_scn\":2000,\"bootstrap_catchup_source_index\":1," + "\"bootstrap_snapshot_source_index\":1}");
    ckpt_cu.assertCheckpoint();
    long numRowsReadFromDb = 1;
    long maxRowsPerFetch = 2;
    long windowScn = 1;
    // result is null, phaseCompleted == false ( irrespective of snapshot / catchup phase )
    numRowsReadFromDb = 0;
    boolean pc = bp.computeIsPhaseCompleted(null, ckpt_ss, numRowsReadFromDb, maxRowsPerFetch, windowScn);
    Assert.assertTrue(pc);
    pc = bp.computeIsPhaseCompleted(null, ckpt_cu, numRowsReadFromDb, maxRowsPerFetch, windowScn);
    Assert.assertTrue(pc);
    numRowsReadFromDb = 1;
    // numRowsReadFromDb < maxRowsPerFetch, in SNAPSHOT mode
    pc = bp.computeIsPhaseCompleted(result_ok, ckpt_ss, numRowsReadFromDb, maxRowsPerFetch, windowScn);
    Assert.assertTrue(pc);
    // Same as above, but result.isError == true ( overrides all other conditions )
    pc = bp.computeIsPhaseCompleted(result_err, ckpt_ss, numRowsReadFromDb, maxRowsPerFetch, windowScn);
    Assert.assertFalse(pc);
    // Same as above, but result.isLimitExceeded == true ( overrides all other conditions )
    pc = bp.computeIsPhaseCompleted(result_ile, ckpt_ss, numRowsReadFromDb, maxRowsPerFetch, windowScn);
    Assert.assertFalse(pc);
    // numRowsReadFromDb == maxRowsPerFetch, in SNAPSHOT mode
    numRowsReadFromDb = 2;
    pc = bp.computeIsPhaseCompleted(result_ok, ckpt_ss, numRowsReadFromDb, maxRowsPerFetch, windowScn);
    Assert.assertFalse(pc);
    // Same as above, but result.isError == true
    pc = bp.computeIsPhaseCompleted(result_err, ckpt_ss, numRowsReadFromDb, maxRowsPerFetch, windowScn);
    Assert.assertFalse(pc);
    // Same as above, but result.isLimitExceeded == true
    pc = bp.computeIsPhaseCompleted(result_ile, ckpt_ss, numRowsReadFromDb, maxRowsPerFetch, windowScn);
    Assert.assertFalse(pc);
    numRowsReadFromDb = 1;
    // numRowsReadFromDb < maxRowsPerFetch, in CATCHUP mode, windowScn != bootstrap_target_scn
    pc = bp.computeIsPhaseCompleted(result_ok, ckpt_ss, numRowsReadFromDb, maxRowsPerFetch, windowScn);
    Assert.assertTrue(pc);
    windowScn = ckpt_cu.getBootstrapTargetScn();
    pc = bp.computeIsPhaseCompleted(result_ok, ckpt_cu, numRowsReadFromDb, maxRowsPerFetch, windowScn);
    Assert.assertTrue(pc);
    // Same as above, but result.isError == true ( overrides all other conditions )
    pc = bp.computeIsPhaseCompleted(result_err, ckpt_ss, numRowsReadFromDb, maxRowsPerFetch, windowScn);
    Assert.assertFalse(pc);
    // Same as above, but result.isLimitExceeded == true ( overrides result being null )
    pc = bp.computeIsPhaseCompleted(result_ile, ckpt_ss, numRowsReadFromDb, maxRowsPerFetch, windowScn);
    Assert.assertFalse(pc);
    // numRowsReadFromDb == maxRowsPerFetch, in CATCHUP mode
    numRowsReadFromDb = 2;
    // not equal to bootstrap_target_scn
    windowScn = 10;
    pc = bp.computeIsPhaseCompleted(result_ok, ckpt_cu, numRowsReadFromDb, maxRowsPerFetch, windowScn);
    Assert.assertFalse(pc);
    // equal to bootstrap_target_scn, but does not matter still
    windowScn = ckpt_cu.getBootstrapTargetScn();
    pc = bp.computeIsPhaseCompleted(result_ok, ckpt_cu, numRowsReadFromDb, maxRowsPerFetch, windowScn);
    Assert.assertFalse(pc);
    // Same as above, but result.isError == true
    pc = bp.computeIsPhaseCompleted(result_err, ckpt_cu, numRowsReadFromDb, maxRowsPerFetch, windowScn);
    Assert.assertFalse(pc);
    // Same as above, but result.isLimitExceeded == true
    pc = bp.computeIsPhaseCompleted(result_ile, ckpt_cu, numRowsReadFromDb, maxRowsPerFetch, windowScn);
    Assert.assertFalse(pc);
}