Examples with BootstrapDBTimedQuery com.linkedin.databus.bootstrap.common.BootstrapDBTimedQuery used on opensource projects

Example 1 with BootstrapDBTimedQuery

use of com.linkedin.databus.bootstrap.common.BootstrapDBTimedQuery in project databus by linkedin.

the class BootstrapProcessor method streamSnapShotRows.

// Get specificed number of snapshot rows
public boolean streamSnapShotRows(Checkpoint currState, BootstrapEventCallback callBack) throws SQLException, BootstrapProcessingException, BootstrapDatabaseTooOldException, BootstrapDatabaseTooYoungException {
    assert (currState.getConsumptionMode() == DbusClientMode.BOOTSTRAP_SNAPSHOT);
    boolean phaseCompleted = false;
    long startSCN = currState.getBootstrapStartScn();
    long sinceSCN = currState.getBootstrapSinceScn();
    if (startSCN <= sinceSCN) {
        LOG.info("StartSCN is less than or equal to sinceSCN. Bypassing snapshot phase !! startSCN:" + startSCN + ",sinceSCN:" + sinceSCN);
        return true;
    }
    Connection conn = _dbDao.getBootstrapConn().getDBConn();
    BootstrapDBMetaDataDAO.SourceStatusInfo srcIdStatusPair = _dbDao.getSrcIdStatusFromDB(currState.getSnapshotSource(), true);
    if (!srcIdStatusPair.isValidSource())
        throw new BootstrapProcessingException("Bootstrap DB not servicing source :" + currState.getCatchupSource());
    PreparedStatement stmt = null;
    ResultSet rs = null;
    try {
        if (config.isEnableMinScnCheck()) {
            long minScn = _dbDao.getMinScnOfSnapshots(srcIdStatusPair.getSrcId());
            LOG.info("Min scn for tab tables is: " + minScn);
            if (minScn == BootstrapDBMetaDataDAO.DEFAULT_WINDOWSCN) {
                throw new BootstrapDatabaseTooYoungException("BootstrapDB has no minScn for these sources, but minScn check is enabled! minScn=" + minScn);
            }
            // sinceSCN should be greater than minScn, except when sinceSCN == minScn == 0.
            if ((sinceSCN <= minScn) && !(sinceSCN == 0 && minScn == 0)) {
                LOG.error("Bootstrap Snapshot doesn't have requested data . sinceScn too old! sinceScn is " + sinceSCN + " but minScn available is " + minScn);
                throw new BootstrapDatabaseTooYoungException("Min scn=" + minScn + " Since scn=" + sinceSCN);
            }
        } else {
            LOG.debug("Bypassing minScn check!");
        }
        String snapshotSQL = getSnapshotSQLString(_dbDao.getBootstrapConn().getSrcTableName(srcIdStatusPair.getSrcId()), currState.getSnapshotSource());
        stmt = conn.prepareStatement(snapshotSQL);
        long offset = currState.getSnapshotOffset();
        int i = 1;
        stmt.setLong(i++, offset);
        stmt.setLong(i++, currState.getBootstrapStartScn());
        stmt.setLong(i++, currState.getBootstrapSinceScn());
        stmt.setLong(i++, _maxSnapshotRowsPerFetch);
        LOG.info("SnapshotSQL string: " + snapshotSQL + ", " + offset + ", " + currState.getBootstrapStartScn() + ", " + currState.getBootstrapSinceScn() + ", " + _maxSnapshotRowsPerFetch);
        rs = new BootstrapDBTimedQuery(stmt, _queryTimeInSec).executeQuery();
        phaseCompleted = streamOutRows(currState, rs, callBack, _maxSnapshotRowsPerFetch);
    } catch (SQLException e) {
        DBHelper.close(rs, stmt, null);
        LOG.error("Exception occurred when getting snapshot rows" + e);
        throw e;
    } finally {
        if (stmt != null) {
            stmt.close();
            stmt = null;
        }
        mergeAndResetStats();
    }
    return phaseCompleted;
}

Example 2 with BootstrapDBTimedQuery

use of com.linkedin.databus.bootstrap.common.BootstrapDBTimedQuery in project databus by linkedin.

the class BootstrapProcessor method streamCatchupRows.

// Get specified number of catchup rows
public boolean streamCatchupRows(Checkpoint currState, BootstrapEventCallback callBack) throws SQLException, BootstrapProcessingException, BootstrapDatabaseTooOldException {
    assert (currState.getConsumptionMode() == DbusClientMode.BOOTSTRAP_CATCHUP);
    boolean foundRows = false;
    BootstrapDBMetaDataDAO.SourceStatusInfo srcIdStatusPair = _dbDao.getSrcIdStatusFromDB(currState.getCatchupSource(), true);
    if (!srcIdStatusPair.isValidSource())
        throw new BootstrapProcessingException("Bootstrap DB not servicing source :" + currState.getCatchupSource());
    int curSrcId = srcIdStatusPair.getSrcId();
    int curLogId = _dbDao.getLogIdToCatchup(curSrcId, currState.getWindowScn());
    int targetLogId = _dbDao.getLogIdToCatchup(curSrcId, currState.getBootstrapTargetScn());
    boolean phaseCompleted = false;
    PreparedStatement stmt = null;
    try {
        ResultSet rs = null;
        while (!foundRows && curLogId <= targetLogId) {
            stmt = createCatchupStatement(curSrcId, curLogId, currState);
            rs = new BootstrapDBTimedQuery(stmt, _queryTimeInSec).executeQuery();
            foundRows = rs.isBeforeFirst();
            if (!foundRows) {
                // move to next log
                curLogId++;
                // reset rid to 0 for the next log
                currState.setCatchupOffset(0);
                LOG.info("Moving to next log table log_" + curSrcId + "_" + curLogId + " because current log table exhausted!");
            }
        }
        phaseCompleted = streamOutRows(currState, rs, callBack, _maxCatchupRowsPerFetch);
    } catch (SQLException e) {
        LOG.error("Exception occured during fetching catchup rows" + e);
        throw e;
    } finally {
        if (stmt != null) {
            stmt.close();
            stmt = null;
        }
        mergeAndResetStats();
    }
    return phaseCompleted;
}

Example 3 with BootstrapDBTimedQuery

use of com.linkedin.databus.bootstrap.common.BootstrapDBTimedQuery in project databus by linkedin.

the class BootstrapSCNProcessor method getSourceTargetScn.

public long getSourceTargetScn(int srcId) throws SQLException {
    long scn = 0;
    ResultSet rs = null;
    PreparedStatement targetScnStmt = getTargetScnStmt();
    try {
        targetScnStmt.setInt(1, srcId);
        rs = new BootstrapDBTimedQuery(targetScnStmt, _config.getQueryTimeoutInSec()).executeQuery();
        while (rs.next()) {
            scn = rs.getLong(1);
            LOG.info("target scn for source " + srcId + " is " + scn);
        }
        rs.close();
    } catch (SQLException e) {
        LOG.error("Error encountered while selecting target scn for bootstrap_producer_state:", e);
        throw e;
    } finally {
        DBHelper.close(rs, targetScnStmt, null);
    }
    return scn;
}

Example 4 with BootstrapDBTimedQuery

use of com.linkedin.databus.bootstrap.common.BootstrapDBTimedQuery in project databus by linkedin.

the class BootstrapSCNProcessor method getMinApplierWindowScn.

/**
 * Note: for snapshoting each source, we get the min(windowscn) so that we
 * can guarantee not to deliver events later than the scn all prior sources
 * are consistent on. We may end up doing a bit more unnecessary catch up.
 * But it's an optimization we can investigate later.
 * @return startScn
 * @throws SQLException
 */
public long getMinApplierWindowScn(long sinceScn, List<SourceStatusInfo> sourceList) throws BootstrapDatabaseTooOldException, BootstrapProcessingException, SQLException {
    long terminationTime = System.currentTimeMillis() + START_SCN_QUERY_WAIT_TIME;
    long startScn = -1;
    long producerScn = -1;
    ResultSet rs = null;
    Connection conn = _dbDao.getBootstrapConn().getDBConn();
    PreparedStatement getScnStmt = null;
    StringBuffer buf = new StringBuffer();
    // get src id from db - make sure BootstrapDB is not too old
    boolean first = true;
    for (SourceStatusInfo pair : sourceList) {
        if (!pair.isValidSource())
            throw new BootstrapProcessingException("Bootstrap DB not servicing source :" + pair.getSrcId());
        if (!first)
            buf.append(",");
        buf.append(pair.getSrcId());
        first = false;
    }
    String sources = buf.toString();
    while (producerScn < sinceScn && System.currentTimeMillis() < terminationTime) {
        try {
            String applierSql = START_SCN_STMT_SQL_PREFIX + sources + START_SCN_STMT_SQL_SUFFIX;
            String producerSql = PRODUCER_SCN_STMT_SQL_PREFIX + sources + PRODUCER_SCN_STMT_SQL_SUFFIX;
            // Get Applier SCN
            LOG.info("Executing Applier SCN Query :" + applierSql);
            getScnStmt = conn.prepareStatement(applierSql);
            rs = new BootstrapDBTimedQuery(getScnStmt, _config.getQueryTimeoutInSec()).executeQuery();
            if (rs.next()) {
                startScn = rs.getLong(1);
            }
            DBHelper.close(rs, getScnStmt, null);
            rs = null;
            getScnStmt = null;
            // Get ProducerSCN
            LOG.info("Executing Producer SCN Query :" + producerSql);
            getScnStmt = conn.prepareStatement(producerSql);
            rs = new BootstrapDBTimedQuery(getScnStmt, _config.getQueryTimeoutInSec()).executeQuery();
            if (rs.next()) {
                producerScn = rs.getLong(1);
            }
            if (producerScn < startScn) {
                String msg = "Bootstrap Producer has lower SCN than Applier SCN. This is unexpected !! Producer SCN :" + producerScn + ", Applier SCN :" + startScn;
                LOG.fatal(msg);
                throw new BootstrapDatabaseTooOldException(msg);
            }
            if (producerScn < sinceScn) {
                // bootstrap producer needs sometime to consumer events in the buffer, wait a bit.
                LOG.warn("Bootstrap producer has not caught up to all events in its buffer yet to server client properly");
                Thread.sleep(QUERY_WAIT_TIME_SLICE);
            }
        } catch (InterruptedException e) {
        // keeps on sleeping until timed out
        } catch (SQLException e) {
            LOG.warn("SQLException encountered while querying for start scn", e);
        } finally {
            DBHelper.close(rs, getScnStmt, null);
        }
    }
    // Slow Producer case
    if (producerScn < sinceScn) {
        String msg = "Bootstrap producer is slower than the client. Client is at SCN :" + sinceScn + ", Producer is at SCN :" + producerScn + ", Applier is at SCN :" + startScn;
        LOG.error(msg);
        throw new BootstrapDatabaseTooOldException(msg);
    }
    LOG.info("StartSCN Request for sources :" + sources + ",Client SCN :" + sinceScn + ",Producer SCN :" + producerScn + ", Applier SCN :" + startScn);
    return startScn;
}