Examples with BootstrapProcessingException com.linkedin.databus.bootstrap.api.BootstrapProcessingException used on opensource projects

Example 1 with BootstrapProcessingException

use of com.linkedin.databus.bootstrap.api.BootstrapProcessingException in project databus by linkedin.

the class BootstrapDBMetaDataDAO method getLogIdToCatchup.

public int getLogIdToCatchup(int srcId, long sinceScn) throws SQLException, BootstrapProcessingException {
    int logid = -1;
    int deleted = 0;
    Connection conn = _bootstrapConn.getDBConn();
    PreparedStatement stmt = null;
    ResultSet rs = null;
    try {
        if (0 >= sinceScn) {
            // special handling for invalid scn, e.g. 0 or -1:
            // in this case, we want to ruturn the logid corresponding to the earliest log
            stmt = conn.prepareStatement("select logid, deleted from bootstrap_loginfo where srcid = ? and minwindowscn = (select min(minwindowscn) from bootstrap_loginfo where srcid = ? and deleted != 1 and minwindowscn >= 0)  order by logid asc limit 1");
            stmt.setInt(1, srcId);
            stmt.setInt(2, srcId);
        } else {
            stmt = conn.prepareStatement("SELECT logid, deleted from bootstrap_loginfo where srcid = ? and minwindowscn <= ? and maxwindowscn >= ? order by logid asc limit 1");
            stmt.setInt(1, srcId);
            stmt.setLong(2, sinceScn);
            stmt.setLong(3, sinceScn);
        }
        rs = stmt.executeQuery();
        if (rs.next()) {
            logid = rs.getInt(1);
            deleted = rs.getInt(2);
            LOG.info("logid for catchup:" + logid + ", Deleted :" + deleted);
        }
        if ((0 > logid) || (1 == deleted)) {
            throw new BootstrapProcessingException("Log file with logid=" + logid + " ,srcid=" + srcId + " is either deleted or not found in bootstrap_loginfo for since scn: " + sinceScn);
        }
    } catch (SQLException e) {
        LOG.error("Error encountered while selecting logid from bootstrap_loginfo", e);
        throw e;
    } finally {
        DBHelper.close(rs, stmt, null);
    }
    return logid;
}

Example 2 with BootstrapProcessingException

use of com.linkedin.databus.bootstrap.api.BootstrapProcessingException in project databus by linkedin.

the class BootstrapProcessor method streamSnapShotRows.

// Get specificed number of snapshot rows
public boolean streamSnapShotRows(Checkpoint currState, BootstrapEventCallback callBack) throws SQLException, BootstrapProcessingException, BootstrapDatabaseTooOldException, BootstrapDatabaseTooYoungException {
    assert (currState.getConsumptionMode() == DbusClientMode.BOOTSTRAP_SNAPSHOT);
    boolean phaseCompleted = false;
    long startSCN = currState.getBootstrapStartScn();
    long sinceSCN = currState.getBootstrapSinceScn();
    if (startSCN <= sinceSCN) {
        LOG.info("StartSCN is less than or equal to sinceSCN. Bypassing snapshot phase !! startSCN:" + startSCN + ",sinceSCN:" + sinceSCN);
        return true;
    }
    Connection conn = _dbDao.getBootstrapConn().getDBConn();
    BootstrapDBMetaDataDAO.SourceStatusInfo srcIdStatusPair = _dbDao.getSrcIdStatusFromDB(currState.getSnapshotSource(), true);
    if (!srcIdStatusPair.isValidSource())
        throw new BootstrapProcessingException("Bootstrap DB not servicing source :" + currState.getCatchupSource());
    PreparedStatement stmt = null;
    ResultSet rs = null;
    try {
        if (config.isEnableMinScnCheck()) {
            long minScn = _dbDao.getMinScnOfSnapshots(srcIdStatusPair.getSrcId());
            LOG.info("Min scn for tab tables is: " + minScn);
            if (minScn == BootstrapDBMetaDataDAO.DEFAULT_WINDOWSCN) {
                throw new BootstrapDatabaseTooYoungException("BootstrapDB has no minScn for these sources, but minScn check is enabled! minScn=" + minScn);
            }
            //sinceSCN should be greater than minScn, except when sinceSCN == minScn == 0.
            if ((sinceSCN <= minScn) && !(sinceSCN == 0 && minScn == 0)) {
                LOG.error("Bootstrap Snapshot doesn't have requested data . sinceScn too old! sinceScn is " + sinceSCN + " but minScn available is " + minScn);
                throw new BootstrapDatabaseTooYoungException("Min scn=" + minScn + " Since scn=" + sinceSCN);
            }
        } else {
            LOG.debug("Bypassing minScn check!");
        }
        String snapshotSQL = getSnapshotSQLString(_dbDao.getBootstrapConn().getSrcTableName(srcIdStatusPair.getSrcId()), currState.getSnapshotSource());
        stmt = conn.prepareStatement(snapshotSQL);
        long offset = currState.getSnapshotOffset();
        int i = 1;
        stmt.setLong(i++, offset);
        stmt.setLong(i++, currState.getBootstrapStartScn());
        stmt.setLong(i++, currState.getBootstrapSinceScn());
        stmt.setLong(i++, _maxSnapshotRowsPerFetch);
        LOG.info("SnapshotSQL string: " + snapshotSQL + ", " + offset + ", " + currState.getBootstrapStartScn() + ", " + currState.getBootstrapSinceScn() + ", " + _maxSnapshotRowsPerFetch);
        rs = new BootstrapDBTimedQuery(stmt, _queryTimeInSec).executeQuery();
        phaseCompleted = streamOutRows(currState, rs, callBack, _maxSnapshotRowsPerFetch);
    } catch (SQLException e) {
        DBHelper.close(rs, stmt, null);
        LOG.error("Exception occurred when getting snapshot rows" + e);
        throw e;
    } finally {
        if (stmt != null) {
            stmt.close();
            stmt = null;
        }
        mergeAndResetStats();
    }
    return phaseCompleted;
}

Example 3 with BootstrapProcessingException

use of com.linkedin.databus.bootstrap.api.BootstrapProcessingException in project databus by linkedin.

the class BootstrapProcessor method streamCatchupRows.

// Get specified number of catchup rows
public boolean streamCatchupRows(Checkpoint currState, BootstrapEventCallback callBack) throws SQLException, BootstrapProcessingException, BootstrapDatabaseTooOldException {
    assert (currState.getConsumptionMode() == DbusClientMode.BOOTSTRAP_CATCHUP);
    boolean foundRows = false;
    BootstrapDBMetaDataDAO.SourceStatusInfo srcIdStatusPair = _dbDao.getSrcIdStatusFromDB(currState.getCatchupSource(), true);
    if (!srcIdStatusPair.isValidSource())
        throw new BootstrapProcessingException("Bootstrap DB not servicing source :" + currState.getCatchupSource());
    int curSrcId = srcIdStatusPair.getSrcId();
    int curLogId = _dbDao.getLogIdToCatchup(curSrcId, currState.getWindowScn());
    int targetLogId = _dbDao.getLogIdToCatchup(curSrcId, currState.getBootstrapTargetScn());
    boolean phaseCompleted = false;
    PreparedStatement stmt = null;
    try {
        ResultSet rs = null;
        while (!foundRows && curLogId <= targetLogId) {
            stmt = createCatchupStatement(curSrcId, curLogId, currState);
            rs = new BootstrapDBTimedQuery(stmt, _queryTimeInSec).executeQuery();
            foundRows = rs.isBeforeFirst();
            if (!foundRows) {
                // move to next log
                curLogId++;
                // reset rid to 0 for the next log
                currState.setCatchupOffset(0);
                LOG.info("Moving to next log table log_" + curSrcId + "_" + curLogId + " because current log table exhausted!");
            }
        }
        phaseCompleted = streamOutRows(currState, rs, callBack, _maxCatchupRowsPerFetch);
    } catch (SQLException e) {
        LOG.error("Exception occured during fetching catchup rows" + e);
        throw e;
    } finally {
        if (stmt != null) {
            stmt.close();
            stmt = null;
        }
        mergeAndResetStats();
    }
    return phaseCompleted;
}

Example 4 with BootstrapProcessingException

use of com.linkedin.databus.bootstrap.api.BootstrapProcessingException in project databus by linkedin.

the class BootstrapSCNProcessor method getMinApplierWindowScn.

/**
	   * Note: for snapshoting each source, we get the min(windowscn) so that we
	   * can guarantee not to deliver events later than the scn all prior sources
	   * are consistent on. We may end up doing a bit more unnecessary catch up.
	   * But it's an optimization we can investigate later.
	   * @return startScn
	   * @throws SQLException
	   */
public long getMinApplierWindowScn(long sinceScn, List<SourceStatusInfo> sourceList) throws BootstrapDatabaseTooOldException, BootstrapProcessingException, SQLException {
    long terminationTime = System.currentTimeMillis() + START_SCN_QUERY_WAIT_TIME;
    long startScn = -1;
    long producerScn = -1;
    ResultSet rs = null;
    Connection conn = _dbDao.getBootstrapConn().getDBConn();
    PreparedStatement getScnStmt = null;
    StringBuffer buf = new StringBuffer();
    // get src id from db - make sure BootstrapDB is not too old
    boolean first = true;
    for (SourceStatusInfo pair : sourceList) {
        if (!pair.isValidSource())
            throw new BootstrapProcessingException("Bootstrap DB not servicing source :" + pair.getSrcId());
        if (!first)
            buf.append(",");
        buf.append(pair.getSrcId());
        first = false;
    }
    String sources = buf.toString();
    while (producerScn < sinceScn && System.currentTimeMillis() < terminationTime) {
        try {
            String applierSql = START_SCN_STMT_SQL_PREFIX + sources + START_SCN_STMT_SQL_SUFFIX;
            String producerSql = PRODUCER_SCN_STMT_SQL_PREFIX + sources + PRODUCER_SCN_STMT_SQL_SUFFIX;
            // Get Applier SCN
            LOG.info("Executing Applier SCN Query :" + applierSql);
            getScnStmt = conn.prepareStatement(applierSql);
            rs = new BootstrapDBTimedQuery(getScnStmt, _config.getQueryTimeoutInSec()).executeQuery();
            if (rs.next()) {
                startScn = rs.getLong(1);
            }
            DBHelper.close(rs, getScnStmt, null);
            rs = null;
            getScnStmt = null;
            // Get ProducerSCN
            LOG.info("Executing Producer SCN Query :" + producerSql);
            getScnStmt = conn.prepareStatement(producerSql);
            rs = new BootstrapDBTimedQuery(getScnStmt, _config.getQueryTimeoutInSec()).executeQuery();
            if (rs.next()) {
                producerScn = rs.getLong(1);
            }
            if (producerScn < startScn) {
                String msg = "Bootstrap Producer has lower SCN than Applier SCN. This is unexpected !! Producer SCN :" + producerScn + ", Applier SCN :" + startScn;
                LOG.fatal(msg);
                throw new BootstrapDatabaseTooOldException(msg);
            }
            if (producerScn < sinceScn) {
                // bootstrap producer needs sometime to consumer events in the buffer, wait a bit.
                LOG.warn("Bootstrap producer has not caught up to all events in its buffer yet to server client properly");
                Thread.sleep(QUERY_WAIT_TIME_SLICE);
            }
        } catch (InterruptedException e) {
        // keeps on sleeping until timed out
        } catch (SQLException e) {
            LOG.warn("SQLException encountered while querying for start scn", e);
        } finally {
            DBHelper.close(rs, getScnStmt, null);
        }
    }
    // Slow Producer case
    if (producerScn < sinceScn) {
        String msg = "Bootstrap producer is slower than the client. Client is at SCN :" + sinceScn + ", Producer is at SCN :" + producerScn + ", Applier is at SCN :" + startScn;
        LOG.error(msg);
        throw new BootstrapDatabaseTooOldException(msg);
    }
    LOG.info("StartSCN Request for sources :" + sources + ",Client SCN :" + sinceScn + ",Producer SCN :" + producerScn + ", Applier SCN :" + startScn);
    return startScn;
}

Example 5 with BootstrapProcessingException

use of com.linkedin.databus.bootstrap.api.BootstrapProcessingException in project databus by linkedin.

the class TargetSCNRequestProcessor method doProcess.

@Override
protected DatabusRequest doProcess(DatabusRequest request) throws IOException, RequestProcessingException {
    BootstrapHttpStatsCollector bootstrapStatsCollector = _bootstrapServer.getBootstrapStatsCollector();
    long startTime = System.currentTimeMillis();
    int srcId = -1;
    long targetScn = -1;
    String source = request.getRequiredStringParam(SOURCE_PARAM);
    BootstrapSCNProcessor processor = null;
    try {
        processor = new BootstrapSCNProcessor(_config, _bootstrapServer.getInboundEventStatisticsCollector());
        try {
            // get src id from db
            BootstrapDBMetaDataDAO.SourceStatusInfo srcIdStatus = processor.getSrcIdStatusFromDB(source, true);
            if (!srcIdStatus.isValidSource())
                throw new BootstrapProcessingException("Bootstrap DB not servicing source :" + source);
            srcId = srcIdStatus.getSrcId();
            // select target scn
            targetScn = processor.getSourceTargetScn(srcId);
        } catch (BootstrapDatabaseTooOldException tooOldException) {
            if (bootstrapStatsCollector != null) {
                bootstrapStatsCollector.registerErrTargetSCN();
                bootstrapStatsCollector.registerErrDatabaseTooOld();
            }
            LOG.error("The bootstrap database is too old!", tooOldException);
            throw new RequestProcessingException(tooOldException);
        } catch (SQLException e) {
            if (bootstrapStatsCollector != null) {
                bootstrapStatsCollector.registerErrTargetSCN();
                bootstrapStatsCollector.registerErrSqlException();
            }
            LOG.error("Error encountered while fetching targetSCN from database.", e);
            throw new RequestProcessingException(e);
        }
        ObjectMapper mapper = new ObjectMapper();
        StringWriter out = new StringWriter(1024);
        mapper.writeValue(out, String.valueOf(targetScn));
        byte[] resultBytes = out.toString().getBytes(Charset.defaultCharset());
        request.getResponseContent().write(ByteBuffer.wrap(resultBytes));
        LOG.info("targetSCN: " + targetScn);
    } catch (Exception ex) {
        LOG.error("Got exception while calculating targetSCN", ex);
        throw new RequestProcessingException(ex);
    } finally {
        if (null != processor)
            processor.shutdown();
    }
    if (bootstrapStatsCollector != null) {
        bootstrapStatsCollector.registerTargetSCNReq(System.currentTimeMillis() - startTime);
    }
    return request;
}