Examples with BootstrapDatabaseTooOldException com.linkedin.databus2.core.container.request.BootstrapDatabaseTooOldException used on opensource projects

Example 1 with BootstrapDatabaseTooOldException

use of com.linkedin.databus2.core.container.request.BootstrapDatabaseTooOldException in project databus by linkedin.

the class BootstrapProcessor method streamSnapShotRows.

// Get specificed number of snapshot rows
public boolean streamSnapShotRows(Checkpoint currState, BootstrapEventCallback callBack) throws SQLException, BootstrapProcessingException, BootstrapDatabaseTooOldException, BootstrapDatabaseTooYoungException {
    assert (currState.getConsumptionMode() == DbusClientMode.BOOTSTRAP_SNAPSHOT);
    boolean phaseCompleted = false;
    long startSCN = currState.getBootstrapStartScn();
    long sinceSCN = currState.getBootstrapSinceScn();
    if (startSCN <= sinceSCN) {
        LOG.info("StartSCN is less than or equal to sinceSCN. Bypassing snapshot phase !! startSCN:" + startSCN + ",sinceSCN:" + sinceSCN);
        return true;
    }
    Connection conn = _dbDao.getBootstrapConn().getDBConn();
    BootstrapDBMetaDataDAO.SourceStatusInfo srcIdStatusPair = _dbDao.getSrcIdStatusFromDB(currState.getSnapshotSource(), true);
    if (!srcIdStatusPair.isValidSource())
        throw new BootstrapProcessingException("Bootstrap DB not servicing source :" + currState.getCatchupSource());
    PreparedStatement stmt = null;
    ResultSet rs = null;
    try {
        if (config.isEnableMinScnCheck()) {
            long minScn = _dbDao.getMinScnOfSnapshots(srcIdStatusPair.getSrcId());
            LOG.info("Min scn for tab tables is: " + minScn);
            if (minScn == BootstrapDBMetaDataDAO.DEFAULT_WINDOWSCN) {
                throw new BootstrapDatabaseTooYoungException("BootstrapDB has no minScn for these sources, but minScn check is enabled! minScn=" + minScn);
            }
            // sinceSCN should be greater than minScn, except when sinceSCN == minScn == 0.
            if ((sinceSCN <= minScn) && !(sinceSCN == 0 && minScn == 0)) {
                LOG.error("Bootstrap Snapshot doesn't have requested data . sinceScn too old! sinceScn is " + sinceSCN + " but minScn available is " + minScn);
                throw new BootstrapDatabaseTooYoungException("Min scn=" + minScn + " Since scn=" + sinceSCN);
            }
        } else {
            LOG.debug("Bypassing minScn check!");
        }
        String snapshotSQL = getSnapshotSQLString(_dbDao.getBootstrapConn().getSrcTableName(srcIdStatusPair.getSrcId()), currState.getSnapshotSource());
        stmt = conn.prepareStatement(snapshotSQL);
        long offset = currState.getSnapshotOffset();
        int i = 1;
        stmt.setLong(i++, offset);
        stmt.setLong(i++, currState.getBootstrapStartScn());
        stmt.setLong(i++, currState.getBootstrapSinceScn());
        stmt.setLong(i++, _maxSnapshotRowsPerFetch);
        LOG.info("SnapshotSQL string: " + snapshotSQL + ", " + offset + ", " + currState.getBootstrapStartScn() + ", " + currState.getBootstrapSinceScn() + ", " + _maxSnapshotRowsPerFetch);
        rs = new BootstrapDBTimedQuery(stmt, _queryTimeInSec).executeQuery();
        phaseCompleted = streamOutRows(currState, rs, callBack, _maxSnapshotRowsPerFetch);
    } catch (SQLException e) {
        DBHelper.close(rs, stmt, null);
        LOG.error("Exception occurred when getting snapshot rows" + e);
        throw e;
    } finally {
        if (stmt != null) {
            stmt.close();
            stmt = null;
        }
        mergeAndResetStats();
    }
    return phaseCompleted;
}

Example 2 with BootstrapDatabaseTooOldException

use of com.linkedin.databus2.core.container.request.BootstrapDatabaseTooOldException in project databus by linkedin.

the class StartSCNRequestProcessor method doProcess.

@Override
protected DatabusRequest doProcess(DatabusRequest request) throws IOException, RequestProcessingException {
    BootstrapHttpStatsCollector bootstrapStatsCollector = _bootstrapServer.getBootstrapStatsCollector();
    long startTime = System.currentTimeMillis();
    String sources = request.getRequiredStringParam(SOURCES_PARAM);
    List<String> srcList = getSources(sources);
    Checkpoint ckpt = new Checkpoint(request.getRequiredStringParam(CHECKPOINT_PARAM));
    LOG.info("StartSCN requested for sources : (" + sources + "). CheckPoint is :" + ckpt);
    long sinceScn = ckpt.getBootstrapSinceScn();
    ObjectMapper mapper = new ObjectMapper();
    StringWriter out = new StringWriter(1024);
    long startSCN = -1;
    BootstrapSCNProcessor processor = null;
    try {
        processor = new BootstrapSCNProcessor(_config, _bootstrapServer.getInboundEventStatisticsCollector());
        List<SourceStatusInfo> srcStatusPairs = null;
        try {
            srcStatusPairs = processor.getSourceIdAndStatusFromName(srcList);
            startSCN = processor.getMinApplierWindowScn(sinceScn, srcStatusPairs);
            if (processor.shouldBypassSnapshot(sinceScn, startSCN, srcStatusPairs)) {
                LOG.info("Bootstrap Snapshot phase will be bypassed for startScn request :" + request);
                LOG.info("Original startSCN is:" + startSCN + ", Setting startSCN to the sinceSCN:" + sinceScn);
                startSCN = sinceScn;
            } else {
                if (startSCN == BootstrapDBMetaDataDAO.DEFAULT_WINDOWSCN) {
                    throw new RequestProcessingException("Bootstrap DB is being initialized! startSCN=" + startSCN);
                }
                if (_config.isEnableMinScnCheck()) {
                    // snapshot isn't bypassed. Check if snapshot is possible from sinceScn by checking minScn
                    long minScn = processor.getBootstrapMetaDataDAO().getMinScnOfSnapshots(srcStatusPairs);
                    LOG.info("Min scn for tab tables is: " + minScn);
                    if (minScn == BootstrapDBMetaDataDAO.DEFAULT_WINDOWSCN) {
                        throw new BootstrapDatabaseTooYoungException("BootstrapDB has no minScn for these sources, but minScn check is enabled! minScn=" + minScn);
                    }
                    // sinceSCN should be greater than minScn, unless sinceScn=minScn=0
                    if ((sinceScn <= minScn) && !(sinceScn == 0 && minScn == 0)) {
                        LOG.error("Bootstrap Snapshot doesn't have requested data . sinceScn too old! sinceScn is " + sinceScn + " but minScn available is " + minScn);
                        throw new BootstrapDatabaseTooYoungException("Min scn=" + minScn + " Since scn=" + sinceScn);
                    }
                } else {
                    LOG.debug("Bypassing minScn check! ");
                }
            }
        } catch (BootstrapDatabaseTooOldException tooOldException) {
            if (bootstrapStatsCollector != null) {
                bootstrapStatsCollector.registerErrStartSCN();
                bootstrapStatsCollector.registerErrDatabaseTooOld();
            }
            LOG.error("The bootstrap database is too old!", tooOldException);
            throw new RequestProcessingException(tooOldException);
        } catch (BootstrapDatabaseTooYoungException e) {
            if (bootstrapStatsCollector != null) {
                bootstrapStatsCollector.registerErrStartSCN();
                bootstrapStatsCollector.registerErrBootstrap();
            }
            LOG.error("The bootstrap database is too young!", e);
            throw new RequestProcessingException(e);
        } catch (SQLException e) {
            if (bootstrapStatsCollector != null) {
                bootstrapStatsCollector.registerErrStartSCN();
                bootstrapStatsCollector.registerErrSqlException();
            }
            LOG.error("Error encountered while fetching startSCN from database.", e);
            throw new RequestProcessingException(e);
        }
        mapper.writeValue(out, String.valueOf(startSCN));
        byte[] resultBytes = out.toString().getBytes(Charset.defaultCharset());
        request.getResponseContent().write(ByteBuffer.wrap(resultBytes));
        LOG.info("startSCN: " + startSCN + " with server Info :" + _serverHostPort);
    } catch (RequestProcessingException ex) {
        LOG.error("Got exception while calculating startSCN", ex);
        throw ex;
    } catch (Exception ex) {
        LOG.error("Got exception while calculating startSCN", ex);
        throw new RequestProcessingException(ex);
    } finally {
        if (null != processor)
            processor.shutdown();
    }
    if (bootstrapStatsCollector != null) {
        bootstrapStatsCollector.registerStartSCNReq(System.currentTimeMillis() - startTime);
    }
    return request;
}

Example 3 with BootstrapDatabaseTooOldException

use of com.linkedin.databus2.core.container.request.BootstrapDatabaseTooOldException in project databus by linkedin.

the class DatabusBootstrapProducer method initBootstrapDBMetadata.

private void initBootstrapDBMetadata() throws SQLException, BootstrapDatabaseTooOldException {
    DatabusHttpClientImpl.RuntimeConfig clientRtConfig = getClientConfigManager().getReadOnlyConfig();
    // create source list
    for (ServerInfo relayInfo : clientRtConfig.getRelays()) {
        _registeredSources.addAll(relayInfo.getSources());
        for (String source : _registeredSources) {
            BootstrapDBMetaDataDAO.SourceStatusInfo srcIdStatus = _dbDao.getSrcIdStatusFromDB(source, false);
            if (0 > srcIdStatus.getSrcId()) {
                int newState = BootstrapProducerStatus.NEW;
                if (!_bootstrapProducerStaticConfig.isBootstrapDBStateCheck()) {
                    // TO allow test framework to listen to relay directly,DBStateCheck
                    // flag is used
                    newState = BootstrapProducerStatus.ACTIVE;
                }
                _dbDao.addNewSourceInDB(source, newState);
            }
            srcIdStatus = _dbDao.getSrcIdStatusFromDB(source, false);
            _srcNameIdMap.put(source, srcIdStatus.getSrcId());
            if (_bootstrapProducerStaticConfig.isBootstrapDBStateCheck()) {
                if (!BootstrapProducerStatus.isReadyForConsumption(srcIdStatus.getStatus()))
                    throw new BootstrapDatabaseTooOldException("Bootstrap DB is not ready to read from relay !! Status :" + srcIdStatus);
            }
        }
    }
}

Example 4 with BootstrapDatabaseTooOldException

use of com.linkedin.databus2.core.container.request.BootstrapDatabaseTooOldException in project databus by linkedin.

the class BootstrapSCNProcessor method getMinApplierWindowScn.

/**
 * Note: for snapshoting each source, we get the min(windowscn) so that we
 * can guarantee not to deliver events later than the scn all prior sources
 * are consistent on. We may end up doing a bit more unnecessary catch up.
 * But it's an optimization we can investigate later.
 * @return startScn
 * @throws SQLException
 */
public long getMinApplierWindowScn(long sinceScn, List<SourceStatusInfo> sourceList) throws BootstrapDatabaseTooOldException, BootstrapProcessingException, SQLException {
    long terminationTime = System.currentTimeMillis() + START_SCN_QUERY_WAIT_TIME;
    long startScn = -1;
    long producerScn = -1;
    ResultSet rs = null;
    Connection conn = _dbDao.getBootstrapConn().getDBConn();
    PreparedStatement getScnStmt = null;
    StringBuffer buf = new StringBuffer();
    // get src id from db - make sure BootstrapDB is not too old
    boolean first = true;
    for (SourceStatusInfo pair : sourceList) {
        if (!pair.isValidSource())
            throw new BootstrapProcessingException("Bootstrap DB not servicing source :" + pair.getSrcId());
        if (!first)
            buf.append(",");
        buf.append(pair.getSrcId());
        first = false;
    }
    String sources = buf.toString();
    while (producerScn < sinceScn && System.currentTimeMillis() < terminationTime) {
        try {
            String applierSql = START_SCN_STMT_SQL_PREFIX + sources + START_SCN_STMT_SQL_SUFFIX;
            String producerSql = PRODUCER_SCN_STMT_SQL_PREFIX + sources + PRODUCER_SCN_STMT_SQL_SUFFIX;
            // Get Applier SCN
            LOG.info("Executing Applier SCN Query :" + applierSql);
            getScnStmt = conn.prepareStatement(applierSql);
            rs = new BootstrapDBTimedQuery(getScnStmt, _config.getQueryTimeoutInSec()).executeQuery();
            if (rs.next()) {
                startScn = rs.getLong(1);
            }
            DBHelper.close(rs, getScnStmt, null);
            rs = null;
            getScnStmt = null;
            // Get ProducerSCN
            LOG.info("Executing Producer SCN Query :" + producerSql);
            getScnStmt = conn.prepareStatement(producerSql);
            rs = new BootstrapDBTimedQuery(getScnStmt, _config.getQueryTimeoutInSec()).executeQuery();
            if (rs.next()) {
                producerScn = rs.getLong(1);
            }
            if (producerScn < startScn) {
                String msg = "Bootstrap Producer has lower SCN than Applier SCN. This is unexpected !! Producer SCN :" + producerScn + ", Applier SCN :" + startScn;
                LOG.fatal(msg);
                throw new BootstrapDatabaseTooOldException(msg);
            }
            if (producerScn < sinceScn) {
                // bootstrap producer needs sometime to consumer events in the buffer, wait a bit.
                LOG.warn("Bootstrap producer has not caught up to all events in its buffer yet to server client properly");
                Thread.sleep(QUERY_WAIT_TIME_SLICE);
            }
        } catch (InterruptedException e) {
        // keeps on sleeping until timed out
        } catch (SQLException e) {
            LOG.warn("SQLException encountered while querying for start scn", e);
        } finally {
            DBHelper.close(rs, getScnStmt, null);
        }
    }
    // Slow Producer case
    if (producerScn < sinceScn) {
        String msg = "Bootstrap producer is slower than the client. Client is at SCN :" + sinceScn + ", Producer is at SCN :" + producerScn + ", Applier is at SCN :" + startScn;
        LOG.error(msg);
        throw new BootstrapDatabaseTooOldException(msg);
    }
    LOG.info("StartSCN Request for sources :" + sources + ",Client SCN :" + sinceScn + ",Producer SCN :" + producerScn + ", Applier SCN :" + startScn);
    return startScn;
}

Example 5 with BootstrapDatabaseTooOldException

use of com.linkedin.databus2.core.container.request.BootstrapDatabaseTooOldException in project databus by linkedin.

the class BootstrapStartScnHttpResponseProcessor method finishResponse.

@Override
public void finishResponse() throws Exception {
    super.finishResponse();
    if (_errorHandled) {
        return;
    }
    try {
        String exceptionName = RemoteExceptionHandler.getExceptionName(_decorated);
        Throwable remoteException = _remoteExceptionHandler.getException(_decorated);
        if (null != remoteException && remoteException instanceof BootstrapDatabaseTooOldException) {
            _remoteExceptionHandler.handleException(remoteException);
        } else if (null != exceptionName) {
            LOG.error("/startScn response error: " + RemoteExceptionHandler.getExceptionMessage(_decorated));
            _stateReuse.switchToStartScnResponseError();
            LOG.error("Failed to process /startscn response");
        } else {
            String hostHdr = DbusConstants.UNKNOWN_HOST;
            String svcHdr = DbusConstants.UNKNOWN_SERVICE_ID;
            if (null != getParent()) {
                hostHdr = getParent().getRemoteHost();
                svcHdr = getParent().getRemoteService();
                LOG.info("initiated bootstrap sesssion to host " + hostHdr + " service " + svcHdr);
            }
            InputStream bodyStream = Channels.newInputStream(_decorated);
            ObjectMapper mapper = new ObjectMapper();
            String scnString = mapper.readValue(bodyStream, String.class);
            ServerInfo serverInfo = null;
            String serverHostPort = _decorated.getMetadata(DbusConstants.SERVER_INFO_HOSTPORT_HEADER_PARAM);
            try {
                serverInfo = ServerInfo.buildServerInfoFromHostPort(serverHostPort, DbusConstants.HOSTPORT_DELIMITER);
            } catch (Exception ex) {
                LOG.error("Unable to extract Boostrap Server info from StartSCN response. ServerInfo was :" + serverHostPort, ex);
            }
            LOG.info("Response startScn:" + scnString + ", from bootstrap Server :" + serverHostPort);
            long startScn = Long.parseLong(scnString);
            Checkpoint ckpt = _checkpoint;
            if (startScn < 0) {
                LOG.error("unexpected value for startSCN: " + startScn);
                _stateReuse.switchToStartScnResponseError();
            } else if (ckpt.getConsumptionMode() != DbusClientMode.BOOTSTRAP_SNAPSHOT) {
                LOG.error("StartScnResponseProcessor:" + " expecting in client mode: " + DbusClientMode.BOOTSTRAP_SNAPSHOT + " while in the incorrect mode: " + ckpt.getConsumptionMode());
            } else {
                LOG.info("Start SCN " + startScn + " received for bootstrap snapshot source " + ckpt.getSnapshotSource());
                ckpt.setBootstrapStartScn(startScn);
                ckpt.setBootstrapServerInfo(serverHostPort);
                /*
           * No need to create a seperate BootstrapConnection as we are guaranteed to have a bootstrap Connection
           * at this point.
           */
                _stateReuse.switchToStartScnSuccess(_checkpoint, null, serverInfo);
            }
        }
    } catch (Exception ex) {
        LOG.error("Failed to process /startscn response", ex);
        _stateReuse.switchToStartScnResponseError();
    }
    _callback.enqueueMessage(_stateReuse);
}