Examples with SourceStatusInfo com.linkedin.databus.bootstrap.common.BootstrapDBMetaDataDAO.SourceStatusInfo used on opensource projects

Example 1 with SourceStatusInfo

use of com.linkedin.databus.bootstrap.common.BootstrapDBMetaDataDAO.SourceStatusInfo in project databus by linkedin.

the class BootstrapSCNProcessor method getMinApplierWindowScn.

/**
	   * Note: for snapshoting each source, we get the min(windowscn) so that we
	   * can guarantee not to deliver events later than the scn all prior sources
	   * are consistent on. We may end up doing a bit more unnecessary catch up.
	   * But it's an optimization we can investigate later.
	   * @return startScn
	   * @throws SQLException
	   */
public long getMinApplierWindowScn(long sinceScn, List<SourceStatusInfo> sourceList) throws BootstrapDatabaseTooOldException, BootstrapProcessingException, SQLException {
    long terminationTime = System.currentTimeMillis() + START_SCN_QUERY_WAIT_TIME;
    long startScn = -1;
    long producerScn = -1;
    ResultSet rs = null;
    Connection conn = _dbDao.getBootstrapConn().getDBConn();
    PreparedStatement getScnStmt = null;
    StringBuffer buf = new StringBuffer();
    // get src id from db - make sure BootstrapDB is not too old
    boolean first = true;
    for (SourceStatusInfo pair : sourceList) {
        if (!pair.isValidSource())
            throw new BootstrapProcessingException("Bootstrap DB not servicing source :" + pair.getSrcId());
        if (!first)
            buf.append(",");
        buf.append(pair.getSrcId());
        first = false;
    }
    String sources = buf.toString();
    while (producerScn < sinceScn && System.currentTimeMillis() < terminationTime) {
        try {
            String applierSql = START_SCN_STMT_SQL_PREFIX + sources + START_SCN_STMT_SQL_SUFFIX;
            String producerSql = PRODUCER_SCN_STMT_SQL_PREFIX + sources + PRODUCER_SCN_STMT_SQL_SUFFIX;
            // Get Applier SCN
            LOG.info("Executing Applier SCN Query :" + applierSql);
            getScnStmt = conn.prepareStatement(applierSql);
            rs = new BootstrapDBTimedQuery(getScnStmt, _config.getQueryTimeoutInSec()).executeQuery();
            if (rs.next()) {
                startScn = rs.getLong(1);
            }
            DBHelper.close(rs, getScnStmt, null);
            rs = null;
            getScnStmt = null;
            // Get ProducerSCN
            LOG.info("Executing Producer SCN Query :" + producerSql);
            getScnStmt = conn.prepareStatement(producerSql);
            rs = new BootstrapDBTimedQuery(getScnStmt, _config.getQueryTimeoutInSec()).executeQuery();
            if (rs.next()) {
                producerScn = rs.getLong(1);
            }
            if (producerScn < startScn) {
                String msg = "Bootstrap Producer has lower SCN than Applier SCN. This is unexpected !! Producer SCN :" + producerScn + ", Applier SCN :" + startScn;
                LOG.fatal(msg);
                throw new BootstrapDatabaseTooOldException(msg);
            }
            if (producerScn < sinceScn) {
                // bootstrap producer needs sometime to consumer events in the buffer, wait a bit.
                LOG.warn("Bootstrap producer has not caught up to all events in its buffer yet to server client properly");
                Thread.sleep(QUERY_WAIT_TIME_SLICE);
            }
        } catch (InterruptedException e) {
        // keeps on sleeping until timed out
        } catch (SQLException e) {
            LOG.warn("SQLException encountered while querying for start scn", e);
        } finally {
            DBHelper.close(rs, getScnStmt, null);
        }
    }
    // Slow Producer case
    if (producerScn < sinceScn) {
        String msg = "Bootstrap producer is slower than the client. Client is at SCN :" + sinceScn + ", Producer is at SCN :" + producerScn + ", Applier is at SCN :" + startScn;
        LOG.error(msg);
        throw new BootstrapDatabaseTooOldException(msg);
    }
    LOG.info("StartSCN Request for sources :" + sources + ",Client SCN :" + sinceScn + ",Producer SCN :" + producerScn + ", Applier SCN :" + startScn);
    return startScn;
}

Example 2 with SourceStatusInfo

use of com.linkedin.databus.bootstrap.common.BootstrapDBMetaDataDAO.SourceStatusInfo in project databus by linkedin.

the class BootstrapSCNProcessor method shouldBypassSnapshot.

/*
     * Business Logic to bypass snapshot phase.
     * Here are the rules that each source requested by client must satisfy:
     * 1. Events with windowscn = Min(sinceScn,startScn) MUST still be available in log tables (not purged).
     * 2. Approx NumRows between sinceSCN and startSCN MUST be less than or equal to RowsThresholdForSnapshotBypass
     * 3. It is not mandatory for startScn < sinceScn to bypass snapshot. That is, we can have startScn >= sinceScn,
     *    but the log tables may still have events from sinceScn in the log tables.
     * 4. sinceSCN must be > 0.
     * @param startScn : The max scn in the snapshot table for the source ( windowscn in the bootstrap applier state )
     * @param sinceScn : The scn from which the client wants to bootstrap from
     */
public boolean shouldBypassSnapshot(long sinceScn, long startScn, List<SourceStatusInfo> srcList) throws SQLException, BootstrapProcessingException {
    if ((srcList == null) || (srcList.isEmpty()))
        return false;
    if (sinceScn <= 0) {
        // client is requesting full bootstrap. Cannot guarantee with just log tables.
        LOG.info("Client requesting from SCN (" + sinceScn + "). Bootstrap Snapshot will not be bypassed !!");
        return false;
    }
    for (SourceStatusInfo pair : srcList) {
        boolean disableSnapshotBypass = _config.isBypassSnapshotDisabled(pair.getSrcName());
        if (disableSnapshotBypass)
            return false;
        boolean canCatchupFromLog = validateIfCanCatchupFromLog(sinceScn, startScn, pair.getSrcId());
        if (!canCatchupFromLog)
            return false;
        long threshold = _config.getRowsThresholdForSnapshotBypass(pair.getSrcName());
        if (threshold == Long.MAX_VALUE) {
            // for this source, as it could potentially be an expensive query
            continue;
        }
        long currRows = getRowDiff(sinceScn, startScn, pair.getSrcId());
        if (currRows > threshold) {
            LOG.info("Threshold check failed for source (" + pair.getSrcName() + ") Threshold:" + threshold + ",Approx Rows:" + currRows);
            return false;
        }
    }
    return true;
}

Example 3 with SourceStatusInfo

use of com.linkedin.databus.bootstrap.common.BootstrapDBMetaDataDAO.SourceStatusInfo in project databus by linkedin.

the class BootstrapMetadata method main.

/**
   */
public static void main(String[] args) {
    int exitStatus = 0;
    GnuParser cmdLineParser = new GnuParser();
    Options options = new Options();
    options.addOption("d", "dbname", true, "database name : [bootstrap]").addOption("s", "dbhost", true, "database hostname: [localhost]").addOption("p", "dbpassword", true, "database password: [bootstrap]").addOption("u", "dbuser", true, "database user: [bootstrap] ").addOption("S", "sources", true, " <srcid,..>|<srcName substring,..,>  [all]").addOption("m", "minScnType", true, "type of minScn algorithm: [catchup-forever] catchup-forever|catchup-purged|snapshot. " + "'snapshot':        use when snapshot table has been purged and the size of snapshot is not large.Times-out in 5 minutes if query cannot complete. " + "'catchup-purged':  use when snapshot table has been purged and a conservative value of minScn is acceptable or when 'snapshot' isn't feasible. " + "'catchup-forever': use when snapshot table has *never* been purged . " + "Note: If a source has been seeded, regardless of the minScnType, the value returned is 0.").addOption("h", "help", false, "help");
    // add descriptions used in usage() 'functionName' , 'description-pairs'
    String[] functionNames = { "getSources", "get sourceName, sourceId, sourceStatus from bootstrap db", "createMinScnTable", "create metadata required for minScn feature in bootstrap db if none exists", "createAllTables", "create all metadata in bootstrap db if they don't exist", "getMinScn", "[--sources <>] return minScn as seen by bootstrap service for specified sources", "computeMinScn", "[--sources <>] [--minScnType <>] return minScn using one of the algorithms specified. " + "INIT could mean failure to find a minSCN ", "computeAndSetMinScn", "[--sources <>] [--minScnType<>] set minScn using one of the algorithms specified. ", "setMinScn", "[--sources <>] INFINITY|INIT|<scn> set minScn to a desired value. " };
    try {
        CommandLine cmdLineArgs = cmdLineParser.parse(options, args, false);
        if (cmdLineArgs.hasOption('h')) {
            usage(options, functionNames);
            System.exit(0);
        }
        String[] fns = cmdLineArgs.getArgs();
        if (fns.length < 1) {
            usage(options, functionNames);
            throw new Exception("Missing argument");
        }
        String function = fns[0];
        String arg1 = (fns.length > 1) ? fns[1] : null;
        if (!legitFunction(functionNames, function)) {
            usage(options, functionNames);
            throw new Exception("Unknown function");
        }
        String bootstrapHost = cmdLineArgs.getOptionValue("dbhost", DEFAULT_BOOTSTRAP_HOST);
        String bootstrapDbName = cmdLineArgs.getOptionValue("dbname", DEFAULT_BOOTSTRAP_DATABASE);
        String bootstrapPassword = cmdLineArgs.getOptionValue("dbpassword", DEFAULT_BOOTSTRAP_PASSWORD);
        String bootstrapUser = cmdLineArgs.getOptionValue("dbuser", DEFAULT_BOOTSTRAP_USER);
        _bsConn = new BootstrapConn();
        _bsConn.initBootstrapConn(false, bootstrapUser, bootstrapPassword, bootstrapHost, bootstrapDbName);
        _dbao = new BootstrapDBMetaDataDAO(_bsConn, bootstrapHost, bootstrapUser, bootstrapPassword, bootstrapDbName, false);
        String minScnType = cmdLineArgs.getOptionValue("minScnType", DEFAULT_MINSCN_TYPE);
        if (function.equals("createMinScnTable")) {
            createBootstrapMinScn();
            return;
        } else if (function.equals("createAllTables")) {
            _dbao.setupDB();
            return;
        }
        List<SourceStatusInfo> sourceInfoList = _dbao.getSrcStatusInfoFromDB();
        if (function.equalsIgnoreCase("getSources")) {
            for (SourceStatusInfo s : sourceInfoList) {
                StringBuilder sb = new StringBuilder();
                sb.append(s.getSrcName()).append('\t').append(s.getSrcId()).append('\t').append(s.getStatus());
                System.out.println(sb.toString());
            }
        } else {
            // not getSources()
            // expect srcId list; for brevity;
            String mySources = cmdLineArgs.getOptionValue("sources");
            String[] mySrcList = null;
            if (mySources != null) {
                mySrcList = mySources.split(",");
            }
            List<SourceStatusInfo> finalSrcIdList = (mySrcList != null) ? getFinalSrcIdList(sourceInfoList, mySrcList) : sourceInfoList;
            if (function.equalsIgnoreCase("getMinScn")) {
                // read minScn from minscn table;
                createBootstrapMinScn();
                for (SourceStatusInfo sInfo : finalSrcIdList) {
                    long minScn = _dbao.getMinScnOfSnapshots(sInfo.getSrcId());
                    DBHelper.commit(_bsConn.getDBConn());
                    if (minScn == Long.MIN_VALUE) {
                        System.err.println("Error: Cannot get minScn for " + sInfo.getSrcId());
                    } else {
                        printScn(minScn, sInfo, '\t');
                    }
                }
            } else if (function.equalsIgnoreCase("computeMinScn")) {
                // database not affected
                for (SourceStatusInfo sInfo : finalSrcIdList) {
                    long minScn = computeMinScn(minScnType, sInfo.getSrcId());
                    // commit select;
                    DBHelper.commit(_bsConn.getDBConn());
                    if (minScn == Long.MIN_VALUE) {
                        System.err.println("Error: Cannot get minScn for " + sInfo.getSrcId());
                    } else {
                        printScn(minScn, sInfo, '\t');
                    }
                }
            } else if (function.equalsIgnoreCase("setMinScn")) {
                if (arg1 == null) {
                    throw new Exception("Missing argument for setMinScn: <scn>, INFINITY or INIT");
                }
                // setMinScn <scn>
                // override existing scn; never fail unless update fails ; show
                // warning that you should know what you are doing
                createBootstrapMinScn();
                long scn;
                if (arg1.equalsIgnoreCase("INFINITY")) {
                    scn = Long.MAX_VALUE;
                } else if (arg1.equalsIgnoreCase("INIT")) {
                    scn = BootstrapDBMetaDataDAO.DEFAULT_WINDOWSCN;
                } else {
                    scn = Long.parseLong(arg1);
                }
                for (SourceStatusInfo sInfo : finalSrcIdList) {
                    // calls commit
                    _dbao.updateMinScnOfSnapshot(sInfo.getSrcId(), scn);
                }
            } else if (function.equalsIgnoreCase("computeAndSetMinScn")) {
                // set scn iff minScn has been initialized; ensure minScn is updated
                // safely according to --type
                // if seeded; --type is ignored;
                createBootstrapMinScn();
                for (SourceStatusInfo sInfo : finalSrcIdList) {
                    long minScn = computeMinScn(minScnType, sInfo.getSrcId());
                    if (minScn != Long.MIN_VALUE) {
                        _dbao.updateMinScnOfSnapshot(sInfo.getSrcId(), minScn);
                    }
                    DBHelper.commit(_bsConn.getDBConn());
                }
            }
        }
    } catch (ParseException e) {
        usage(options, functionNames);
        exitStatus = 1;
    } catch (Exception e) {
        System.err.println("Error: " + e);
        exitStatus = 1;
    } finally {
        if (_dbao != null)
            _dbao.close();
        System.exit(exitStatus);
    }
}

Example 4 with SourceStatusInfo

use of com.linkedin.databus.bootstrap.common.BootstrapDBMetaDataDAO.SourceStatusInfo in project databus by linkedin.

the class StartSCNRequestProcessor method doProcess.

@Override
protected DatabusRequest doProcess(DatabusRequest request) throws IOException, RequestProcessingException {
    BootstrapHttpStatsCollector bootstrapStatsCollector = _bootstrapServer.getBootstrapStatsCollector();
    long startTime = System.currentTimeMillis();
    String sources = request.getRequiredStringParam(SOURCES_PARAM);
    List<String> srcList = getSources(sources);
    Checkpoint ckpt = new Checkpoint(request.getRequiredStringParam(CHECKPOINT_PARAM));
    LOG.info("StartSCN requested for sources : (" + sources + "). CheckPoint is :" + ckpt);
    long sinceScn = ckpt.getBootstrapSinceScn();
    ObjectMapper mapper = new ObjectMapper();
    StringWriter out = new StringWriter(1024);
    long startSCN = -1;
    BootstrapSCNProcessor processor = null;
    try {
        processor = new BootstrapSCNProcessor(_config, _bootstrapServer.getInboundEventStatisticsCollector());
        List<SourceStatusInfo> srcStatusPairs = null;
        try {
            srcStatusPairs = processor.getSourceIdAndStatusFromName(srcList);
            startSCN = processor.getMinApplierWindowScn(sinceScn, srcStatusPairs);
            if (processor.shouldBypassSnapshot(sinceScn, startSCN, srcStatusPairs)) {
                LOG.info("Bootstrap Snapshot phase will be bypassed for startScn request :" + request);
                LOG.info("Original startSCN is:" + startSCN + ", Setting startSCN to the sinceSCN:" + sinceScn);
                startSCN = sinceScn;
            } else {
                if (startSCN == BootstrapDBMetaDataDAO.DEFAULT_WINDOWSCN) {
                    throw new RequestProcessingException("Bootstrap DB is being initialized! startSCN=" + startSCN);
                }
                if (_config.isEnableMinScnCheck()) {
                    //snapshot isn't bypassed. Check if snapshot is possible from sinceScn by checking minScn
                    long minScn = processor.getBootstrapMetaDataDAO().getMinScnOfSnapshots(srcStatusPairs);
                    LOG.info("Min scn for tab tables is: " + minScn);
                    if (minScn == BootstrapDBMetaDataDAO.DEFAULT_WINDOWSCN) {
                        throw new BootstrapDatabaseTooYoungException("BootstrapDB has no minScn for these sources, but minScn check is enabled! minScn=" + minScn);
                    }
                    //sinceSCN should be greater than minScn, unless sinceScn=minScn=0
                    if ((sinceScn <= minScn) && !(sinceScn == 0 && minScn == 0)) {
                        LOG.error("Bootstrap Snapshot doesn't have requested data . sinceScn too old! sinceScn is " + sinceScn + " but minScn available is " + minScn);
                        throw new BootstrapDatabaseTooYoungException("Min scn=" + minScn + " Since scn=" + sinceScn);
                    }
                } else {
                    LOG.debug("Bypassing minScn check! ");
                }
            }
        } catch (BootstrapDatabaseTooOldException tooOldException) {
            if (bootstrapStatsCollector != null) {
                bootstrapStatsCollector.registerErrStartSCN();
                bootstrapStatsCollector.registerErrDatabaseTooOld();
            }
            LOG.error("The bootstrap database is too old!", tooOldException);
            throw new RequestProcessingException(tooOldException);
        } catch (BootstrapDatabaseTooYoungException e) {
            if (bootstrapStatsCollector != null) {
                bootstrapStatsCollector.registerErrStartSCN();
                bootstrapStatsCollector.registerErrBootstrap();
            }
            LOG.error("The bootstrap database is too young!", e);
            throw new RequestProcessingException(e);
        } catch (SQLException e) {
            if (bootstrapStatsCollector != null) {
                bootstrapStatsCollector.registerErrStartSCN();
                bootstrapStatsCollector.registerErrSqlException();
            }
            LOG.error("Error encountered while fetching startSCN from database.", e);
            throw new RequestProcessingException(e);
        }
        mapper.writeValue(out, String.valueOf(startSCN));
        byte[] resultBytes = out.toString().getBytes(Charset.defaultCharset());
        request.getResponseContent().write(ByteBuffer.wrap(resultBytes));
        LOG.info("startSCN: " + startSCN + " with server Info :" + _serverHostPort);
    } catch (RequestProcessingException ex) {
        LOG.error("Got exception while calculating startSCN", ex);
        throw ex;
    } catch (Exception ex) {
        LOG.error("Got exception while calculating startSCN", ex);
        throw new RequestProcessingException(ex);
    } finally {
        if (null != processor)
            processor.shutdown();
    }
    if (bootstrapStatsCollector != null) {
        bootstrapStatsCollector.registerStartSCNReq(System.currentTimeMillis() - startTime);
    }
    return request;
}

Example 5 with SourceStatusInfo

use of com.linkedin.databus.bootstrap.common.BootstrapDBMetaDataDAO.SourceStatusInfo in project databus by linkedin.

the class TestBootstrap method testBootstrapService.

@Test
public void testBootstrapService() throws InstantiationException, IllegalAccessException, ClassNotFoundException, SQLException, IOException, BootstrapProcessingException, DatabusException, BootstrapDatabaseTooOldException, BootstrapDBException {
    final Logger log = Logger.getLogger("TestBootstrap.testBootstrapService");
    EventProcessor processorCallback = new EventProcessor();
    BootstrapConfig config = new BootstrapConfig();
    BootstrapReadOnlyConfig staticConfig = config.build();
    String[] sources = new String[4];
    sources[0] = "TestBootstrap.testBootstrapService.event";
    sources[1] = "TestBootstrap.testBootstrapService.event1";
    sources[2] = "TestBootstrap.testBootstrapService.event2";
    sources[3] = "TestBootstrap.testBootstrapService.event3";
    // Create the tables for all the sources before starting up the threads
    BootstrapConn _bootstrapConn = new BootstrapConn();
    final boolean autoCommit = true;
    _bootstrapConn.initBootstrapConn(autoCommit, staticConfig.getBootstrapDBUsername(), staticConfig.getBootstrapDBPassword(), staticConfig.getBootstrapDBHostname(), staticConfig.getBootstrapDBName());
    BootstrapDBMetaDataDAO dao = new BootstrapDBMetaDataDAO(_bootstrapConn, staticConfig.getBootstrapDBHostname(), staticConfig.getBootstrapDBUsername(), staticConfig.getBootstrapDBPassword(), staticConfig.getBootstrapDBName(), autoCommit);
    for (String source : sources) {
        SourceStatusInfo srcStatusInfo = dao.getSrcIdStatusFromDB(source, false);
        if (srcStatusInfo.getSrcId() >= 0) {
            dao.dropSourceInDB(srcStatusInfo.getSrcId());
        }
        dao.addNewSourceInDB(source, BootstrapProducerStatus.ACTIVE);
    }
    setBootstrapLoginfoTable(_bootstrapConn, 1, 1);
    DatabusBootstrapClient s = new DatabusBootstrapClient(sources);
    Checkpoint cp;
    while ((cp = s.getNextBatch(10, processorCallback)).getConsumptionMode() != DbusClientMode.ONLINE_CONSUMPTION) {
        log.debug(cp);
    }
}