Examples with SourceStatusInfo com.linkedin.databus.bootstrap.common.BootstrapDBMetaDataDAO.SourceStatusInfo used on opensource projects

Example 6 with SourceStatusInfo

use of com.linkedin.databus.bootstrap.common.BootstrapDBMetaDataDAO.SourceStatusInfo in project databus by linkedin.

the class BootstrapDBSingleSourceCleaner method doClean.

private void doClean() {
    try {
        incCleanerStats();
        SourceStatusInfo s = _sourceStatusInfo;
        {
            assert (s.getSrcName().equals(_source));
            BootstrapDBType type = _bootstrapCleanerStaticConfig.getBootstrapType(s.getSrcName());
            LOG.info("Cleaner running for source :" + s.getSrcName() + "(" + s.getSrcId() + ") with bootstrapDB type :" + type);
            BootstrapLogInfo logInfo = _bootstrapDBCleanerQueryExecutor.getThresholdWindowSCN(type, s.getSrcId());
            if (null == logInfo) {
                LOG.info("No WindowSCN. Nothing to cleanup for source : " + s.getSrcName());
                return;
            }
            LOG.info("LOG info with lowest windowSCN :" + logInfo);
            LOG.info("Begin phase 1 : Gather candidate loginfo :");
            List<BootstrapLogInfo> candidateLogsInfo = _bootstrapDBCleanerQueryExecutor.getCandidateLogsInfo(logInfo.getMinWindowSCN(), (short) (s.getSrcId()));
            if ((null == candidateLogsInfo) || (candidateLogsInfo.isEmpty())) {
                LOG.info("No logs to cleanup for source :" + s.getSrcName() + "(" + s.getSrcId() + ")");
                return;
            }
            LOG.info("End phase 1 : Gather candidate loginfo :");
            LOG.info("Initial Candidate Set for Source :" + s.getSrcName() + " is :" + candidateLogsInfo);
            RetentionStaticConfig rConf = _bootstrapCleanerStaticConfig.getRetentionConfig(s.getSrcName());
            LOG.info("Retention Config for source :" + s.getSrcName() + " is :" + rConf);
            LOG.info("Begin phase 2 : Filter based on retention config :");
            long scn = filterCandidateLogInfo((short) s.getSrcId(), candidateLogsInfo, _bootstrapCleanerStaticConfig.getRetentionConfig(s.getSrcName()));
            LOG.info("Log tables to be deleted for source :" + s.getSrcName() + "(" + s.getSrcId() + ") are :" + candidateLogsInfo + ", Max SCN of deleted logs:" + scn);
            LOG.info("End phase 2 : Filter based on retention config :");
            if ((scn <= 0) || (candidateLogsInfo.isEmpty())) {
                LOG.info("Source :" + s.getSrcName() + "(" + s.getSrcId() + ") No log tables to be deleted !! MaxSCN : " + scn + ", candidateLogs :" + candidateLogsInfo);
                return;
            }
            LOG.info("Begin phase 3 : Updating Meta Info :");
            BootstrapLogInfo firstValidLog = _bootstrapDBCleanerQueryExecutor.getFirstLogTableWithGreaterSCN((short) s.getSrcId(), scn);
            _bootstrapDBCleanerQueryExecutor.updateSource(firstValidLog);
            LOG.info("End phase 3 : Updating Meta Info :");
            LOG.info("Begin phase 4 : Deleting Log tables :");
            // marking logs as done; if any failures; there is a chance that the
            // logs have to be cleaned up later
            _bootstrapDBCleanerQueryExecutor.markDeleted(candidateLogsInfo);
            _bootstrapDBCleanerQueryExecutor.dropTables(candidateLogsInfo);
            LOG.info("End phase 4 : Deleting Log tables :");
            if ((_bootstrapCleanerStaticConfig.getBootstrapType(s.getSrcName()) == BootstrapDBType.BOOTSTRAP_CATCHUP_APPLIER_RUNNING) && ((_applier != null) || _bootstrapCleanerStaticConfig.forceTabTableCleanup(s.getSrcName()))) {
                LOG.info("Source :" + s.getSrcName() + "(" + s.getSrcId() + ") is running in catchup_applier_running mode. " + "Will delete all rows whose scn is less than or equal to " + scn);
                if ((null != _applier) && (_applier.isAlive())) {
                    LOG.info("Begin phase 5 : Pausing Applier and deleting Rows from tab table :");
                    LOG.info("Requesting applier to pause !!");
                    _applier.pause();
                    LOG.info("Applier paused !!");
                }
                try {
                    // mark ahead of time; if this doesn't work this time; it will next
                    // cycle
                    _bootstrapDao.updateMinScnOfSnapshot(s.getSrcId(), scn);
                    String srcTable = _bootstrapDBCleanerQueryHelper.getSrcTable(s.getSrcId());
                    int numRowsDeleted = _bootstrapDBCleanerQueryExecutor.deleteTable(srcTable, scn);
                    LOG.info("Number of Rows deleted for source  :" + s.getSrcName() + "(" + s.getSrcId() + ") :" + numRowsDeleted);
                    if (numRowsDeleted > 0 && _bootstrapCleanerStaticConfig.isOptimizeTableEnabled(s.getSrcName())) {
                        LOG.info("Optimizing table to reclaim space for source :" + s.getSrcName() + "(" + s.getSrcId() + ")");
                        _bootstrapDBCleanerQueryExecutor.optimizeTable(srcTable);
                    }
                } finally {
                    if ((null != _applier) && (_applier.isAlive())) {
                        LOG.info("Requesting applier to resume !!");
                        _applier.unpause();
                        LOG.info("Applier resumed !!");
                    }
                }
                LOG.info("End phase 5 : Deleting Rows from tab table :");
            }
            LOG.info("Cleaner done for source :" + s.getSrcName() + "(" + s.getSrcId() + ")");
        }
    } catch (SQLException ex) {
        LOG.error("Got SQL exception while cleaning bootstrapDB !!", ex);
    } catch (InterruptedException ie) {
        LOG.error("Got interrupted exception while cleaning bootstrapDB !!", ie);
    } finally {
        decCleanerStats();
    }
}

Example 7 with SourceStatusInfo

use of com.linkedin.databus.bootstrap.common.BootstrapDBMetaDataDAO.SourceStatusInfo in project databus by linkedin.

the class TestBootstrapSCNProcessor method testShouldBypassSnapshot.

@Test
public void testShouldBypassSnapshot() throws SQLException, BootstrapProcessingException, NoSuchFieldException, IllegalAccessException, NoSuchMethodException {
    BootstrapSCNProcessor bsp = new BootstrapSCNProcessor();
    Long defaultRowsThresholdForSnapshotBypass = Long.MAX_VALUE;
    Map<String, Long> rowsThresholdForSnapshotBypass = new HashMap<String, Long>();
    Map<String, Boolean> disableSnapshotBypass = new HashMap<String, Boolean>();
    boolean predicatePushDown = false;
    Map<String, Boolean> predicatePushDownBypass = new HashMap<String, Boolean>();
    int queryTimeoutInSec = 10;
    BootstrapReadOnlyConfig db = null;
    boolean enableMinScnCheck = false;
    final long longestDbTxnTimeMins = 240L;
    BootstrapServerStaticConfig bssc = new BootstrapServerStaticConfig(defaultRowsThresholdForSnapshotBypass, rowsThresholdForSnapshotBypass, disableSnapshotBypass, predicatePushDown, predicatePushDownBypass, queryTimeoutInSec, enableMinScnCheck, db, longestDbTxnTimeMins);
    Field field = bsp.getClass().getDeclaredField("_config");
    field.setAccessible(true);
    field.set(bsp, bssc);
    int srcId = 101;
    long sinceScn = 5;
    long startScn = 10;
    BootstrapDBMetaDataDAO bmdd = EasyMock.createMock(BootstrapDBMetaDataDAO.class);
    EasyMock.expect(bmdd.getLogIdToCatchup(srcId, startScn)).andReturn(0).anyTimes();
    EasyMock.expect(bmdd.getLogIdToCatchup(srcId, sinceScn)).andReturn(0).anyTimes();
    EasyMock.replay(bmdd);
    Field dbDaoField = bsp.getClass().getDeclaredField("_dbDao");
    dbDaoField.setAccessible(true);
    dbDaoField.set(bsp, bmdd);
    List<SourceStatusInfo> srcList = new ArrayList<SourceStatusInfo>();
    String name = "foo";
    SourceStatusInfo ssi = new SourceStatusInfo(name, srcId, 4);
    srcList.add(ssi);
    // case 1. Single source, defaultRowsThresholdForSnapshotBypass set to Long.MAX_VALUE,
    // individual overrides not set
    boolean sbs = bsp.shouldBypassSnapshot(sinceScn, startScn, srcList);
    Assert.assertEquals(true, sbs);
    // case 2. Single source, defaultRowsThresholdForSnapshotBypass set to finite value,
    // individual overrides set for the source
    rowsThresholdForSnapshotBypass.put(name, Long.MAX_VALUE);
    BootstrapServerStaticConfig bssc2 = new BootstrapServerStaticConfig(defaultRowsThresholdForSnapshotBypass, rowsThresholdForSnapshotBypass, disableSnapshotBypass, predicatePushDown, predicatePushDownBypass, queryTimeoutInSec, enableMinScnCheck, db, longestDbTxnTimeMins);
    field.set(bsp, bssc2);
    sbs = bsp.shouldBypassSnapshot(sinceScn, startScn, srcList);
    Assert.assertEquals(true, sbs);
    // Case 3:  Detect case when the log is not available on log tables ( so it should NOT bypass snapshot )
    BootstrapDBMetaDataDAO bmdd2 = EasyMock.createMock(BootstrapDBMetaDataDAO.class);
    EasyMock.expect(bmdd2.getLogIdToCatchup(srcId, startScn)).andReturn(2).anyTimes();
    EasyMock.expect(bmdd2.getLogIdToCatchup(srcId, sinceScn)).andThrow(new BootstrapProcessingException(""));
    EasyMock.replay(bmdd2);
    Field dbDaoField2 = bsp.getClass().getDeclaredField("_dbDao");
    dbDaoField2.setAccessible(true);
    dbDaoField2.set(bsp, bmdd2);
    sbs = bsp.shouldBypassSnapshot(sinceScn, startScn, srcList);
    Assert.assertEquals(false, sbs);
}

Example 8 with SourceStatusInfo

use of com.linkedin.databus.bootstrap.common.BootstrapDBMetaDataDAO.SourceStatusInfo in project databus by linkedin.

the class TestBootstrap method testBootstrapProcessor.

@Test
public void testBootstrapProcessor() throws InstantiationException, IllegalAccessException, ClassNotFoundException, SQLException, IOException, BootstrapProcessingException, DatabusException, BootstrapDatabaseTooOldException, BootstrapDBException {
    EventProcessor processorCallback = new EventProcessor();
    BootstrapConfig config = new BootstrapConfig();
    BootstrapReadOnlyConfig staticConfig = config.build();
    BootstrapServerConfig configBuilder = new BootstrapServerConfig();
    configBuilder.setEnableMinScnCheck(false);
    BootstrapServerStaticConfig staticServerConfig = configBuilder.build();
    BootstrapProcessor processor = new BootstrapProcessor(staticServerConfig, null);
    String sourceName = "TestBootstrap.testBootstrapProcessor.events";
    // Create the tables for all the sources before starting up the threads
    BootstrapConn _bootstrapConn = new BootstrapConn();
    boolean autoCommit = true;
    _bootstrapConn.initBootstrapConn(autoCommit, staticConfig.getBootstrapDBUsername(), staticConfig.getBootstrapDBPassword(), staticConfig.getBootstrapDBHostname(), staticConfig.getBootstrapDBName());
    BootstrapDBMetaDataDAO dao = new BootstrapDBMetaDataDAO(_bootstrapConn, staticConfig.getBootstrapDBHostname(), staticConfig.getBootstrapDBUsername(), staticConfig.getBootstrapDBPassword(), staticConfig.getBootstrapDBName(), autoCommit);
    SourceStatusInfo srcStatusInfo = dao.getSrcIdStatusFromDB(sourceName, false);
    if (srcStatusInfo.getSrcId() >= 0) {
        dao.dropSourceInDB(srcStatusInfo.getSrcId());
    }
    dao.addNewSourceInDB(sourceName, BootstrapProducerStatus.ACTIVE);
    srcStatusInfo = dao.getSrcIdStatusFromDB(sourceName, false);
    setBootstrapLoginfoTable(_bootstrapConn, 0, Long.MAX_VALUE);
    //insert one row
    insertOneSnapshotEvent(dao, srcStatusInfo.getSrcId(), Long.MAX_VALUE - 10, Long.MAX_VALUE - 100, "check", "test_payload_data");
    BootstrapCheckpointHandler bstCheckpointHandler = new BootstrapCheckpointHandler(sourceName);
    Checkpoint c = bstCheckpointHandler.createInitialBootstrapCheckpoint(null, 0L);
    c.setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);
    c.setBootstrapStartScn(Long.MAX_VALUE - 10);
    c.setSnapshotOffset(Long.MAX_VALUE - 1000);
    // System.out.println("Snapshot");
    processor.streamSnapShotRows(c, processorCallback);
    Assert.assertEquals(processorCallback.getrIds().size(), 1);
    Assert.assertEquals(Long.MAX_VALUE - 10, processorCallback.getrIds().get(0).longValue());
    Assert.assertEquals(Long.MAX_VALUE - 100, processorCallback.getSequences().get(0).longValue());
    Assert.assertEquals("check", processorCallback.getSrcKeys().get(0));
    Assert.assertEquals("test_payload_data", new String(processorCallback.getValues().get(0)));
    processorCallback.reset();
    c.setSnapshotOffset(Checkpoint.FULLY_CONSUMED_WINDOW_OFFSET);
    c.setBootstrapTargetScn(c.getBootstrapStartScn().longValue());
    bstCheckpointHandler.advanceAfterTargetScn(c);
    bstCheckpointHandler.advanceAfterSnapshotPhase(c);
    //		  c.setConsumptionMode(DbusClientMode.BOOTSTRAP_CATCHUP);
    //		  c.setCatchupSource(sourceName);
    //		  c.setWindowScn(0L);
    //		  c.setWindowOffset(0);
    // System.out.println("Catchup");
    boolean phaseCompleted = processor.streamCatchupRows(c, processorCallback);
    Assert.assertEquals(true, phaseCompleted);
}