Examples with RetentionStaticConfig com.linkedin.databus.bootstrap.common.BootstrapCleanerStaticConfig.RetentionStaticConfig used on opensource projects

Example 1 with RetentionStaticConfig

use of com.linkedin.databus.bootstrap.common.BootstrapCleanerStaticConfig.RetentionStaticConfig in project databus by linkedin.

the class BootstrapDBSingleSourceCleaner method doClean.

private void doClean() {
    try {
        incCleanerStats();
        SourceStatusInfo s = _sourceStatusInfo;
        {
            assert (s.getSrcName().equals(_source));
            BootstrapDBType type = _bootstrapCleanerStaticConfig.getBootstrapType(s.getSrcName());
            LOG.info("Cleaner running for source :" + s.getSrcName() + "(" + s.getSrcId() + ") with bootstrapDB type :" + type);
            BootstrapLogInfo logInfo = _bootstrapDBCleanerQueryExecutor.getThresholdWindowSCN(type, s.getSrcId());
            if (null == logInfo) {
                LOG.info("No WindowSCN. Nothing to cleanup for source : " + s.getSrcName());
                return;
            }
            LOG.info("LOG info with lowest windowSCN :" + logInfo);
            LOG.info("Begin phase 1 : Gather candidate loginfo :");
            List<BootstrapLogInfo> candidateLogsInfo = _bootstrapDBCleanerQueryExecutor.getCandidateLogsInfo(logInfo.getMinWindowSCN(), (short) (s.getSrcId()));
            if ((null == candidateLogsInfo) || (candidateLogsInfo.isEmpty())) {
                LOG.info("No logs to cleanup for source :" + s.getSrcName() + "(" + s.getSrcId() + ")");
                return;
            }
            LOG.info("End phase 1 : Gather candidate loginfo :");
            LOG.info("Initial Candidate Set for Source :" + s.getSrcName() + " is :" + candidateLogsInfo);
            RetentionStaticConfig rConf = _bootstrapCleanerStaticConfig.getRetentionConfig(s.getSrcName());
            LOG.info("Retention Config for source :" + s.getSrcName() + " is :" + rConf);
            LOG.info("Begin phase 2 : Filter based on retention config :");
            long scn = filterCandidateLogInfo((short) s.getSrcId(), candidateLogsInfo, _bootstrapCleanerStaticConfig.getRetentionConfig(s.getSrcName()));
            LOG.info("Log tables to be deleted for source :" + s.getSrcName() + "(" + s.getSrcId() + ") are :" + candidateLogsInfo + ", Max SCN of deleted logs:" + scn);
            LOG.info("End phase 2 : Filter based on retention config :");
            if ((scn <= 0) || (candidateLogsInfo.isEmpty())) {
                LOG.info("Source :" + s.getSrcName() + "(" + s.getSrcId() + ") No log tables to be deleted !! MaxSCN : " + scn + ", candidateLogs :" + candidateLogsInfo);
                return;
            }
            LOG.info("Begin phase 3 : Updating Meta Info :");
            BootstrapLogInfo firstValidLog = _bootstrapDBCleanerQueryExecutor.getFirstLogTableWithGreaterSCN((short) s.getSrcId(), scn);
            _bootstrapDBCleanerQueryExecutor.updateSource(firstValidLog);
            LOG.info("End phase 3 : Updating Meta Info :");
            LOG.info("Begin phase 4 : Deleting Log tables :");
            // marking logs as done; if any failures; there is a chance that the
            // logs have to be cleaned up later
            _bootstrapDBCleanerQueryExecutor.markDeleted(candidateLogsInfo);
            _bootstrapDBCleanerQueryExecutor.dropTables(candidateLogsInfo);
            LOG.info("End phase 4 : Deleting Log tables :");
            if ((_bootstrapCleanerStaticConfig.getBootstrapType(s.getSrcName()) == BootstrapDBType.BOOTSTRAP_CATCHUP_APPLIER_RUNNING) && ((_applier != null) || _bootstrapCleanerStaticConfig.forceTabTableCleanup(s.getSrcName()))) {
                LOG.info("Source :" + s.getSrcName() + "(" + s.getSrcId() + ") is running in catchup_applier_running mode. " + "Will delete all rows whose scn is less than or equal to " + scn);
                if ((null != _applier) && (_applier.isAlive())) {
                    LOG.info("Begin phase 5 : Pausing Applier and deleting Rows from tab table :");
                    LOG.info("Requesting applier to pause !!");
                    _applier.pause();
                    LOG.info("Applier paused !!");
                }
                try {
                    // mark ahead of time; if this doesn't work this time; it will next
                    // cycle
                    _bootstrapDao.updateMinScnOfSnapshot(s.getSrcId(), scn);
                    String srcTable = _bootstrapDBCleanerQueryHelper.getSrcTable(s.getSrcId());
                    int numRowsDeleted = _bootstrapDBCleanerQueryExecutor.deleteTable(srcTable, scn);
                    LOG.info("Number of Rows deleted for source  :" + s.getSrcName() + "(" + s.getSrcId() + ") :" + numRowsDeleted);
                    if (numRowsDeleted > 0 && _bootstrapCleanerStaticConfig.isOptimizeTableEnabled(s.getSrcName())) {
                        LOG.info("Optimizing table to reclaim space for source :" + s.getSrcName() + "(" + s.getSrcId() + ")");
                        _bootstrapDBCleanerQueryExecutor.optimizeTable(srcTable);
                    }
                } finally {
                    if ((null != _applier) && (_applier.isAlive())) {
                        LOG.info("Requesting applier to resume !!");
                        _applier.unpause();
                        LOG.info("Applier resumed !!");
                    }
                }
                LOG.info("End phase 5 : Deleting Rows from tab table :");
            }
            LOG.info("Cleaner done for source :" + s.getSrcName() + "(" + s.getSrcId() + ")");
        }
    } catch (SQLException ex) {
        LOG.error("Got SQL exception while cleaning bootstrapDB !!", ex);
    } catch (InterruptedException ie) {
        LOG.error("Got interrupted exception while cleaning bootstrapDB !!", ie);
    } finally {
        decCleanerStats();
    }
}