Examples with SweepMetricsManager com.palantir.atlasdb.sweep.metrics.SweepMetricsManager used on opensource projects

Example 1 with SweepMetricsManager

use of com.palantir.atlasdb.sweep.metrics.SweepMetricsManager in project atlasdb by palantir.

the class TransactionManagers method initializeSweepEndpointAndBackgroundProcess.

private static BackgroundSweeperImpl initializeSweepEndpointAndBackgroundProcess(AtlasDbConfig config, Supplier<AtlasDbRuntimeConfig> runtimeConfigSupplier, Consumer<Object> env, KeyValueService kvs, TransactionService transactionService, SweepStrategyManager sweepStrategyManager, CleanupFollower follower, SerializableTransactionManager transactionManager, PersistentLockManager persistentLockManager) {
    CellsSweeper cellsSweeper = new CellsSweeper(transactionManager, kvs, persistentLockManager, ImmutableList.of(follower));
    SweepMetricsManager sweepMetricsManager = new SweepMetricsManager();
    SweepTaskRunner sweepRunner = new SweepTaskRunner(kvs, transactionManager::getUnreadableTimestamp, transactionManager::getImmutableTimestamp, transactionService, sweepStrategyManager, cellsSweeper, sweepMetricsManager);
    BackgroundSweeperPerformanceLogger sweepPerfLogger = new NoOpBackgroundSweeperPerformanceLogger();
    AdjustableSweepBatchConfigSource sweepBatchConfigSource = AdjustableSweepBatchConfigSource.create(() -> getSweepBatchConfig(runtimeConfigSupplier.get().sweep(), config.keyValueService()));
    SweepMetricsManager sweepMetrics = new SweepMetricsManager();
    SpecificTableSweeper specificTableSweeper = initializeSweepEndpoint(env, kvs, transactionManager, sweepRunner, sweepPerfLogger, sweepMetrics, config.initializeAsync(), sweepBatchConfigSource);
    BackgroundSweeperImpl backgroundSweeper = BackgroundSweeperImpl.create(sweepBatchConfigSource, () -> runtimeConfigSupplier.get().sweep().enabled(), () -> runtimeConfigSupplier.get().sweep().pauseMillis(), persistentLockManager, specificTableSweeper);
    transactionManager.registerClosingCallback(backgroundSweeper::shutdown);
    backgroundSweeper.runInBackground();
    return backgroundSweeper;
}

Example 2 with SweepMetricsManager

use of com.palantir.atlasdb.sweep.metrics.SweepMetricsManager in project atlasdb by palantir.

the class SweepTaskRunner method doRun.

private SweepResults doRun(TableReference tableRef, SweepBatchConfig batchConfig, byte[] startRow, RunType runType, Sweeper sweeper) {
    Stopwatch watch = Stopwatch.createStarted();
    long timeSweepStarted = System.currentTimeMillis();
    log.info("Beginning iteration of sweep for table {} starting at row {}", LoggingArgs.tableRef(tableRef), UnsafeArg.of("startRow", PtBytes.encodeHexString(startRow)));
    // Earliest start timestamp of any currently open transaction, with two caveats:
    // (1) unreadableTimestamps are calculated via wall-clock time, and so may not be correct
    // under pathological clock conditions
    // (2) immutableTimestamps do not account for locks have timed out after checking their locks;
    // such a transaction may have a start timestamp less than the immutableTimestamp, and it
    // could still get successfully committed (its commit timestamp may or may not be less than
    // the immutableTimestamp
    // Note that this is fine, because we'll either
    // (1) force old readers to abort (if they read a garbage collection sentinel), or
    // (2) force old writers to retry (note that we must roll back any uncommitted transactions that
    // we encounter
    long sweepTs = sweeper.getSweepTimestampSupplier().getSweepTimestamp(unreadableTimestampSupplier, immutableTimestampSupplier);
    CandidateCellForSweepingRequest request = ImmutableCandidateCellForSweepingRequest.builder().startRowInclusive(startRow).batchSizeHint(batchConfig.candidateBatchSize()).maxTimestampExclusive(sweepTs).shouldCheckIfLatestValueIsEmpty(sweeper.shouldSweepLastCommitted()).shouldDeleteGarbageCollectionSentinels(!sweeper.shouldAddSentinels()).build();
    SweepableCellFilter sweepableCellFilter = new SweepableCellFilter(transactionService, sweeper, sweepTs);
    try (ClosableIterator<List<CandidateCellForSweeping>> candidates = keyValueService.getCandidateCellsForSweeping(tableRef, request)) {
        ExaminedCellLimit limit = new ExaminedCellLimit(startRow, batchConfig.maxCellTsPairsToExamine());
        Iterator<BatchOfCellsToSweep> batchesToSweep = getBatchesToSweep(candidates, batchConfig, sweepableCellFilter, limit);
        long totalCellTsPairsExamined = 0;
        long totalCellTsPairsDeleted = 0;
        metricsManager.ifPresent(SweepMetricsManager::resetBeforeDeleteBatch);
        byte[] lastRow = startRow;
        while (batchesToSweep.hasNext()) {
            BatchOfCellsToSweep batch = batchesToSweep.next();
            /*
                 * At this point cells were merged in batches of at least deleteBatchSize blocks per batch. Therefore we
                 * expect most batches to have slightly more than deleteBatchSize blocks. Partitioning such batches with
                 * deleteBatchSize as a limit results in a small second batch, which is bad for performance reasons.
                 * Therefore, deleteBatchSize is doubled.
                 */
            long cellsDeleted = sweepBatch(tableRef, batch.cells(), runType, 2 * batchConfig.deleteBatchSize());
            totalCellTsPairsDeleted += cellsDeleted;
            long cellsExamined = batch.numCellTsPairsExamined();
            totalCellTsPairsExamined += cellsExamined;
            metricsManager.ifPresent(manager -> manager.updateAfterDeleteBatch(cellsExamined, cellsDeleted));
            lastRow = batch.lastCellExamined().getRowName();
        }
        return SweepResults.builder().previousStartRow(Optional.of(startRow)).nextStartRow(Arrays.equals(startRow, lastRow) ? Optional.empty() : Optional.of(lastRow)).cellTsPairsExamined(totalCellTsPairsExamined).staleValuesDeleted(totalCellTsPairsDeleted).minSweptTimestamp(sweepTs).timeInMillis(watch.elapsed(TimeUnit.MILLISECONDS)).timeSweepStarted(timeSweepStarted).build();
    }
}

Example 3 with SweepMetricsManager

use of com.palantir.atlasdb.sweep.metrics.SweepMetricsManager in project atlasdb by palantir.

the class AbstractBackgroundSweeperIntegrationTest method setup.

@Before
public void setup() {
    TimestampService tsService = new InMemoryTimestampService();
    kvs = SweepStatsKeyValueService.create(getKeyValueService(), tsService, () -> AtlasDbConstants.DEFAULT_SWEEP_WRITE_THRESHOLD, () -> AtlasDbConstants.DEFAULT_SWEEP_WRITE_SIZE_THRESHOLD);
    SweepStrategyManager ssm = SweepStrategyManagers.createDefault(kvs);
    txService = TransactionServices.createTransactionService(kvs);
    txManager = SweepTestUtils.setupTxManager(kvs, tsService, ssm, txService);
    LongSupplier tsSupplier = sweepTimestamp::get;
    PersistentLockManager persistentLockManager = new PersistentLockManager(SweepTestUtils.getPersistentLockService(kvs), AtlasDbConstants.DEFAULT_SWEEP_PERSISTENT_LOCK_WAIT_MILLIS);
    CellsSweeper cellsSweeper = new CellsSweeper(txManager, kvs, persistentLockManager, ImmutableList.of());
    SweepTaskRunner sweepRunner = new SweepTaskRunner(kvs, tsSupplier, tsSupplier, txService, ssm, cellsSweeper);
    SweepMetricsManager sweepMetricsManager = new SweepMetricsManager();
    specificTableSweeper = SpecificTableSweeper.create(txManager, kvs, sweepRunner, SweepTableFactory.of(), new NoOpBackgroundSweeperPerformanceLogger(), sweepMetricsManager, false);
    sweepBatchConfigSource = AdjustableSweepBatchConfigSource.create(() -> sweepBatchConfig);
    backgroundSweeper = BackgroundSweeperImpl.create(sweepBatchConfigSource, // sweepEnabled
    () -> true, // sweepPauseMillis
    () -> 10L, persistentLockManager, specificTableSweeper);
}