Examples with SweepResults com.palantir.atlasdb.keyvalue.api.SweepResults used on opensource projects

Example 21 with SweepResults

use of com.palantir.atlasdb.keyvalue.api.SweepResults in project atlasdb by palantir.

the class AbstractSweepTaskRunnerTest method testSweepManyValuesThorough.

@Test(timeout = 50000)
public void testSweepManyValuesThorough() {
    createTable(SweepStrategy.THOROUGH);
    putIntoDefaultColumn("foo", "bar", 50);
    putUncommitted("foo", "bad", 75);
    putIntoDefaultColumn("foo", "baz", 100);
    putIntoDefaultColumn("foo", "buzz", 125);
    putUncommitted("foo", "foo", 150);
    SweepResults results = completeSweep(175).get();
    assertEquals(4, results.getStaleValuesDeleted());
    assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(5);
    assertEquals("buzz", getFromDefaultColumn("foo", 200));
    assertNull(getFromDefaultColumn("foo", 124));
    assertEquals(ImmutableSet.of(125L), getAllTsFromDefaultColumn("foo"));
}

Example 22 with SweepResults

use of com.palantir.atlasdb.keyvalue.api.SweepResults in project atlasdb by palantir.

the class AbstractSweepTaskRunnerTest method partialSweep.

private SweepResults partialSweep(long ts) {
    sweepTimestamp.set(ts);
    SweepResults results = sweepRunner.run(TABLE_NAME, ImmutableSweepBatchConfig.builder().deleteBatchSize(1).candidateBatchSize(1).maxCellTsPairsToExamine(1).build(), PtBytes.EMPTY_BYTE_ARRAY);
    assertTrue(results.getNextStartRow().isPresent());
    return results;
}

Example 23 with SweepResults

use of com.palantir.atlasdb.keyvalue.api.SweepResults in project atlasdb by palantir.

the class AbstractSweepTaskRunnerTest method testSweepBatchesInDifferentRows.

@Test(timeout = 50000)
public void testSweepBatchesInDifferentRows() {
    CellsSweeper cellsSweeper = Mockito.mock(CellsSweeper.class);
    SweepTaskRunner spiedSweepRunner = new SweepTaskRunner(kvs, tsSupplier, tsSupplier, txService, ssm, cellsSweeper);
    putTwoValuesInEachCell(BIG_LIST_OF_CELLS_IN_DIFFERENT_ROWS);
    int deleteBatchSize = 2;
    Pair<List<List<Cell>>, SweepResults> sweptCellsAndSweepResults = runSweep(cellsSweeper, spiedSweepRunner, 10, 1, deleteBatchSize);
    List<List<Cell>> sweptCells = sweptCellsAndSweepResults.getLhSide();
    SweepResults sweepResults = sweptCellsAndSweepResults.getRhSide();
    assertThat(Iterables.concat(sweptCells)).containsExactlyElementsOf(BIG_LIST_OF_CELLS_IN_DIFFERENT_ROWS);
    for (List<Cell> sweptBatch : sweptCells.subList(0, sweptCells.size() - 1)) {
        // We requested deleteBatchSize = 2, so we expect between 2 and 4 timestamps deleted at a time.
        // We also expect a single timestamp to be swept per each cell.
        assertThat(sweptBatch.size()).isBetween(deleteBatchSize, 2 * deleteBatchSize);
    }
    // The last batch can be smaller than deleteBatchSize
    assertThat(sweptCells.get(sweptCells.size() - 1).size()).isLessThanOrEqualTo(2 * deleteBatchSize);
    assertEquals("Expected Ts Pairs Examined should add up to entire table (2 values in each cell)", 2 * BIG_LIST_OF_CELLS_IN_DIFFERENT_ROWS.size(), sweepResults.getCellTsPairsExamined());
}

Example 24 with SweepResults

use of com.palantir.atlasdb.keyvalue.api.SweepResults in project atlasdb by palantir.

the class AbstractSweepTest method testSweepOnMixedCaseTable.

@Test(timeout = 50000)
public void testSweepOnMixedCaseTable() {
    TableReference mixedCaseTable = TableReference.create(Namespace.create("someNamespace"), "someTable");
    createTable(mixedCaseTable, SweepStrategy.CONSERVATIVE);
    put(mixedCaseTable, "row", "col", "val", 10);
    put(mixedCaseTable, "row", "col", "val", 20);
    Optional<SweepResults> optResults = completeSweep(mixedCaseTable, 30);
    optResults.ifPresent(results -> {
        assertEquals(1, results.getStaleValuesDeleted());
        assertThat(results.getCellTsPairsExamined()).isGreaterThanOrEqualTo(2);
    });
}

Example 25 with SweepResults

use of com.palantir.atlasdb.keyvalue.api.SweepResults in project atlasdb by palantir.

the class SweepCommand method execute.

@Override
public int execute(final AtlasDbServices services) {
    SweepTaskRunner sweepRunner = services.getSweepTaskRunner();
    if (!((namespace != null) ^ (table != null) ^ sweepAllTables)) {
        printer.error("Specify one of --namespace, --table, or --all options.");
        return 1;
    }
    if ((namespace != null) && (row != null)) {
        printer.error("Cannot specify a start row (" + row + ") when sweeping multiple tables (in namespace " + namespace + ")");
        return 1;
    }
    Map<TableReference, byte[]> tableToStartRow = Maps.newHashMap();
    if (table != null) {
        TableReference tableToSweep = TableReference.createUnsafe(table);
        if (!services.getKeyValueService().getAllTableNames().contains(tableToSweep)) {
            printer.info("The table {} passed in to sweep does not exist", LoggingArgs.tableRef(tableToSweep));
            return 1;
        }
        byte[] startRow = PtBytes.EMPTY_BYTE_ARRAY;
        if (row != null) {
            startRow = decodeStartRow(row);
        }
        tableToStartRow.put(tableToSweep, startRow);
    } else if (namespace != null) {
        Set<TableReference> tablesInNamespace = services.getKeyValueService().getAllTableNames().stream().filter(tableRef -> tableRef.getNamespace().getName().equals(namespace)).collect(Collectors.toSet());
        for (TableReference tableInNamespace : tablesInNamespace) {
            tableToStartRow.put(tableInNamespace, new byte[0]);
        }
    } else if (sweepAllTables) {
        tableToStartRow.putAll(Maps.asMap(Sets.difference(services.getKeyValueService().getAllTableNames(), AtlasDbConstants.hiddenTables), Functions.constant(new byte[0])));
    }
    SweepBatchConfig batchConfig = getSweepBatchConfig();
    for (Map.Entry<TableReference, byte[]> entry : tableToStartRow.entrySet()) {
        final TableReference tableToSweep = entry.getKey();
        SweepResults accumulatedResults = SweepResults.createEmptySweepResult(Optional.of(entry.getValue()));
        while (accumulatedResults.getNextStartRow().isPresent()) {
            SweepResults newResults = dryRun ? sweepRunner.dryRun(tableToSweep, batchConfig, accumulatedResults.getNextStartRow().get()) : sweepRunner.run(tableToSweep, batchConfig, accumulatedResults.getNextStartRow().get());
            accumulatedResults = accumulatedResults.accumulateWith(newResults);
            printer.info("{} Swept from {} to {} in table {} in {} ms, examined {} cell values," + " deleted {} stale versions of those cells. Time elapsed since started sweeping:" + " {} ms. Total time sweeping this table: {} ms.", SafeArg.of("isDryRun", dryRun ? "[DRY RUN]" : ""), UnsafeArg.of("startRow", encodeStartRow(accumulatedResults.getNextStartRow())), UnsafeArg.of("exclusiveEndRow", encodeEndRow(newResults.getNextStartRow())), LoggingArgs.tableRef(tableToSweep), SafeArg.of("time taken millis", newResults.getTimeInMillis()), SafeArg.of("cellTs pairs examined", newResults.getCellTsPairsExamined()), SafeArg.of("cellTs pairs deleted", newResults.getStaleValuesDeleted()), SafeArg.of("time elapsed", accumulatedResults.getTimeElapsedSinceStartedSweeping()), SafeArg.of("time sweeping", accumulatedResults.getTimeInMillis()));
            maybeSleep();
        }
        SweepResults finalAccumulatedResults = accumulatedResults;
        if (!dryRun) {
            services.getTransactionManager().runTaskWithRetry((TxTask) t -> {
                SweepPriorityTable priorityTable = SweepTableFactory.of().getSweepPriorityTable(t);
                SweepPriorityTable.SweepPriorityRow row1 = SweepPriorityTable.SweepPriorityRow.of(tableToSweep.getQualifiedName());
                priorityTable.putWriteCount(row1, 0L);
                priorityTable.putCellsExamined(row1, finalAccumulatedResults.getCellTsPairsExamined());
                priorityTable.putCellsDeleted(row1, finalAccumulatedResults.getStaleValuesDeleted());
                priorityTable.putLastSweepTime(row1, System.currentTimeMillis());
                return null;
            });
        }
        printer.info("{} Finished sweeping {}, examined {} cell values, deleted {} stale versions of those cells.", SafeArg.of("isDryRun", dryRun ? "[DRY RUN]" : ""), LoggingArgs.tableRef(tableToSweep), SafeArg.of("cellTs pairs examined", finalAccumulatedResults.getCellTsPairsExamined()), SafeArg.of("cellTs pairs deleted", finalAccumulatedResults.getStaleValuesDeleted()));
        if (!dryRun && finalAccumulatedResults.getStaleValuesDeleted() > 0) {
            Stopwatch watch = Stopwatch.createStarted();
            services.getKeyValueService().compactInternally(tableToSweep);
            printer.info("Finished performing compactInternally on {} in {} ms.", LoggingArgs.tableRef(tableToSweep), SafeArg.of("time taken", watch.elapsed(TimeUnit.MILLISECONDS)));
        }
    }
    return 0;
}