Search in sources :

Example 1 with AtlasDbServices

use of com.palantir.atlasdb.services.AtlasDbServices in project atlasdb by palantir.

the class KvsMigrationCommand method call.

@Override
public Integer call() throws Exception {
    if (inlineConfig == null && toConfigFile == null) {
        printer.error("Argument -mc/--migrateConfig is required when not running as a dropwizard CLI");
        return 1;
    }
    AtlasDbServices fromServices = connectFromServices();
    AtlasDbServices toServices = connectToServices();
    return execute(fromServices, toServices);
}
Also used : DaggerAtlasDbServices(com.palantir.atlasdb.services.DaggerAtlasDbServices) AtlasDbServices(com.palantir.atlasdb.services.AtlasDbServices)

Example 2 with AtlasDbServices

use of com.palantir.atlasdb.services.AtlasDbServices in project atlasdb by palantir.

the class TestKvsMigrationCommand method doesNotSweepDuringMigration.

@Test
public void doesNotSweepDuringMigration() throws Exception {
    KvsMigrationCommand cmd = getCommand(new String[] { "-smv" });
    AtlasDbServices fromServices = cmd.connectFromServices();
    assertFalse(fromServices.getAtlasDbRuntimeConfig().sweep().enabled());
    AtlasDbServices toServices = cmd.connectToServices();
    assertFalse(toServices.getAtlasDbRuntimeConfig().sweep().enabled());
}
Also used : AtlasDbServices(com.palantir.atlasdb.services.AtlasDbServices) Test(org.junit.Test)

Example 3 with AtlasDbServices

use of com.palantir.atlasdb.services.AtlasDbServices in project atlasdb by palantir.

the class KeyValueServiceMigrators method setupMigrator.

public static KeyValueServiceMigrator setupMigrator(MigratorSpec migratorSpec) {
    AtlasDbServices fromServices = migratorSpec.fromServices();
    long migrationStartTimestamp = fromServices.getTimestampService().getFreshTimestamp();
    long migrationCommitTimestamp = fromServices.getTimestampService().getFreshTimestamp();
    AtlasDbServices toServices = migratorSpec.toServices();
    TimestampManagementService toTimestampManagementService = getTimestampManagementService(toServices);
    toServices.getTransactionService().putUnlessExists(migrationStartTimestamp, migrationCommitTimestamp);
    toTimestampManagementService.fastForwardTimestamp(migrationCommitTimestamp + 1);
    return new KeyValueServiceMigrator(CHECKPOINT_NAMESPACE, fromServices.getTransactionManager(), toServices.getTransactionManager(), fromServices.getKeyValueService(), toServices.getKeyValueService(), Suppliers.ofInstance(migrationStartTimestamp), migratorSpec.threads(), migratorSpec.batchSize(), ImmutableMap.of(), (String message, KeyValueServiceMigrator.KvsMigrationMessageLevel level) -> printer.info(level.toString() + ": " + message), new TaskProgress() {

        @Override
        public void beginTask(String message, int tasks) {
            printer.info(message);
        }

        @Override
        public void subTaskComplete() {
        // 
        }

        @Override
        public void taskComplete() {
        // 
        }
    }, ImmutableSet.of());
}
Also used : KeyValueServiceMigrator(com.palantir.atlasdb.schema.KeyValueServiceMigrator) TimestampManagementService(com.palantir.timestamp.TimestampManagementService) AtlasDbServices(com.palantir.atlasdb.services.AtlasDbServices) TaskProgress(com.palantir.atlasdb.schema.TaskProgress)

Example 4 with AtlasDbServices

use of com.palantir.atlasdb.services.AtlasDbServices in project atlasdb by palantir.

the class SweepCommand method execute.

@Override
public int execute(final AtlasDbServices services) {
    SweepTaskRunner sweepRunner = services.getSweepTaskRunner();
    if (!((namespace != null) ^ (table != null) ^ sweepAllTables)) {
        printer.error("Specify one of --namespace, --table, or --all options.");
        return 1;
    }
    if ((namespace != null) && (row != null)) {
        printer.error("Cannot specify a start row (" + row + ") when sweeping multiple tables (in namespace " + namespace + ")");
        return 1;
    }
    Map<TableReference, byte[]> tableToStartRow = Maps.newHashMap();
    if (table != null) {
        TableReference tableToSweep = TableReference.createUnsafe(table);
        if (!services.getKeyValueService().getAllTableNames().contains(tableToSweep)) {
            printer.info("The table {} passed in to sweep does not exist", LoggingArgs.tableRef(tableToSweep));
            return 1;
        }
        byte[] startRow = PtBytes.EMPTY_BYTE_ARRAY;
        if (row != null) {
            startRow = decodeStartRow(row);
        }
        tableToStartRow.put(tableToSweep, startRow);
    } else if (namespace != null) {
        Set<TableReference> tablesInNamespace = services.getKeyValueService().getAllTableNames().stream().filter(tableRef -> tableRef.getNamespace().getName().equals(namespace)).collect(Collectors.toSet());
        for (TableReference tableInNamespace : tablesInNamespace) {
            tableToStartRow.put(tableInNamespace, new byte[0]);
        }
    } else if (sweepAllTables) {
        tableToStartRow.putAll(Maps.asMap(Sets.difference(services.getKeyValueService().getAllTableNames(), AtlasDbConstants.hiddenTables), Functions.constant(new byte[0])));
    }
    SweepBatchConfig batchConfig = getSweepBatchConfig();
    for (Map.Entry<TableReference, byte[]> entry : tableToStartRow.entrySet()) {
        final TableReference tableToSweep = entry.getKey();
        SweepResults accumulatedResults = SweepResults.createEmptySweepResult(Optional.of(entry.getValue()));
        while (accumulatedResults.getNextStartRow().isPresent()) {
            SweepResults newResults = dryRun ? sweepRunner.dryRun(tableToSweep, batchConfig, accumulatedResults.getNextStartRow().get()) : sweepRunner.run(tableToSweep, batchConfig, accumulatedResults.getNextStartRow().get());
            accumulatedResults = accumulatedResults.accumulateWith(newResults);
            printer.info("{} Swept from {} to {} in table {} in {} ms, examined {} cell values," + " deleted {} stale versions of those cells. Time elapsed since started sweeping:" + " {} ms. Total time sweeping this table: {} ms.", SafeArg.of("isDryRun", dryRun ? "[DRY RUN]" : ""), UnsafeArg.of("startRow", encodeStartRow(accumulatedResults.getNextStartRow())), UnsafeArg.of("exclusiveEndRow", encodeEndRow(newResults.getNextStartRow())), LoggingArgs.tableRef(tableToSweep), SafeArg.of("time taken millis", newResults.getTimeInMillis()), SafeArg.of("cellTs pairs examined", newResults.getCellTsPairsExamined()), SafeArg.of("cellTs pairs deleted", newResults.getStaleValuesDeleted()), SafeArg.of("time elapsed", accumulatedResults.getTimeElapsedSinceStartedSweeping()), SafeArg.of("time sweeping", accumulatedResults.getTimeInMillis()));
            maybeSleep();
        }
        SweepResults finalAccumulatedResults = accumulatedResults;
        if (!dryRun) {
            services.getTransactionManager().runTaskWithRetry((TxTask) t -> {
                SweepPriorityTable priorityTable = SweepTableFactory.of().getSweepPriorityTable(t);
                SweepPriorityTable.SweepPriorityRow row1 = SweepPriorityTable.SweepPriorityRow.of(tableToSweep.getQualifiedName());
                priorityTable.putWriteCount(row1, 0L);
                priorityTable.putCellsExamined(row1, finalAccumulatedResults.getCellTsPairsExamined());
                priorityTable.putCellsDeleted(row1, finalAccumulatedResults.getStaleValuesDeleted());
                priorityTable.putLastSweepTime(row1, System.currentTimeMillis());
                return null;
            });
        }
        printer.info("{} Finished sweeping {}, examined {} cell values, deleted {} stale versions of those cells.", SafeArg.of("isDryRun", dryRun ? "[DRY RUN]" : ""), LoggingArgs.tableRef(tableToSweep), SafeArg.of("cellTs pairs examined", finalAccumulatedResults.getCellTsPairsExamined()), SafeArg.of("cellTs pairs deleted", finalAccumulatedResults.getStaleValuesDeleted()));
        if (!dryRun && finalAccumulatedResults.getStaleValuesDeleted() > 0) {
            Stopwatch watch = Stopwatch.createStarted();
            services.getKeyValueService().compactInternally(tableToSweep);
            printer.info("Finished performing compactInternally on {} in {} ms.", LoggingArgs.tableRef(tableToSweep), SafeArg.of("time taken", watch.elapsed(TimeUnit.MILLISECONDS)));
        }
    }
    return 0;
}
Also used : Arrays(java.util.Arrays) OutputPrinter(com.palantir.atlasdb.cli.output.OutputPrinter) Stopwatch(com.google.common.base.Stopwatch) Throwables(com.palantir.common.base.Throwables) LoggerFactory(org.slf4j.LoggerFactory) PtBytes(com.palantir.atlasdb.encoding.PtBytes) SafeArg(com.palantir.logsafe.SafeArg) ImmutableSweepBatchConfig(com.palantir.atlasdb.sweep.ImmutableSweepBatchConfig) Option(io.airlift.airline.Option) Map(java.util.Map) LoggingArgs(com.palantir.atlasdb.logging.LoggingArgs) TableReference(com.palantir.atlasdb.keyvalue.api.TableReference) SweepTaskRunner(com.palantir.atlasdb.sweep.SweepTaskRunner) Nullable(javax.annotation.Nullable) AtlasDbConstants(com.palantir.atlasdb.AtlasDbConstants) Functions(com.google.common.base.Functions) BaseEncoding(com.google.common.io.BaseEncoding) Set(java.util.Set) SweepResults(com.palantir.atlasdb.keyvalue.api.SweepResults) Collectors(java.util.stream.Collectors) Maps(com.google.common.collect.Maps) Sets(com.google.common.collect.Sets) SweepPriorityTable(com.palantir.atlasdb.schema.generated.SweepPriorityTable) TimeUnit(java.util.concurrent.TimeUnit) SweepBatchConfig(com.palantir.atlasdb.sweep.SweepBatchConfig) Command(io.airlift.airline.Command) SweepTableFactory(com.palantir.atlasdb.schema.generated.SweepTableFactory) TxTask(com.palantir.atlasdb.transaction.impl.TxTask) UnsafeArg(com.palantir.logsafe.UnsafeArg) Optional(java.util.Optional) AtlasDbServices(com.palantir.atlasdb.services.AtlasDbServices) SweepResults(com.palantir.atlasdb.keyvalue.api.SweepResults) Set(java.util.Set) Stopwatch(com.google.common.base.Stopwatch) SweepTaskRunner(com.palantir.atlasdb.sweep.SweepTaskRunner) ImmutableSweepBatchConfig(com.palantir.atlasdb.sweep.ImmutableSweepBatchConfig) SweepBatchConfig(com.palantir.atlasdb.sweep.SweepBatchConfig) TableReference(com.palantir.atlasdb.keyvalue.api.TableReference) SweepPriorityTable(com.palantir.atlasdb.schema.generated.SweepPriorityTable) Map(java.util.Map)

Example 5 with AtlasDbServices

use of com.palantir.atlasdb.services.AtlasDbServices in project atlasdb by palantir.

the class KeyValueServiceMigratorsTest method createMockAtlasDbServices.

private static AtlasDbServices createMockAtlasDbServices() {
    TimestampService timestampService = new InMemoryTimestampService();
    AtlasDbServices mockServices = mock(AtlasDbServices.class);
    when(mockServices.getTimestampService()).thenReturn(timestampService);
    when(mockServices.getTransactionService()).thenReturn(mock(TransactionService.class));
    return mockServices;
}
Also used : TransactionService(com.palantir.atlasdb.transaction.service.TransactionService) InMemoryTimestampService(com.palantir.timestamp.InMemoryTimestampService) AtlasDbServices(com.palantir.atlasdb.services.AtlasDbServices) InMemoryTimestampService(com.palantir.timestamp.InMemoryTimestampService) TimestampService(com.palantir.timestamp.TimestampService)

Aggregations

AtlasDbServices (com.palantir.atlasdb.services.AtlasDbServices)8 InMemoryTestRunner (com.palantir.atlasdb.cli.runner.InMemoryTestRunner)2 DaggerTestAtlasDbServices (com.palantir.atlasdb.services.test.DaggerTestAtlasDbServices)2 TestAtlasDbServices (com.palantir.atlasdb.services.test.TestAtlasDbServices)2 Functions (com.google.common.base.Functions)1 Stopwatch (com.google.common.base.Stopwatch)1 Maps (com.google.common.collect.Maps)1 Sets (com.google.common.collect.Sets)1 BaseEncoding (com.google.common.io.BaseEncoding)1 AtlasDbConstants (com.palantir.atlasdb.AtlasDbConstants)1 OutputPrinter (com.palantir.atlasdb.cli.output.OutputPrinter)1 PtBytes (com.palantir.atlasdb.encoding.PtBytes)1 SweepResults (com.palantir.atlasdb.keyvalue.api.SweepResults)1 TableReference (com.palantir.atlasdb.keyvalue.api.TableReference)1 LoggingArgs (com.palantir.atlasdb.logging.LoggingArgs)1 KeyValueServiceMigrator (com.palantir.atlasdb.schema.KeyValueServiceMigrator)1 TaskProgress (com.palantir.atlasdb.schema.TaskProgress)1 SweepPriorityTable (com.palantir.atlasdb.schema.generated.SweepPriorityTable)1 SweepTableFactory (com.palantir.atlasdb.schema.generated.SweepTableFactory)1 DaggerAtlasDbServices (com.palantir.atlasdb.services.DaggerAtlasDbServices)1