use of com.palantir.atlasdb.transaction.impl.SweepStrategyManager in project atlasdb by palantir.
the class TableTasksTest method setup.
@Before
public void setup() {
kvs = new InMemoryKeyValueService(true);
TimestampService tsService = new InMemoryTimestampService();
LockClient lockClient = LockClient.of("sweep client");
lockService = LockServiceImpl.create(LockServerOptions.builder().isStandaloneServer(false).build());
TransactionService txService = TransactionServices.createTransactionService(kvs);
Supplier<AtlasDbConstraintCheckingMode> constraints = Suppliers.ofInstance(AtlasDbConstraintCheckingMode.NO_CONSTRAINT_CHECKING);
ConflictDetectionManager cdm = ConflictDetectionManagers.createWithoutWarmingCache(kvs);
SweepStrategyManager ssm = SweepStrategyManagers.createDefault(kvs);
Cleaner cleaner = new NoOpCleaner();
SerializableTransactionManager transactionManager = SerializableTransactionManager.createForTest(kvs, tsService, lockClient, lockService, txService, constraints, cdm, ssm, cleaner, AbstractTransactionTest.GET_RANGES_THREAD_POOL_SIZE, AbstractTransactionTest.DEFAULT_GET_RANGES_CONCURRENCY, () -> AtlasDbConstants.DEFAULT_TIMESTAMP_CACHE_SIZE, MultiTableSweepQueueWriter.NO_OP);
txManager = transactionManager;
}
use of com.palantir.atlasdb.transaction.impl.SweepStrategyManager in project atlasdb by palantir.
the class TransactionManagers method serializableInternal.
private SerializableTransactionManager serializableInternal(@Output List<AutoCloseable> closeables) {
AtlasDbMetrics.setMetricRegistries(globalMetricsRegistry(), globalTaggedMetricRegistry());
final AtlasDbConfig config = config();
checkInstallConfig(config);
AtlasDbRuntimeConfig defaultRuntime = AtlasDbRuntimeConfig.defaultRuntimeConfig();
Supplier<AtlasDbRuntimeConfig> runtimeConfigSupplier = () -> runtimeConfigSupplier().get().orElse(defaultRuntime);
QosClient qosClient = initializeCloseable(() -> getQosClient(JavaSuppliers.compose(AtlasDbRuntimeConfig::qos, runtimeConfigSupplier)), closeables);
ServiceDiscoveringAtlasSupplier atlasFactory = new ServiceDiscoveringAtlasSupplier(config.keyValueService(), JavaSuppliers.compose(AtlasDbRuntimeConfig::keyValueService, runtimeConfigSupplier), config.leader(), config.namespace(), config.initializeAsync(), qosClient);
LockRequest.setDefaultLockTimeout(SimpleTimeDuration.of(config.getDefaultLockTimeoutSeconds(), TimeUnit.SECONDS));
LockAndTimestampServices lockAndTimestampServices = createLockAndTimestampServices(config, runtimeConfigSupplier, registrar(), () -> LockServiceImpl.create(lockServerOptions()), atlasFactory::getTimestampService, atlasFactory.getTimestampStoreInvalidator(), userAgent());
KvsProfilingLogger.setSlowLogThresholdMillis(config.getKvsSlowLogThresholdMillis());
Supplier<SweepConfig> sweepConfig = JavaSuppliers.compose(AtlasDbRuntimeConfig::sweep, runtimeConfigSupplier);
KeyValueService keyValueService = initializeCloseable(() -> {
KeyValueService kvs = atlasFactory.getKeyValueService();
kvs = ProfilingKeyValueService.create(kvs);
kvs = SweepStatsKeyValueService.create(kvs, new TimelockTimestampServiceAdapter(lockAndTimestampServices.timelock()), JavaSuppliers.compose(SweepConfig::writeThreshold, sweepConfig), JavaSuppliers.compose(SweepConfig::writeSizeThreshold, sweepConfig));
kvs = TracingKeyValueService.create(kvs);
kvs = AtlasDbMetrics.instrument(KeyValueService.class, kvs, MetricRegistry.name(KeyValueService.class));
return ValidatingQueryRewritingKeyValueService.create(kvs);
}, closeables);
SchemaMetadataService schemaMetadataService = SchemaMetadataServiceImpl.create(keyValueService, config.initializeAsync());
TransactionManagersInitializer initializer = TransactionManagersInitializer.createInitialTables(keyValueService, schemas(), schemaMetadataService, config.initializeAsync());
PersistentLockService persistentLockService = createAndRegisterPersistentLockService(keyValueService, registrar(), config.initializeAsync());
TransactionService transactionService = AtlasDbMetrics.instrument(TransactionService.class, TransactionServices.createTransactionService(keyValueService));
ConflictDetectionManager conflictManager = ConflictDetectionManagers.create(keyValueService);
SweepStrategyManager sweepStrategyManager = SweepStrategyManagers.createDefault(keyValueService);
CleanupFollower follower = CleanupFollower.create(schemas());
Cleaner cleaner = initializeCloseable(() -> new DefaultCleanerBuilder(keyValueService, lockAndTimestampServices.timelock(), ImmutableList.of(follower), transactionService).setBackgroundScrubAggressively(config.backgroundScrubAggressively()).setBackgroundScrubBatchSize(config.getBackgroundScrubBatchSize()).setBackgroundScrubFrequencyMillis(config.getBackgroundScrubFrequencyMillis()).setBackgroundScrubThreads(config.getBackgroundScrubThreads()).setPunchIntervalMillis(config.getPunchIntervalMillis()).setTransactionReadTimeout(config.getTransactionReadTimeoutMillis()).setInitializeAsync(config.initializeAsync()).buildCleaner(), closeables);
SerializableTransactionManager transactionManager = initializeCloseable(() -> SerializableTransactionManager.create(keyValueService, lockAndTimestampServices.timelock(), lockAndTimestampServices.lock(), transactionService, Suppliers.ofInstance(AtlasDbConstraintCheckingMode.FULL_CONSTRAINT_CHECKING_THROWS_EXCEPTIONS), conflictManager, sweepStrategyManager, cleaner, () -> areTransactionManagerInitializationPrerequisitesSatisfied(initializer, lockAndTimestampServices), allowHiddenTableAccess(), () -> runtimeConfigSupplier.get().transaction().getLockAcquireTimeoutMillis(), config.keyValueService().concurrentGetRangesThreadPoolSize(), config.keyValueService().defaultGetRangesConcurrency(), config.initializeAsync(), () -> runtimeConfigSupplier.get().getTimestampCacheSize(), MultiTableSweepQueueWriter.NO_OP, wrapInitializationCallbackAndAddConsistencyChecks(config, runtimeConfigSupplier.get(), lockAndTimestampServices, asyncInitializationCallback())), closeables);
PersistentLockManager persistentLockManager = initializeCloseable(() -> new PersistentLockManager(persistentLockService, config.getSweepPersistentLockWaitMillis()), closeables);
initializeCloseable(() -> initializeSweepEndpointAndBackgroundProcess(config, runtimeConfigSupplier, registrar(), keyValueService, transactionService, sweepStrategyManager, follower, transactionManager, persistentLockManager), closeables);
initializeCloseable(initializeCompactBackgroundProcess(lockAndTimestampServices, keyValueService, transactionManager, JavaSuppliers.compose(AtlasDbRuntimeConfig::compact, runtimeConfigSupplier)), closeables);
return transactionManager;
}
use of com.palantir.atlasdb.transaction.impl.SweepStrategyManager in project atlasdb by palantir.
the class InMemoryAtlasDbFactory method createInMemoryTransactionManagerInternal.
private static SerializableTransactionManager createInMemoryTransactionManagerInternal(Set<Schema> schemas) {
TimestampService ts = new InMemoryTimestampService();
KeyValueService keyValueService = new InMemoryKeyValueService(false);
schemas.forEach(s -> Schemas.createTablesAndIndexes(s, keyValueService));
TransactionTables.createTables(keyValueService);
TransactionService transactionService = TransactionServices.createTransactionService(keyValueService);
LockService lock = LockRefreshingLockService.create(LockServiceImpl.create(LockServerOptions.builder().isStandaloneServer(false).build()));
LockClient client = LockClient.of("in memory atlasdb instance");
ConflictDetectionManager conflictManager = ConflictDetectionManagers.createWithoutWarmingCache(keyValueService);
SweepStrategyManager sweepStrategyManager = SweepStrategyManagers.createDefault(keyValueService);
CleanupFollower follower = CleanupFollower.create(schemas);
Cleaner cleaner = new DefaultCleanerBuilder(keyValueService, lock, ts, client, ImmutableList.of(follower), transactionService).buildCleaner();
SerializableTransactionManager ret = SerializableTransactionManager.createForTest(keyValueService, ts, client, lock, transactionService, Suppliers.ofInstance(AtlasDbConstraintCheckingMode.FULL_CONSTRAINT_CHECKING_THROWS_EXCEPTIONS), conflictManager, sweepStrategyManager, cleaner, DEFAULT_MAX_CONCURRENT_RANGES, DEFAULT_GET_RANGES_CONCURRENCY, () -> DEFAULT_TIMESTAMP_CACHE_SIZE, MultiTableSweepQueueWriter.NO_OP);
cleaner.start(ret);
return ret;
}
use of com.palantir.atlasdb.transaction.impl.SweepStrategyManager in project atlasdb by palantir.
the class AbstractBackgroundSweeperIntegrationTest method setup.
@Before
public void setup() {
TimestampService tsService = new InMemoryTimestampService();
kvs = SweepStatsKeyValueService.create(getKeyValueService(), tsService, () -> AtlasDbConstants.DEFAULT_SWEEP_WRITE_THRESHOLD, () -> AtlasDbConstants.DEFAULT_SWEEP_WRITE_SIZE_THRESHOLD);
SweepStrategyManager ssm = SweepStrategyManagers.createDefault(kvs);
txService = TransactionServices.createTransactionService(kvs);
txManager = SweepTestUtils.setupTxManager(kvs, tsService, ssm, txService);
LongSupplier tsSupplier = sweepTimestamp::get;
PersistentLockManager persistentLockManager = new PersistentLockManager(SweepTestUtils.getPersistentLockService(kvs), AtlasDbConstants.DEFAULT_SWEEP_PERSISTENT_LOCK_WAIT_MILLIS);
CellsSweeper cellsSweeper = new CellsSweeper(txManager, kvs, persistentLockManager, ImmutableList.of());
SweepTaskRunner sweepRunner = new SweepTaskRunner(kvs, tsSupplier, tsSupplier, txService, ssm, cellsSweeper);
SweepMetricsManager sweepMetricsManager = new SweepMetricsManager();
specificTableSweeper = SpecificTableSweeper.create(txManager, kvs, sweepRunner, SweepTableFactory.of(), new NoOpBackgroundSweeperPerformanceLogger(), sweepMetricsManager, false);
sweepBatchConfigSource = AdjustableSweepBatchConfigSource.create(() -> sweepBatchConfig);
backgroundSweeper = BackgroundSweeperImpl.create(sweepBatchConfigSource, // sweepEnabled
() -> true, // sweepPauseMillis
() -> 10L, persistentLockManager, specificTableSweeper);
}
use of com.palantir.atlasdb.transaction.impl.SweepStrategyManager in project atlasdb by palantir.
the class TableMigratorTest method testMigrationToDifferentKvs.
// Table/IndexDefinition syntax
@SuppressWarnings({ "checkstyle:Indentation", "checkstyle:RightCurly" })
@Test
public void testMigrationToDifferentKvs() throws TableMappingNotFoundException {
final TableReference tableRef = TableReference.create(Namespace.DEFAULT_NAMESPACE, "table");
final TableReference namespacedTableRef = TableReference.createFromFullyQualifiedName("namespace." + tableRef.getTablename());
TableDefinition definition = new TableDefinition() {
{
rowName();
rowComponent("r", ValueType.BLOB);
columns();
column("c", "c", ValueType.BLOB);
}
};
keyValueService.createTable(tableRef, definition.toTableMetadata().persistToBytes());
keyValueService.createTable(namespacedTableRef, definition.toTableMetadata().persistToBytes());
keyValueService.putMetadataForTable(namespacedTableRef, definition.toTableMetadata().persistToBytes());
final Cell theCell = Cell.create(PtBytes.toBytes("r1"), PtBytes.toBytes("c"));
final byte[] theValue = PtBytes.toBytes("v1");
txManager.runTaskWithRetry((TransactionTask<Void, RuntimeException>) txn -> {
Map<Cell, byte[]> values = ImmutableMap.of(theCell, theValue);
txn.put(tableRef, values);
txn.put(namespacedTableRef, values);
return null;
});
final InMemoryKeyValueService kvs2 = new InMemoryKeyValueService(false);
final ConflictDetectionManager cdm2 = ConflictDetectionManagers.createWithNoConflictDetection();
final SweepStrategyManager ssm2 = SweepStrategyManagers.completelyConservative(kvs2);
final TestTransactionManagerImpl txManager2 = new TestTransactionManagerImpl(kvs2, timestampService, lockClient, lockService, transactionService, cdm2, ssm2, MultiTableSweepQueueWriter.NO_OP);
kvs2.createTable(tableRef, definition.toTableMetadata().persistToBytes());
kvs2.createTable(namespacedTableRef, definition.toTableMetadata().persistToBytes());
TableReference checkpointTable = TableReference.create(Namespace.DEFAULT_NAMESPACE, "checkpoint");
GeneralTaskCheckpointer checkpointer = new GeneralTaskCheckpointer(checkpointTable, kvs2, txManager2);
for (final TableReference name : Lists.newArrayList(tableRef, namespacedTableRef)) {
TransactionRangeMigrator rangeMigrator = new TransactionRangeMigratorBuilder().srcTable(name).readTxManager(txManager).txManager(txManager2).checkpointer(checkpointer).build();
TableMigratorBuilder builder = new TableMigratorBuilder().srcTable(name).partitions(1).executor(PTExecutors.newSingleThreadExecutor()).checkpointer(checkpointer).rangeMigrator(rangeMigrator);
TableMigrator migrator = builder.build();
migrator.migrate();
}
checkpointer.deleteCheckpoints();
final ConflictDetectionManager verifyCdm = ConflictDetectionManagers.createWithNoConflictDetection();
final SweepStrategyManager verifySsm = SweepStrategyManagers.completelyConservative(kvs2);
final TestTransactionManagerImpl verifyTxManager = new TestTransactionManagerImpl(kvs2, timestampService, lockClient, lockService, transactionService, verifyCdm, verifySsm, MultiTableSweepQueueWriter.NO_OP);
final MutableLong count = new MutableLong();
for (final TableReference name : Lists.newArrayList(tableRef, namespacedTableRef)) {
verifyTxManager.runTaskReadOnly((TransactionTask<Void, RuntimeException>) txn -> {
BatchingVisitable<RowResult<byte[]>> bv = txn.getRange(name, RangeRequest.all());
bv.batchAccept(1000, AbortingVisitors.batching(new AbortingVisitor<RowResult<byte[]>, RuntimeException>() {
@Override
public boolean visit(RowResult<byte[]> item) throws RuntimeException {
Iterable<Entry<Cell, byte[]>> cells = item.getCells();
Entry<Cell, byte[]> entry = Iterables.getOnlyElement(cells);
Assert.assertEquals(theCell, entry.getKey());
Assert.assertArrayEquals(theValue, entry.getValue());
count.increment();
return true;
}
}));
return null;
});
}
Assert.assertEquals(2L, count.longValue());
}
Aggregations