Examples with MetricsManager com.palantir.atlasdb.util.MetricsManager used on opensource projects

Example 16 with MetricsManager

use of com.palantir.atlasdb.util.MetricsManager in project atlasdb by palantir.

the class TransactionManagers method serializableInternal.

@SuppressWarnings("MethodLength")
private TransactionManager serializableInternal(@Output List<AutoCloseable> closeables) {
    MetricsManager metricsManager = setUpMetricsAndGetMetricsManager();
    AtlasDbRuntimeConfigRefreshable runtimeConfigRefreshable = initializeCloseable(() -> AtlasDbRuntimeConfigRefreshable.create(this), closeables);
    Refreshable<AtlasDbRuntimeConfig> runtime = runtimeConfigRefreshable.config();
    Optional<TimeLockFeedbackBackgroundTask> timeLockFeedbackBackgroundTask = getTimeLockFeedbackBackgroundTask(metricsManager, closeables, config(), runtime);
    FreshTimestampSupplierAdapter adapter = new FreshTimestampSupplierAdapter();
    KeyValueServiceConfig installConfig = config().keyValueService();
    ServiceDiscoveringAtlasSupplier atlasFactory = new ServiceDiscoveringAtlasSupplier(metricsManager, installConfig, runtime.map(AtlasDbRuntimeConfig::keyValueService), config().leader(), config().namespace(), Optional.empty(), config().initializeAsync(), adapter);
    DerivedSnapshotConfig derivedSnapshotConfig = atlasFactory.getDerivedSnapshotConfig();
    LockRequest.setDefaultLockTimeout(SimpleTimeDuration.of(config().getDefaultLockTimeoutSeconds(), TimeUnit.SECONDS));
    LockAndTimestampServiceFactory factory = lockAndTimestampServiceFactory().orElseGet(() -> new DefaultLockAndTimestampServiceFactory(metricsManager, config(), runtime, registrar(), () -> LockServiceImpl.create(lockServerOptions()), atlasFactory::getManagedTimestampService, atlasFactory.getTimestampStoreInvalidator(), userAgent(), lockDiagnosticComponents(), reloadingFactory(), timeLockFeedbackBackgroundTask, timelockRequestBatcherProviders(), schemas()));
    LockAndTimestampServices lockAndTimestampServices = factory.createLockAndTimestampServices();
    adapter.setTimestampService(lockAndTimestampServices.managedTimestampService());
    KvsProfilingLogger.setSlowLogThresholdMillis(config().getKvsSlowLogThresholdMillis());
    Refreshable<SweepConfig> sweepConfig = runtime.map(AtlasDbRuntimeConfig::sweep);
    KeyValueService keyValueService = initializeCloseable(() -> {
        KeyValueService kvs = atlasFactory.getKeyValueService();
        kvs = ProfilingKeyValueService.create(kvs);
        kvs = new SafeTableClearerKeyValueService(lockAndTimestampServices.timelock()::getImmutableTimestamp, kvs);
        // table.
        if (!targetedSweepIsFullyEnabled(config(), runtime)) {
            kvs = SweepStatsKeyValueService.create(kvs, new TimelockTimestampServiceAdapter(lockAndTimestampServices.timelock()), sweepConfig.map(SweepConfig::writeThreshold), sweepConfig.map(SweepConfig::writeSizeThreshold), () -> true);
        }
        kvs = TracingKeyValueService.create(kvs);
        kvs = AtlasDbMetrics.instrumentTimed(metricsManager.getRegistry(), KeyValueService.class, kvs, MetricRegistry.name(KeyValueService.class));
        return ValidatingQueryRewritingKeyValueService.create(kvs);
    }, closeables);
    if (config().targetedSweep().enableSweepQueueWrites()) {
        initializeCloseable(() -> OldestTargetedSweepTrackedTimestamp.createStarted(keyValueService, lockAndTimestampServices.timestamp(), PTExecutors.newSingleThreadScheduledExecutor(new NamedThreadFactory("OldestTargetedSweepTrackedTimestamp", true))), closeables);
    }
    TransactionManagersInitializer initializer = TransactionManagersInitializer.createInitialTables(keyValueService, schemas(), config().initializeAsync(), allSafeForLogging());
    TransactionComponents components = createTransactionComponents(closeables, metricsManager, lockAndTimestampServices, keyValueService, runtime);
    TransactionService transactionService = components.transactionService();
    ConflictDetectionManager conflictManager = ConflictDetectionManagers.create(keyValueService);
    SweepStrategyManager sweepStrategyManager = SweepStrategyManagers.createDefault(keyValueService);
    CleanupFollower follower = CleanupFollower.create(schemas());
    Cleaner cleaner = initializeCloseable(() -> new DefaultCleanerBuilder(keyValueService, lockAndTimestampServices.timelock(), ImmutableList.of(follower), transactionService, metricsManager).setBackgroundScrubAggressively(config().backgroundScrubAggressively()).setBackgroundScrubBatchSize(config().getBackgroundScrubBatchSize()).setBackgroundScrubFrequencyMillis(config().getBackgroundScrubFrequencyMillis()).setBackgroundScrubThreads(config().getBackgroundScrubThreads()).setPunchIntervalMillis(config().getPunchIntervalMillis()).setTransactionReadTimeout(config().getTransactionReadTimeoutMillis()).setInitializeAsync(config().initializeAsync()).buildCleaner(), closeables);
    MultiTableSweepQueueWriter targetedSweep = initializeCloseable(() -> uninitializedTargetedSweeper(metricsManager, config().targetedSweep(), follower, runtime.map(AtlasDbRuntimeConfig::targetedSweep)), closeables);
    Supplier<TransactionConfig> transactionConfigSupplier = runtime.map(AtlasDbRuntimeConfig::transaction).map(this::withConsolidatedGrabImmutableTsLockFlag);
    TimestampCache timestampCache = config().timestampCache().orElseGet(() -> new DefaultTimestampCache(metricsManager.getRegistry(), () -> runtime.get().getTimestampCacheSize()));
    ConflictTracer conflictTracer = lockDiagnosticComponents().map(LockDiagnosticComponents::clientLockDiagnosticCollector).<ConflictTracer>map(Function.identity()).orElse(ConflictTracer.NO_OP);
    Callback<TransactionManager> callbacks = new Callback.CallChain<>(timelockConsistencyCheckCallback(config(), runtime.get(), lockAndTimestampServices), targetedSweep.singleAttemptCallback(), asyncInitializationCallback(), createClearsTable());
    TransactionManager transactionManager = initializeCloseable(() -> SerializableTransactionManager.createInstrumented(metricsManager, keyValueService, lockAndTimestampServices.timelock(), lockAndTimestampServices.lockWatcher(), lockAndTimestampServices.managedTimestampService(), lockAndTimestampServices.lock(), transactionService, () -> AtlasDbConstraintCheckingMode.FULL_CONSTRAINT_CHECKING_THROWS_EXCEPTIONS, conflictManager, sweepStrategyManager, cleaner, () -> areTransactionManagerInitializationPrerequisitesSatisfied(initializer, lockAndTimestampServices), allowHiddenTableAccess(), derivedSnapshotConfig.concurrentGetRangesThreadPoolSize(), derivedSnapshotConfig.defaultGetRangesConcurrency(), config().initializeAsync(), timestampCache, targetedSweep, callbacks, validateLocksOnReads(), transactionConfigSupplier, conflictTracer, metricsFilterEvaluationContext(), installConfig.sharedResourcesConfig().map(SharedResourcesConfig::sharedGetRangesPoolSize)), closeables);
    transactionManager.registerClosingCallback(runtimeConfigRefreshable::close);
    timeLockFeedbackBackgroundTask.ifPresent(task -> transactionManager.registerClosingCallback(task::close));
    lockAndTimestampServices.resources().forEach(transactionManager::registerClosingCallback);
    transactionManager.registerClosingCallback(transactionService::close);
    components.schemaInstaller().ifPresent(installer -> transactionManager.registerClosingCallback(installer::close));
    transactionManager.registerClosingCallback(targetedSweep::close);
    initializeCloseable(() -> initializeSweepEndpointAndBackgroundProcess(metricsManager, config(), runtime, registrar(), keyValueService, transactionService, follower, transactionManager, runBackgroundSweepProcess()), closeables);
    initializeCloseable(initializeCompactBackgroundProcess(metricsManager, lockAndTimestampServices, keyValueService, transactionManager, runtime.map(AtlasDbRuntimeConfig::compact)), closeables);
    log.info("Successfully created, and now returning a transaction manager: this may not be fully initialised.");
    return transactionManager;
}

Example 17 with MetricsManager

use of com.palantir.atlasdb.util.MetricsManager in project atlasdb by palantir.

the class FeedbackHandlerTest method publishesSizeMetricToMetricManager.

@Test
@SuppressWarnings("unchecked")
public void publishesSizeMetricToMetricManager() {
    MetricsManager metricsManager = MetricsManagers.createForTests();
    FeedbackHandler handler = new FeedbackHandler(metricsManager, () -> true);
    handler.handle(FEEDBACK);
    assertThat(metricsManager.getPublishableMetrics().getMetrics()).containsKey(METRIC_NAME).satisfies(map -> assertThat(((Gauge<Long>) map.get(METRIC_NAME)).getValue()).isEqualTo(1L));
}

Example 18 with MetricsManager

use of com.palantir.atlasdb.util.MetricsManager in project atlasdb by palantir.

the class InMemoryTimelockServices method before.

@Override
protected void before() throws IOException {
    PaxosInstallConfiguration paxos = PaxosInstallConfiguration.builder().dataDirectory(tryCreateSubFolder(tempFolder)).leaderMode(PaxosLeaderMode.SINGLE_LEADER).sqlitePersistence(ImmutableSqlitePaxosPersistenceConfiguration.builder().dataDirectory(tryCreateSubFolder(tempFolder, client)).build()).isNewService(false).build();
    ClusterInstallConfiguration cluster = ImmutableClusterInstallConfiguration.builder().enableNonstandardAndPossiblyDangerousTopology(true).build();
    TimeLockInstallConfiguration install = TimeLockInstallConfiguration.builder().paxos(paxos).cluster(cluster).build();
    ImmutableDefaultClusterConfiguration clusterConfig = ImmutableDefaultClusterConfiguration.builder().localServer("local").cluster(PartialServiceConfiguration.of(List.of("local"), Optional.empty())).build();
    TimeLockRuntimeConfiguration runtime = ImmutableTimeLockRuntimeConfiguration.builder().permittedBackupToken(BEARER_TOKEN).clusterSnapshot(clusterConfig).build();
    MetricsManager metricsManager = MetricsManagers.createForTests();
    timeLockAgent = TimeLockAgent.create(metricsManager, install, // This won't actually live reload.
    Refreshable.only(runtime), runtime.clusterSnapshot(), USER_AGENT, THREAD_POOL_SIZE, BLOCKING_TIMEOUT_MS, _unused -> {
    }, Optional.empty(), OrderableSlsVersion.valueOf("0.0.0"), ObjectMappers.newServerObjectMapper(), () -> System.exit(0));
    delegate = timeLockAgent.createInvalidatingTimeLockServices(client);
    createHelperServices(metricsManager);
    // Wait for leadership
    Awaitility.await().atMost(Duration.ofSeconds(30L)).pollInterval(Duration.ofMillis(50)).ignoreExceptions().until(() -> delegate.getTimestampService().getFreshTimestamp() > 0);
}

Example 19 with MetricsManager

use of com.palantir.atlasdb.util.MetricsManager in project atlasdb by palantir.

the class TableMigratorTest method testMigrationToDifferentKvs.

// Table/IndexDefinition syntax
@SuppressWarnings({ "checkstyle:Indentation", "checkstyle:RightCurly" })
@Test
public void testMigrationToDifferentKvs() throws TableMappingNotFoundException {
    final TableReference tableRef = TableReference.create(Namespace.DEFAULT_NAMESPACE, "table");
    final TableReference namespacedTableRef = TableReference.createFromFullyQualifiedName("namespace." + tableRef.getTableName());
    TableDefinition definition = new TableDefinition() {

        {
            rowName();
            rowComponent("r", ValueType.BLOB);
            columns();
            column("c", "c", ValueType.BLOB);
        }
    };
    keyValueService.createTable(tableRef, definition.toTableMetadata().persistToBytes());
    keyValueService.createTable(namespacedTableRef, definition.toTableMetadata().persistToBytes());
    keyValueService.putMetadataForTable(namespacedTableRef, definition.toTableMetadata().persistToBytes());
    final Cell theCell = Cell.create(PtBytes.toBytes("r1"), PtBytes.toBytes("c"));
    final byte[] theValue = PtBytes.toBytes("v1");
    txManager.runTaskWithRetry((TransactionTask<Void, RuntimeException>) txn -> {
        Map<Cell, byte[]> values = ImmutableMap.of(theCell, theValue);
        txn.put(tableRef, values);
        txn.put(namespacedTableRef, values);
        return null;
    });
    final InMemoryKeyValueService kvs2 = new InMemoryKeyValueService(false);
    final ConflictDetectionManager cdm2 = ConflictDetectionManagers.createWithNoConflictDetection();
    final SweepStrategyManager ssm2 = SweepStrategyManagers.completelyConservative();
    final MetricsManager metricsManager = MetricsManagers.createForTests();
    final TestTransactionManagerImpl txManager2 = new TestTransactionManagerImpl(metricsManager, kvs2, inMemoryTimeLockRule.get(), lockService, transactionService, cdm2, ssm2, DefaultTimestampCache.createForTests(), MultiTableSweepQueueWriter.NO_OP, MoreExecutors.newDirectExecutorService());
    kvs2.createTable(tableRef, definition.toTableMetadata().persistToBytes());
    kvs2.createTable(namespacedTableRef, definition.toTableMetadata().persistToBytes());
    TableReference checkpointTable = TableReference.create(Namespace.DEFAULT_NAMESPACE, "checkpoint");
    GeneralTaskCheckpointer checkpointer = new GeneralTaskCheckpointer(checkpointTable, kvs2, txManager2);
    for (final TableReference name : Lists.newArrayList(tableRef, namespacedTableRef)) {
        TransactionRangeMigrator rangeMigrator = new TransactionRangeMigratorBuilder().srcTable(name).readTxManager(txManager).txManager(txManager2).checkpointer(checkpointer).build();
        TableMigratorBuilder builder = new TableMigratorBuilder().srcTable(name).partitions(1).executor(PTExecutors.newSingleThreadExecutor()).checkpointer(checkpointer).rangeMigrator(rangeMigrator);
        TableMigrator migrator = builder.build();
        migrator.migrate();
    }
    checkpointer.deleteCheckpoints();
    final ConflictDetectionManager verifyCdm = ConflictDetectionManagers.createWithNoConflictDetection();
    final SweepStrategyManager verifySsm = SweepStrategyManagers.completelyConservative();
    final TestTransactionManagerImpl verifyTxManager = new TestTransactionManagerImpl(metricsManager, kvs2, inMemoryTimeLockRule.get(), lockService, transactionService, verifyCdm, verifySsm, DefaultTimestampCache.createForTests(), MultiTableSweepQueueWriter.NO_OP, MoreExecutors.newDirectExecutorService());
    final MutableLong count = new MutableLong();
    for (final TableReference name : Lists.newArrayList(tableRef, namespacedTableRef)) {
        verifyTxManager.runTaskReadOnly((TransactionTask<Void, RuntimeException>) txn -> {
            BatchingVisitable<RowResult<byte[]>> bv = txn.getRange(name, RangeRequest.all());
            bv.batchAccept(1000, AbortingVisitors.batching(new AbortingVisitor<RowResult<byte[]>, RuntimeException>() {

                @Override
                public boolean visit(RowResult<byte[]> item) throws RuntimeException {
                    Iterable<Map.Entry<Cell, byte[]>> cells = item.getCells();
                    Map.Entry<Cell, byte[]> entry = Iterables.getOnlyElement(cells);
                    assertThat(entry.getKey()).isEqualTo(theCell);
                    assertThat(entry.getValue()).isEqualTo(theValue);
                    count.increment();
                    return true;
                }
            }));
            return null;
        });
    }
    assertThat(count.longValue()).isEqualTo(2L);
}

Example 20 with MetricsManager

use of com.palantir.atlasdb.util.MetricsManager in project atlasdb by palantir.

the class CassandraKeyValueServiceImpl method createForTesting.

public static CassandraKeyValueService createForTesting(CassandraKeyValueServiceConfig config, Refreshable<CassandraKeyValueServiceRuntimeConfig> runtimeConfig) {
    MetricsManager metricsManager = MetricsManagers.createForTests();
    CassandraClientPool clientPool = CassandraClientPoolImpl.createImplForTest(metricsManager, config, runtimeConfig, StartupChecks.RUN, new Blacklist(config, runtimeConfig.map(CassandraKeyValueServiceRuntimeConfig::unresponsiveHostBackoffTimeSeconds)));
    return createOrShutdownClientPool(metricsManager, config, runtimeConfig, clientPool, CassandraMutationTimestampProviders.legacyModeForTestsOnly(), LoggerFactory.getLogger(CassandraKeyValueService.class), AtlasDbConstants.DEFAULT_INITIALIZE_ASYNC);
}