Examples with LatencyTrackingStateConfig org.apache.flink.runtime.state.metrics.LatencyTrackingStateConfig used on opensource projects

Example 1 with LatencyTrackingStateConfig

use of org.apache.flink.runtime.state.metrics.LatencyTrackingStateConfig in project flink by apache.

the class MemoryStateBackend method createKeyedStateBackend.

@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(Environment env, JobID jobID, String operatorIdentifier, TypeSerializer<K> keySerializer, int numberOfKeyGroups, KeyGroupRange keyGroupRange, TaskKvStateRegistry kvStateRegistry, TtlTimeProvider ttlTimeProvider, MetricGroup metricGroup, @Nonnull Collection<KeyedStateHandle> stateHandles, CloseableRegistry cancelStreamRegistry) throws BackendBuildingException {
    TaskStateManager taskStateManager = env.getTaskStateManager();
    HeapPriorityQueueSetFactory priorityQueueSetFactory = new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128);
    LatencyTrackingStateConfig latencyTrackingStateConfig = latencyTrackingConfigBuilder.setMetricGroup(metricGroup).build();
    return new HeapKeyedStateBackendBuilder<>(kvStateRegistry, keySerializer, env.getUserCodeClassLoader().asClassLoader(), numberOfKeyGroups, keyGroupRange, env.getExecutionConfig(), ttlTimeProvider, latencyTrackingStateConfig, stateHandles, AbstractStateBackend.getCompressionDecorator(env.getExecutionConfig()), taskStateManager.createLocalRecoveryConfig(), priorityQueueSetFactory, isUsingAsynchronousSnapshots(), cancelStreamRegistry).build();
}

Example 2 with LatencyTrackingStateConfig

use of org.apache.flink.runtime.state.metrics.LatencyTrackingStateConfig in project flink by apache.

the class FsStateBackend method createKeyedStateBackend.

// ------------------------------------------------------------------------
// state holding structures
// ------------------------------------------------------------------------
@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(Environment env, JobID jobID, String operatorIdentifier, TypeSerializer<K> keySerializer, int numberOfKeyGroups, KeyGroupRange keyGroupRange, TaskKvStateRegistry kvStateRegistry, TtlTimeProvider ttlTimeProvider, MetricGroup metricGroup, @Nonnull Collection<KeyedStateHandle> stateHandles, CloseableRegistry cancelStreamRegistry) throws BackendBuildingException {
    TaskStateManager taskStateManager = env.getTaskStateManager();
    LocalRecoveryConfig localRecoveryConfig = taskStateManager.createLocalRecoveryConfig();
    HeapPriorityQueueSetFactory priorityQueueSetFactory = new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128);
    LatencyTrackingStateConfig latencyTrackingStateConfig = latencyTrackingConfigBuilder.setMetricGroup(metricGroup).build();
    return new HeapKeyedStateBackendBuilder<>(kvStateRegistry, keySerializer, env.getUserCodeClassLoader().asClassLoader(), numberOfKeyGroups, keyGroupRange, env.getExecutionConfig(), ttlTimeProvider, latencyTrackingStateConfig, stateHandles, AbstractStateBackend.getCompressionDecorator(env.getExecutionConfig()), localRecoveryConfig, priorityQueueSetFactory, isUsingAsynchronousSnapshots(), cancelStreamRegistry).build();
}

Example 3 with LatencyTrackingStateConfig

use of org.apache.flink.runtime.state.metrics.LatencyTrackingStateConfig in project flink by apache.

the class HashMapStateBackend method createKeyedStateBackend.

@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(Environment env, JobID jobID, String operatorIdentifier, TypeSerializer<K> keySerializer, int numberOfKeyGroups, KeyGroupRange keyGroupRange, TaskKvStateRegistry kvStateRegistry, TtlTimeProvider ttlTimeProvider, MetricGroup metricGroup, @Nonnull Collection<KeyedStateHandle> stateHandles, CloseableRegistry cancelStreamRegistry) throws IOException {
    TaskStateManager taskStateManager = env.getTaskStateManager();
    LocalRecoveryConfig localRecoveryConfig = taskStateManager.createLocalRecoveryConfig();
    HeapPriorityQueueSetFactory priorityQueueSetFactory = new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128);
    LatencyTrackingStateConfig latencyTrackingStateConfig = latencyTrackingConfigBuilder.setMetricGroup(metricGroup).build();
    return new HeapKeyedStateBackendBuilder<>(kvStateRegistry, keySerializer, env.getUserCodeClassLoader().asClassLoader(), numberOfKeyGroups, keyGroupRange, env.getExecutionConfig(), ttlTimeProvider, latencyTrackingStateConfig, stateHandles, getCompressionDecorator(env.getExecutionConfig()), localRecoveryConfig, priorityQueueSetFactory, true, cancelStreamRegistry).build();
}

Example 4 with LatencyTrackingStateConfig

use of org.apache.flink.runtime.state.metrics.LatencyTrackingStateConfig in project flink by apache.

the class EmbeddedRocksDBStateBackend method createKeyedStateBackend.

@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(Environment env, JobID jobID, String operatorIdentifier, TypeSerializer<K> keySerializer, int numberOfKeyGroups, KeyGroupRange keyGroupRange, TaskKvStateRegistry kvStateRegistry, TtlTimeProvider ttlTimeProvider, MetricGroup metricGroup, @Nonnull Collection<KeyedStateHandle> stateHandles, CloseableRegistry cancelStreamRegistry, double managedMemoryFraction) throws IOException {
    // first, make sure that the RocksDB JNI library is loaded
    // we do this explicitly here to have better error handling
    String tempDir = env.getTaskManagerInfo().getTmpWorkingDirectory().getAbsolutePath();
    ensureRocksDBIsLoaded(tempDir);
    // replace all characters that are not legal for filenames with underscore
    String fileCompatibleIdentifier = operatorIdentifier.replaceAll("[^a-zA-Z0-9\\-]", "_");
    lazyInitializeForJob(env, fileCompatibleIdentifier);
    File instanceBasePath = new File(getNextStoragePath(), "job_" + jobId + "_op_" + fileCompatibleIdentifier + "_uuid_" + UUID.randomUUID());
    LocalRecoveryConfig localRecoveryConfig = env.getTaskStateManager().createLocalRecoveryConfig();
    final OpaqueMemoryResource<RocksDBSharedResources> sharedResources = RocksDBOperationUtils.allocateSharedCachesIfConfigured(memoryConfiguration, env.getMemoryManager(), managedMemoryFraction, LOG);
    if (sharedResources != null) {
        LOG.info("Obtained shared RocksDB cache of size {} bytes", sharedResources.getSize());
    }
    final RocksDBResourceContainer resourceContainer = createOptionsAndResourceContainer(sharedResources);
    ExecutionConfig executionConfig = env.getExecutionConfig();
    StreamCompressionDecorator keyGroupCompressionDecorator = getCompressionDecorator(executionConfig);
    LatencyTrackingStateConfig latencyTrackingStateConfig = latencyTrackingConfigBuilder.setMetricGroup(metricGroup).build();
    RocksDBKeyedStateBackendBuilder<K> builder = new RocksDBKeyedStateBackendBuilder<>(operatorIdentifier, env.getUserCodeClassLoader().asClassLoader(), instanceBasePath, resourceContainer, stateName -> resourceContainer.getColumnOptions(), kvStateRegistry, keySerializer, numberOfKeyGroups, keyGroupRange, executionConfig, localRecoveryConfig, getPriorityQueueStateType(), ttlTimeProvider, latencyTrackingStateConfig, metricGroup, stateHandles, keyGroupCompressionDecorator, cancelStreamRegistry).setEnableIncrementalCheckpointing(isIncrementalCheckpointsEnabled()).setNumberOfTransferingThreads(getNumberOfTransferThreads()).setNativeMetricOptions(resourceContainer.getMemoryWatcherOptions(defaultMetricOptions)).setWriteBatchSize(getWriteBatchSize());
    return builder.build();
}