use of org.apache.flink.runtime.state.LocalRecoveryConfig in project flink by apache.
the class LocalStateForwardingTest method testReportingFromTaskStateManagerToResponderAndTaskLocalStateStore.
/**
* This tests that state that was reported to the {@link
* org.apache.flink.runtime.state.TaskStateManager} is also reported to {@link
* org.apache.flink.runtime.taskmanager.CheckpointResponder} and {@link
* TaskLocalStateStoreImpl}.
*/
@Test
public void testReportingFromTaskStateManagerToResponderAndTaskLocalStateStore() throws Exception {
final JobID jobID = new JobID();
final AllocationID allocationID = new AllocationID();
final ExecutionAttemptID executionAttemptID = new ExecutionAttemptID();
final CheckpointMetaData checkpointMetaData = new CheckpointMetaData(42L, 4711L);
final CheckpointMetrics checkpointMetrics = new CheckpointMetrics();
final int subtaskIdx = 42;
JobVertexID jobVertexID = new JobVertexID();
TaskStateSnapshot jmSnapshot = new TaskStateSnapshot();
TaskStateSnapshot tmSnapshot = new TaskStateSnapshot();
final AtomicBoolean jmReported = new AtomicBoolean(false);
final AtomicBoolean tmReported = new AtomicBoolean(false);
TestCheckpointResponder checkpointResponder = new TestCheckpointResponder() {
@Override
public void acknowledgeCheckpoint(JobID lJobID, ExecutionAttemptID lExecutionAttemptID, long lCheckpointId, CheckpointMetrics lCheckpointMetrics, TaskStateSnapshot lSubtaskState) {
Assert.assertEquals(jobID, lJobID);
Assert.assertEquals(executionAttemptID, lExecutionAttemptID);
Assert.assertEquals(checkpointMetaData.getCheckpointId(), lCheckpointId);
Assert.assertEquals(checkpointMetrics, lCheckpointMetrics);
jmReported.set(true);
}
};
Executor executor = Executors.directExecutor();
LocalRecoveryDirectoryProviderImpl directoryProvider = new LocalRecoveryDirectoryProviderImpl(temporaryFolder.newFolder(), jobID, jobVertexID, subtaskIdx);
LocalRecoveryConfig localRecoveryConfig = new LocalRecoveryConfig(directoryProvider);
TaskLocalStateStore taskLocalStateStore = new TaskLocalStateStoreImpl(jobID, allocationID, jobVertexID, subtaskIdx, localRecoveryConfig, executor) {
@Override
public void storeLocalState(@Nonnegative long checkpointId, @Nullable TaskStateSnapshot localState) {
Assert.assertEquals(tmSnapshot, localState);
tmReported.set(true);
}
};
StateChangelogStorage<?> stateChangelogStorage = new InMemoryStateChangelogStorage();
TaskStateManagerImpl taskStateManager = new TaskStateManagerImpl(jobID, executionAttemptID, taskLocalStateStore, stateChangelogStorage, null, checkpointResponder);
taskStateManager.reportTaskStateSnapshots(checkpointMetaData, checkpointMetrics, jmSnapshot, tmSnapshot);
Assert.assertTrue("Reporting for JM state was not called.", jmReported.get());
Assert.assertTrue("Reporting for TM state was not called.", tmReported.get());
}
use of org.apache.flink.runtime.state.LocalRecoveryConfig in project flink by apache.
the class StateBackendBenchmarkUtils method createRocksDBKeyedStateBackend.
private static RocksDBKeyedStateBackend<Long> createRocksDBKeyedStateBackend(File rootDir) throws IOException {
File recoveryBaseDir = prepareDirectory(recoveryDirName, rootDir);
File dbPathFile = prepareDirectory(dbDirName, rootDir);
ExecutionConfig executionConfig = new ExecutionConfig();
RocksDBResourceContainer resourceContainer = new RocksDBResourceContainer();
RocksDBKeyedStateBackendBuilder<Long> builder = new RocksDBKeyedStateBackendBuilder<>("Test", Thread.currentThread().getContextClassLoader(), dbPathFile, resourceContainer, stateName -> resourceContainer.getColumnOptions(), null, LongSerializer.INSTANCE, 2, new KeyGroupRange(0, 1), executionConfig, new LocalRecoveryConfig(null), EmbeddedRocksDBStateBackend.PriorityQueueStateType.ROCKSDB, TtlTimeProvider.DEFAULT, LatencyTrackingStateConfig.disabled(), new UnregisteredMetricsGroup(), Collections.emptyList(), AbstractStateBackend.getCompressionDecorator(executionConfig), new CloseableRegistry());
try {
return builder.build();
} catch (Exception e) {
IOUtils.closeQuietly(resourceContainer);
throw e;
}
}
use of org.apache.flink.runtime.state.LocalRecoveryConfig in project flink by apache.
the class StateBackendBenchmarkUtils method createHeapKeyedStateBackend.
private static HeapKeyedStateBackend<Long> createHeapKeyedStateBackend(File rootDir) throws IOException {
File recoveryBaseDir = prepareDirectory(recoveryDirName, rootDir);
KeyGroupRange keyGroupRange = new KeyGroupRange(0, 1);
int numberOfKeyGroups = keyGroupRange.getNumberOfKeyGroups();
ExecutionConfig executionConfig = new ExecutionConfig();
HeapPriorityQueueSetFactory priorityQueueSetFactory = new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128);
HeapKeyedStateBackendBuilder<Long> backendBuilder = new HeapKeyedStateBackendBuilder<>(null, new LongSerializer(), Thread.currentThread().getContextClassLoader(), numberOfKeyGroups, keyGroupRange, executionConfig, TtlTimeProvider.DEFAULT, LatencyTrackingStateConfig.disabled(), Collections.emptyList(), AbstractStateBackend.getCompressionDecorator(executionConfig), new LocalRecoveryConfig(null), priorityQueueSetFactory, false, new CloseableRegistry());
return backendBuilder.build();
}
use of org.apache.flink.runtime.state.LocalRecoveryConfig in project flink by apache.
the class FsStateBackend method createKeyedStateBackend.
// ------------------------------------------------------------------------
// state holding structures
// ------------------------------------------------------------------------
@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(Environment env, JobID jobID, String operatorIdentifier, TypeSerializer<K> keySerializer, int numberOfKeyGroups, KeyGroupRange keyGroupRange, TaskKvStateRegistry kvStateRegistry, TtlTimeProvider ttlTimeProvider, MetricGroup metricGroup, @Nonnull Collection<KeyedStateHandle> stateHandles, CloseableRegistry cancelStreamRegistry) throws BackendBuildingException {
TaskStateManager taskStateManager = env.getTaskStateManager();
LocalRecoveryConfig localRecoveryConfig = taskStateManager.createLocalRecoveryConfig();
HeapPriorityQueueSetFactory priorityQueueSetFactory = new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128);
LatencyTrackingStateConfig latencyTrackingStateConfig = latencyTrackingConfigBuilder.setMetricGroup(metricGroup).build();
return new HeapKeyedStateBackendBuilder<>(kvStateRegistry, keySerializer, env.getUserCodeClassLoader().asClassLoader(), numberOfKeyGroups, keyGroupRange, env.getExecutionConfig(), ttlTimeProvider, latencyTrackingStateConfig, stateHandles, AbstractStateBackend.getCompressionDecorator(env.getExecutionConfig()), localRecoveryConfig, priorityQueueSetFactory, isUsingAsynchronousSnapshots(), cancelStreamRegistry).build();
}
use of org.apache.flink.runtime.state.LocalRecoveryConfig in project flink by apache.
the class HashMapStateBackend method createKeyedStateBackend.
@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(Environment env, JobID jobID, String operatorIdentifier, TypeSerializer<K> keySerializer, int numberOfKeyGroups, KeyGroupRange keyGroupRange, TaskKvStateRegistry kvStateRegistry, TtlTimeProvider ttlTimeProvider, MetricGroup metricGroup, @Nonnull Collection<KeyedStateHandle> stateHandles, CloseableRegistry cancelStreamRegistry) throws IOException {
TaskStateManager taskStateManager = env.getTaskStateManager();
LocalRecoveryConfig localRecoveryConfig = taskStateManager.createLocalRecoveryConfig();
HeapPriorityQueueSetFactory priorityQueueSetFactory = new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128);
LatencyTrackingStateConfig latencyTrackingStateConfig = latencyTrackingConfigBuilder.setMetricGroup(metricGroup).build();
return new HeapKeyedStateBackendBuilder<>(kvStateRegistry, keySerializer, env.getUserCodeClassLoader().asClassLoader(), numberOfKeyGroups, keyGroupRange, env.getExecutionConfig(), ttlTimeProvider, latencyTrackingStateConfig, stateHandles, getCompressionDecorator(env.getExecutionConfig()), localRecoveryConfig, priorityQueueSetFactory, true, cancelStreamRegistry).build();
}
Aggregations