use of org.apache.flink.runtime.util.BlockerCheckpointStreamFactory in project flink by apache.
the class StateBackendTestBase method testAsyncSnapshotCancellation.
@Test
public void testAsyncSnapshotCancellation() throws Exception {
OneShotLatch blocker = new OneShotLatch();
OneShotLatch waiter = new OneShotLatch();
BlockerCheckpointStreamFactory streamFactory = new BlockerCheckpointStreamFactory(1024 * 1024);
streamFactory.setWaiterLatch(waiter);
streamFactory.setBlockerLatch(blocker);
streamFactory.setAfterNumberInvocations(100);
AbstractKeyedStateBackend<Integer> backend = null;
try {
backend = createKeyedBackend(IntSerializer.INSTANCE);
if (!backend.supportsAsynchronousSnapshots()) {
return;
}
InternalValueState<VoidNamespace, Integer> valueState = backend.createValueState(VoidNamespaceSerializer.INSTANCE, new ValueStateDescriptor<>("test", IntSerializer.INSTANCE));
valueState.setCurrentNamespace(VoidNamespace.INSTANCE);
for (int i = 0; i < 10; ++i) {
backend.setCurrentKey(i);
valueState.update(i);
}
RunnableFuture<KeyGroupsStateHandle> snapshot = backend.snapshot(0L, 0L, streamFactory, CheckpointOptions.forFullCheckpoint());
Thread runner = new Thread(snapshot);
runner.start();
// wait until the code reached some stream read
waiter.await();
// close the backend to see if the close is propagated to the stream
backend.close();
//unblock the stream so that it can run into the IOException
blocker.trigger();
//dispose the backend
backend.dispose();
runner.join();
try {
snapshot.get();
fail("Close was not propagated.");
} catch (ExecutionException ex) {
//ignore
}
} finally {
if (null != backend) {
IOUtils.closeQuietly(backend);
backend.dispose();
}
}
}
use of org.apache.flink.runtime.util.BlockerCheckpointStreamFactory in project flink by apache.
the class RocksDBStateBackendTest method setupRocksKeyedStateBackend.
public void setupRocksKeyedStateBackend() throws Exception {
blocker = new OneShotLatch();
waiter = new OneShotLatch();
testStreamFactory = new BlockerCheckpointStreamFactory(1024 * 1024);
testStreamFactory.setBlockerLatch(blocker);
testStreamFactory.setWaiterLatch(waiter);
testStreamFactory.setAfterNumberInvocations(100);
RocksDBStateBackend backend = getStateBackend();
Environment env = new DummyEnvironment("TestTask", 1, 0);
keyedStateBackend = (RocksDBKeyedStateBackend<Integer>) backend.createKeyedStateBackend(env, new JobID(), "Test", IntSerializer.INSTANCE, 2, new KeyGroupRange(0, 1), mock(TaskKvStateRegistry.class));
testState1 = keyedStateBackend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, new ValueStateDescriptor<>("TestState-1", Integer.class, 0));
testState2 = keyedStateBackend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, new ValueStateDescriptor<>("TestState-2", String.class, ""));
allCreatedCloseables = new ArrayList<>();
keyedStateBackend.db = spy(keyedStateBackend.db);
doAnswer(new Answer<Object>() {
@Override
public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
RocksIterator rocksIterator = spy((RocksIterator) invocationOnMock.callRealMethod());
allCreatedCloseables.add(rocksIterator);
return rocksIterator;
}
}).when(keyedStateBackend.db).newIterator(any(ColumnFamilyHandle.class), any(ReadOptions.class));
doAnswer(new Answer<Object>() {
@Override
public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
Snapshot snapshot = spy((Snapshot) invocationOnMock.callRealMethod());
allCreatedCloseables.add(snapshot);
return snapshot;
}
}).when(keyedStateBackend.db).getSnapshot();
doAnswer(new Answer<Object>() {
@Override
public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
ColumnFamilyHandle snapshot = spy((ColumnFamilyHandle) invocationOnMock.callRealMethod());
allCreatedCloseables.add(snapshot);
return snapshot;
}
}).when(keyedStateBackend.db).createColumnFamily(any(ColumnFamilyDescriptor.class));
for (int i = 0; i < 100; ++i) {
keyedStateBackend.setCurrentKey(i);
testState1.update(4200 + i);
testState2.update("S-" + (4200 + i));
}
}
use of org.apache.flink.runtime.util.BlockerCheckpointStreamFactory in project flink by apache.
the class StateBackendTestBase method testAsyncSnapshot.
@Test
public void testAsyncSnapshot() throws Exception {
OneShotLatch waiter = new OneShotLatch();
BlockerCheckpointStreamFactory streamFactory = new BlockerCheckpointStreamFactory(1024 * 1024);
streamFactory.setWaiterLatch(waiter);
AbstractKeyedStateBackend<Integer> backend = null;
KeyGroupsStateHandle stateHandle = null;
try {
backend = createKeyedBackend(IntSerializer.INSTANCE);
InternalValueState<VoidNamespace, Integer> valueState = backend.createValueState(VoidNamespaceSerializer.INSTANCE, new ValueStateDescriptor<>("test", IntSerializer.INSTANCE));
valueState.setCurrentNamespace(VoidNamespace.INSTANCE);
for (int i = 0; i < 10; ++i) {
backend.setCurrentKey(i);
valueState.update(i);
}
RunnableFuture<KeyGroupsStateHandle> snapshot = backend.snapshot(0L, 0L, streamFactory, CheckpointOptions.forFullCheckpoint());
Thread runner = new Thread(snapshot);
runner.start();
for (int i = 0; i < 20; ++i) {
backend.setCurrentKey(i);
valueState.update(i + 1);
if (10 == i) {
waiter.await();
}
}
runner.join();
stateHandle = snapshot.get();
// test isolation
for (int i = 0; i < 20; ++i) {
backend.setCurrentKey(i);
Assert.assertEquals(i + 1, (int) valueState.value());
}
} finally {
if (null != backend) {
IOUtils.closeQuietly(backend);
backend.dispose();
}
}
Assert.assertNotNull(stateHandle);
backend = createKeyedBackend(IntSerializer.INSTANCE);
try {
backend.restore(Collections.singleton(stateHandle));
InternalValueState<VoidNamespace, Integer> valueState = backend.createValueState(VoidNamespaceSerializer.INSTANCE, new ValueStateDescriptor<>("test", IntSerializer.INSTANCE));
valueState.setCurrentNamespace(VoidNamespace.INSTANCE);
for (int i = 0; i < 10; ++i) {
backend.setCurrentKey(i);
Assert.assertEquals(i, (int) valueState.value());
}
backend.setCurrentKey(11);
Assert.assertEquals(null, valueState.value());
} finally {
if (null != backend) {
IOUtils.closeQuietly(backend);
backend.dispose();
}
}
}
Aggregations