use of org.rocksdb.Snapshot in project flink by apache.
the class RocksDBStateBackendTest method testRunningSnapshotAfterBackendClosed.
@Test
public void testRunningSnapshotAfterBackendClosed() throws Exception {
setupRocksKeyedStateBackend();
RunnableFuture<KeyGroupsStateHandle> snapshot = keyedStateBackend.snapshot(0L, 0L, testStreamFactory, CheckpointOptions.forFullCheckpoint());
RocksDB spyDB = keyedStateBackend.db;
verify(spyDB, times(1)).getSnapshot();
verify(spyDB, times(0)).releaseSnapshot(any(Snapshot.class));
this.keyedStateBackend.dispose();
verify(spyDB, times(1)).close();
assertEquals(null, keyedStateBackend.db);
//Ensure every RocksObjects not closed yet
for (RocksObject rocksCloseable : allCreatedCloseables) {
verify(rocksCloseable, times(0)).close();
}
Thread asyncSnapshotThread = new Thread(snapshot);
asyncSnapshotThread.start();
try {
snapshot.get();
fail();
} catch (Exception ignored) {
}
asyncSnapshotThread.join();
//Ensure every RocksObject was closed exactly once
for (RocksObject rocksCloseable : allCreatedCloseables) {
verify(rocksCloseable, times(1)).close();
}
}
use of org.rocksdb.Snapshot in project flink by apache.
the class RocksDBStateBackendTest method setupRocksKeyedStateBackend.
public void setupRocksKeyedStateBackend() throws Exception {
blocker = new OneShotLatch();
waiter = new OneShotLatch();
testStreamFactory = new BlockerCheckpointStreamFactory(1024 * 1024);
testStreamFactory.setBlockerLatch(blocker);
testStreamFactory.setWaiterLatch(waiter);
testStreamFactory.setAfterNumberInvocations(100);
RocksDBStateBackend backend = getStateBackend();
Environment env = new DummyEnvironment("TestTask", 1, 0);
keyedStateBackend = (RocksDBKeyedStateBackend<Integer>) backend.createKeyedStateBackend(env, new JobID(), "Test", IntSerializer.INSTANCE, 2, new KeyGroupRange(0, 1), mock(TaskKvStateRegistry.class));
testState1 = keyedStateBackend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, new ValueStateDescriptor<>("TestState-1", Integer.class, 0));
testState2 = keyedStateBackend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, new ValueStateDescriptor<>("TestState-2", String.class, ""));
allCreatedCloseables = new ArrayList<>();
keyedStateBackend.db = spy(keyedStateBackend.db);
doAnswer(new Answer<Object>() {
@Override
public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
RocksIterator rocksIterator = spy((RocksIterator) invocationOnMock.callRealMethod());
allCreatedCloseables.add(rocksIterator);
return rocksIterator;
}
}).when(keyedStateBackend.db).newIterator(any(ColumnFamilyHandle.class), any(ReadOptions.class));
doAnswer(new Answer<Object>() {
@Override
public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
Snapshot snapshot = spy((Snapshot) invocationOnMock.callRealMethod());
allCreatedCloseables.add(snapshot);
return snapshot;
}
}).when(keyedStateBackend.db).getSnapshot();
doAnswer(new Answer<Object>() {
@Override
public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
ColumnFamilyHandle snapshot = spy((ColumnFamilyHandle) invocationOnMock.callRealMethod());
allCreatedCloseables.add(snapshot);
return snapshot;
}
}).when(keyedStateBackend.db).createColumnFamily(any(ColumnFamilyDescriptor.class));
for (int i = 0; i < 100; ++i) {
keyedStateBackend.setCurrentKey(i);
testState1.update(4200 + i);
testState2.update("S-" + (4200 + i));
}
}
use of org.rocksdb.Snapshot in project flink by apache.
the class RocksDBStateBackendTest method testReleasingSnapshotAfterBackendClosed.
@Test
public void testReleasingSnapshotAfterBackendClosed() throws Exception {
setupRocksKeyedStateBackend();
RunnableFuture<KeyGroupsStateHandle> snapshot = keyedStateBackend.snapshot(0L, 0L, testStreamFactory, CheckpointOptions.forFullCheckpoint());
RocksDB spyDB = keyedStateBackend.db;
verify(spyDB, times(1)).getSnapshot();
verify(spyDB, times(0)).releaseSnapshot(any(Snapshot.class));
this.keyedStateBackend.dispose();
verify(spyDB, times(1)).close();
assertEquals(null, keyedStateBackend.db);
//Ensure every RocksObjects not closed yet
for (RocksObject rocksCloseable : allCreatedCloseables) {
verify(rocksCloseable, times(0)).close();
}
snapshot.cancel(true);
//Ensure every RocksObjects was closed exactly once
for (RocksObject rocksCloseable : allCreatedCloseables) {
verify(rocksCloseable, times(1)).close();
}
}
Aggregations