Examples with BlockerCheckpointStreamFactory org.apache.flink.runtime.util.BlockerCheckpointStreamFactory used on opensource projects

Example 1 with BlockerCheckpointStreamFactory

use of org.apache.flink.runtime.util.BlockerCheckpointStreamFactory in project flink by apache.

the class StateBackendTestBase method testAsyncSnapshotCancellation.

@Test
public void testAsyncSnapshotCancellation() throws Exception {
    OneShotLatch blocker = new OneShotLatch();
    OneShotLatch waiter = new OneShotLatch();
    BlockerCheckpointStreamFactory streamFactory = new BlockerCheckpointStreamFactory(1024 * 1024);
    streamFactory.setWaiterLatch(waiter);
    streamFactory.setBlockerLatch(blocker);
    streamFactory.setAfterNumberInvocations(100);
    AbstractKeyedStateBackend<Integer> backend = null;
    try {
        backend = createKeyedBackend(IntSerializer.INSTANCE);
        if (!backend.supportsAsynchronousSnapshots()) {
            return;
        }
        InternalValueState<VoidNamespace, Integer> valueState = backend.createValueState(VoidNamespaceSerializer.INSTANCE, new ValueStateDescriptor<>("test", IntSerializer.INSTANCE));
        valueState.setCurrentNamespace(VoidNamespace.INSTANCE);
        for (int i = 0; i < 10; ++i) {
            backend.setCurrentKey(i);
            valueState.update(i);
        }
        RunnableFuture<KeyGroupsStateHandle> snapshot = backend.snapshot(0L, 0L, streamFactory, CheckpointOptions.forFullCheckpoint());
        Thread runner = new Thread(snapshot);
        runner.start();
        // wait until the code reached some stream read
        waiter.await();
        // close the backend to see if the close is propagated to the stream
        backend.close();
        //unblock the stream so that it can run into the IOException
        blocker.trigger();
        //dispose the backend
        backend.dispose();
        runner.join();
        try {
            snapshot.get();
            fail("Close was not propagated.");
        } catch (ExecutionException ex) {
        //ignore
        }
    } finally {
        if (null != backend) {
            IOUtils.closeQuietly(backend);
            backend.dispose();
        }
    }
}

Example 2 with BlockerCheckpointStreamFactory

use of org.apache.flink.runtime.util.BlockerCheckpointStreamFactory in project flink by apache.

the class RocksDBStateBackendTest method setupRocksKeyedStateBackend.

public void setupRocksKeyedStateBackend() throws Exception {
    blocker = new OneShotLatch();
    waiter = new OneShotLatch();
    testStreamFactory = new BlockerCheckpointStreamFactory(1024 * 1024);
    testStreamFactory.setBlockerLatch(blocker);
    testStreamFactory.setWaiterLatch(waiter);
    testStreamFactory.setAfterNumberInvocations(100);
    RocksDBStateBackend backend = getStateBackend();
    Environment env = new DummyEnvironment("TestTask", 1, 0);
    keyedStateBackend = (RocksDBKeyedStateBackend<Integer>) backend.createKeyedStateBackend(env, new JobID(), "Test", IntSerializer.INSTANCE, 2, new KeyGroupRange(0, 1), mock(TaskKvStateRegistry.class));
    testState1 = keyedStateBackend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, new ValueStateDescriptor<>("TestState-1", Integer.class, 0));
    testState2 = keyedStateBackend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, new ValueStateDescriptor<>("TestState-2", String.class, ""));
    allCreatedCloseables = new ArrayList<>();
    keyedStateBackend.db = spy(keyedStateBackend.db);
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            RocksIterator rocksIterator = spy((RocksIterator) invocationOnMock.callRealMethod());
            allCreatedCloseables.add(rocksIterator);
            return rocksIterator;
        }
    }).when(keyedStateBackend.db).newIterator(any(ColumnFamilyHandle.class), any(ReadOptions.class));
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            Snapshot snapshot = spy((Snapshot) invocationOnMock.callRealMethod());
            allCreatedCloseables.add(snapshot);
            return snapshot;
        }
    }).when(keyedStateBackend.db).getSnapshot();
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            ColumnFamilyHandle snapshot = spy((ColumnFamilyHandle) invocationOnMock.callRealMethod());
            allCreatedCloseables.add(snapshot);
            return snapshot;
        }
    }).when(keyedStateBackend.db).createColumnFamily(any(ColumnFamilyDescriptor.class));
    for (int i = 0; i < 100; ++i) {
        keyedStateBackend.setCurrentKey(i);
        testState1.update(4200 + i);
        testState2.update("S-" + (4200 + i));
    }
}

Example 3 with BlockerCheckpointStreamFactory

use of org.apache.flink.runtime.util.BlockerCheckpointStreamFactory in project flink by apache.

the class StateBackendTestBase method testAsyncSnapshot.

@Test
public void testAsyncSnapshot() throws Exception {
    OneShotLatch waiter = new OneShotLatch();
    BlockerCheckpointStreamFactory streamFactory = new BlockerCheckpointStreamFactory(1024 * 1024);
    streamFactory.setWaiterLatch(waiter);
    AbstractKeyedStateBackend<Integer> backend = null;
    KeyGroupsStateHandle stateHandle = null;
    try {
        backend = createKeyedBackend(IntSerializer.INSTANCE);
        InternalValueState<VoidNamespace, Integer> valueState = backend.createValueState(VoidNamespaceSerializer.INSTANCE, new ValueStateDescriptor<>("test", IntSerializer.INSTANCE));
        valueState.setCurrentNamespace(VoidNamespace.INSTANCE);
        for (int i = 0; i < 10; ++i) {
            backend.setCurrentKey(i);
            valueState.update(i);
        }
        RunnableFuture<KeyGroupsStateHandle> snapshot = backend.snapshot(0L, 0L, streamFactory, CheckpointOptions.forFullCheckpoint());
        Thread runner = new Thread(snapshot);
        runner.start();
        for (int i = 0; i < 20; ++i) {
            backend.setCurrentKey(i);
            valueState.update(i + 1);
            if (10 == i) {
                waiter.await();
            }
        }
        runner.join();
        stateHandle = snapshot.get();
        // test isolation
        for (int i = 0; i < 20; ++i) {
            backend.setCurrentKey(i);
            Assert.assertEquals(i + 1, (int) valueState.value());
        }
    } finally {
        if (null != backend) {
            IOUtils.closeQuietly(backend);
            backend.dispose();
        }
    }
    Assert.assertNotNull(stateHandle);
    backend = createKeyedBackend(IntSerializer.INSTANCE);
    try {
        backend.restore(Collections.singleton(stateHandle));
        InternalValueState<VoidNamespace, Integer> valueState = backend.createValueState(VoidNamespaceSerializer.INSTANCE, new ValueStateDescriptor<>("test", IntSerializer.INSTANCE));
        valueState.setCurrentNamespace(VoidNamespace.INSTANCE);
        for (int i = 0; i < 10; ++i) {
            backend.setCurrentKey(i);
            Assert.assertEquals(i, (int) valueState.value());
        }
        backend.setCurrentKey(11);
        Assert.assertEquals(null, valueState.value());
    } finally {
        if (null != backend) {
            IOUtils.closeQuietly(backend);
            backend.dispose();
        }
    }
}