Examples with SharedStateRegistryImpl org.apache.flink.runtime.state.SharedStateRegistryImpl used on opensource projects

Example 16 with SharedStateRegistryImpl

use of org.apache.flink.runtime.state.SharedStateRegistryImpl in project flink by apache.

the class ZooKeeperCompletedCheckpointStoreTest method testDiscardingSubsumedCheckpoints.

/**
 * Tests that subsumed checkpoints are discarded.
 */
@Test
public void testDiscardingSubsumedCheckpoints() throws Exception {
    final SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    final Configuration configuration = new Configuration();
    configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zooKeeperResource.getConnectString());
    final CuratorFrameworkWithUnhandledErrorListener curatorFrameworkWrapper = ZooKeeperUtils.startCuratorFramework(configuration, NoOpFatalErrorHandler.INSTANCE);
    final CompletedCheckpointStore checkpointStore = createZooKeeperCheckpointStore(curatorFrameworkWrapper.asCuratorFramework());
    try {
        final CompletedCheckpointStoreTest.TestCompletedCheckpoint checkpoint1 = CompletedCheckpointStoreTest.createCheckpoint(0, sharedStateRegistry);
        checkpointStore.addCheckpointAndSubsumeOldestOne(checkpoint1, new CheckpointsCleaner(), () -> {
        });
        assertThat(checkpointStore.getAllCheckpoints(), Matchers.contains(checkpoint1));
        final CompletedCheckpointStoreTest.TestCompletedCheckpoint checkpoint2 = CompletedCheckpointStoreTest.createCheckpoint(1, sharedStateRegistry);
        checkpointStore.addCheckpointAndSubsumeOldestOne(checkpoint2, new CheckpointsCleaner(), () -> {
        });
        final List<CompletedCheckpoint> allCheckpoints = checkpointStore.getAllCheckpoints();
        assertThat(allCheckpoints, Matchers.contains(checkpoint2));
        assertThat(allCheckpoints, Matchers.not(Matchers.contains(checkpoint1)));
        // verify that the subsumed checkpoint is discarded
        CompletedCheckpointStoreTest.verifyCheckpointDiscarded(checkpoint1);
    } finally {
        curatorFrameworkWrapper.close();
    }
}

Example 17 with SharedStateRegistryImpl

use of org.apache.flink.runtime.state.SharedStateRegistryImpl in project flink by apache.

the class StandaloneCompletedCheckpointStoreTest method testSuspendDiscardsCheckpoints.

/**
 * Tests that suspends discards all checkpoints (as they cannot be recovered later in standalone
 * recovery mode).
 */
@Test
public void testSuspendDiscardsCheckpoints() throws Exception {
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    CompletedCheckpointStore store = createRecoveredCompletedCheckpointStore(1);
    TestCompletedCheckpoint checkpoint = createCheckpoint(0, sharedStateRegistry);
    Collection<OperatorState> taskStates = checkpoint.getOperatorStates().values();
    store.addCheckpointAndSubsumeOldestOne(checkpoint, new CheckpointsCleaner(), () -> {
    });
    assertEquals(1, store.getNumberOfRetainedCheckpoints());
    verifyCheckpointRegistered(taskStates, sharedStateRegistry);
    store.shutdown(JobStatus.SUSPENDED, new CheckpointsCleaner());
    assertEquals(0, store.getNumberOfRetainedCheckpoints());
    assertTrue(checkpoint.isDiscarded());
    verifyCheckpointDiscarded(taskStates);
}

Example 18 with SharedStateRegistryImpl

use of org.apache.flink.runtime.state.SharedStateRegistryImpl in project flink by apache.

the class EmbeddedRocksDBStateBackendTest method testSharedIncrementalStateDeRegistration.

@Test
public void testSharedIncrementalStateDeRegistration() throws Exception {
    if (enableIncrementalCheckpointing) {
        CheckpointableKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
        try {
            ValueStateDescriptor<String> kvId = new ValueStateDescriptor<>("id", String.class, null);
            kvId.initializeSerializerUnlessSet(new ExecutionConfig());
            ValueState<String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
            Queue<IncrementalRemoteKeyedStateHandle> previousStateHandles = new LinkedList<>();
            SharedStateRegistry sharedStateRegistry = spy(new SharedStateRegistryImpl());
            for (int checkpointId = 0; checkpointId < 3; ++checkpointId) {
                reset(sharedStateRegistry);
                backend.setCurrentKey(checkpointId);
                state.update("Hello-" + checkpointId);
                RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = backend.snapshot(checkpointId, checkpointId, createStreamFactory(), CheckpointOptions.forCheckpointWithDefaultLocation());
                snapshot.run();
                SnapshotResult<KeyedStateHandle> snapshotResult = snapshot.get();
                IncrementalRemoteKeyedStateHandle stateHandle = (IncrementalRemoteKeyedStateHandle) snapshotResult.getJobManagerOwnedSnapshot();
                Map<StateHandleID, StreamStateHandle> sharedState = new HashMap<>(stateHandle.getSharedState());
                stateHandle.registerSharedStates(sharedStateRegistry, checkpointId);
                for (Map.Entry<StateHandleID, StreamStateHandle> e : sharedState.entrySet()) {
                    verify(sharedStateRegistry).registerReference(stateHandle.createSharedStateRegistryKeyFromFileName(e.getKey()), e.getValue(), checkpointId);
                }
                previousStateHandles.add(stateHandle);
                ((CheckpointListener) backend).notifyCheckpointComplete(checkpointId);
                if (previousStateHandles.size() > 1) {
                    previousStateHandles.remove().discardState();
                }
            }
            while (!previousStateHandles.isEmpty()) {
                reset(sharedStateRegistry);
                previousStateHandles.remove().discardState();
            }
        } finally {
            IOUtils.closeQuietly(backend);
            backend.dispose();
        }
    }
}

Example 19 with SharedStateRegistryImpl

use of org.apache.flink.runtime.state.SharedStateRegistryImpl in project flink by apache.

the class CompletedCheckpointStoreTest method testAcquireLatestCompletedCheckpointId.

@Test
public void testAcquireLatestCompletedCheckpointId() throws Exception {
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    CompletedCheckpointStore checkpoints = createRecoveredCompletedCheckpointStore(1);
    assertEquals(0, checkpoints.getLatestCheckpointId());
    checkpoints.addCheckpointAndSubsumeOldestOne(createCheckpoint(2, sharedStateRegistry), new CheckpointsCleaner(), () -> {
    });
    assertEquals(2, checkpoints.getLatestCheckpointId());
    checkpoints.addCheckpointAndSubsumeOldestOne(createCheckpoint(4, sharedStateRegistry), new CheckpointsCleaner(), () -> {
    });
    assertEquals(4, checkpoints.getLatestCheckpointId());
}

Example 20 with SharedStateRegistryImpl

use of org.apache.flink.runtime.state.SharedStateRegistryImpl in project flink by apache.

the class CompletedCheckpointStoreTest method testAddCheckpointMoreThanMaxRetained.

/**
 * Tests that adding more checkpoints than retained discards the correct checkpoints (using the
 * correct class loader).
 */
@Test
public void testAddCheckpointMoreThanMaxRetained() throws Exception {
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    CompletedCheckpointStore checkpoints = createRecoveredCompletedCheckpointStore(1);
    TestCompletedCheckpoint[] expected = new TestCompletedCheckpoint[] { createCheckpoint(0, sharedStateRegistry), createCheckpoint(1, sharedStateRegistry), createCheckpoint(2, sharedStateRegistry), createCheckpoint(3, sharedStateRegistry) };
    // Add checkpoints
    checkpoints.addCheckpointAndSubsumeOldestOne(expected[0], new CheckpointsCleaner(), () -> {
    });
    assertEquals(1, checkpoints.getNumberOfRetainedCheckpoints());
    for (int i = 1; i < expected.length; i++) {
        checkpoints.addCheckpointAndSubsumeOldestOne(expected[i], new CheckpointsCleaner(), () -> {
        });
        // The ZooKeeper implementation discards asynchronously
        expected[i - 1].awaitDiscard();
        assertTrue(expected[i - 1].isDiscarded());
        assertEquals(1, checkpoints.getNumberOfRetainedCheckpoints());
    }
}