Examples with SharedStateRegistry org.apache.flink.runtime.state.SharedStateRegistry used on opensource projects

Example 6 with SharedStateRegistry

use of org.apache.flink.runtime.state.SharedStateRegistry in project flink by apache.

the class ZooKeeperCompletedCheckpointStoreITCase method testSuspendKeepsCheckpoints.

/**
 * Tests that suspends keeps all checkpoints (so that they can be recovered later by the
 * ZooKeeper store). Furthermore, suspending a job should release all locks.
 */
@Test
public void testSuspendKeepsCheckpoints() throws Exception {
    CuratorFramework client = ZOOKEEPER.getClient();
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    CompletedCheckpointStore store = createRecoveredCompletedCheckpointStore(1);
    TestCompletedCheckpoint checkpoint = createCheckpoint(0, sharedStateRegistry);
    store.addCheckpointAndSubsumeOldestOne(checkpoint, new CheckpointsCleaner(), () -> {
    });
    assertEquals(1, store.getNumberOfRetainedCheckpoints());
    assertNotNull(client.checkExists().forPath(CHECKPOINT_PATH + checkpointStoreUtil.checkpointIDToName(checkpoint.getCheckpointID())));
    store.shutdown(JobStatus.SUSPENDED, new CheckpointsCleaner());
    assertEquals(0, store.getNumberOfRetainedCheckpoints());
    final String checkpointPath = CHECKPOINT_PATH + checkpointStoreUtil.checkpointIDToName(checkpoint.getCheckpointID());
    Stat stat = client.checkExists().forPath(checkpointPath);
    assertNotNull("The checkpoint node should exist.", stat);
    assertEquals("The checkpoint node should not be locked.", 0, stat.getNumChildren());
    // Recover again
    sharedStateRegistry.close();
    store = createRecoveredCompletedCheckpointStore(1);
    CompletedCheckpoint recovered = store.getLatestCheckpoint();
    assertEquals(checkpoint, recovered);
}

Example 7 with SharedStateRegistry

use of org.apache.flink.runtime.state.SharedStateRegistry in project flink by apache.

the class ZooKeeperCompletedCheckpointStoreITCase method testLatestCheckpointRecovery.

/**
 * FLINK-6284.
 *
 * <p>Tests that the latest recovered checkpoint is the one with the highest checkpoint id
 */
@Test
public void testLatestCheckpointRecovery() throws Exception {
    final int numCheckpoints = 3;
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    CompletedCheckpointStore checkpointStore = createRecoveredCompletedCheckpointStore(numCheckpoints);
    List<CompletedCheckpoint> checkpoints = new ArrayList<>(numCheckpoints);
    checkpoints.add(createCheckpoint(9, sharedStateRegistry));
    checkpoints.add(createCheckpoint(10, sharedStateRegistry));
    checkpoints.add(createCheckpoint(11, sharedStateRegistry));
    for (CompletedCheckpoint checkpoint : checkpoints) {
        checkpointStore.addCheckpointAndSubsumeOldestOne(checkpoint, new CheckpointsCleaner(), () -> {
        });
    }
    sharedStateRegistry.close();
    final CompletedCheckpoint latestCheckpoint = createRecoveredCompletedCheckpointStore(numCheckpoints).getLatestCheckpoint();
    assertEquals(checkpoints.get(checkpoints.size() - 1), latestCheckpoint);
}

Example 8 with SharedStateRegistry

use of org.apache.flink.runtime.state.SharedStateRegistry in project flink by apache.

the class CompletedCheckpointTest method testCleanUpOnSubsume.

/**
 * Tests that the garbage collection properties are respected when subsuming checkpoints.
 */
@Test
public void testCleanUpOnSubsume() throws Exception {
    OperatorState state = mock(OperatorState.class);
    Map<OperatorID, OperatorState> operatorStates = new HashMap<>();
    operatorStates.put(new OperatorID(), state);
    EmptyStreamStateHandle metadata = new EmptyStreamStateHandle();
    TestCompletedCheckpointStorageLocation location = new TestCompletedCheckpointStorageLocation(metadata, "ptr");
    CheckpointProperties props = new CheckpointProperties(false, CheckpointType.CHECKPOINT, true, false, false, false, false, false);
    CompletedCheckpoint checkpoint = new CompletedCheckpoint(new JobID(), 0, 0, 1, operatorStates, Collections.emptyList(), props, location);
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    checkpoint.registerSharedStatesAfterRestored(sharedStateRegistry);
    verify(state, times(1)).registerSharedStates(sharedStateRegistry, 0L);
    // Subsume
    checkpoint.discardOnSubsume();
    verify(state, times(1)).discardState();
    assertTrue(location.isDisposed());
    assertTrue(metadata.isDisposed());
}

Example 9 with SharedStateRegistry

use of org.apache.flink.runtime.state.SharedStateRegistry in project flink by apache.

the class StandaloneCompletedCheckpointStoreTest method testShutdownDiscardsCheckpoints.

/**
 * Tests that shutdown discards all checkpoints.
 */
@Test
public void testShutdownDiscardsCheckpoints() throws Exception {
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    CompletedCheckpointStore store = createRecoveredCompletedCheckpointStore(1);
    TestCompletedCheckpoint checkpoint = createCheckpoint(0, sharedStateRegistry);
    Collection<OperatorState> operatorStates = checkpoint.getOperatorStates().values();
    store.addCheckpointAndSubsumeOldestOne(checkpoint, new CheckpointsCleaner(), () -> {
    });
    assertEquals(1, store.getNumberOfRetainedCheckpoints());
    verifyCheckpointRegistered(operatorStates, sharedStateRegistry);
    store.shutdown(JobStatus.FINISHED, new CheckpointsCleaner());
    assertEquals(0, store.getNumberOfRetainedCheckpoints());
    assertTrue(checkpoint.isDiscarded());
    verifyCheckpointDiscarded(operatorStates);
}

Example 10 with SharedStateRegistry

use of org.apache.flink.runtime.state.SharedStateRegistry in project flink by apache.

the class SchedulerUtilsTest method testSharedStateRegistration.

/**
 * Check that a {@link SharedStateRegistryFactory} used by {@link SchedulerUtils} registers
 * shared checkpoint state on restore.
 */
@Test
public void testSharedStateRegistration() throws Exception {
    UUID backendId = UUID.randomUUID();
    StateHandleID key = new StateHandleID("k0");
    StreamStateHandle handle = new ByteStreamStateHandle("h0", new byte[] { 1, 2, 3 });
    CheckpointRecoveryFactory recoveryFactory = buildRecoveryFactory(buildCheckpoint(buildIncrementalHandle(key, handle, backendId)));
    CompletedCheckpointStore checkpointStore = SchedulerUtils.createCompletedCheckpointStore(new Configuration(), recoveryFactory, Executors.directExecutor(), log, new JobID());
    SharedStateRegistry sharedStateRegistry = checkpointStore.getSharedStateRegistry();
    IncrementalRemoteKeyedStateHandle newHandle = buildIncrementalHandle(key, new PlaceholderStreamStateHandle(1L), backendId);
    newHandle.registerSharedStates(sharedStateRegistry, 1L);
    assertSame(handle, newHandle.getSharedState().get(key));
}