Examples with SharedStateRegistry org.apache.flink.runtime.state.SharedStateRegistry used on opensource projects

Example 11 with SharedStateRegistry

use of org.apache.flink.runtime.state.SharedStateRegistry in project flink by apache.

the class ZooKeeperCompletedCheckpointStoreITCase method testShutdownDiscardsCheckpoints.

/**
 * Tests that shutdown discards all checkpoints.
 */
@Test
public void testShutdownDiscardsCheckpoints() throws Exception {
    CuratorFramework client = ZOOKEEPER.getClient();
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    CompletedCheckpointStore store = createRecoveredCompletedCheckpointStore(1);
    TestCompletedCheckpoint checkpoint = createCheckpoint(0, sharedStateRegistry);
    store.addCheckpointAndSubsumeOldestOne(checkpoint, new CheckpointsCleaner(), () -> {
    });
    assertEquals(1, store.getNumberOfRetainedCheckpoints());
    assertNotNull(client.checkExists().forPath(CHECKPOINT_PATH + checkpointStoreUtil.checkpointIDToName(checkpoint.getCheckpointID())));
    store.shutdown(JobStatus.FINISHED, new CheckpointsCleaner());
    assertEquals(0, store.getNumberOfRetainedCheckpoints());
    assertNull(client.checkExists().forPath(CHECKPOINT_PATH + checkpointStoreUtil.checkpointIDToName(checkpoint.getCheckpointID())));
    sharedStateRegistry.close();
    assertEquals(0, createRecoveredCompletedCheckpointStore(1).getNumberOfRetainedCheckpoints());
}

Example 12 with SharedStateRegistry

use of org.apache.flink.runtime.state.SharedStateRegistry in project flink by apache.

the class ZooKeeperCompletedCheckpointStoreITCase method testConcurrentCheckpointOperations.

/**
 * FLINK-6612
 *
 * <p>Checks that a concurrent checkpoint completion won't discard a checkpoint which has been
 * recovered by a different completed checkpoint store.
 */
@Test
public void testConcurrentCheckpointOperations() throws Exception {
    final int numberOfCheckpoints = 1;
    final long waitingTimeout = 50L;
    final CompletedCheckpointStore zkCheckpointStore1 = createRecoveredCompletedCheckpointStore(numberOfCheckpoints);
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    TestCompletedCheckpoint completedCheckpoint = createCheckpoint(1, sharedStateRegistry);
    // complete the first checkpoint
    zkCheckpointStore1.addCheckpointAndSubsumeOldestOne(completedCheckpoint, new CheckpointsCleaner(), () -> {
    });
    // recover the checkpoint by a different checkpoint store
    sharedStateRegistry.close();
    sharedStateRegistry = new SharedStateRegistryImpl();
    final CompletedCheckpointStore zkCheckpointStore2 = createRecoveredCompletedCheckpointStore(numberOfCheckpoints);
    CompletedCheckpoint recoveredCheckpoint = zkCheckpointStore2.getLatestCheckpoint();
    assertTrue(recoveredCheckpoint instanceof TestCompletedCheckpoint);
    TestCompletedCheckpoint recoveredTestCheckpoint = (TestCompletedCheckpoint) recoveredCheckpoint;
    // Check that the recovered checkpoint is not yet discarded
    assertFalse(recoveredTestCheckpoint.isDiscarded());
    // complete another checkpoint --> this should remove the first checkpoint from the store
    // because the number of retained checkpoints == 1
    TestCompletedCheckpoint completedCheckpoint2 = createCheckpoint(2, sharedStateRegistry);
    zkCheckpointStore1.addCheckpointAndSubsumeOldestOne(completedCheckpoint2, new CheckpointsCleaner(), () -> {
    });
    List<CompletedCheckpoint> allCheckpoints = zkCheckpointStore1.getAllCheckpoints();
    // check that we have removed the first checkpoint from zkCompletedStore1
    assertEquals(Collections.singletonList(completedCheckpoint2), allCheckpoints);
    // lets wait a little bit to see that no discard operation will be executed
    assertFalse("The checkpoint should not have been discarded.", recoveredTestCheckpoint.awaitDiscard(waitingTimeout));
    // check that we have not discarded the first completed checkpoint
    assertFalse(recoveredTestCheckpoint.isDiscarded());
    TestCompletedCheckpoint completedCheckpoint3 = createCheckpoint(3, sharedStateRegistry);
    // this should release the last lock on completedCheckpoint and thus discard it
    zkCheckpointStore2.addCheckpointAndSubsumeOldestOne(completedCheckpoint3, new CheckpointsCleaner(), () -> {
    });
    // the checkpoint should be discarded eventually because there is no lock on it anymore
    recoveredTestCheckpoint.awaitDiscard();
}

Example 13 with SharedStateRegistry

use of org.apache.flink.runtime.state.SharedStateRegistry in project flink by apache.

the class CompletedCheckpointTest method testRegisterStatesAtRegistry.

@Test
public void testRegisterStatesAtRegistry() {
    OperatorState state = mock(OperatorState.class);
    Map<OperatorID, OperatorState> operatorStates = new HashMap<>();
    operatorStates.put(new OperatorID(), state);
    CompletedCheckpoint checkpoint = new CompletedCheckpoint(new JobID(), 0, 0, 1, operatorStates, Collections.emptyList(), CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.RETAIN_ON_FAILURE), new TestCompletedCheckpointStorageLocation());
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    checkpoint.registerSharedStatesAfterRestored(sharedStateRegistry);
    verify(state, times(1)).registerSharedStates(sharedStateRegistry, 0L);
}

Example 14 with SharedStateRegistry

use of org.apache.flink.runtime.state.SharedStateRegistry in project flink by apache.

the class ZooKeeperCompletedCheckpointStoreTest method testDiscardingSubsumedCheckpoints.

/**
 * Tests that subsumed checkpoints are discarded.
 */
@Test
public void testDiscardingSubsumedCheckpoints() throws Exception {
    final SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    final Configuration configuration = new Configuration();
    configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zooKeeperResource.getConnectString());
    final CuratorFrameworkWithUnhandledErrorListener curatorFrameworkWrapper = ZooKeeperUtils.startCuratorFramework(configuration, NoOpFatalErrorHandler.INSTANCE);
    final CompletedCheckpointStore checkpointStore = createZooKeeperCheckpointStore(curatorFrameworkWrapper.asCuratorFramework());
    try {
        final CompletedCheckpointStoreTest.TestCompletedCheckpoint checkpoint1 = CompletedCheckpointStoreTest.createCheckpoint(0, sharedStateRegistry);
        checkpointStore.addCheckpointAndSubsumeOldestOne(checkpoint1, new CheckpointsCleaner(), () -> {
        });
        assertThat(checkpointStore.getAllCheckpoints(), Matchers.contains(checkpoint1));
        final CompletedCheckpointStoreTest.TestCompletedCheckpoint checkpoint2 = CompletedCheckpointStoreTest.createCheckpoint(1, sharedStateRegistry);
        checkpointStore.addCheckpointAndSubsumeOldestOne(checkpoint2, new CheckpointsCleaner(), () -> {
        });
        final List<CompletedCheckpoint> allCheckpoints = checkpointStore.getAllCheckpoints();
        assertThat(allCheckpoints, Matchers.contains(checkpoint2));
        assertThat(allCheckpoints, Matchers.not(Matchers.contains(checkpoint1)));
        // verify that the subsumed checkpoint is discarded
        CompletedCheckpointStoreTest.verifyCheckpointDiscarded(checkpoint1);
    } finally {
        curatorFrameworkWrapper.close();
    }
}

Example 15 with SharedStateRegistry

use of org.apache.flink.runtime.state.SharedStateRegistry in project flink by apache.

the class StandaloneCompletedCheckpointStoreTest method testSuspendDiscardsCheckpoints.

/**
 * Tests that suspends discards all checkpoints (as they cannot be recovered later in standalone
 * recovery mode).
 */
@Test
public void testSuspendDiscardsCheckpoints() throws Exception {
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    CompletedCheckpointStore store = createRecoveredCompletedCheckpointStore(1);
    TestCompletedCheckpoint checkpoint = createCheckpoint(0, sharedStateRegistry);
    Collection<OperatorState> taskStates = checkpoint.getOperatorStates().values();
    store.addCheckpointAndSubsumeOldestOne(checkpoint, new CheckpointsCleaner(), () -> {
    });
    assertEquals(1, store.getNumberOfRetainedCheckpoints());
    verifyCheckpointRegistered(taskStates, sharedStateRegistry);
    store.shutdown(JobStatus.SUSPENDED, new CheckpointsCleaner());
    assertEquals(0, store.getNumberOfRetainedCheckpoints());
    assertTrue(checkpoint.isDiscarded());
    verifyCheckpointDiscarded(taskStates);
}