Examples with SharedStateRegistryImpl org.apache.flink.runtime.state.SharedStateRegistryImpl used on opensource projects

Example 1 with SharedStateRegistryImpl

use of org.apache.flink.runtime.state.SharedStateRegistryImpl in project flink by apache.

the class DefaultCompletedCheckpointStoreTest method testShutdownFailsAnyFutureCallsToAddCheckpoint.

@Test
public void testShutdownFailsAnyFutureCallsToAddCheckpoint() throws Exception {
    final CheckpointsCleaner checkpointsCleaner = new CheckpointsCleaner();
    for (JobStatus status : JobStatus.values()) {
        final CompletedCheckpointStore completedCheckpointStore = createCompletedCheckpointStore(builder.build());
        completedCheckpointStore.shutdown(status, checkpointsCleaner);
        assertThrows(IllegalStateException.class, () -> completedCheckpointStore.addCheckpointAndSubsumeOldestOne(CompletedCheckpointStoreTest.createCheckpoint(0L, new SharedStateRegistryImpl()), checkpointsCleaner, () -> {
        // No-op.
        }));
    }
}

Example 2 with SharedStateRegistryImpl

use of org.apache.flink.runtime.state.SharedStateRegistryImpl in project flink by apache.

the class CompletedCheckpointStoreTest method testGetAllCheckpoints.

/**
 * Tests that all added checkpoints are returned.
 */
@Test
public void testGetAllCheckpoints() throws Exception {
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    CompletedCheckpointStore checkpoints = createRecoveredCompletedCheckpointStore(4);
    TestCompletedCheckpoint[] expected = new TestCompletedCheckpoint[] { createCheckpoint(0, sharedStateRegistry), createCheckpoint(1, sharedStateRegistry), createCheckpoint(2, sharedStateRegistry), createCheckpoint(3, sharedStateRegistry) };
    for (TestCompletedCheckpoint checkpoint : expected) {
        checkpoints.addCheckpointAndSubsumeOldestOne(checkpoint, new CheckpointsCleaner(), () -> {
        });
    }
    List<CompletedCheckpoint> actual = checkpoints.getAllCheckpoints();
    assertEquals(expected.length, actual.size());
    for (int i = 0; i < expected.length; i++) {
        assertEquals(expected[i], actual.get(i));
    }
}

Example 3 with SharedStateRegistryImpl

use of org.apache.flink.runtime.state.SharedStateRegistryImpl in project flink by apache.

the class CompletedCheckpointStoreTest method testDiscardAllCheckpoints.

/**
 * Tests that all checkpoints are discarded (using the correct class loader).
 */
@Test
public void testDiscardAllCheckpoints() throws Exception {
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    CompletedCheckpointStore checkpoints = createRecoveredCompletedCheckpointStore(4);
    TestCompletedCheckpoint[] expected = new TestCompletedCheckpoint[] { createCheckpoint(0, sharedStateRegistry), createCheckpoint(1, sharedStateRegistry), createCheckpoint(2, sharedStateRegistry), createCheckpoint(3, sharedStateRegistry) };
    for (TestCompletedCheckpoint checkpoint : expected) {
        checkpoints.addCheckpointAndSubsumeOldestOne(checkpoint, new CheckpointsCleaner(), () -> {
        });
    }
    checkpoints.shutdown(JobStatus.FINISHED, new CheckpointsCleaner());
    // Empty state
    assertNull(checkpoints.getLatestCheckpoint());
    assertEquals(0, checkpoints.getAllCheckpoints().size());
    assertEquals(0, checkpoints.getNumberOfRetainedCheckpoints());
    // All have been discarded
    for (TestCompletedCheckpoint checkpoint : expected) {
        // The ZooKeeper implementation discards asynchronously
        checkpoint.awaitDiscard();
        assertTrue(checkpoint.isDiscarded());
    }
}

Example 4 with SharedStateRegistryImpl

use of org.apache.flink.runtime.state.SharedStateRegistryImpl in project flink by apache.

the class ZooKeeperCompletedCheckpointStoreTest method testDiscardingCheckpointsAtShutDown.

/**
 * Tests that checkpoints are discarded when the completed checkpoint store is shut down with a
 * globally terminal state.
 */
@Test
public void testDiscardingCheckpointsAtShutDown() throws Exception {
    final SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    final Configuration configuration = new Configuration();
    configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zooKeeperResource.getConnectString());
    final CuratorFrameworkWithUnhandledErrorListener curatorFrameworkWrapper = ZooKeeperUtils.startCuratorFramework(configuration, NoOpFatalErrorHandler.INSTANCE);
    final CompletedCheckpointStore checkpointStore = createZooKeeperCheckpointStore(curatorFrameworkWrapper.asCuratorFramework());
    try {
        final CompletedCheckpointStoreTest.TestCompletedCheckpoint checkpoint1 = CompletedCheckpointStoreTest.createCheckpoint(0, sharedStateRegistry);
        checkpointStore.addCheckpointAndSubsumeOldestOne(checkpoint1, new CheckpointsCleaner(), () -> {
        });
        assertThat(checkpointStore.getAllCheckpoints(), Matchers.contains(checkpoint1));
        checkpointStore.shutdown(JobStatus.FINISHED, new CheckpointsCleaner());
        // verify that the checkpoint is discarded
        CompletedCheckpointStoreTest.verifyCheckpointDiscarded(checkpoint1);
    } finally {
        curatorFrameworkWrapper.close();
    }
}

Example 5 with SharedStateRegistryImpl

use of org.apache.flink.runtime.state.SharedStateRegistryImpl in project flink by apache.

the class ZooKeeperCompletedCheckpointStoreITCase method testRecover.

// ---------------------------------------------------------------------------------------------
/**
 * Tests that older checkpoints are not cleaned up right away when recovering. Only after
 * another checkpoint has been completed the old checkpoints exceeding the number of checkpoints
 * to retain will be removed.
 */
@Test
public void testRecover() throws Exception {
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    CompletedCheckpointStore checkpoints = createRecoveredCompletedCheckpointStore(3);
    TestCompletedCheckpoint[] expected = new TestCompletedCheckpoint[] { createCheckpoint(0, sharedStateRegistry), createCheckpoint(1, sharedStateRegistry), createCheckpoint(2, sharedStateRegistry) };
    // Add multiple checkpoints
    checkpoints.addCheckpointAndSubsumeOldestOne(expected[0], new CheckpointsCleaner(), () -> {
    });
    checkpoints.addCheckpointAndSubsumeOldestOne(expected[1], new CheckpointsCleaner(), () -> {
    });
    checkpoints.addCheckpointAndSubsumeOldestOne(expected[2], new CheckpointsCleaner(), () -> {
    });
    verifyCheckpointRegistered(expected[0].getOperatorStates().values(), sharedStateRegistry);
    verifyCheckpointRegistered(expected[1].getOperatorStates().values(), sharedStateRegistry);
    verifyCheckpointRegistered(expected[2].getOperatorStates().values(), sharedStateRegistry);
    // All three should be in ZK
    assertEquals(3, ZOOKEEPER.getClient().getChildren().forPath(CHECKPOINT_PATH).size());
    assertEquals(3, checkpoints.getNumberOfRetainedCheckpoints());
    // Recover
    sharedStateRegistry.close();
    sharedStateRegistry = new SharedStateRegistryImpl();
    assertEquals(3, ZOOKEEPER.getClient().getChildren().forPath(CHECKPOINT_PATH).size());
    assertEquals(3, checkpoints.getNumberOfRetainedCheckpoints());
    assertEquals(expected[2], checkpoints.getLatestCheckpoint());
    List<CompletedCheckpoint> expectedCheckpoints = new ArrayList<>(3);
    expectedCheckpoints.add(expected[1]);
    expectedCheckpoints.add(expected[2]);
    expectedCheckpoints.add(createCheckpoint(3, sharedStateRegistry));
    checkpoints.addCheckpointAndSubsumeOldestOne(expectedCheckpoints.get(2), new CheckpointsCleaner(), () -> {
    });
    List<CompletedCheckpoint> actualCheckpoints = checkpoints.getAllCheckpoints();
    assertEquals(expectedCheckpoints, actualCheckpoints);
    for (CompletedCheckpoint actualCheckpoint : actualCheckpoints) {
        verifyCheckpointRegistered(actualCheckpoint.getOperatorStates().values(), sharedStateRegistry);
    }
}