Examples with TestingLongStateHandleHelper org.apache.flink.runtime.persistence.TestingLongStateHandleHelper used on opensource projects

Example 1 with TestingLongStateHandleHelper

use of org.apache.flink.runtime.persistence.TestingLongStateHandleHelper in project flink by apache.

the class KubernetesStateHandleStoreITCase method testMultipleKubernetesStateHandleStores.

@Test
public void testMultipleKubernetesStateHandleStores() throws Exception {
    final Configuration configuration = kubernetesResource.getConfiguration();
    final String leaderConfigMapName = LEADER_CONFIGMAP_NAME + System.currentTimeMillis();
    final int leaderNum = 3;
    final KubernetesLeaderElector[] leaderElectors = new KubernetesLeaderElector[leaderNum];
    final FlinkKubeClient[] kubeClients = new FlinkKubeClient[leaderNum];
    final TestingLeaderCallbackHandler[] leaderCallbackHandlers = new TestingLeaderCallbackHandler[leaderNum];
    @SuppressWarnings("unchecked") final KubernetesStateHandleStore<TestingLongStateHandleHelper.LongStateHandle>[] stateHandleStores = new KubernetesStateHandleStore[leaderNum];
    try {
        for (int i = 0; i < leaderNum; i++) {
            final String lockIdentity = UUID.randomUUID().toString();
            kubeClients[i] = kubeClientFactory.fromConfiguration(configuration, "testing");
            leaderCallbackHandlers[i] = new TestingLeaderCallbackHandler(lockIdentity);
            leaderElectors[i] = kubeClients[i].createLeaderElector(new KubernetesLeaderElectionConfiguration(leaderConfigMapName, lockIdentity, configuration), leaderCallbackHandlers[i]);
            stateHandleStores[i] = new KubernetesStateHandleStore<>(kubeClients[i], leaderConfigMapName, new TestingLongStateHandleHelper(), (ignore) -> true, lockIdentity);
            leaderElectors[i].run();
        }
        // Wait for the leader
        final String lockIdentity = TestingLeaderCallbackHandler.waitUntilNewLeaderAppears(TIMEOUT);
        Long expectedState = null;
        for (int i = 0; i < leaderNum; i++) {
            // leader
            if (leaderCallbackHandlers[i].getLockIdentity().equals(lockIdentity)) {
                expectedState = (long) i;
            }
            stateHandleStores[i].addAndLock(KEY, new TestingLongStateHandleHelper.LongStateHandle(i));
        }
        // Only the leader could add successfully
        assertThat(expectedState, is(notNullValue()));
        assertThat(stateHandleStores[0].getAllAndLock().size(), is(1));
        assertThat(stateHandleStores[0].getAllAndLock().get(0).f0.retrieveState().getValue(), is(expectedState));
        assertThat(stateHandleStores[0].getAllAndLock().get(0).f1, is(KEY));
    } finally {
        TestingLongStateHandleHelper.clearGlobalState();
        // Cleanup the resources
        for (int i = 0; i < leaderNum; i++) {
            if (leaderElectors[i] != null) {
                leaderElectors[i].stop();
            }
            if (kubeClients[i] != null) {
                kubeClients[i].close();
            }
        }
        kubernetesResource.getFlinkKubeClient().deleteConfigMap(leaderConfigMapName).get();
    }
}

Example 2 with TestingLongStateHandleHelper

use of org.apache.flink.runtime.persistence.TestingLongStateHandleHelper in project flink by apache.

the class ZooKeeperStateHandleStoreTest method testReleaseAll.

/**
 * FLINK-6612
 *
 * <p>Tests that we can release all locked state handles in the ZooKeeperStateHandleStore
 */
@Test
public void testReleaseAll() throws Exception {
    final TestingLongStateHandleHelper longStateStorage = new TestingLongStateHandleHelper();
    ZooKeeperStateHandleStore<TestingLongStateHandleHelper.LongStateHandle> zkStore = new ZooKeeperStateHandleStore<>(ZOOKEEPER.getClient(), longStateStorage);
    final Collection<String> paths = Arrays.asList("/state1", "/state2", "/state3");
    for (String path : paths) {
        zkStore.addAndLock(path, new TestingLongStateHandleHelper.LongStateHandle(42L));
    }
    for (String path : paths) {
        Stat stat = ZOOKEEPER.getClient().checkExists().forPath(zkStore.getLockPath(path));
        assertNotNull("Expecte and existing lock.", stat);
    }
    zkStore.releaseAll();
    for (String path : paths) {
        Stat stat = ZOOKEEPER.getClient().checkExists().forPath(path);
        assertEquals(0, stat.getNumChildren());
    }
    zkStore.releaseAndTryRemoveAll();
    Stat stat = ZOOKEEPER.getClient().checkExists().forPath("/");
    assertEquals(0, stat.getNumChildren());
}

Example 3 with TestingLongStateHandleHelper

use of org.apache.flink.runtime.persistence.TestingLongStateHandleHelper in project flink by apache.

the class ZooKeeperStateHandleStoreTest method testReplaceDiscardStateHandleAfterFailure.

/**
 * Tests that the replace state handle is discarded if ZooKeeper setData fails.
 */
@Test
public void testReplaceDiscardStateHandleAfterFailure() throws Exception {
    // Setup
    final TestingLongStateHandleHelper stateHandleProvider = new TestingLongStateHandleHelper();
    CuratorFramework client = spy(ZOOKEEPER.getClient());
    when(client.setData()).thenThrow(new RuntimeException("Expected test Exception."));
    ZooKeeperStateHandleStore<TestingLongStateHandleHelper.LongStateHandle> store = new ZooKeeperStateHandleStore<>(client, stateHandleProvider);
    // Config
    final String pathInZooKeeper = "/testReplaceDiscardStateHandleAfterFailure";
    final long initialState = 30968470898L;
    final long replaceState = 88383776661L;
    // Test
    store.addAndLock(pathInZooKeeper, new TestingLongStateHandleHelper.LongStateHandle(initialState));
    try {
        store.replace(pathInZooKeeper, IntegerResourceVersion.valueOf(0), new TestingLongStateHandleHelper.LongStateHandle(replaceState));
        fail("Did not throw expected exception");
    } catch (Exception ignored) {
    }
    // Verify
    // State handle created and discarded
    assertEquals(2, TestingLongStateHandleHelper.getGlobalStorageSize());
    assertEquals(initialState, TestingLongStateHandleHelper.getStateHandleValueByIndex(0));
    assertEquals(replaceState, TestingLongStateHandleHelper.getStateHandleValueByIndex(1));
    assertThat(TestingLongStateHandleHelper.getDiscardCallCountForStateHandleByIndex(0), is(0));
    assertThat(TestingLongStateHandleHelper.getDiscardCallCountForStateHandleByIndex(1), is(0));
    // Initial value
    @SuppressWarnings("unchecked") final long actual = ((RetrievableStateHandle<TestingLongStateHandleHelper.LongStateHandle>) InstantiationUtil.deserializeObject(ZOOKEEPER.getClient().getData().forPath(pathInZooKeeper), ClassLoader.getSystemClassLoader())).retrieveState().getValue();
    assertEquals(initialState, actual);
}

Example 4 with TestingLongStateHandleHelper

use of org.apache.flink.runtime.persistence.TestingLongStateHandleHelper in project flink by apache.

the class ZooKeeperStateHandleStoreTest method testLockCleanupWhenGetAndLockFails.

/**
 * FLINK-6612
 *
 * <p>Tests that getAndLock removes a created lock if the RetrievableStateHandle cannot be
 * retrieved (e.g. deserialization problem).
 */
@Test
public void testLockCleanupWhenGetAndLockFails() throws Exception {
    final TestingLongStateHandleHelper longStateStorage = new TestingLongStateHandleHelper();
    ZooKeeperStateHandleStore<TestingLongStateHandleHelper.LongStateHandle> zkStore1 = new ZooKeeperStateHandleStore<>(ZOOKEEPER.getClient(), longStateStorage);
    ZooKeeperStateHandleStore<TestingLongStateHandleHelper.LongStateHandle> zkStore2 = new ZooKeeperStateHandleStore<>(ZOOKEEPER.getClient(), longStateStorage);
    final String path = "/state";
    zkStore1.addAndLock(path, new TestingLongStateHandleHelper.LongStateHandle(42L));
    final byte[] corruptedData = { 1, 2 };
    // corrupt the data
    ZOOKEEPER.getClient().setData().forPath(path, corruptedData);
    try {
        zkStore2.getAndLock(path);
        fail("Should fail because we cannot deserialize the node's data");
    } catch (IOException ignored) {
    // expected to fail
    }
    // check that there is no lock node left
    String lockNodePath = zkStore2.getLockPath(path);
    Stat stat = ZOOKEEPER.getClient().checkExists().forPath(lockNodePath);
    // zkStore2 should not have created a lock node
    assertNull("zkStore2 should not have created a lock node.", stat);
    Collection<String> children = ZOOKEEPER.getClient().getChildren().forPath(path);
    // there should be exactly one lock node from zkStore1
    assertEquals(1, children.size());
    zkStore1.releaseAndTryRemove(path);
    stat = ZOOKEEPER.getClient().checkExists().forPath(path);
    assertNull("The state node should have been removed.", stat);
}

Example 5 with TestingLongStateHandleHelper

use of org.apache.flink.runtime.persistence.TestingLongStateHandleHelper in project flink by apache.

the class ZooKeeperStateHandleStoreTest method testFailingAddWithStateDiscardTriggeredFor.

private static void testFailingAddWithStateDiscardTriggeredFor(Exception actualException, Class<? extends Throwable> expectedException) {
    final TestingLongStateHandleHelper stateHandleProvider = new TestingLongStateHandleHelper();
    ZooKeeperStateHandleStore<TestingLongStateHandleHelper.LongStateHandle> store = new ZooKeeperStateHandleStore<TestingLongStateHandleHelper.LongStateHandle>(ZOOKEEPER.getClient(), stateHandleProvider) {

        @Override
        protected void writeStoreHandleTransactionally(String path, byte[] serializedStoreHandle) throws Exception {
            throw actualException;
        }
    };
    // Config
    final String pathInZooKeeper = "/testAddDiscardStateHandleAfterFailure-" + expectedException.getSimpleName();
    final long state = 81282227L;
    try {
        // Test
        store.addAndLock(pathInZooKeeper, new TestingLongStateHandleHelper.LongStateHandle(state));
        fail(expectedException.getSimpleName() + " should have been thrown.");
    } catch (Exception ex) {
        assertThat(ex, IsInstanceOf.instanceOf(expectedException));
    }
    // State handle created and discarded
    assertEquals(1, TestingLongStateHandleHelper.getGlobalStorageSize());
    assertEquals(state, TestingLongStateHandleHelper.getStateHandleValueByIndex(0));
    assertEquals(1, TestingLongStateHandleHelper.getDiscardCallCountForStateHandleByIndex(0));
}