Examples with StringResourceVersion org.apache.flink.runtime.persistence.StringResourceVersion used on opensource projects

Example 1 with StringResourceVersion

use of org.apache.flink.runtime.persistence.StringResourceVersion in project flink by apache.

the class KubernetesStateHandleStoreTest method testReplaceFailedWithPossiblyInconsistentState.

@Test
public void testReplaceFailedWithPossiblyInconsistentState() throws Exception {
    final PossibleInconsistentStateException updateException = new PossibleInconsistentStateException();
    new Context() {

        {
            runTest(() -> {
                leaderCallbackGrantLeadership();
                final KubernetesStateHandleStore<TestingLongStateHandleHelper.LongStateHandle> store = new KubernetesStateHandleStore<>(flinkKubeClient, LEADER_CONFIGMAP_NAME, longStateStorage, filter, LOCK_IDENTITY);
                store.addAndLock(key, state);
                final FlinkKubeClient anotherFlinkKubeClient = createFlinkKubeClientBuilder().setCheckAndUpdateConfigMapFunction((configMapName, function) -> FutureUtils.completedExceptionally(updateException)).build();
                final KubernetesStateHandleStore<TestingLongStateHandleHelper.LongStateHandle> anotherStore = new KubernetesStateHandleStore<>(anotherFlinkKubeClient, LEADER_CONFIGMAP_NAME, longStateStorage, filter, LOCK_IDENTITY);
                final StringResourceVersion resourceVersion = anotherStore.exists(key);
                assertThat(resourceVersion.isExisting(), is(true));
                try {
                    anotherStore.replace(key, resourceVersion, new TestingLongStateHandleHelper.LongStateHandle(23456L));
                    fail("An exception having a PossibleInconsistentStateException as its cause should have been thrown.");
                } catch (Exception ex) {
                    assertThat(ex, is(updateException));
                }
                assertThat(anotherStore.getAllAndLock().size(), is(1));
                // The state does not change
                assertThat(anotherStore.getAndLock(key).retrieveState(), is(state));
                assertThat(TestingLongStateHandleHelper.getGlobalStorageSize(), is(2));
                // no state was discarded
                assertThat(TestingLongStateHandleHelper.getDiscardCallCountForStateHandleByIndex(0), is(0));
                assertThat(TestingLongStateHandleHelper.getDiscardCallCountForStateHandleByIndex(1), is(0));
            });
        }
    };
}

Example 2 with StringResourceVersion

use of org.apache.flink.runtime.persistence.StringResourceVersion in project flink by apache.

the class KubernetesStateHandleStoreTest method testReplaceFailedAndDiscardState.

@Test
public void testReplaceFailedAndDiscardState() throws Exception {
    final FlinkRuntimeException updateException = new FlinkRuntimeException("Failed to update");
    new Context() {

        {
            runTest(() -> {
                leaderCallbackGrantLeadership();
                final KubernetesStateHandleStore<TestingLongStateHandleHelper.LongStateHandle> store = new KubernetesStateHandleStore<>(flinkKubeClient, LEADER_CONFIGMAP_NAME, longStateStorage, filter, LOCK_IDENTITY);
                store.addAndLock(key, state);
                final FlinkKubeClient anotherFlinkKubeClient = createFlinkKubeClientBuilder().setCheckAndUpdateConfigMapFunction((configMapName, function) -> {
                    throw updateException;
                }).build();
                final KubernetesStateHandleStore<TestingLongStateHandleHelper.LongStateHandle> anotherStore = new KubernetesStateHandleStore<>(anotherFlinkKubeClient, LEADER_CONFIGMAP_NAME, longStateStorage, filter, LOCK_IDENTITY);
                final TestingLongStateHandleHelper.LongStateHandle newState = new TestingLongStateHandleHelper.LongStateHandle(23456L);
                final StringResourceVersion resourceVersion = anotherStore.exists(key);
                assertThat(resourceVersion.isExisting(), is(true));
                try {
                    anotherStore.replace(key, resourceVersion, newState);
                    fail("We should get an exception when kube client failed to update.");
                } catch (Exception ex) {
                    assertThat(ex, FlinkMatchers.containsCause(updateException));
                }
                assertThat(anotherStore.getAllAndLock().size(), is(1));
                // The state do not change
                assertThat(anotherStore.getAndLock(key).retrieveState(), is(state));
                assertThat(TestingLongStateHandleHelper.getGlobalStorageSize(), is(2));
                assertThat(TestingLongStateHandleHelper.getDiscardCallCountForStateHandleByIndex(0), is(0));
                assertThat(TestingLongStateHandleHelper.getDiscardCallCountForStateHandleByIndex(1), is(1));
            });
        }
    };
}

Example 3 with StringResourceVersion

use of org.apache.flink.runtime.persistence.StringResourceVersion in project flink by apache.

the class KubernetesStateHandleStoreTest method testReplace.

@Test
public void testReplace() throws Exception {
    new Context() {

        {
            runTest(() -> {
                leaderCallbackGrantLeadership();
                final KubernetesStateHandleStore<TestingLongStateHandleHelper.LongStateHandle> store = new KubernetesStateHandleStore<>(flinkKubeClient, LEADER_CONFIGMAP_NAME, longStateStorage, filter, LOCK_IDENTITY);
                store.addAndLock(key, state);
                final TestingLongStateHandleHelper.LongStateHandle newState = new TestingLongStateHandleHelper.LongStateHandle(23456L);
                final StringResourceVersion resourceVersion = store.exists(key);
                store.replace(key, resourceVersion, newState);
                assertThat(store.getAllAndLock().size(), is(1));
                assertThat(store.getAndLock(key).retrieveState(), is(newState));
            });
        }
    };
}

Example 4 with StringResourceVersion

use of org.apache.flink.runtime.persistence.StringResourceVersion in project flink by apache.

the class KubernetesStateHandleStoreTest method testReplaceWithNoLeadershipAndDiscardState.

@Test
public void testReplaceWithNoLeadershipAndDiscardState() throws Exception {
    new Context() {

        {
            runTest(() -> {
                leaderCallbackGrantLeadership();
                final KubernetesStateHandleStore<TestingLongStateHandleHelper.LongStateHandle> store = new KubernetesStateHandleStore<>(flinkKubeClient, LEADER_CONFIGMAP_NAME, longStateStorage, filter, LOCK_IDENTITY);
                final TestingLongStateHandleHelper.LongStateHandle newState = new TestingLongStateHandleHelper.LongStateHandle(23456L);
                store.addAndLock(key, state);
                // Lost leadership
                getLeaderCallback().notLeader();
                electionEventHandler.waitForRevokeLeader(TIMEOUT);
                getLeaderConfigMap().getAnnotations().remove(KubernetesLeaderElector.LEADER_ANNOTATION_KEY);
                final StringResourceVersion resourceVersion = store.exists(key);
                store.replace(key, resourceVersion, newState);
                assertThat(store.getAllAndLock().size(), is(1));
                // The state do not change
                assertThat(store.getAndLock(key).retrieveState(), is(state));
                assertThat(TestingLongStateHandleHelper.getGlobalStorageSize(), is(2));
                assertThat(TestingLongStateHandleHelper.getDiscardCallCountForStateHandleByIndex(0), is(0));
                assertThat(TestingLongStateHandleHelper.getDiscardCallCountForStateHandleByIndex(1), is(1));
            });
        }
    };
}

Example 5 with StringResourceVersion

use of org.apache.flink.runtime.persistence.StringResourceVersion in project flink by apache.

the class KubernetesStateHandleStore method replace.

/**
 * Replaces a state handle in ConfigMap and discards the old state handle. Wo do not lock
 * resource version and then replace in Kubernetes. Since the ConfigMap is periodically updated
 * by leader, the resource version changes very fast. We use a "check-existence and update"
 * transactional operation instead.
 *
 * @param key Key in ConfigMap
 * @param resourceVersion resource version when checking existence via {@link #exists}.
 * @param state State to be added
 * @throws NotExistException if the name does not exist
 * @throws PossibleInconsistentStateException if a failure occurred during the update operation.
 *     It's unclear whether the operation actually succeeded or not. No state was discarded. The
 *     method's caller should handle this case properly.
 * @throws Exception if persisting state or writing state handle failed
 */
@Override
public void replace(String key, StringResourceVersion resourceVersion, T state) throws Exception {
    checkNotNull(key, "Key in ConfigMap.");
    checkNotNull(state, "State.");
    final RetrievableStateHandle<T> oldStateHandle = getAndLock(key);
    final RetrievableStateHandle<T> newStateHandle = storage.store(state);
    final byte[] serializedStateHandle = serializeOrDiscard(newStateHandle);
    // initialize flags to serve the failure case
    boolean discardOldState = false;
    boolean discardNewState = true;
    try {
        boolean success = kubeClient.checkAndUpdateConfigMap(configMapName, c -> {
            if (isValidOperation(c)) {
                // Check the existence
                if (c.getData().containsKey(key)) {
                    c.getData().put(key, encodeStateHandle(serializedStateHandle));
                } else {
                    throw new CompletionException(getKeyNotExistException(key));
                }
                return Optional.of(c);
            }
            return Optional.empty();
        }).get();
        // swap subject for deletion in case of success
        discardOldState = success;
        discardNewState = !success;
    } catch (Exception ex) {
        final Optional<PossibleInconsistentStateException> possibleInconsistentStateException = ExceptionUtils.findThrowable(ex, PossibleInconsistentStateException.class);
        if (possibleInconsistentStateException.isPresent()) {
            // it's unclear whether the state handle metadata was written to the ConfigMap -
            // hence, we don't discard any data
            discardNewState = false;
            throw possibleInconsistentStateException.get();
        }
        throw ExceptionUtils.findThrowable(ex, NotExistException.class).orElseThrow(() -> ex);
    } finally {
        if (discardNewState) {
            newStateHandle.discardState();
        }
        if (discardOldState) {
            oldStateHandle.discardState();
        }
    }
}