Examples with ValueStateDescriptor org.apache.flink.api.common.state.ValueStateDescriptor used on opensource projects

Example 1 with ValueStateDescriptor

use of org.apache.flink.api.common.state.ValueStateDescriptor in project flink by apache.

the class StateBackendTestBase method testValueStateRace.

/**
	 * Tests {@link ValueState#value()} and {@link InternalKvState#getSerializedValue(byte[])}
	 * accessing the state concurrently. They should not get in the way of each
	 * other.
	 */
@Test
@SuppressWarnings("unchecked")
public void testValueStateRace() throws Exception {
    final AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
    final Integer namespace = 1;
    final ValueStateDescriptor<String> kvId = new ValueStateDescriptor<>("id", String.class);
    kvId.initializeSerializerUnlessSet(new ExecutionConfig());
    final TypeSerializer<Integer> keySerializer = IntSerializer.INSTANCE;
    final TypeSerializer<Integer> namespaceSerializer = IntSerializer.INSTANCE;
    final TypeSerializer<String> valueSerializer = kvId.getSerializer();
    final ValueState<String> state = backend.getPartitionedState(namespace, IntSerializer.INSTANCE, kvId);
    @SuppressWarnings("unchecked") final InternalKvState<Integer> kvState = (InternalKvState<Integer>) state;
    /**
		 * 1) Test that ValueState#value() before and after
		 * KvState#getSerializedValue(byte[]) return the same value.
		 */
    // set some key and namespace
    final int key1 = 1;
    backend.setCurrentKey(key1);
    kvState.setCurrentNamespace(2);
    state.update("2");
    assertEquals("2", state.value());
    // query another key and namespace
    assertNull(getSerializedValue(kvState, 3, keySerializer, namespace, IntSerializer.INSTANCE, valueSerializer));
    // the state should not have changed!
    assertEquals("2", state.value());
    // re-set values
    kvState.setCurrentNamespace(namespace);
    /**
		 * 2) Test two threads concurrently using ValueState#value() and
		 * KvState#getSerializedValue(byte[]).
		 */
    // some modifications to the state
    final int key2 = 10;
    backend.setCurrentKey(key2);
    assertNull(state.value());
    assertNull(getSerializedValue(kvState, key2, keySerializer, namespace, namespaceSerializer, valueSerializer));
    state.update("1");
    final CheckedThread getter = new CheckedThread("State getter") {

        @Override
        public void go() throws Exception {
            while (!isInterrupted()) {
                assertEquals("1", state.value());
            }
        }
    };
    final CheckedThread serializedGetter = new CheckedThread("Serialized state getter") {

        @Override
        public void go() throws Exception {
            while (!isInterrupted() && getter.isAlive()) {
                final String serializedValue = getSerializedValue(kvState, key2, keySerializer, namespace, namespaceSerializer, valueSerializer);
                assertEquals("1", serializedValue);
            }
        }
    };
    getter.start();
    serializedGetter.start();
    // run both threads for max 100ms
    Timer t = new Timer("stopper");
    t.schedule(new TimerTask() {

        @Override
        public void run() {
            getter.interrupt();
            serializedGetter.interrupt();
            this.cancel();
        }
    }, 100);
    // wait for both threads to finish
    try {
        // serializedGetter will finish if its assertion fails or if
        // getter is not alive any more
        serializedGetter.sync();
        // if serializedGetter crashed, getter will not know -> interrupt just in case
        getter.interrupt();
        getter.sync();
        // if not executed yet
        t.cancel();
    } finally {
        // clean up
        backend.dispose();
    }
}

Example 2 with ValueStateDescriptor

use of org.apache.flink.api.common.state.ValueStateDescriptor in project flink by apache.

the class StateBackendTestBase method testValueStateNullAsDefaultValue.

/**
	 * Verify that {@link ValueStateDescriptor} allows {@code null} as default.
	 */
@Test
public void testValueStateNullAsDefaultValue() throws Exception {
    AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
    ValueStateDescriptor<String> kvId = new ValueStateDescriptor<>("id", String.class, null);
    kvId.initializeSerializerUnlessSet(new ExecutionConfig());
    ValueState<String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
    backend.setCurrentKey(1);
    assertEquals(null, state.value());
    state.update("Ciao");
    assertEquals("Ciao", state.value());
    state.clear();
    assertEquals(null, state.value());
    backend.dispose();
}

Example 3 with ValueStateDescriptor

use of org.apache.flink.api.common.state.ValueStateDescriptor in project flink by apache.

the class StateBackendTestBase method testKeyGroupSnapshotRestore.

/**
	 * This test verifies that state is correctly assigned to key groups and that restore
	 * restores the relevant key groups in the backend.
	 *
	 * <p>We have ten key groups. Initially, one backend is responsible for all ten key groups.
	 * Then we snapshot, split up the state and restore in to backends where each is responsible
	 * for five key groups. Then we make sure that the state is only available in the correct
	 * backend.
	 * @throws Exception
	 */
@Test
public void testKeyGroupSnapshotRestore() throws Exception {
    final int MAX_PARALLELISM = 10;
    CheckpointStreamFactory streamFactory = createStreamFactory();
    AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE, MAX_PARALLELISM, new KeyGroupRange(0, MAX_PARALLELISM - 1), new DummyEnvironment("test", 1, 0));
    ValueStateDescriptor<String> kvId = new ValueStateDescriptor<>("id", String.class);
    kvId.initializeSerializerUnlessSet(new ExecutionConfig());
    ValueState<String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
    // keys that fall into the first half/second half of the key groups, respectively
    int keyInFirstHalf = 17;
    int keyInSecondHalf = 42;
    Random rand = new Random(0);
    // for each key, determine into which half of the key-group space they fall
    int firstKeyHalf = KeyGroupRangeAssignment.assignKeyToParallelOperator(keyInFirstHalf, MAX_PARALLELISM, 2);
    int secondKeyHalf = KeyGroupRangeAssignment.assignKeyToParallelOperator(keyInFirstHalf, MAX_PARALLELISM, 2);
    while (firstKeyHalf == secondKeyHalf) {
        keyInSecondHalf = rand.nextInt();
        secondKeyHalf = KeyGroupRangeAssignment.assignKeyToParallelOperator(keyInSecondHalf, MAX_PARALLELISM, 2);
    }
    backend.setCurrentKey(keyInFirstHalf);
    state.update("ShouldBeInFirstHalf");
    backend.setCurrentKey(keyInSecondHalf);
    state.update("ShouldBeInSecondHalf");
    KeyGroupsStateHandle snapshot = FutureUtil.runIfNotDoneAndGet(backend.snapshot(0, 0, streamFactory, CheckpointOptions.forFullCheckpoint()));
    List<KeyGroupsStateHandle> firstHalfKeyGroupStates = StateAssignmentOperation.getKeyGroupsStateHandles(Collections.singletonList(snapshot), KeyGroupRangeAssignment.computeKeyGroupRangeForOperatorIndex(MAX_PARALLELISM, 2, 0));
    List<KeyGroupsStateHandle> secondHalfKeyGroupStates = StateAssignmentOperation.getKeyGroupsStateHandles(Collections.singletonList(snapshot), KeyGroupRangeAssignment.computeKeyGroupRangeForOperatorIndex(MAX_PARALLELISM, 2, 1));
    backend.dispose();
    // backend for the first half of the key group range
    AbstractKeyedStateBackend<Integer> firstHalfBackend = restoreKeyedBackend(IntSerializer.INSTANCE, MAX_PARALLELISM, new KeyGroupRange(0, 4), firstHalfKeyGroupStates, new DummyEnvironment("test", 1, 0));
    // backend for the second half of the key group range
    AbstractKeyedStateBackend<Integer> secondHalfBackend = restoreKeyedBackend(IntSerializer.INSTANCE, MAX_PARALLELISM, new KeyGroupRange(5, 9), secondHalfKeyGroupStates, new DummyEnvironment("test", 1, 0));
    ValueState<String> firstHalfState = firstHalfBackend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
    firstHalfBackend.setCurrentKey(keyInFirstHalf);
    assertTrue(firstHalfState.value().equals("ShouldBeInFirstHalf"));
    firstHalfBackend.setCurrentKey(keyInSecondHalf);
    assertTrue(firstHalfState.value() == null);
    ValueState<String> secondHalfState = secondHalfBackend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
    secondHalfBackend.setCurrentKey(keyInFirstHalf);
    assertTrue(secondHalfState.value() == null);
    secondHalfBackend.setCurrentKey(keyInSecondHalf);
    assertTrue(secondHalfState.value().equals("ShouldBeInSecondHalf"));
    firstHalfBackend.dispose();
    secondHalfBackend.dispose();
}

Example 4 with ValueStateDescriptor

use of org.apache.flink.api.common.state.ValueStateDescriptor in project flink by apache.

the class StateBackendTestBase method testValueStateNullUpdate.

/**
	 * This test verifies that passing {@code null} to {@link ValueState#update(Object)} acts
	 * the same as {@link ValueState#clear()}.
	 *
	 * @throws Exception
	 */
@Test
@SuppressWarnings("unchecked")
public void testValueStateNullUpdate() throws Exception {
    // later if null values where actually stored in the state instead of acting as clear()
    try {
        LongSerializer.INSTANCE.serialize(null, new DataOutputViewStreamWrapper(new ByteArrayOutputStream()));
        fail("Should fail with NullPointerException");
    } catch (NullPointerException e) {
    // alrighty
    }
    CheckpointStreamFactory streamFactory = createStreamFactory();
    AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
    ValueStateDescriptor<Long> kvId = new ValueStateDescriptor<>("id", LongSerializer.INSTANCE, 42L);
    kvId.initializeSerializerUnlessSet(new ExecutionConfig());
    ValueState<Long> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
    // some modifications to the state
    backend.setCurrentKey(1);
    // verify default value
    assertEquals(42L, (long) state.value());
    state.update(1L);
    assertEquals(1L, (long) state.value());
    backend.setCurrentKey(2);
    assertEquals(42L, (long) state.value());
    backend.setCurrentKey(1);
    state.clear();
    assertEquals(42L, (long) state.value());
    state.update(17L);
    assertEquals(17L, (long) state.value());
    state.update(null);
    assertEquals(42L, (long) state.value());
    // draw a snapshot
    KeyGroupsStateHandle snapshot1 = FutureUtil.runIfNotDoneAndGet(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forFullCheckpoint()));
    backend.dispose();
    backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot1);
    snapshot1.discardState();
    backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
    backend.dispose();
}

Example 5 with ValueStateDescriptor

use of org.apache.flink.api.common.state.ValueStateDescriptor in project flink by apache.

the class StateBackendTestBase method testRequireNonNullNamespace.

/**
	 * Previously, it was possible to create partitioned state with
	 * <code>null</code> namespace. This test makes sure that this is
	 * prohibited now.
	 */
@Test
public void testRequireNonNullNamespace() throws Exception {
    AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
    ValueStateDescriptor<IntValue> kvId = new ValueStateDescriptor<>("id", IntValue.class, new IntValue(-1));
    kvId.initializeSerializerUnlessSet(new ExecutionConfig());
    try {
        backend.getPartitionedState(null, VoidNamespaceSerializer.INSTANCE, kvId);
        fail("Did not throw expected NullPointerException");
    } catch (NullPointerException ignored) {
    }
    try {
        backend.getPartitionedState(VoidNamespace.INSTANCE, null, kvId);
        fail("Did not throw expected NullPointerException");
    } catch (NullPointerException ignored) {
    }
    try {
        backend.getPartitionedState(null, null, kvId);
        fail("Did not throw expected NullPointerException");
    } catch (NullPointerException ignored) {
    }
    backend.dispose();
}