Examples with ExecutionConfig org.apache.flink.api.common.ExecutionConfig used on opensource projects

Example 26 with ExecutionConfig

use of org.apache.flink.api.common.ExecutionConfig in project flink by apache.

the class StateBackendTestBase method testFoldingState.

@Test
@SuppressWarnings("unchecked,rawtypes")
public void testFoldingState() {
    try {
        CheckpointStreamFactory streamFactory = createStreamFactory();
        AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
        FoldingStateDescriptor<Integer, String> kvId = new FoldingStateDescriptor<>("id", "Fold-Initial:", new AppendingFold(), String.class);
        kvId.initializeSerializerUnlessSet(new ExecutionConfig());
        TypeSerializer<Integer> keySerializer = IntSerializer.INSTANCE;
        TypeSerializer<VoidNamespace> namespaceSerializer = VoidNamespaceSerializer.INSTANCE;
        TypeSerializer<String> valueSerializer = kvId.getSerializer();
        FoldingState<Integer, String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        @SuppressWarnings("unchecked") InternalKvState<VoidNamespace> kvState = (InternalKvState<VoidNamespace>) state;
        // some modifications to the state
        backend.setCurrentKey(1);
        assertEquals(null, state.get());
        assertEquals(null, getSerializedValue(kvState, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
        state.add(1);
        backend.setCurrentKey(2);
        assertEquals(null, state.get());
        assertEquals(null, getSerializedValue(kvState, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
        state.add(2);
        backend.setCurrentKey(1);
        assertEquals("Fold-Initial:,1", state.get());
        assertEquals("Fold-Initial:,1", getSerializedValue(kvState, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
        // draw a snapshot
        KeyGroupsStateHandle snapshot1 = FutureUtil.runIfNotDoneAndGet(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forFullCheckpoint()));
        // make some more modifications
        backend.setCurrentKey(1);
        state.clear();
        state.add(101);
        backend.setCurrentKey(2);
        state.add(102);
        backend.setCurrentKey(3);
        state.add(103);
        // draw another snapshot
        KeyGroupsStateHandle snapshot2 = FutureUtil.runIfNotDoneAndGet(backend.snapshot(682375462379L, 4, streamFactory, CheckpointOptions.forFullCheckpoint()));
        // validate the original state
        backend.setCurrentKey(1);
        assertEquals("Fold-Initial:,101", state.get());
        assertEquals("Fold-Initial:,101", getSerializedValue(kvState, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
        backend.setCurrentKey(2);
        assertEquals("Fold-Initial:,2,102", state.get());
        assertEquals("Fold-Initial:,2,102", getSerializedValue(kvState, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
        backend.setCurrentKey(3);
        assertEquals("Fold-Initial:,103", state.get());
        assertEquals("Fold-Initial:,103", getSerializedValue(kvState, 3, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
        backend.dispose();
        // restore the first snapshot and validate it
        backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot1);
        snapshot1.discardState();
        FoldingState<Integer, String> restored1 = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        @SuppressWarnings("unchecked") InternalKvState<VoidNamespace> restoredKvState1 = (InternalKvState<VoidNamespace>) restored1;
        backend.setCurrentKey(1);
        assertEquals("Fold-Initial:,1", restored1.get());
        assertEquals("Fold-Initial:,1", getSerializedValue(restoredKvState1, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
        backend.setCurrentKey(2);
        assertEquals("Fold-Initial:,2", restored1.get());
        assertEquals("Fold-Initial:,2", getSerializedValue(restoredKvState1, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
        backend.dispose();
        // restore the second snapshot and validate it
        backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot2);
        snapshot1.discardState();
        @SuppressWarnings("unchecked") FoldingState<Integer, String> restored2 = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        @SuppressWarnings("unchecked") InternalKvState<VoidNamespace> restoredKvState2 = (InternalKvState<VoidNamespace>) restored2;
        backend.setCurrentKey(1);
        assertEquals("Fold-Initial:,101", restored2.get());
        assertEquals("Fold-Initial:,101", getSerializedValue(restoredKvState2, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
        backend.setCurrentKey(2);
        assertEquals("Fold-Initial:,2,102", restored2.get());
        assertEquals("Fold-Initial:,2,102", getSerializedValue(restoredKvState2, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
        backend.setCurrentKey(3);
        assertEquals("Fold-Initial:,103", restored2.get());
        assertEquals("Fold-Initial:,103", getSerializedValue(restoredKvState2, 3, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
        backend.dispose();
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 27 with ExecutionConfig

use of org.apache.flink.api.common.ExecutionConfig in project flink by apache.

the class StateBackendTestBase method testKeyGroupSnapshotRestore.

/**
	 * This test verifies that state is correctly assigned to key groups and that restore
	 * restores the relevant key groups in the backend.
	 *
	 * <p>We have ten key groups. Initially, one backend is responsible for all ten key groups.
	 * Then we snapshot, split up the state and restore in to backends where each is responsible
	 * for five key groups. Then we make sure that the state is only available in the correct
	 * backend.
	 * @throws Exception
	 */
@Test
public void testKeyGroupSnapshotRestore() throws Exception {
    final int MAX_PARALLELISM = 10;
    CheckpointStreamFactory streamFactory = createStreamFactory();
    AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE, MAX_PARALLELISM, new KeyGroupRange(0, MAX_PARALLELISM - 1), new DummyEnvironment("test", 1, 0));
    ValueStateDescriptor<String> kvId = new ValueStateDescriptor<>("id", String.class);
    kvId.initializeSerializerUnlessSet(new ExecutionConfig());
    ValueState<String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
    // keys that fall into the first half/second half of the key groups, respectively
    int keyInFirstHalf = 17;
    int keyInSecondHalf = 42;
    Random rand = new Random(0);
    // for each key, determine into which half of the key-group space they fall
    int firstKeyHalf = KeyGroupRangeAssignment.assignKeyToParallelOperator(keyInFirstHalf, MAX_PARALLELISM, 2);
    int secondKeyHalf = KeyGroupRangeAssignment.assignKeyToParallelOperator(keyInFirstHalf, MAX_PARALLELISM, 2);
    while (firstKeyHalf == secondKeyHalf) {
        keyInSecondHalf = rand.nextInt();
        secondKeyHalf = KeyGroupRangeAssignment.assignKeyToParallelOperator(keyInSecondHalf, MAX_PARALLELISM, 2);
    }
    backend.setCurrentKey(keyInFirstHalf);
    state.update("ShouldBeInFirstHalf");
    backend.setCurrentKey(keyInSecondHalf);
    state.update("ShouldBeInSecondHalf");
    KeyGroupsStateHandle snapshot = FutureUtil.runIfNotDoneAndGet(backend.snapshot(0, 0, streamFactory, CheckpointOptions.forFullCheckpoint()));
    List<KeyGroupsStateHandle> firstHalfKeyGroupStates = StateAssignmentOperation.getKeyGroupsStateHandles(Collections.singletonList(snapshot), KeyGroupRangeAssignment.computeKeyGroupRangeForOperatorIndex(MAX_PARALLELISM, 2, 0));
    List<KeyGroupsStateHandle> secondHalfKeyGroupStates = StateAssignmentOperation.getKeyGroupsStateHandles(Collections.singletonList(snapshot), KeyGroupRangeAssignment.computeKeyGroupRangeForOperatorIndex(MAX_PARALLELISM, 2, 1));
    backend.dispose();
    // backend for the first half of the key group range
    AbstractKeyedStateBackend<Integer> firstHalfBackend = restoreKeyedBackend(IntSerializer.INSTANCE, MAX_PARALLELISM, new KeyGroupRange(0, 4), firstHalfKeyGroupStates, new DummyEnvironment("test", 1, 0));
    // backend for the second half of the key group range
    AbstractKeyedStateBackend<Integer> secondHalfBackend = restoreKeyedBackend(IntSerializer.INSTANCE, MAX_PARALLELISM, new KeyGroupRange(5, 9), secondHalfKeyGroupStates, new DummyEnvironment("test", 1, 0));
    ValueState<String> firstHalfState = firstHalfBackend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
    firstHalfBackend.setCurrentKey(keyInFirstHalf);
    assertTrue(firstHalfState.value().equals("ShouldBeInFirstHalf"));
    firstHalfBackend.setCurrentKey(keyInSecondHalf);
    assertTrue(firstHalfState.value() == null);
    ValueState<String> secondHalfState = secondHalfBackend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
    secondHalfBackend.setCurrentKey(keyInFirstHalf);
    assertTrue(secondHalfState.value() == null);
    secondHalfBackend.setCurrentKey(keyInSecondHalf);
    assertTrue(secondHalfState.value().equals("ShouldBeInSecondHalf"));
    firstHalfBackend.dispose();
    secondHalfBackend.dispose();
}

Example 28 with ExecutionConfig

use of org.apache.flink.api.common.ExecutionConfig in project flink by apache.

the class StateBackendTestBase method testValueStateNullUpdate.

/**
	 * This test verifies that passing {@code null} to {@link ValueState#update(Object)} acts
	 * the same as {@link ValueState#clear()}.
	 *
	 * @throws Exception
	 */
@Test
@SuppressWarnings("unchecked")
public void testValueStateNullUpdate() throws Exception {
    // later if null values where actually stored in the state instead of acting as clear()
    try {
        LongSerializer.INSTANCE.serialize(null, new DataOutputViewStreamWrapper(new ByteArrayOutputStream()));
        fail("Should fail with NullPointerException");
    } catch (NullPointerException e) {
    // alrighty
    }
    CheckpointStreamFactory streamFactory = createStreamFactory();
    AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
    ValueStateDescriptor<Long> kvId = new ValueStateDescriptor<>("id", LongSerializer.INSTANCE, 42L);
    kvId.initializeSerializerUnlessSet(new ExecutionConfig());
    ValueState<Long> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
    // some modifications to the state
    backend.setCurrentKey(1);
    // verify default value
    assertEquals(42L, (long) state.value());
    state.update(1L);
    assertEquals(1L, (long) state.value());
    backend.setCurrentKey(2);
    assertEquals(42L, (long) state.value());
    backend.setCurrentKey(1);
    state.clear();
    assertEquals(42L, (long) state.value());
    state.update(17L);
    assertEquals(17L, (long) state.value());
    state.update(null);
    assertEquals(42L, (long) state.value());
    // draw a snapshot
    KeyGroupsStateHandle snapshot1 = FutureUtil.runIfNotDoneAndGet(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forFullCheckpoint()));
    backend.dispose();
    backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot1);
    snapshot1.discardState();
    backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
    backend.dispose();
}

Example 29 with ExecutionConfig

use of org.apache.flink.api.common.ExecutionConfig in project flink by apache.

the class StateBackendTestBase method testRequireNonNullNamespace.

/**
	 * Previously, it was possible to create partitioned state with
	 * <code>null</code> namespace. This test makes sure that this is
	 * prohibited now.
	 */
@Test
public void testRequireNonNullNamespace() throws Exception {
    AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
    ValueStateDescriptor<IntValue> kvId = new ValueStateDescriptor<>("id", IntValue.class, new IntValue(-1));
    kvId.initializeSerializerUnlessSet(new ExecutionConfig());
    try {
        backend.getPartitionedState(null, VoidNamespaceSerializer.INSTANCE, kvId);
        fail("Did not throw expected NullPointerException");
    } catch (NullPointerException ignored) {
    }
    try {
        backend.getPartitionedState(VoidNamespace.INSTANCE, null, kvId);
        fail("Did not throw expected NullPointerException");
    } catch (NullPointerException ignored) {
    }
    try {
        backend.getPartitionedState(null, null, kvId);
        fail("Did not throw expected NullPointerException");
    } catch (NullPointerException ignored) {
    }
    backend.dispose();
}

Example 30 with ExecutionConfig

use of org.apache.flink.api.common.ExecutionConfig in project flink by apache.

the class MemoryStateBackendTest method testNumStateEntries.

@Test
@SuppressWarnings("unchecked")
public void testNumStateEntries() throws Exception {
    KeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
    ValueStateDescriptor<String> kvId = new ValueStateDescriptor<>("id", String.class, null);
    kvId.initializeSerializerUnlessSet(new ExecutionConfig());
    HeapKeyedStateBackend<Integer> heapBackend = (HeapKeyedStateBackend<Integer>) backend;
    assertEquals(0, heapBackend.numStateEntries());
    ValueState<String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
    backend.setCurrentKey(0);
    state.update("hello");
    state.update("ciao");
    assertEquals(1, heapBackend.numStateEntries());
    backend.setCurrentKey(42);
    state.update("foo");
    assertEquals(2, heapBackend.numStateEntries());
    backend.setCurrentKey(0);
    state.clear();
    assertEquals(1, heapBackend.numStateEntries());
    backend.setCurrentKey(42);
    state.clear();
    assertEquals(0, heapBackend.numStateEntries());
    backend.dispose();
}