Examples with ExecutionConfig org.apache.flink.api.common.ExecutionConfig used on opensource projects

Example 21 with ExecutionConfig

use of org.apache.flink.api.common.ExecutionConfig in project flink by apache.

the class HeapListStateTest method testMerging.

@Test
public void testMerging() throws Exception {
    final ListStateDescriptor<Long> stateDescr = new ListStateDescriptor<>("my-state", Long.class);
    stateDescr.initializeSerializerUnlessSet(new ExecutionConfig());
    final Integer namespace1 = 1;
    final Integer namespace2 = 2;
    final Integer namespace3 = 3;
    final Set<Long> expectedResult = new HashSet<>(asList(11L, 22L, 33L, 44L, 55L));
    final HeapKeyedStateBackend<String> keyedBackend = createKeyedBackend();
    try {
        InternalListState<Integer, Long> state = keyedBackend.createListState(IntSerializer.INSTANCE, stateDescr);
        // populate the different namespaces
        //  - abc spreads the values over three namespaces
        //  - def spreads teh values over two namespaces (one empty)
        //  - ghi is empty
        //  - jkl has all elements already in the target namespace
        //  - mno has all elements already in one source namespace
        keyedBackend.setCurrentKey("abc");
        state.setCurrentNamespace(namespace1);
        state.add(33L);
        state.add(55L);
        state.setCurrentNamespace(namespace2);
        state.add(22L);
        state.add(11L);
        state.setCurrentNamespace(namespace3);
        state.add(44L);
        keyedBackend.setCurrentKey("def");
        state.setCurrentNamespace(namespace1);
        state.add(11L);
        state.add(44L);
        state.setCurrentNamespace(namespace3);
        state.add(22L);
        state.add(55L);
        state.add(33L);
        keyedBackend.setCurrentKey("jkl");
        state.setCurrentNamespace(namespace1);
        state.add(11L);
        state.add(22L);
        state.add(33L);
        state.add(44L);
        state.add(55L);
        keyedBackend.setCurrentKey("mno");
        state.setCurrentNamespace(namespace3);
        state.add(11L);
        state.add(22L);
        state.add(33L);
        state.add(44L);
        state.add(55L);
        keyedBackend.setCurrentKey("abc");
        state.mergeNamespaces(namespace1, asList(namespace2, namespace3));
        state.setCurrentNamespace(namespace1);
        validateResult(state.get(), expectedResult);
        keyedBackend.setCurrentKey("def");
        state.mergeNamespaces(namespace1, asList(namespace2, namespace3));
        state.setCurrentNamespace(namespace1);
        validateResult(state.get(), expectedResult);
        keyedBackend.setCurrentKey("ghi");
        state.mergeNamespaces(namespace1, asList(namespace2, namespace3));
        state.setCurrentNamespace(namespace1);
        assertNull(state.get());
        keyedBackend.setCurrentKey("jkl");
        state.mergeNamespaces(namespace1, asList(namespace2, namespace3));
        state.setCurrentNamespace(namespace1);
        validateResult(state.get(), expectedResult);
        keyedBackend.setCurrentKey("mno");
        state.mergeNamespaces(namespace1, asList(namespace2, namespace3));
        state.setCurrentNamespace(namespace1);
        validateResult(state.get(), expectedResult);
        // make sure all lists / maps are cleared
        keyedBackend.setCurrentKey("abc");
        state.setCurrentNamespace(namespace1);
        state.clear();
        keyedBackend.setCurrentKey("def");
        state.setCurrentNamespace(namespace1);
        state.clear();
        keyedBackend.setCurrentKey("ghi");
        state.setCurrentNamespace(namespace1);
        state.clear();
        keyedBackend.setCurrentKey("jkl");
        state.setCurrentNamespace(namespace1);
        state.clear();
        keyedBackend.setCurrentKey("mno");
        state.setCurrentNamespace(namespace1);
        state.clear();
        StateTable<String, Integer, ArrayList<Long>> stateTable = ((HeapListState<String, Integer, Long>) state).stateTable;
        assertTrue(stateTable.isEmpty());
    } finally {
        keyedBackend.close();
        keyedBackend.dispose();
    }
}

Example 22 with ExecutionConfig

use of org.apache.flink.api.common.ExecutionConfig in project flink by apache.

the class HeapReducingStateTest method testMerging.

@Test
public void testMerging() throws Exception {
    final ReducingStateDescriptor<Long> stateDescr = new ReducingStateDescriptor<>("my-state", new AddingFunction(), Long.class);
    stateDescr.initializeSerializerUnlessSet(new ExecutionConfig());
    final Integer namespace1 = 1;
    final Integer namespace2 = 2;
    final Integer namespace3 = 3;
    final Long expectedResult = 165L;
    final HeapKeyedStateBackend<String> keyedBackend = createKeyedBackend();
    try {
        final InternalReducingState<Integer, Long> state = keyedBackend.createReducingState(IntSerializer.INSTANCE, stateDescr);
        // populate the different namespaces
        //  - abc spreads the values over three namespaces
        //  - def spreads teh values over two namespaces (one empty)
        //  - ghi is empty
        //  - jkl has all elements already in the target namespace
        //  - mno has all elements already in one source namespace
        keyedBackend.setCurrentKey("abc");
        state.setCurrentNamespace(namespace1);
        state.add(33L);
        state.add(55L);
        state.setCurrentNamespace(namespace2);
        state.add(22L);
        state.add(11L);
        state.setCurrentNamespace(namespace3);
        state.add(44L);
        keyedBackend.setCurrentKey("def");
        state.setCurrentNamespace(namespace1);
        state.add(11L);
        state.add(44L);
        state.setCurrentNamespace(namespace3);
        state.add(22L);
        state.add(55L);
        state.add(33L);
        keyedBackend.setCurrentKey("jkl");
        state.setCurrentNamespace(namespace1);
        state.add(11L);
        state.add(22L);
        state.add(33L);
        state.add(44L);
        state.add(55L);
        keyedBackend.setCurrentKey("mno");
        state.setCurrentNamespace(namespace3);
        state.add(11L);
        state.add(22L);
        state.add(33L);
        state.add(44L);
        state.add(55L);
        keyedBackend.setCurrentKey("abc");
        state.mergeNamespaces(namespace1, asList(namespace2, namespace3));
        state.setCurrentNamespace(namespace1);
        assertEquals(expectedResult, state.get());
        keyedBackend.setCurrentKey("def");
        state.mergeNamespaces(namespace1, asList(namespace2, namespace3));
        state.setCurrentNamespace(namespace1);
        assertEquals(expectedResult, state.get());
        keyedBackend.setCurrentKey("ghi");
        state.mergeNamespaces(namespace1, asList(namespace2, namespace3));
        state.setCurrentNamespace(namespace1);
        assertNull(state.get());
        keyedBackend.setCurrentKey("jkl");
        state.mergeNamespaces(namespace1, asList(namespace2, namespace3));
        state.setCurrentNamespace(namespace1);
        assertEquals(expectedResult, state.get());
        keyedBackend.setCurrentKey("mno");
        state.mergeNamespaces(namespace1, asList(namespace2, namespace3));
        state.setCurrentNamespace(namespace1);
        assertEquals(expectedResult, state.get());
        // make sure all lists / maps are cleared
        keyedBackend.setCurrentKey("abc");
        state.setCurrentNamespace(namespace1);
        state.clear();
        keyedBackend.setCurrentKey("def");
        state.setCurrentNamespace(namespace1);
        state.clear();
        keyedBackend.setCurrentKey("ghi");
        state.setCurrentNamespace(namespace1);
        state.clear();
        keyedBackend.setCurrentKey("jkl");
        state.setCurrentNamespace(namespace1);
        state.clear();
        keyedBackend.setCurrentKey("mno");
        state.setCurrentNamespace(namespace1);
        state.clear();
        StateTable<String, Integer, Long> stateTable = ((HeapReducingState<String, Integer, Long>) state).stateTable;
        assertTrue(stateTable.isEmpty());
    } finally {
        keyedBackend.close();
        keyedBackend.dispose();
    }
}

Example 23 with ExecutionConfig

use of org.apache.flink.api.common.ExecutionConfig in project flink by apache.

the class StateBackendTestBase method testValueStateRace.

/**
	 * Tests {@link ValueState#value()} and {@link InternalKvState#getSerializedValue(byte[])}
	 * accessing the state concurrently. They should not get in the way of each
	 * other.
	 */
@Test
@SuppressWarnings("unchecked")
public void testValueStateRace() throws Exception {
    final AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
    final Integer namespace = 1;
    final ValueStateDescriptor<String> kvId = new ValueStateDescriptor<>("id", String.class);
    kvId.initializeSerializerUnlessSet(new ExecutionConfig());
    final TypeSerializer<Integer> keySerializer = IntSerializer.INSTANCE;
    final TypeSerializer<Integer> namespaceSerializer = IntSerializer.INSTANCE;
    final TypeSerializer<String> valueSerializer = kvId.getSerializer();
    final ValueState<String> state = backend.getPartitionedState(namespace, IntSerializer.INSTANCE, kvId);
    @SuppressWarnings("unchecked") final InternalKvState<Integer> kvState = (InternalKvState<Integer>) state;
    /**
		 * 1) Test that ValueState#value() before and after
		 * KvState#getSerializedValue(byte[]) return the same value.
		 */
    // set some key and namespace
    final int key1 = 1;
    backend.setCurrentKey(key1);
    kvState.setCurrentNamespace(2);
    state.update("2");
    assertEquals("2", state.value());
    // query another key and namespace
    assertNull(getSerializedValue(kvState, 3, keySerializer, namespace, IntSerializer.INSTANCE, valueSerializer));
    // the state should not have changed!
    assertEquals("2", state.value());
    // re-set values
    kvState.setCurrentNamespace(namespace);
    /**
		 * 2) Test two threads concurrently using ValueState#value() and
		 * KvState#getSerializedValue(byte[]).
		 */
    // some modifications to the state
    final int key2 = 10;
    backend.setCurrentKey(key2);
    assertNull(state.value());
    assertNull(getSerializedValue(kvState, key2, keySerializer, namespace, namespaceSerializer, valueSerializer));
    state.update("1");
    final CheckedThread getter = new CheckedThread("State getter") {

        @Override
        public void go() throws Exception {
            while (!isInterrupted()) {
                assertEquals("1", state.value());
            }
        }
    };
    final CheckedThread serializedGetter = new CheckedThread("Serialized state getter") {

        @Override
        public void go() throws Exception {
            while (!isInterrupted() && getter.isAlive()) {
                final String serializedValue = getSerializedValue(kvState, key2, keySerializer, namespace, namespaceSerializer, valueSerializer);
                assertEquals("1", serializedValue);
            }
        }
    };
    getter.start();
    serializedGetter.start();
    // run both threads for max 100ms
    Timer t = new Timer("stopper");
    t.schedule(new TimerTask() {

        @Override
        public void run() {
            getter.interrupt();
            serializedGetter.interrupt();
            this.cancel();
        }
    }, 100);
    // wait for both threads to finish
    try {
        // serializedGetter will finish if its assertion fails or if
        // getter is not alive any more
        serializedGetter.sync();
        // if serializedGetter crashed, getter will not know -> interrupt just in case
        getter.interrupt();
        getter.sync();
        // if not executed yet
        t.cancel();
    } finally {
        // clean up
        backend.dispose();
    }
}

Example 24 with ExecutionConfig

use of org.apache.flink.api.common.ExecutionConfig in project flink by apache.

the class StateBackendTestBase method testValueStateNullAsDefaultValue.

/**
	 * Verify that {@link ValueStateDescriptor} allows {@code null} as default.
	 */
@Test
public void testValueStateNullAsDefaultValue() throws Exception {
    AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
    ValueStateDescriptor<String> kvId = new ValueStateDescriptor<>("id", String.class, null);
    kvId.initializeSerializerUnlessSet(new ExecutionConfig());
    ValueState<String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
    backend.setCurrentKey(1);
    assertEquals(null, state.value());
    state.update("Ciao");
    assertEquals("Ciao", state.value());
    state.clear();
    assertEquals(null, state.value());
    backend.dispose();
}

Example 25 with ExecutionConfig

use of org.apache.flink.api.common.ExecutionConfig in project flink by apache.

the class StateBackendTestBase method testListState.

@Test
@SuppressWarnings("unchecked,rawtypes")
public void testListState() {
    try {
        CheckpointStreamFactory streamFactory = createStreamFactory();
        AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
        ListStateDescriptor<String> kvId = new ListStateDescriptor<>("id", String.class);
        kvId.initializeSerializerUnlessSet(new ExecutionConfig());
        TypeSerializer<Integer> keySerializer = IntSerializer.INSTANCE;
        TypeSerializer<VoidNamespace> namespaceSerializer = VoidNamespaceSerializer.INSTANCE;
        TypeSerializer<String> valueSerializer = kvId.getElementSerializer();
        ListState<String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        @SuppressWarnings("unchecked") InternalKvState<VoidNamespace> kvState = (InternalKvState<VoidNamespace>) state;
        Joiner joiner = Joiner.on(",");
        // some modifications to the state
        backend.setCurrentKey(1);
        assertEquals(null, state.get());
        assertEquals(null, getSerializedList(kvState, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
        state.add("1");
        backend.setCurrentKey(2);
        assertEquals(null, state.get());
        assertEquals(null, getSerializedList(kvState, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
        state.add("2");
        backend.setCurrentKey(1);
        assertEquals("1", joiner.join(state.get()));
        assertEquals("1", joiner.join(getSerializedList(kvState, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer)));
        // draw a snapshot
        KeyGroupsStateHandle snapshot1 = FutureUtil.runIfNotDoneAndGet(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forFullCheckpoint()));
        // make some more modifications
        backend.setCurrentKey(1);
        state.add("u1");
        backend.setCurrentKey(2);
        state.add("u2");
        backend.setCurrentKey(3);
        state.add("u3");
        // draw another snapshot
        KeyGroupsStateHandle snapshot2 = FutureUtil.runIfNotDoneAndGet(backend.snapshot(682375462379L, 4, streamFactory, CheckpointOptions.forFullCheckpoint()));
        // validate the original state
        backend.setCurrentKey(1);
        assertEquals("1,u1", joiner.join(state.get()));
        assertEquals("1,u1", joiner.join(getSerializedList(kvState, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer)));
        backend.setCurrentKey(2);
        assertEquals("2,u2", joiner.join(state.get()));
        assertEquals("2,u2", joiner.join(getSerializedList(kvState, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer)));
        backend.setCurrentKey(3);
        assertEquals("u3", joiner.join(state.get()));
        assertEquals("u3", joiner.join(getSerializedList(kvState, 3, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer)));
        backend.dispose();
        // restore the first snapshot and validate it
        backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot1);
        snapshot1.discardState();
        ListState<String> restored1 = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        @SuppressWarnings("unchecked") InternalKvState<VoidNamespace> restoredKvState1 = (InternalKvState<VoidNamespace>) restored1;
        backend.setCurrentKey(1);
        assertEquals("1", joiner.join(restored1.get()));
        assertEquals("1", joiner.join(getSerializedList(restoredKvState1, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer)));
        backend.setCurrentKey(2);
        assertEquals("2", joiner.join(restored1.get()));
        assertEquals("2", joiner.join(getSerializedList(restoredKvState1, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer)));
        backend.dispose();
        // restore the second snapshot and validate it
        backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot2);
        snapshot2.discardState();
        ListState<String> restored2 = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        @SuppressWarnings("unchecked") InternalKvState<VoidNamespace> restoredKvState2 = (InternalKvState<VoidNamespace>) restored2;
        backend.setCurrentKey(1);
        assertEquals("1,u1", joiner.join(restored2.get()));
        assertEquals("1,u1", joiner.join(getSerializedList(restoredKvState2, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer)));
        backend.setCurrentKey(2);
        assertEquals("2,u2", joiner.join(restored2.get()));
        assertEquals("2,u2", joiner.join(getSerializedList(restoredKvState2, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer)));
        backend.setCurrentKey(3);
        assertEquals("u3", joiner.join(restored2.get()));
        assertEquals("u3", joiner.join(getSerializedList(restoredKvState2, 3, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer)));
        backend.dispose();
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}