Examples with ValueStateDescriptor org.apache.flink.api.common.state.ValueStateDescriptor used on opensource projects

Example 16 with ValueStateDescriptor

use of org.apache.flink.api.common.state.ValueStateDescriptor in project flink by apache.

the class KvStateServerHandlerTest method testQueryExecutorShutDown.

/**
	 * Tests the failure response on a rejected execution, because the query
	 * executor has been closed.
	 */
@Test
public void testQueryExecutorShutDown() throws Exception {
    KvStateRegistry registry = new KvStateRegistry();
    AtomicKvStateRequestStats stats = new AtomicKvStateRequestStats();
    ExecutorService closedExecutor = Executors.newSingleThreadExecutor();
    closedExecutor.shutdown();
    assertTrue(closedExecutor.isShutdown());
    KvStateServerHandler handler = new KvStateServerHandler(registry, closedExecutor, stats);
    EmbeddedChannel channel = new EmbeddedChannel(getFrameDecoder(), handler);
    int numKeyGroups = 1;
    AbstractStateBackend abstractBackend = new MemoryStateBackend();
    DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
    dummyEnv.setKvStateRegistry(registry);
    KeyedStateBackend<Integer> backend = abstractBackend.createKeyedStateBackend(dummyEnv, new JobID(), "test_op", IntSerializer.INSTANCE, numKeyGroups, new KeyGroupRange(0, 0), registry.createTaskRegistry(dummyEnv.getJobID(), dummyEnv.getJobVertexId()));
    final TestRegistryListener registryListener = new TestRegistryListener();
    registry.registerListener(registryListener);
    // Register state
    ValueStateDescriptor<Integer> desc = new ValueStateDescriptor<>("any", IntSerializer.INSTANCE);
    desc.setQueryable("vanilla");
    backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
    assertTrue(registryListener.registrationName.equals("vanilla"));
    ByteBuf request = KvStateRequestSerializer.serializeKvStateRequest(channel.alloc(), 282872, registryListener.kvStateId, new byte[0]);
    // Write the request and wait for the response
    channel.writeInbound(request);
    ByteBuf buf = (ByteBuf) readInboundBlocking(channel);
    // skip frame length
    buf.skipBytes(4);
    // Verify the response
    assertEquals(KvStateRequestType.REQUEST_FAILURE, KvStateRequestSerializer.deserializeHeader(buf));
    KvStateRequestFailure response = KvStateRequestSerializer.deserializeKvStateRequestFailure(buf);
    assertTrue(response.getCause().getMessage().contains("RejectedExecutionException"));
    assertEquals(1, stats.getNumRequests());
    assertEquals(1, stats.getNumFailed());
}

Example 17 with ValueStateDescriptor

use of org.apache.flink.api.common.state.ValueStateDescriptor in project flink by apache.

the class KvStateServerHandlerTest method testSimpleQuery.

/**
	 * Tests a simple successful query via an EmbeddedChannel.
	 */
@Test
public void testSimpleQuery() throws Exception {
    KvStateRegistry registry = new KvStateRegistry();
    AtomicKvStateRequestStats stats = new AtomicKvStateRequestStats();
    KvStateServerHandler handler = new KvStateServerHandler(registry, TEST_THREAD_POOL, stats);
    EmbeddedChannel channel = new EmbeddedChannel(getFrameDecoder(), handler);
    // Register state
    ValueStateDescriptor<Integer> desc = new ValueStateDescriptor<>("any", IntSerializer.INSTANCE);
    desc.setQueryable("vanilla");
    int numKeyGroups = 1;
    AbstractStateBackend abstractBackend = new MemoryStateBackend();
    DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
    dummyEnv.setKvStateRegistry(registry);
    AbstractKeyedStateBackend<Integer> backend = abstractBackend.createKeyedStateBackend(dummyEnv, new JobID(), "test_op", IntSerializer.INSTANCE, numKeyGroups, new KeyGroupRange(0, 0), registry.createTaskRegistry(dummyEnv.getJobID(), dummyEnv.getJobVertexId()));
    final TestRegistryListener registryListener = new TestRegistryListener();
    registry.registerListener(registryListener);
    // Update the KvState and request it
    int expectedValue = 712828289;
    int key = 99812822;
    backend.setCurrentKey(key);
    ValueState<Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
    state.update(expectedValue);
    byte[] serializedKeyAndNamespace = KvStateRequestSerializer.serializeKeyAndNamespace(key, IntSerializer.INSTANCE, VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE);
    long requestId = Integer.MAX_VALUE + 182828L;
    assertTrue(registryListener.registrationName.equals("vanilla"));
    ByteBuf request = KvStateRequestSerializer.serializeKvStateRequest(channel.alloc(), requestId, registryListener.kvStateId, serializedKeyAndNamespace);
    // Write the request and wait for the response
    channel.writeInbound(request);
    ByteBuf buf = (ByteBuf) readInboundBlocking(channel);
    // skip frame length
    buf.skipBytes(4);
    // Verify the response
    assertEquals(KvStateRequestType.REQUEST_RESULT, KvStateRequestSerializer.deserializeHeader(buf));
    KvStateRequestResult response = KvStateRequestSerializer.deserializeKvStateRequestResult(buf);
    assertEquals(requestId, response.getRequestId());
    int actualValue = KvStateRequestSerializer.deserializeValue(response.getSerializedResult(), IntSerializer.INSTANCE);
    assertEquals(expectedValue, actualValue);
    assertEquals(stats.toString(), 1, stats.getNumRequests());
    // Wait for async successful request report
    long deadline = System.nanoTime() + TimeUnit.NANOSECONDS.convert(30, TimeUnit.SECONDS);
    while (stats.getNumSuccessful() != 1 && System.nanoTime() <= deadline) {
        Thread.sleep(10);
    }
    assertEquals(stats.toString(), 1, stats.getNumSuccessful());
}

Example 18 with ValueStateDescriptor

use of org.apache.flink.api.common.state.ValueStateDescriptor in project flink by apache.

the class StateBackendTestBase method testQueryableStateRegistration.

/**
	 * Tests registration with the KvStateRegistry.
	 */
@Test
public void testQueryableStateRegistration() throws Exception {
    DummyEnvironment env = new DummyEnvironment("test", 1, 0);
    KvStateRegistry registry = env.getKvStateRegistry();
    CheckpointStreamFactory streamFactory = createStreamFactory();
    AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE, env);
    KeyGroupRange expectedKeyGroupRange = backend.getKeyGroupRange();
    KvStateRegistryListener listener = mock(KvStateRegistryListener.class);
    registry.registerListener(listener);
    ValueStateDescriptor<Integer> desc = new ValueStateDescriptor<>("test", IntSerializer.INSTANCE);
    desc.setQueryable("banana");
    backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
    // Verify registered
    verify(listener, times(1)).notifyKvStateRegistered(eq(env.getJobID()), eq(env.getJobVertexId()), eq(expectedKeyGroupRange), eq("banana"), any(KvStateID.class));
    KeyGroupsStateHandle snapshot = FutureUtil.runIfNotDoneAndGet(backend.snapshot(682375462379L, 4, streamFactory, CheckpointOptions.forFullCheckpoint()));
    backend.dispose();
    verify(listener, times(1)).notifyKvStateUnregistered(eq(env.getJobID()), eq(env.getJobVertexId()), eq(expectedKeyGroupRange), eq("banana"));
    backend.dispose();
    // Initialize again
    backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot, env);
    snapshot.discardState();
    backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
    // Verify registered again
    verify(listener, times(2)).notifyKvStateRegistered(eq(env.getJobID()), eq(env.getJobVertexId()), eq(expectedKeyGroupRange), eq("banana"), any(KvStateID.class));
    backend.dispose();
}

Example 19 with ValueStateDescriptor

use of org.apache.flink.api.common.state.ValueStateDescriptor in project flink by apache.

the class StateBackendTestBase method testMultipleValueStates.

@Test
@SuppressWarnings("unchecked")
public void testMultipleValueStates() throws Exception {
    CheckpointStreamFactory streamFactory = createStreamFactory();
    AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE, 1, new KeyGroupRange(0, 0), new DummyEnvironment("test_op", 1, 0));
    ValueStateDescriptor<String> desc1 = new ValueStateDescriptor<>("a-string", StringSerializer.INSTANCE);
    ValueStateDescriptor<Integer> desc2 = new ValueStateDescriptor<>("an-integer", IntSerializer.INSTANCE);
    desc1.initializeSerializerUnlessSet(new ExecutionConfig());
    desc2.initializeSerializerUnlessSet(new ExecutionConfig());
    ValueState<String> state1 = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc1);
    ValueState<Integer> state2 = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc2);
    // some modifications to the state
    backend.setCurrentKey(1);
    assertNull(state1.value());
    assertNull(state2.value());
    state1.update("1");
    // state2 should still have nothing
    assertEquals("1", state1.value());
    assertNull(state2.value());
    state2.update(13);
    // both have some state now
    assertEquals("1", state1.value());
    assertEquals(13, (int) state2.value());
    // draw a snapshot
    KeyGroupsStateHandle snapshot1 = FutureUtil.runIfNotDoneAndGet(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forFullCheckpoint()));
    backend.dispose();
    backend = restoreKeyedBackend(IntSerializer.INSTANCE, 1, new KeyGroupRange(0, 0), Collections.singletonList(snapshot1), new DummyEnvironment("test_op", 1, 0));
    snapshot1.discardState();
    backend.setCurrentKey(1);
    state1 = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc1);
    state2 = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc2);
    // verify that they are still the same
    assertEquals("1", state1.value());
    assertEquals(13, (int) state2.value());
    backend.dispose();
}

Example 20 with ValueStateDescriptor

use of org.apache.flink.api.common.state.ValueStateDescriptor in project flink by apache.

the class StateBackendTestBase method testConcurrentMapIfQueryable.

/**
	 * Tests that {@link AbstractHeapState} instances respect the queryable
	 * flag and create concurrent variants for internal state structures.
	 */
@SuppressWarnings("unchecked")
protected void testConcurrentMapIfQueryable() throws Exception {
    final int numberOfKeyGroups = 1;
    AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE, numberOfKeyGroups, new KeyGroupRange(0, 0), new DummyEnvironment("test_op", 1, 0));
    {
        // ValueState
        ValueStateDescriptor<Integer> desc = new ValueStateDescriptor<>("value-state", Integer.class, -1);
        desc.setQueryable("my-query");
        desc.initializeSerializerUnlessSet(new ExecutionConfig());
        ValueState<Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
        InternalKvState<VoidNamespace> kvState = (InternalKvState<VoidNamespace>) state;
        assertTrue(kvState instanceof AbstractHeapState);
        kvState.setCurrentNamespace(VoidNamespace.INSTANCE);
        backend.setCurrentKey(1);
        state.update(121818273);
        StateTable<?, ?, ?> stateTable = ((AbstractHeapState<?, ?, ?, ?, ?>) kvState).getStateTable();
        checkConcurrentStateTable(stateTable, numberOfKeyGroups);
    }
    {
        // ListState
        ListStateDescriptor<Integer> desc = new ListStateDescriptor<>("list-state", Integer.class);
        desc.setQueryable("my-query");
        desc.initializeSerializerUnlessSet(new ExecutionConfig());
        ListState<Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
        InternalKvState<VoidNamespace> kvState = (InternalKvState<VoidNamespace>) state;
        assertTrue(kvState instanceof AbstractHeapState);
        kvState.setCurrentNamespace(VoidNamespace.INSTANCE);
        backend.setCurrentKey(1);
        state.add(121818273);
        StateTable<?, ?, ?> stateTable = ((AbstractHeapState<?, ?, ?, ?, ?>) kvState).getStateTable();
        checkConcurrentStateTable(stateTable, numberOfKeyGroups);
    }
    {
        // ReducingState
        ReducingStateDescriptor<Integer> desc = new ReducingStateDescriptor<>("reducing-state", new ReduceFunction<Integer>() {

            @Override
            public Integer reduce(Integer value1, Integer value2) throws Exception {
                return value1 + value2;
            }
        }, Integer.class);
        desc.setQueryable("my-query");
        desc.initializeSerializerUnlessSet(new ExecutionConfig());
        ReducingState<Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
        InternalKvState<VoidNamespace> kvState = (InternalKvState<VoidNamespace>) state;
        assertTrue(kvState instanceof AbstractHeapState);
        kvState.setCurrentNamespace(VoidNamespace.INSTANCE);
        backend.setCurrentKey(1);
        state.add(121818273);
        StateTable<?, ?, ?> stateTable = ((AbstractHeapState<?, ?, ?, ?, ?>) kvState).getStateTable();
        checkConcurrentStateTable(stateTable, numberOfKeyGroups);
    }
    {
        // FoldingState
        FoldingStateDescriptor<Integer, Integer> desc = new FoldingStateDescriptor<>("folding-state", 0, new FoldFunction<Integer, Integer>() {

            @Override
            public Integer fold(Integer accumulator, Integer value) throws Exception {
                return accumulator + value;
            }
        }, Integer.class);
        desc.setQueryable("my-query");
        desc.initializeSerializerUnlessSet(new ExecutionConfig());
        FoldingState<Integer, Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
        InternalKvState<VoidNamespace> kvState = (InternalKvState<VoidNamespace>) state;
        assertTrue(kvState instanceof AbstractHeapState);
        kvState.setCurrentNamespace(VoidNamespace.INSTANCE);
        backend.setCurrentKey(1);
        state.add(121818273);
        StateTable<?, ?, ?> stateTable = ((AbstractHeapState<?, ?, ?, ?, ?>) kvState).getStateTable();
        checkConcurrentStateTable(stateTable, numberOfKeyGroups);
    }
    {
        // MapState
        MapStateDescriptor<Integer, String> desc = new MapStateDescriptor<>("map-state", Integer.class, String.class);
        desc.setQueryable("my-query");
        desc.initializeSerializerUnlessSet(new ExecutionConfig());
        MapState<Integer, String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
        InternalKvState<VoidNamespace> kvState = (InternalKvState<VoidNamespace>) state;
        assertTrue(kvState instanceof AbstractHeapState);
        kvState.setCurrentNamespace(VoidNamespace.INSTANCE);
        backend.setCurrentKey(1);
        state.put(121818273, "121818273");
        int keyGroupIndex = KeyGroupRangeAssignment.assignToKeyGroup(1, numberOfKeyGroups);
        StateTable stateTable = ((AbstractHeapState) kvState).getStateTable();
        assertNotNull("State not set", stateTable.get(keyGroupIndex));
        checkConcurrentStateTable(stateTable, numberOfKeyGroups);
    }
    backend.dispose();
}