Examples with KvStateResponse org.apache.flink.queryablestate.messages.KvStateResponse used on opensource projects

Example 31 with KvStateResponse

use of org.apache.flink.queryablestate.messages.KvStateResponse in project flink by splunk.

the class KvStateServerHandlerTest method testChunkedResponse.

/**
 * Tests that large responses are chunked.
 */
@Test
public void testChunkedResponse() throws Exception {
    KvStateRegistry registry = new KvStateRegistry();
    KvStateRequestStats stats = new AtomicKvStateRequestStats();
    MessageSerializer<KvStateInternalRequest, KvStateResponse> serializer = new MessageSerializer<>(new KvStateInternalRequest.KvStateInternalRequestDeserializer(), new KvStateResponse.KvStateResponseDeserializer());
    KvStateServerHandler handler = new KvStateServerHandler(testServer, registry, serializer, stats);
    EmbeddedChannel channel = new EmbeddedChannel(getFrameDecoder(), handler);
    int numKeyGroups = 1;
    AbstractStateBackend abstractBackend = new MemoryStateBackend();
    DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
    dummyEnv.setKvStateRegistry(registry);
    AbstractKeyedStateBackend<Integer> backend = createKeyedStateBackend(registry, numKeyGroups, abstractBackend, dummyEnv);
    final TestRegistryListener registryListener = new TestRegistryListener();
    registry.registerListener(dummyEnv.getJobID(), registryListener);
    // Register state
    ValueStateDescriptor<byte[]> desc = new ValueStateDescriptor<>("any", BytePrimitiveArraySerializer.INSTANCE);
    desc.setQueryable("vanilla");
    ValueState<byte[]> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
    // Update KvState
    byte[] bytes = new byte[2 * channel.config().getWriteBufferHighWaterMark()];
    byte current = 0;
    for (int i = 0; i < bytes.length; i++) {
        bytes[i] = current++;
    }
    int key = 99812822;
    backend.setCurrentKey(key);
    state.update(bytes);
    // Request
    byte[] serializedKeyAndNamespace = KvStateSerializer.serializeKeyAndNamespace(key, IntSerializer.INSTANCE, VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE);
    long requestId = Integer.MAX_VALUE + 182828L;
    assertTrue(registryListener.registrationName.equals("vanilla"));
    KvStateInternalRequest request = new KvStateInternalRequest(registryListener.kvStateId, serializedKeyAndNamespace);
    ByteBuf serRequest = MessageSerializer.serializeRequest(channel.alloc(), requestId, request);
    // Write the request and wait for the response
    channel.writeInbound(serRequest);
    Object msg = readInboundBlocking(channel);
    assertTrue("Not ChunkedByteBuf", msg instanceof ChunkedByteBuf);
    ((ChunkedByteBuf) msg).close();
}

Example 32 with KvStateResponse

use of org.apache.flink.queryablestate.messages.KvStateResponse in project flink by splunk.

the class KvStateServerHandlerTest method testSimpleQuery.

/**
 * Tests a simple successful query via an EmbeddedChannel.
 */
@Test
public void testSimpleQuery() throws Exception {
    KvStateRegistry registry = new KvStateRegistry();
    AtomicKvStateRequestStats stats = new AtomicKvStateRequestStats();
    MessageSerializer<KvStateInternalRequest, KvStateResponse> serializer = new MessageSerializer<>(new KvStateInternalRequest.KvStateInternalRequestDeserializer(), new KvStateResponse.KvStateResponseDeserializer());
    KvStateServerHandler handler = new KvStateServerHandler(testServer, registry, serializer, stats);
    EmbeddedChannel channel = new EmbeddedChannel(getFrameDecoder(), handler);
    // Register state
    ValueStateDescriptor<Integer> desc = new ValueStateDescriptor<>("any", IntSerializer.INSTANCE);
    desc.setQueryable("vanilla");
    int numKeyGroups = 1;
    AbstractStateBackend abstractBackend = new MemoryStateBackend();
    DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
    dummyEnv.setKvStateRegistry(registry);
    AbstractKeyedStateBackend<Integer> backend = createKeyedStateBackend(registry, numKeyGroups, abstractBackend, dummyEnv);
    final TestRegistryListener registryListener = new TestRegistryListener();
    registry.registerListener(dummyEnv.getJobID(), registryListener);
    // Update the KvState and request it
    int expectedValue = 712828289;
    int key = 99812822;
    backend.setCurrentKey(key);
    ValueState<Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
    state.update(expectedValue);
    byte[] serializedKeyAndNamespace = KvStateSerializer.serializeKeyAndNamespace(key, IntSerializer.INSTANCE, VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE);
    long requestId = Integer.MAX_VALUE + 182828L;
    assertTrue(registryListener.registrationName.equals("vanilla"));
    KvStateInternalRequest request = new KvStateInternalRequest(registryListener.kvStateId, serializedKeyAndNamespace);
    ByteBuf serRequest = MessageSerializer.serializeRequest(channel.alloc(), requestId, request);
    // Write the request and wait for the response
    channel.writeInbound(serRequest);
    ByteBuf buf = (ByteBuf) readInboundBlocking(channel);
    // skip frame length
    buf.skipBytes(4);
    // Verify the response
    assertEquals(MessageType.REQUEST_RESULT, MessageSerializer.deserializeHeader(buf));
    long deserRequestId = MessageSerializer.getRequestId(buf);
    KvStateResponse response = serializer.deserializeResponse(buf);
    buf.release();
    assertEquals(requestId, deserRequestId);
    int actualValue = KvStateSerializer.deserializeValue(response.getContent(), IntSerializer.INSTANCE);
    assertEquals(expectedValue, actualValue);
    assertEquals(stats.toString(), 1, stats.getNumRequests());
    // Wait for async successful request report
    long deadline = System.nanoTime() + TimeUnit.NANOSECONDS.convert(30, TimeUnit.SECONDS);
    while (stats.getNumSuccessful() != 1L && System.nanoTime() <= deadline) {
        Thread.sleep(10L);
    }
    assertEquals(stats.toString(), 1L, stats.getNumSuccessful());
}

Example 33 with KvStateResponse

use of org.apache.flink.queryablestate.messages.KvStateResponse in project flink by splunk.

the class KvStateServerHandlerTest method testIncomingBufferIsRecycled.

/**
 * Tests that incoming buffer instances are recycled.
 */
@Test
public void testIncomingBufferIsRecycled() throws Exception {
    KvStateRegistry registry = new KvStateRegistry();
    AtomicKvStateRequestStats stats = new AtomicKvStateRequestStats();
    MessageSerializer<KvStateInternalRequest, KvStateResponse> serializer = new MessageSerializer<>(new KvStateInternalRequest.KvStateInternalRequestDeserializer(), new KvStateResponse.KvStateResponseDeserializer());
    KvStateServerHandler handler = new KvStateServerHandler(testServer, registry, serializer, stats);
    EmbeddedChannel channel = new EmbeddedChannel(getFrameDecoder(), handler);
    KvStateInternalRequest request = new KvStateInternalRequest(new KvStateID(), new byte[0]);
    ByteBuf serRequest = MessageSerializer.serializeRequest(channel.alloc(), 282872L, request);
    assertEquals(1L, serRequest.refCnt());
    // Write regular request
    channel.writeInbound(serRequest);
    assertEquals("Buffer not recycled", 0L, serRequest.refCnt());
    // Write unexpected msg
    ByteBuf unexpected = channel.alloc().buffer(8);
    unexpected.writeInt(4);
    unexpected.writeInt(4);
    assertEquals(1L, unexpected.refCnt());
    channel.writeInbound(unexpected);
    assertEquals("Buffer not recycled", 0L, unexpected.refCnt());
    channel.finishAndReleaseAll();
}

Example 34 with KvStateResponse

use of org.apache.flink.queryablestate.messages.KvStateResponse in project flink by splunk.

the class KvStateServerHandlerTest method testUnexpectedMessage.

/**
 * Tests response on unexpected messages.
 */
@Test
public void testUnexpectedMessage() throws Exception {
    KvStateRegistry registry = new KvStateRegistry();
    AtomicKvStateRequestStats stats = new AtomicKvStateRequestStats();
    MessageSerializer<KvStateInternalRequest, KvStateResponse> serializer = new MessageSerializer<>(new KvStateInternalRequest.KvStateInternalRequestDeserializer(), new KvStateResponse.KvStateResponseDeserializer());
    KvStateServerHandler handler = new KvStateServerHandler(testServer, registry, serializer, stats);
    EmbeddedChannel channel = new EmbeddedChannel(getFrameDecoder(), handler);
    // Write the request and wait for the response
    ByteBuf unexpectedMessage = Unpooled.buffer(8);
    unexpectedMessage.writeInt(4);
    unexpectedMessage.writeInt(123238213);
    channel.writeInbound(unexpectedMessage);
    ByteBuf buf = (ByteBuf) readInboundBlocking(channel);
    // skip frame length
    buf.skipBytes(4);
    // Verify the response
    assertEquals(MessageType.SERVER_FAILURE, MessageSerializer.deserializeHeader(buf));
    Throwable response = MessageSerializer.deserializeServerFailure(buf);
    buf.release();
    assertEquals(0L, stats.getNumRequests());
    assertEquals(0L, stats.getNumFailed());
    KvStateResponse stateResponse = new KvStateResponse(new byte[0]);
    unexpectedMessage = MessageSerializer.serializeResponse(channel.alloc(), 192L, stateResponse);
    channel.writeInbound(unexpectedMessage);
    buf = (ByteBuf) readInboundBlocking(channel);
    // skip frame length
    buf.skipBytes(4);
    // Verify the response
    assertEquals(MessageType.SERVER_FAILURE, MessageSerializer.deserializeHeader(buf));
    response = MessageSerializer.deserializeServerFailure(buf);
    buf.release();
    assertTrue("Unexpected failure cause " + response.getClass().getName(), response instanceof IllegalArgumentException);
    assertEquals(0L, stats.getNumRequests());
    assertEquals(0L, stats.getNumFailed());
}

Example 35 with KvStateResponse

use of org.apache.flink.queryablestate.messages.KvStateResponse in project flink by splunk.

the class KvStateServerHandler method handleRequest.

@Override
public CompletableFuture<KvStateResponse> handleRequest(final long requestId, final KvStateInternalRequest request) {
    final CompletableFuture<KvStateResponse> responseFuture = new CompletableFuture<>();
    try {
        final KvStateEntry<?, ?, ?> kvState = registry.getKvState(request.getKvStateId());
        if (kvState == null) {
            responseFuture.completeExceptionally(new UnknownKvStateIdException(getServerName(), request.getKvStateId()));
        } else {
            byte[] serializedKeyAndNamespace = request.getSerializedKeyAndNamespace();
            byte[] serializedResult = getSerializedValue(kvState, serializedKeyAndNamespace);
            if (serializedResult != null) {
                responseFuture.complete(new KvStateResponse(serializedResult));
            } else {
                responseFuture.completeExceptionally(new UnknownKeyOrNamespaceException(getServerName()));
            }
        }
        return responseFuture;
    } catch (Throwable t) {
        String errMsg = "Error while processing request with ID " + requestId + ". Caused by: " + ExceptionUtils.stringifyException(t);
        responseFuture.completeExceptionally(new RuntimeException(errMsg));
        return responseFuture;
    }
}