Search in sources :

Example 16 with KvStateResponse

use of org.apache.flink.queryablestate.messages.KvStateResponse in project flink by apache.

the class KvStateServerHandlerTest method testSimpleQuery.

/**
 * Tests a simple successful query via an EmbeddedChannel.
 */
@Test
public void testSimpleQuery() throws Exception {
    KvStateRegistry registry = new KvStateRegistry();
    AtomicKvStateRequestStats stats = new AtomicKvStateRequestStats();
    MessageSerializer<KvStateInternalRequest, KvStateResponse> serializer = new MessageSerializer<>(new KvStateInternalRequest.KvStateInternalRequestDeserializer(), new KvStateResponse.KvStateResponseDeserializer());
    KvStateServerHandler handler = new KvStateServerHandler(testServer, registry, serializer, stats);
    EmbeddedChannel channel = new EmbeddedChannel(getFrameDecoder(), handler);
    // Register state
    ValueStateDescriptor<Integer> desc = new ValueStateDescriptor<>("any", IntSerializer.INSTANCE);
    desc.setQueryable("vanilla");
    int numKeyGroups = 1;
    AbstractStateBackend abstractBackend = new MemoryStateBackend();
    DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
    dummyEnv.setKvStateRegistry(registry);
    AbstractKeyedStateBackend<Integer> backend = createKeyedStateBackend(registry, numKeyGroups, abstractBackend, dummyEnv);
    final TestRegistryListener registryListener = new TestRegistryListener();
    registry.registerListener(dummyEnv.getJobID(), registryListener);
    // Update the KvState and request it
    int expectedValue = 712828289;
    int key = 99812822;
    backend.setCurrentKey(key);
    ValueState<Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
    state.update(expectedValue);
    byte[] serializedKeyAndNamespace = KvStateSerializer.serializeKeyAndNamespace(key, IntSerializer.INSTANCE, VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE);
    long requestId = Integer.MAX_VALUE + 182828L;
    assertTrue(registryListener.registrationName.equals("vanilla"));
    KvStateInternalRequest request = new KvStateInternalRequest(registryListener.kvStateId, serializedKeyAndNamespace);
    ByteBuf serRequest = MessageSerializer.serializeRequest(channel.alloc(), requestId, request);
    // Write the request and wait for the response
    channel.writeInbound(serRequest);
    ByteBuf buf = (ByteBuf) readInboundBlocking(channel);
    // skip frame length
    buf.skipBytes(4);
    // Verify the response
    assertEquals(MessageType.REQUEST_RESULT, MessageSerializer.deserializeHeader(buf));
    long deserRequestId = MessageSerializer.getRequestId(buf);
    KvStateResponse response = serializer.deserializeResponse(buf);
    buf.release();
    assertEquals(requestId, deserRequestId);
    int actualValue = KvStateSerializer.deserializeValue(response.getContent(), IntSerializer.INSTANCE);
    assertEquals(expectedValue, actualValue);
    assertEquals(stats.toString(), 1, stats.getNumRequests());
    // Wait for async successful request report
    long deadline = System.nanoTime() + TimeUnit.NANOSECONDS.convert(30, TimeUnit.SECONDS);
    while (stats.getNumSuccessful() != 1L && System.nanoTime() <= deadline) {
        Thread.sleep(10L);
    }
    assertEquals(stats.toString(), 1L, stats.getNumSuccessful());
}
Also used : KvStateRegistry(org.apache.flink.runtime.query.KvStateRegistry) MessageSerializer(org.apache.flink.queryablestate.network.messages.MessageSerializer) MemoryStateBackend(org.apache.flink.runtime.state.memory.MemoryStateBackend) EmbeddedChannel(org.apache.flink.shaded.netty4.io.netty.channel.embedded.EmbeddedChannel) DummyEnvironment(org.apache.flink.runtime.operators.testutils.DummyEnvironment) ByteBuf(org.apache.flink.shaded.netty4.io.netty.buffer.ByteBuf) ValueStateDescriptor(org.apache.flink.api.common.state.ValueStateDescriptor) KvStateServerHandler(org.apache.flink.queryablestate.server.KvStateServerHandler) KvStateInternalRequest(org.apache.flink.queryablestate.messages.KvStateInternalRequest) KvStateResponse(org.apache.flink.queryablestate.messages.KvStateResponse) AtomicKvStateRequestStats(org.apache.flink.queryablestate.network.stats.AtomicKvStateRequestStats) AbstractStateBackend(org.apache.flink.runtime.state.AbstractStateBackend) Test(org.junit.Test)

Example 17 with KvStateResponse

use of org.apache.flink.queryablestate.messages.KvStateResponse in project flink by apache.

the class KvStateServerHandlerTest method testFailureOnGetSerializedValue.

/**
 * Tests the failure response on a failure on the {@link
 * InternalKvState#getSerializedValue(byte[], TypeSerializer, TypeSerializer, TypeSerializer)}
 * call.
 */
@Test
public void testFailureOnGetSerializedValue() throws Exception {
    KvStateRegistry registry = new KvStateRegistry();
    AtomicKvStateRequestStats stats = new AtomicKvStateRequestStats();
    MessageSerializer<KvStateInternalRequest, KvStateResponse> serializer = new MessageSerializer<>(new KvStateInternalRequest.KvStateInternalRequestDeserializer(), new KvStateResponse.KvStateResponseDeserializer());
    KvStateServerHandler handler = new KvStateServerHandler(testServer, registry, serializer, stats);
    EmbeddedChannel channel = new EmbeddedChannel(getFrameDecoder(), handler);
    // Failing KvState
    InternalKvState<Integer, VoidNamespace, Long> kvState = new InternalKvState<Integer, VoidNamespace, Long>() {

        @Override
        public TypeSerializer<Integer> getKeySerializer() {
            return IntSerializer.INSTANCE;
        }

        @Override
        public TypeSerializer<VoidNamespace> getNamespaceSerializer() {
            return VoidNamespaceSerializer.INSTANCE;
        }

        @Override
        public TypeSerializer<Long> getValueSerializer() {
            return LongSerializer.INSTANCE;
        }

        @Override
        public void setCurrentNamespace(VoidNamespace namespace) {
        // do nothing
        }

        @Override
        public byte[] getSerializedValue(final byte[] serializedKeyAndNamespace, final TypeSerializer<Integer> safeKeySerializer, final TypeSerializer<VoidNamespace> safeNamespaceSerializer, final TypeSerializer<Long> safeValueSerializer) throws Exception {
            throw new RuntimeException("Expected test Exception");
        }

        @Override
        public StateIncrementalVisitor<Integer, VoidNamespace, Long> getStateIncrementalVisitor(int recommendedMaxNumberOfReturnedRecords) {
            throw new UnsupportedOperationException();
        }

        @Override
        public void clear() {
        }
    };
    KvStateID kvStateId = registry.registerKvState(new JobID(), new JobVertexID(), new KeyGroupRange(0, 0), "vanilla", kvState, getClass().getClassLoader());
    KvStateInternalRequest request = new KvStateInternalRequest(kvStateId, new byte[0]);
    ByteBuf serRequest = MessageSerializer.serializeRequest(channel.alloc(), 282872L, request);
    // Write the request and wait for the response
    channel.writeInbound(serRequest);
    ByteBuf buf = (ByteBuf) readInboundBlocking(channel);
    // skip frame length
    buf.skipBytes(4);
    // Verify the response
    assertEquals(MessageType.REQUEST_FAILURE, MessageSerializer.deserializeHeader(buf));
    RequestFailure response = MessageSerializer.deserializeRequestFailure(buf);
    buf.release();
    assertTrue(response.getCause().getMessage().contains("Expected test Exception"));
    assertEquals(1L, stats.getNumRequests());
    assertEquals(1L, stats.getNumFailed());
}
Also used : KvStateRegistry(org.apache.flink.runtime.query.KvStateRegistry) MessageSerializer(org.apache.flink.queryablestate.network.messages.MessageSerializer) JobVertexID(org.apache.flink.runtime.jobgraph.JobVertexID) KeyGroupRange(org.apache.flink.runtime.state.KeyGroupRange) EmbeddedChannel(org.apache.flink.shaded.netty4.io.netty.channel.embedded.EmbeddedChannel) ByteBuf(org.apache.flink.shaded.netty4.io.netty.buffer.ByteBuf) InternalKvState(org.apache.flink.runtime.state.internal.InternalKvState) KvStateServerHandler(org.apache.flink.queryablestate.server.KvStateServerHandler) TypeSerializer(org.apache.flink.api.common.typeutils.TypeSerializer) KvStateInternalRequest(org.apache.flink.queryablestate.messages.KvStateInternalRequest) KvStateResponse(org.apache.flink.queryablestate.messages.KvStateResponse) VoidNamespace(org.apache.flink.queryablestate.client.VoidNamespace) KvStateID(org.apache.flink.queryablestate.KvStateID) RequestFailure(org.apache.flink.queryablestate.network.messages.RequestFailure) AtomicKvStateRequestStats(org.apache.flink.queryablestate.network.stats.AtomicKvStateRequestStats) JobID(org.apache.flink.api.common.JobID) Test(org.junit.Test)

Example 18 with KvStateResponse

use of org.apache.flink.queryablestate.messages.KvStateResponse in project flink by apache.

the class KvStateServerHandlerTest method testQueryUnknownKvStateID.

/**
 * Tests the failure response with {@link UnknownKvStateIdException} as cause on queries for
 * unregistered KvStateIDs.
 */
@Test
public void testQueryUnknownKvStateID() throws Exception {
    KvStateRegistry registry = new KvStateRegistry();
    AtomicKvStateRequestStats stats = new AtomicKvStateRequestStats();
    MessageSerializer<KvStateInternalRequest, KvStateResponse> serializer = new MessageSerializer<>(new KvStateInternalRequest.KvStateInternalRequestDeserializer(), new KvStateResponse.KvStateResponseDeserializer());
    KvStateServerHandler handler = new KvStateServerHandler(testServer, registry, serializer, stats);
    EmbeddedChannel channel = new EmbeddedChannel(getFrameDecoder(), handler);
    long requestId = Integer.MAX_VALUE + 182828L;
    KvStateInternalRequest request = new KvStateInternalRequest(new KvStateID(), new byte[0]);
    ByteBuf serRequest = MessageSerializer.serializeRequest(channel.alloc(), requestId, request);
    // Write the request and wait for the response
    channel.writeInbound(serRequest);
    ByteBuf buf = (ByteBuf) readInboundBlocking(channel);
    // skip frame length
    buf.skipBytes(4);
    // Verify the response
    assertEquals(MessageType.REQUEST_FAILURE, MessageSerializer.deserializeHeader(buf));
    RequestFailure response = MessageSerializer.deserializeRequestFailure(buf);
    buf.release();
    assertEquals(requestId, response.getRequestId());
    assertTrue("Did not respond with expected failure cause", response.getCause() instanceof UnknownKvStateIdException);
    assertEquals(1L, stats.getNumRequests());
    assertEquals(1L, stats.getNumFailed());
}
Also used : KvStateRegistry(org.apache.flink.runtime.query.KvStateRegistry) MessageSerializer(org.apache.flink.queryablestate.network.messages.MessageSerializer) EmbeddedChannel(org.apache.flink.shaded.netty4.io.netty.channel.embedded.EmbeddedChannel) UnknownKvStateIdException(org.apache.flink.queryablestate.exceptions.UnknownKvStateIdException) ByteBuf(org.apache.flink.shaded.netty4.io.netty.buffer.ByteBuf) KvStateServerHandler(org.apache.flink.queryablestate.server.KvStateServerHandler) KvStateInternalRequest(org.apache.flink.queryablestate.messages.KvStateInternalRequest) KvStateResponse(org.apache.flink.queryablestate.messages.KvStateResponse) KvStateID(org.apache.flink.queryablestate.KvStateID) RequestFailure(org.apache.flink.queryablestate.network.messages.RequestFailure) AtomicKvStateRequestStats(org.apache.flink.queryablestate.network.stats.AtomicKvStateRequestStats) Test(org.junit.Test)

Example 19 with KvStateResponse

use of org.apache.flink.queryablestate.messages.KvStateResponse in project flink by apache.

the class KvStateServerHandlerTest method testIncomingBufferIsRecycled.

/**
 * Tests that incoming buffer instances are recycled.
 */
@Test
public void testIncomingBufferIsRecycled() throws Exception {
    KvStateRegistry registry = new KvStateRegistry();
    AtomicKvStateRequestStats stats = new AtomicKvStateRequestStats();
    MessageSerializer<KvStateInternalRequest, KvStateResponse> serializer = new MessageSerializer<>(new KvStateInternalRequest.KvStateInternalRequestDeserializer(), new KvStateResponse.KvStateResponseDeserializer());
    KvStateServerHandler handler = new KvStateServerHandler(testServer, registry, serializer, stats);
    EmbeddedChannel channel = new EmbeddedChannel(getFrameDecoder(), handler);
    KvStateInternalRequest request = new KvStateInternalRequest(new KvStateID(), new byte[0]);
    ByteBuf serRequest = MessageSerializer.serializeRequest(channel.alloc(), 282872L, request);
    assertEquals(1L, serRequest.refCnt());
    // Write regular request
    channel.writeInbound(serRequest);
    assertEquals("Buffer not recycled", 0L, serRequest.refCnt());
    // Write unexpected msg
    ByteBuf unexpected = channel.alloc().buffer(8);
    unexpected.writeInt(4);
    unexpected.writeInt(4);
    assertEquals(1L, unexpected.refCnt());
    channel.writeInbound(unexpected);
    assertEquals("Buffer not recycled", 0L, unexpected.refCnt());
    channel.finishAndReleaseAll();
}
Also used : KvStateRegistry(org.apache.flink.runtime.query.KvStateRegistry) MessageSerializer(org.apache.flink.queryablestate.network.messages.MessageSerializer) KvStateServerHandler(org.apache.flink.queryablestate.server.KvStateServerHandler) KvStateInternalRequest(org.apache.flink.queryablestate.messages.KvStateInternalRequest) EmbeddedChannel(org.apache.flink.shaded.netty4.io.netty.channel.embedded.EmbeddedChannel) KvStateResponse(org.apache.flink.queryablestate.messages.KvStateResponse) KvStateID(org.apache.flink.queryablestate.KvStateID) ByteBuf(org.apache.flink.shaded.netty4.io.netty.buffer.ByteBuf) AtomicKvStateRequestStats(org.apache.flink.queryablestate.network.stats.AtomicKvStateRequestStats) Test(org.junit.Test)

Example 20 with KvStateResponse

use of org.apache.flink.queryablestate.messages.KvStateResponse in project flink by apache.

the class MessageSerializerTest method testResponseSerializationWithZeroLengthSerializedResult.

/**
 * Tests response serialization with zero-length serialized result.
 */
@Test
public void testResponseSerializationWithZeroLengthSerializedResult() throws Exception {
    byte[] serializedResult = new byte[0];
    final KvStateResponse response = new KvStateResponse(serializedResult);
    final MessageSerializer<KvStateInternalRequest, KvStateResponse> serializer = new MessageSerializer<>(new KvStateInternalRequest.KvStateInternalRequestDeserializer(), new KvStateResponse.KvStateResponseDeserializer());
    ByteBuf buf = MessageSerializer.serializeResponse(alloc, 72727278L, response);
    int frameLength = buf.readInt();
    assertEquals(MessageType.REQUEST_RESULT, MessageSerializer.deserializeHeader(buf));
    assertEquals(72727278L, MessageSerializer.getRequestId(buf));
    KvStateResponse responseDeser = serializer.deserializeResponse(buf);
    assertEquals(buf.readerIndex(), frameLength + 4);
    assertArrayEquals(serializedResult, responseDeser.getContent());
}
Also used : MessageSerializer(org.apache.flink.queryablestate.network.messages.MessageSerializer) KvStateInternalRequest(org.apache.flink.queryablestate.messages.KvStateInternalRequest) KvStateResponse(org.apache.flink.queryablestate.messages.KvStateResponse) ByteBuf(org.apache.flink.shaded.netty4.io.netty.buffer.ByteBuf) Test(org.junit.Test)

Aggregations

KvStateResponse (org.apache.flink.queryablestate.messages.KvStateResponse)23 KvStateInternalRequest (org.apache.flink.queryablestate.messages.KvStateInternalRequest)22 Test (org.junit.Test)22 MessageSerializer (org.apache.flink.queryablestate.network.messages.MessageSerializer)21 ByteBuf (org.apache.flink.shaded.netty4.io.netty.buffer.ByteBuf)19 AtomicKvStateRequestStats (org.apache.flink.queryablestate.network.stats.AtomicKvStateRequestStats)17 KvStateRegistry (org.apache.flink.runtime.query.KvStateRegistry)12 KvStateID (org.apache.flink.queryablestate.KvStateID)11 EmbeddedChannel (org.apache.flink.shaded.netty4.io.netty.channel.embedded.EmbeddedChannel)11 KvStateServerHandler (org.apache.flink.queryablestate.server.KvStateServerHandler)10 ClosedChannelException (java.nio.channels.ClosedChannelException)7 ValueStateDescriptor (org.apache.flink.api.common.state.ValueStateDescriptor)7 DummyEnvironment (org.apache.flink.runtime.operators.testutils.DummyEnvironment)7 AbstractStateBackend (org.apache.flink.runtime.state.AbstractStateBackend)7 MemoryStateBackend (org.apache.flink.runtime.state.memory.MemoryStateBackend)7 ConnectException (java.net.ConnectException)6 InetSocketAddress (java.net.InetSocketAddress)6 UnknownHostException (java.net.UnknownHostException)6 ExecutionException (java.util.concurrent.ExecutionException)6 CompletableFuture (java.util.concurrent.CompletableFuture)5