Search in sources :

Example 66 with KvStateResponse

use of org.apache.flink.queryablestate.messages.KvStateResponse in project flink-mirror by flink-ci.

the class KvStateServerHandlerTest method testUnexpectedMessage.

/**
 * Tests response on unexpected messages.
 */
@Test
public void testUnexpectedMessage() throws Exception {
    KvStateRegistry registry = new KvStateRegistry();
    AtomicKvStateRequestStats stats = new AtomicKvStateRequestStats();
    MessageSerializer<KvStateInternalRequest, KvStateResponse> serializer = new MessageSerializer<>(new KvStateInternalRequest.KvStateInternalRequestDeserializer(), new KvStateResponse.KvStateResponseDeserializer());
    KvStateServerHandler handler = new KvStateServerHandler(testServer, registry, serializer, stats);
    EmbeddedChannel channel = new EmbeddedChannel(getFrameDecoder(), handler);
    // Write the request and wait for the response
    ByteBuf unexpectedMessage = Unpooled.buffer(8);
    unexpectedMessage.writeInt(4);
    unexpectedMessage.writeInt(123238213);
    channel.writeInbound(unexpectedMessage);
    ByteBuf buf = (ByteBuf) readInboundBlocking(channel);
    // skip frame length
    buf.skipBytes(4);
    // Verify the response
    assertEquals(MessageType.SERVER_FAILURE, MessageSerializer.deserializeHeader(buf));
    Throwable response = MessageSerializer.deserializeServerFailure(buf);
    buf.release();
    assertEquals(0L, stats.getNumRequests());
    assertEquals(0L, stats.getNumFailed());
    KvStateResponse stateResponse = new KvStateResponse(new byte[0]);
    unexpectedMessage = MessageSerializer.serializeResponse(channel.alloc(), 192L, stateResponse);
    channel.writeInbound(unexpectedMessage);
    buf = (ByteBuf) readInboundBlocking(channel);
    // skip frame length
    buf.skipBytes(4);
    // Verify the response
    assertEquals(MessageType.SERVER_FAILURE, MessageSerializer.deserializeHeader(buf));
    response = MessageSerializer.deserializeServerFailure(buf);
    buf.release();
    assertTrue("Unexpected failure cause " + response.getClass().getName(), response instanceof IllegalArgumentException);
    assertEquals(0L, stats.getNumRequests());
    assertEquals(0L, stats.getNumFailed());
}
Also used : KvStateRegistry(org.apache.flink.runtime.query.KvStateRegistry) MessageSerializer(org.apache.flink.queryablestate.network.messages.MessageSerializer) EmbeddedChannel(org.apache.flink.shaded.netty4.io.netty.channel.embedded.EmbeddedChannel) ByteBuf(org.apache.flink.shaded.netty4.io.netty.buffer.ByteBuf) KvStateServerHandler(org.apache.flink.queryablestate.server.KvStateServerHandler) KvStateInternalRequest(org.apache.flink.queryablestate.messages.KvStateInternalRequest) KvStateResponse(org.apache.flink.queryablestate.messages.KvStateResponse) AtomicKvStateRequestStats(org.apache.flink.queryablestate.network.stats.AtomicKvStateRequestStats) Test(org.junit.Test)

Example 67 with KvStateResponse

use of org.apache.flink.queryablestate.messages.KvStateResponse in project flink-mirror by flink-ci.

the class KvStateServerTest method testSimpleRequest.

/**
 * Tests a simple successful query via a SocketChannel.
 */
@Test
public void testSimpleRequest() throws Throwable {
    KvStateServerImpl server = null;
    Bootstrap bootstrap = null;
    try {
        KvStateRegistry registry = new KvStateRegistry();
        KvStateRequestStats stats = new AtomicKvStateRequestStats();
        server = new KvStateServerImpl(InetAddress.getLocalHost().getHostName(), Collections.singletonList(0).iterator(), 1, 1, registry, stats);
        server.start();
        InetSocketAddress serverAddress = server.getServerAddress();
        int numKeyGroups = 1;
        AbstractStateBackend abstractBackend = new MemoryStateBackend();
        DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
        dummyEnv.setKvStateRegistry(registry);
        final JobID jobId = new JobID();
        AbstractKeyedStateBackend<Integer> backend = abstractBackend.createKeyedStateBackend(dummyEnv, jobId, "test_op", IntSerializer.INSTANCE, numKeyGroups, new KeyGroupRange(0, 0), registry.createTaskRegistry(jobId, new JobVertexID()), TtlTimeProvider.DEFAULT, new UnregisteredMetricsGroup(), Collections.emptyList(), new CloseableRegistry());
        final KvStateServerHandlerTest.TestRegistryListener registryListener = new KvStateServerHandlerTest.TestRegistryListener();
        registry.registerListener(jobId, registryListener);
        ValueStateDescriptor<Integer> desc = new ValueStateDescriptor<>("any", IntSerializer.INSTANCE);
        desc.setQueryable("vanilla");
        ValueState<Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
        // Update KvState
        int expectedValue = 712828289;
        int key = 99812822;
        backend.setCurrentKey(key);
        state.update(expectedValue);
        // Request
        byte[] serializedKeyAndNamespace = KvStateSerializer.serializeKeyAndNamespace(key, IntSerializer.INSTANCE, VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE);
        // Connect to the server
        final BlockingQueue<ByteBuf> responses = new LinkedBlockingQueue<>();
        bootstrap = createBootstrap(new LengthFieldBasedFrameDecoder(Integer.MAX_VALUE, 0, 4, 0, 4), new ChannelInboundHandlerAdapter() {

            @Override
            public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
                responses.add((ByteBuf) msg);
            }
        });
        Channel channel = bootstrap.connect(serverAddress.getAddress(), serverAddress.getPort()).sync().channel();
        long requestId = Integer.MAX_VALUE + 182828L;
        assertTrue(registryListener.registrationName.equals("vanilla"));
        final KvStateInternalRequest request = new KvStateInternalRequest(registryListener.kvStateId, serializedKeyAndNamespace);
        ByteBuf serializeRequest = MessageSerializer.serializeRequest(channel.alloc(), requestId, request);
        channel.writeAndFlush(serializeRequest);
        ByteBuf buf = responses.poll(TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
        assertEquals(MessageType.REQUEST_RESULT, MessageSerializer.deserializeHeader(buf));
        assertEquals(requestId, MessageSerializer.getRequestId(buf));
        KvStateResponse response = server.getSerializer().deserializeResponse(buf);
        int actualValue = KvStateSerializer.deserializeValue(response.getContent(), IntSerializer.INSTANCE);
        assertEquals(expectedValue, actualValue);
    } finally {
        if (server != null) {
            server.shutdown();
        }
        if (bootstrap != null) {
            EventLoopGroup group = bootstrap.group();
            if (group != null) {
                // note: no "quiet period" to not trigger Netty#4357
                group.shutdownGracefully(0, 10, TimeUnit.SECONDS);
            }
        }
    }
}
Also used : KvStateRegistry(org.apache.flink.runtime.query.KvStateRegistry) UnregisteredMetricsGroup(org.apache.flink.metrics.groups.UnregisteredMetricsGroup) InetSocketAddress(java.net.InetSocketAddress) MemoryStateBackend(org.apache.flink.runtime.state.memory.MemoryStateBackend) JobVertexID(org.apache.flink.runtime.jobgraph.JobVertexID) KeyGroupRange(org.apache.flink.runtime.state.KeyGroupRange) ChannelHandlerContext(org.apache.flink.shaded.netty4.io.netty.channel.ChannelHandlerContext) CloseableRegistry(org.apache.flink.core.fs.CloseableRegistry) ByteBuf(org.apache.flink.shaded.netty4.io.netty.buffer.ByteBuf) LinkedBlockingQueue(java.util.concurrent.LinkedBlockingQueue) AtomicKvStateRequestStats(org.apache.flink.queryablestate.network.stats.AtomicKvStateRequestStats) KvStateRequestStats(org.apache.flink.queryablestate.network.stats.KvStateRequestStats) KvStateServerImpl(org.apache.flink.queryablestate.server.KvStateServerImpl) ValueStateDescriptor(org.apache.flink.api.common.state.ValueStateDescriptor) Bootstrap(org.apache.flink.shaded.netty4.io.netty.bootstrap.Bootstrap) KvStateResponse(org.apache.flink.queryablestate.messages.KvStateResponse) LengthFieldBasedFrameDecoder(org.apache.flink.shaded.netty4.io.netty.handler.codec.LengthFieldBasedFrameDecoder) AbstractStateBackend(org.apache.flink.runtime.state.AbstractStateBackend) NioSocketChannel(org.apache.flink.shaded.netty4.io.netty.channel.socket.nio.NioSocketChannel) SocketChannel(org.apache.flink.shaded.netty4.io.netty.channel.socket.SocketChannel) Channel(org.apache.flink.shaded.netty4.io.netty.channel.Channel) DummyEnvironment(org.apache.flink.runtime.operators.testutils.DummyEnvironment) EventLoopGroup(org.apache.flink.shaded.netty4.io.netty.channel.EventLoopGroup) NioEventLoopGroup(org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoopGroup) KvStateInternalRequest(org.apache.flink.queryablestate.messages.KvStateInternalRequest) AtomicKvStateRequestStats(org.apache.flink.queryablestate.network.stats.AtomicKvStateRequestStats) JobID(org.apache.flink.api.common.JobID) ChannelInboundHandlerAdapter(org.apache.flink.shaded.netty4.io.netty.channel.ChannelInboundHandlerAdapter) Test(org.junit.Test)

Example 68 with KvStateResponse

use of org.apache.flink.queryablestate.messages.KvStateResponse in project flink-mirror by flink-ci.

the class MessageSerializerTest method testResponseSerializationWithZeroLengthSerializedResult.

/**
 * Tests response serialization with zero-length serialized result.
 */
@Test
public void testResponseSerializationWithZeroLengthSerializedResult() throws Exception {
    byte[] serializedResult = new byte[0];
    final KvStateResponse response = new KvStateResponse(serializedResult);
    final MessageSerializer<KvStateInternalRequest, KvStateResponse> serializer = new MessageSerializer<>(new KvStateInternalRequest.KvStateInternalRequestDeserializer(), new KvStateResponse.KvStateResponseDeserializer());
    ByteBuf buf = MessageSerializer.serializeResponse(alloc, 72727278L, response);
    int frameLength = buf.readInt();
    assertEquals(MessageType.REQUEST_RESULT, MessageSerializer.deserializeHeader(buf));
    assertEquals(72727278L, MessageSerializer.getRequestId(buf));
    KvStateResponse responseDeser = serializer.deserializeResponse(buf);
    assertEquals(buf.readerIndex(), frameLength + 4);
    assertArrayEquals(serializedResult, responseDeser.getContent());
}
Also used : MessageSerializer(org.apache.flink.queryablestate.network.messages.MessageSerializer) KvStateInternalRequest(org.apache.flink.queryablestate.messages.KvStateInternalRequest) KvStateResponse(org.apache.flink.queryablestate.messages.KvStateResponse) ByteBuf(org.apache.flink.shaded.netty4.io.netty.buffer.ByteBuf) Test(org.junit.Test)

Example 69 with KvStateResponse

use of org.apache.flink.queryablestate.messages.KvStateResponse in project flink-mirror by flink-ci.

the class MessageSerializerTest method testResponseSerialization.

/**
 * Tests response serialization.
 */
@Test
public void testResponseSerialization() throws Exception {
    long requestId = Integer.MAX_VALUE + 72727278L;
    byte[] serializedResult = randomByteArray(1024);
    final KvStateResponse response = new KvStateResponse(serializedResult);
    final MessageSerializer<KvStateInternalRequest, KvStateResponse> serializer = new MessageSerializer<>(new KvStateInternalRequest.KvStateInternalRequestDeserializer(), new KvStateResponse.KvStateResponseDeserializer());
    ByteBuf buf = MessageSerializer.serializeResponse(alloc, requestId, response);
    int frameLength = buf.readInt();
    assertEquals(MessageType.REQUEST_RESULT, MessageSerializer.deserializeHeader(buf));
    assertEquals(requestId, MessageSerializer.getRequestId(buf));
    KvStateResponse responseDeser = serializer.deserializeResponse(buf);
    assertEquals(buf.readerIndex(), frameLength + 4);
    assertArrayEquals(serializedResult, responseDeser.getContent());
}
Also used : MessageSerializer(org.apache.flink.queryablestate.network.messages.MessageSerializer) KvStateInternalRequest(org.apache.flink.queryablestate.messages.KvStateInternalRequest) KvStateResponse(org.apache.flink.queryablestate.messages.KvStateResponse) ByteBuf(org.apache.flink.shaded.netty4.io.netty.buffer.ByteBuf) Test(org.junit.Test)

Aggregations

KvStateResponse (org.apache.flink.queryablestate.messages.KvStateResponse)69 KvStateInternalRequest (org.apache.flink.queryablestate.messages.KvStateInternalRequest)66 Test (org.junit.Test)66 MessageSerializer (org.apache.flink.queryablestate.network.messages.MessageSerializer)63 ByteBuf (org.apache.flink.shaded.netty4.io.netty.buffer.ByteBuf)57 AtomicKvStateRequestStats (org.apache.flink.queryablestate.network.stats.AtomicKvStateRequestStats)51 KvStateRegistry (org.apache.flink.runtime.query.KvStateRegistry)36 KvStateID (org.apache.flink.queryablestate.KvStateID)33 EmbeddedChannel (org.apache.flink.shaded.netty4.io.netty.channel.embedded.EmbeddedChannel)33 KvStateServerHandler (org.apache.flink.queryablestate.server.KvStateServerHandler)30 ClosedChannelException (java.nio.channels.ClosedChannelException)21 ValueStateDescriptor (org.apache.flink.api.common.state.ValueStateDescriptor)21 DummyEnvironment (org.apache.flink.runtime.operators.testutils.DummyEnvironment)21 AbstractStateBackend (org.apache.flink.runtime.state.AbstractStateBackend)21 MemoryStateBackend (org.apache.flink.runtime.state.memory.MemoryStateBackend)21 ConnectException (java.net.ConnectException)18 InetSocketAddress (java.net.InetSocketAddress)18 UnknownHostException (java.net.UnknownHostException)18 ExecutionException (java.util.concurrent.ExecutionException)18 CompletableFuture (java.util.concurrent.CompletableFuture)15