Examples with MessageSerializer org.apache.flink.queryablestate.network.messages.MessageSerializer used on opensource projects

Example 11 with MessageSerializer

use of org.apache.flink.queryablestate.network.messages.MessageSerializer in project flink by apache.

the class ClientTest method testRequestUnavailableHost.

/**
 * Tests that a request to an unavailable host is failed with ConnectException.
 */
@Test
public void testRequestUnavailableHost() throws Exception {
    AtomicKvStateRequestStats stats = new AtomicKvStateRequestStats();
    MessageSerializer<KvStateInternalRequest, KvStateResponse> serializer = new MessageSerializer<>(new KvStateInternalRequest.KvStateInternalRequestDeserializer(), new KvStateResponse.KvStateResponseDeserializer());
    Client<KvStateInternalRequest, KvStateResponse> client = null;
    try (NetUtils.Port port = NetUtils.getAvailablePort()) {
        client = new Client<>("Test Client", 1, serializer, stats);
        int availablePort = port.getPort();
        InetSocketAddress serverAddress = new InetSocketAddress(InetAddress.getLocalHost(), availablePort);
        KvStateInternalRequest request = new KvStateInternalRequest(new KvStateID(), new byte[0]);
        CompletableFuture<KvStateResponse> future = client.sendRequest(serverAddress, request);
        assertThat(future, FlinkMatchers.futureWillCompleteExceptionally(ConnectException.class, Duration.ofHours(1)));
    } finally {
        if (client != null) {
            try {
                client.shutdown().get();
            } catch (Exception e) {
                e.printStackTrace();
            }
            Assert.assertTrue(client.isEventGroupShutdown());
        }
        assertEquals("Channel leak", 0L, stats.getNumConnections());
    }
}

Example 12 with MessageSerializer

use of org.apache.flink.queryablestate.network.messages.MessageSerializer in project flink by apache.

the class ClientTest method testClientServerIntegration.

/**
 * Tests multiple clients querying multiple servers until 100k queries have been processed. At
 * this point, the client is shut down and its verified that all ongoing requests are failed.
 */
@Test
public void testClientServerIntegration() throws Throwable {
    // Config
    final int numServers = 2;
    final int numServerEventLoopThreads = 2;
    final int numServerQueryThreads = 2;
    final int numClientEventLoopThreads = 4;
    final int numClientsTasks = 8;
    final int batchSize = 16;
    final int numKeyGroups = 1;
    AbstractStateBackend abstractBackend = new MemoryStateBackend();
    KvStateRegistry dummyRegistry = new KvStateRegistry();
    DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
    dummyEnv.setKvStateRegistry(dummyRegistry);
    AbstractKeyedStateBackend<Integer> backend = abstractBackend.createKeyedStateBackend(dummyEnv, new JobID(), "test_op", IntSerializer.INSTANCE, numKeyGroups, new KeyGroupRange(0, 0), dummyRegistry.createTaskRegistry(new JobID(), new JobVertexID()), TtlTimeProvider.DEFAULT, new UnregisteredMetricsGroup(), Collections.emptyList(), new CloseableRegistry());
    AtomicKvStateRequestStats clientStats = new AtomicKvStateRequestStats();
    final MessageSerializer<KvStateInternalRequest, KvStateResponse> serializer = new MessageSerializer<>(new KvStateInternalRequest.KvStateInternalRequestDeserializer(), new KvStateResponse.KvStateResponseDeserializer());
    Client<KvStateInternalRequest, KvStateResponse> client = null;
    ExecutorService clientTaskExecutor = null;
    final KvStateServerImpl[] server = new KvStateServerImpl[numServers];
    try {
        client = new Client<>("Test Client", numClientEventLoopThreads, serializer, clientStats);
        clientTaskExecutor = Executors.newFixedThreadPool(numClientsTasks);
        // Create state
        ValueStateDescriptor<Integer> desc = new ValueStateDescriptor<>("any", IntSerializer.INSTANCE);
        desc.setQueryable("any");
        // Create servers
        KvStateRegistry[] registry = new KvStateRegistry[numServers];
        AtomicKvStateRequestStats[] serverStats = new AtomicKvStateRequestStats[numServers];
        final KvStateID[] ids = new KvStateID[numServers];
        for (int i = 0; i < numServers; i++) {
            registry[i] = new KvStateRegistry();
            serverStats[i] = new AtomicKvStateRequestStats();
            server[i] = new KvStateServerImpl(InetAddress.getLocalHost().getHostName(), Collections.singletonList(0).iterator(), numServerEventLoopThreads, numServerQueryThreads, registry[i], serverStats[i]);
            server[i].start();
            backend.setCurrentKey(1010 + i);
            // Value per server
            ValueState<Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
            state.update(201 + i);
            // we know it must be a KvState but this is not exposed to the user via State
            InternalKvState<Integer, ?, Integer> kvState = (InternalKvState<Integer, ?, Integer>) state;
            // Register KvState (one state instance for all server)
            ids[i] = registry[i].registerKvState(new JobID(), new JobVertexID(), new KeyGroupRange(0, 0), "any", kvState, getClass().getClassLoader());
        }
        final Client<KvStateInternalRequest, KvStateResponse> finalClient = client;
        Callable<Void> queryTask = () -> {
            while (true) {
                if (Thread.interrupted()) {
                    throw new InterruptedException();
                }
                // Random server permutation
                List<Integer> random = new ArrayList<>();
                for (int j = 0; j < batchSize; j++) {
                    random.add(j);
                }
                Collections.shuffle(random);
                // Dispatch queries
                List<CompletableFuture<KvStateResponse>> futures = new ArrayList<>(batchSize);
                for (int j = 0; j < batchSize; j++) {
                    int targetServer = random.get(j) % numServers;
                    byte[] serializedKeyAndNamespace = KvStateSerializer.serializeKeyAndNamespace(1010 + targetServer, IntSerializer.INSTANCE, VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE);
                    KvStateInternalRequest request = new KvStateInternalRequest(ids[targetServer], serializedKeyAndNamespace);
                    futures.add(finalClient.sendRequest(server[targetServer].getServerAddress(), request));
                }
                // Verify results
                for (int j = 0; j < batchSize; j++) {
                    int targetServer = random.get(j) % numServers;
                    Future<KvStateResponse> future = futures.get(j);
                    byte[] buf = future.get().getContent();
                    int value = KvStateSerializer.deserializeValue(buf, IntSerializer.INSTANCE);
                    assertEquals(201L + targetServer, value);
                }
            }
        };
        // Submit tasks
        List<Future<Void>> taskFutures = new ArrayList<>();
        for (int i = 0; i < numClientsTasks; i++) {
            taskFutures.add(clientTaskExecutor.submit(queryTask));
        }
        long numRequests;
        while ((numRequests = clientStats.getNumRequests()) < 100_000L) {
            Thread.sleep(100L);
            LOG.info("Number of requests {}/100_000", numRequests);
        }
        try {
            client.shutdown().get();
        } catch (Exception e) {
            e.printStackTrace();
        }
        Assert.assertTrue(client.isEventGroupShutdown());
        final CombinableMatcher<Throwable> exceptionMatcher = either(FlinkMatchers.containsCause(ClosedChannelException.class)).or(FlinkMatchers.containsCause(IllegalStateException.class));
        for (Future<Void> future : taskFutures) {
            try {
                future.get();
                fail("Did not throw expected Exception after shut down");
            } catch (ExecutionException t) {
                assertThat(t, exceptionMatcher);
            }
        }
        assertEquals("Connection leak (client)", 0L, clientStats.getNumConnections());
        for (int i = 0; i < numServers; i++) {
            boolean success = false;
            int numRetries = 0;
            while (!success) {
                try {
                    assertEquals("Connection leak (server)", 0L, serverStats[i].getNumConnections());
                    success = true;
                } catch (Throwable t) {
                    if (numRetries < 10) {
                        LOG.info("Retrying connection leak check (server)");
                        Thread.sleep((numRetries + 1) * 50L);
                        numRetries++;
                    } else {
                        throw t;
                    }
                }
            }
        }
    } finally {
        if (client != null) {
            try {
                client.shutdown().get();
            } catch (Exception e) {
                e.printStackTrace();
            }
            Assert.assertTrue(client.isEventGroupShutdown());
        }
        for (int i = 0; i < numServers; i++) {
            if (server[i] != null) {
                server[i].shutdown();
            }
        }
        if (clientTaskExecutor != null) {
            clientTaskExecutor.shutdown();
        }
    }
}

Example 13 with MessageSerializer

use of org.apache.flink.queryablestate.network.messages.MessageSerializer in project flink by apache.

the class ClientTest method testSimpleRequests.

/**
 * Tests simple queries, of which half succeed and half fail.
 */
@Test
public void testSimpleRequests() throws Exception {
    AtomicKvStateRequestStats stats = new AtomicKvStateRequestStats();
    MessageSerializer<KvStateInternalRequest, KvStateResponse> serializer = new MessageSerializer<>(new KvStateInternalRequest.KvStateInternalRequestDeserializer(), new KvStateResponse.KvStateResponseDeserializer());
    Client<KvStateInternalRequest, KvStateResponse> client = null;
    Channel serverChannel = null;
    try {
        client = new Client<>("Test Client", 1, serializer, stats);
        // Random result
        final byte[] expected = new byte[1024];
        ThreadLocalRandom.current().nextBytes(expected);
        final LinkedBlockingQueue<ByteBuf> received = new LinkedBlockingQueue<>();
        final AtomicReference<Channel> channel = new AtomicReference<>();
        serverChannel = createServerChannel(new ChannelDataCollectingHandler(channel, received));
        InetSocketAddress serverAddress = getKvStateServerAddress(serverChannel);
        long numQueries = 1024L;
        List<CompletableFuture<KvStateResponse>> futures = new ArrayList<>();
        for (long i = 0L; i < numQueries; i++) {
            KvStateInternalRequest request = new KvStateInternalRequest(new KvStateID(), new byte[0]);
            futures.add(client.sendRequest(serverAddress, request));
        }
        // Respond to messages
        Exception testException = new RuntimeException("Expected test Exception");
        for (long i = 0L; i < numQueries; i++) {
            ByteBuf buf = received.take();
            assertNotNull("Receive timed out", buf);
            Channel ch = channel.get();
            assertNotNull("Channel not active", ch);
            assertEquals(MessageType.REQUEST, MessageSerializer.deserializeHeader(buf));
            long requestId = MessageSerializer.getRequestId(buf);
            KvStateInternalRequest deserRequest = serializer.deserializeRequest(buf);
            buf.release();
            if (i % 2L == 0L) {
                ByteBuf response = MessageSerializer.serializeResponse(serverChannel.alloc(), requestId, new KvStateResponse(expected));
                ch.writeAndFlush(response);
            } else {
                ByteBuf response = MessageSerializer.serializeRequestFailure(serverChannel.alloc(), requestId, testException);
                ch.writeAndFlush(response);
            }
        }
        for (long i = 0L; i < numQueries; i++) {
            if (i % 2L == 0L) {
                KvStateResponse serializedResult = futures.get((int) i).get();
                assertArrayEquals(expected, serializedResult.getContent());
            } else {
                try {
                    futures.get((int) i).get();
                    fail("Did not throw expected Exception");
                } catch (ExecutionException e) {
                    if (!(e.getCause() instanceof RuntimeException)) {
                        fail("Did not throw expected Exception");
                    }
                // else expected
                }
            }
        }
        assertEquals(numQueries, stats.getNumRequests());
        long expectedRequests = numQueries / 2L;
        // Counts can take some time to propagate
        while (stats.getNumSuccessful() != expectedRequests || stats.getNumFailed() != expectedRequests) {
            Thread.sleep(100L);
        }
        assertEquals(expectedRequests, stats.getNumSuccessful());
        assertEquals(expectedRequests, stats.getNumFailed());
    } finally {
        if (client != null) {
            Exception exc = null;
            try {
                // todo here we were seeing this problem:
                // https://github.com/netty/netty/issues/4357 if we do a get().
                // this is why we now simply wait a bit so that everything is
                // shut down and then we check
                client.shutdown().get();
            } catch (Exception e) {
                exc = e;
                LOG.error("An exception occurred while shutting down netty.", e);
            }
            Assert.assertTrue(ExceptionUtils.stringifyException(exc), client.isEventGroupShutdown());
        }
        if (serverChannel != null) {
            serverChannel.close();
        }
        assertEquals("Channel leak", 0L, stats.getNumConnections());
    }
}

Example 14 with MessageSerializer

use of org.apache.flink.queryablestate.network.messages.MessageSerializer in project flink by apache.

the class KvStateServerHandlerTest method testSimpleQuery.

/**
 * Tests a simple successful query via an EmbeddedChannel.
 */
@Test
public void testSimpleQuery() throws Exception {
    KvStateRegistry registry = new KvStateRegistry();
    AtomicKvStateRequestStats stats = new AtomicKvStateRequestStats();
    MessageSerializer<KvStateInternalRequest, KvStateResponse> serializer = new MessageSerializer<>(new KvStateInternalRequest.KvStateInternalRequestDeserializer(), new KvStateResponse.KvStateResponseDeserializer());
    KvStateServerHandler handler = new KvStateServerHandler(testServer, registry, serializer, stats);
    EmbeddedChannel channel = new EmbeddedChannel(getFrameDecoder(), handler);
    // Register state
    ValueStateDescriptor<Integer> desc = new ValueStateDescriptor<>("any", IntSerializer.INSTANCE);
    desc.setQueryable("vanilla");
    int numKeyGroups = 1;
    AbstractStateBackend abstractBackend = new MemoryStateBackend();
    DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
    dummyEnv.setKvStateRegistry(registry);
    AbstractKeyedStateBackend<Integer> backend = createKeyedStateBackend(registry, numKeyGroups, abstractBackend, dummyEnv);
    final TestRegistryListener registryListener = new TestRegistryListener();
    registry.registerListener(dummyEnv.getJobID(), registryListener);
    // Update the KvState and request it
    int expectedValue = 712828289;
    int key = 99812822;
    backend.setCurrentKey(key);
    ValueState<Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
    state.update(expectedValue);
    byte[] serializedKeyAndNamespace = KvStateSerializer.serializeKeyAndNamespace(key, IntSerializer.INSTANCE, VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE);
    long requestId = Integer.MAX_VALUE + 182828L;
    assertTrue(registryListener.registrationName.equals("vanilla"));
    KvStateInternalRequest request = new KvStateInternalRequest(registryListener.kvStateId, serializedKeyAndNamespace);
    ByteBuf serRequest = MessageSerializer.serializeRequest(channel.alloc(), requestId, request);
    // Write the request and wait for the response
    channel.writeInbound(serRequest);
    ByteBuf buf = (ByteBuf) readInboundBlocking(channel);
    // skip frame length
    buf.skipBytes(4);
    // Verify the response
    assertEquals(MessageType.REQUEST_RESULT, MessageSerializer.deserializeHeader(buf));
    long deserRequestId = MessageSerializer.getRequestId(buf);
    KvStateResponse response = serializer.deserializeResponse(buf);
    buf.release();
    assertEquals(requestId, deserRequestId);
    int actualValue = KvStateSerializer.deserializeValue(response.getContent(), IntSerializer.INSTANCE);
    assertEquals(expectedValue, actualValue);
    assertEquals(stats.toString(), 1, stats.getNumRequests());
    // Wait for async successful request report
    long deadline = System.nanoTime() + TimeUnit.NANOSECONDS.convert(30, TimeUnit.SECONDS);
    while (stats.getNumSuccessful() != 1L && System.nanoTime() <= deadline) {
        Thread.sleep(10L);
    }
    assertEquals(stats.toString(), 1L, stats.getNumSuccessful());
}

Example 15 with MessageSerializer

use of org.apache.flink.queryablestate.network.messages.MessageSerializer in project flink by apache.

the class KvStateServerHandlerTest method testFailureOnGetSerializedValue.

/**
 * Tests the failure response on a failure on the {@link
 * InternalKvState#getSerializedValue(byte[], TypeSerializer, TypeSerializer, TypeSerializer)}
 * call.
 */
@Test
public void testFailureOnGetSerializedValue() throws Exception {
    KvStateRegistry registry = new KvStateRegistry();
    AtomicKvStateRequestStats stats = new AtomicKvStateRequestStats();
    MessageSerializer<KvStateInternalRequest, KvStateResponse> serializer = new MessageSerializer<>(new KvStateInternalRequest.KvStateInternalRequestDeserializer(), new KvStateResponse.KvStateResponseDeserializer());
    KvStateServerHandler handler = new KvStateServerHandler(testServer, registry, serializer, stats);
    EmbeddedChannel channel = new EmbeddedChannel(getFrameDecoder(), handler);
    // Failing KvState
    InternalKvState<Integer, VoidNamespace, Long> kvState = new InternalKvState<Integer, VoidNamespace, Long>() {

        @Override
        public TypeSerializer<Integer> getKeySerializer() {
            return IntSerializer.INSTANCE;
        }

        @Override
        public TypeSerializer<VoidNamespace> getNamespaceSerializer() {
            return VoidNamespaceSerializer.INSTANCE;
        }

        @Override
        public TypeSerializer<Long> getValueSerializer() {
            return LongSerializer.INSTANCE;
        }

        @Override
        public void setCurrentNamespace(VoidNamespace namespace) {
        // do nothing
        }

        @Override
        public byte[] getSerializedValue(final byte[] serializedKeyAndNamespace, final TypeSerializer<Integer> safeKeySerializer, final TypeSerializer<VoidNamespace> safeNamespaceSerializer, final TypeSerializer<Long> safeValueSerializer) throws Exception {
            throw new RuntimeException("Expected test Exception");
        }

        @Override
        public StateIncrementalVisitor<Integer, VoidNamespace, Long> getStateIncrementalVisitor(int recommendedMaxNumberOfReturnedRecords) {
            throw new UnsupportedOperationException();
        }

        @Override
        public void clear() {
        }
    };
    KvStateID kvStateId = registry.registerKvState(new JobID(), new JobVertexID(), new KeyGroupRange(0, 0), "vanilla", kvState, getClass().getClassLoader());
    KvStateInternalRequest request = new KvStateInternalRequest(kvStateId, new byte[0]);
    ByteBuf serRequest = MessageSerializer.serializeRequest(channel.alloc(), 282872L, request);
    // Write the request and wait for the response
    channel.writeInbound(serRequest);
    ByteBuf buf = (ByteBuf) readInboundBlocking(channel);
    // skip frame length
    buf.skipBytes(4);
    // Verify the response
    assertEquals(MessageType.REQUEST_FAILURE, MessageSerializer.deserializeHeader(buf));
    RequestFailure response = MessageSerializer.deserializeRequestFailure(buf);
    buf.release();
    assertTrue(response.getCause().getMessage().contains("Expected test Exception"));
    assertEquals(1L, stats.getNumRequests());
    assertEquals(1L, stats.getNumFailed());
}