Examples with EventLoopGroup org.apache.flink.shaded.netty4.io.netty.channel.EventLoopGroup used on opensource projects

Example 6 with EventLoopGroup

use of org.apache.flink.shaded.netty4.io.netty.channel.EventLoopGroup in project pinot by linkedin.

the class ScatterGatherTest method testMultipleServerTimeout.

@Test
public void testMultipleServerTimeout() throws Exception {
    MetricsRegistry registry = new MetricsRegistry();
    // Server start
    int serverPort1 = 7081;
    int serverPort2 = 7082;
    int serverPort3 = 7083;
    // Timeout server
    int serverPort4 = 7084;
    NettyTCPServer server1 = new NettyTCPServer(serverPort1, new TestRequestHandlerFactory(0, 1), null);
    NettyTCPServer server2 = new NettyTCPServer(serverPort2, new TestRequestHandlerFactory(1, 1), null);
    NettyTCPServer server3 = new NettyTCPServer(serverPort3, new TestRequestHandlerFactory(2, 1), null);
    NettyTCPServer server4 = new NettyTCPServer(serverPort4, new TestRequestHandlerFactory(3, 1, 7000, false), null);
    Thread t1 = new Thread(server1);
    Thread t2 = new Thread(server2);
    Thread t3 = new Thread(server3);
    Thread t4 = new Thread(server4);
    t1.start();
    t2.start();
    t3.start();
    t4.start();
    //Client setup
    ScheduledExecutorService timedExecutor = new ScheduledThreadPoolExecutor(1);
    ExecutorService service = new ThreadPoolExecutor(5, 5, 5, TimeUnit.DAYS, new LinkedBlockingDeque<Runnable>());
    EventLoopGroup eventLoopGroup = new NioEventLoopGroup();
    NettyClientMetrics clientMetrics = new NettyClientMetrics(registry, "client_");
    PooledNettyClientResourceManager rm = new PooledNettyClientResourceManager(eventLoopGroup, new HashedWheelTimer(), clientMetrics);
    KeyedPoolImpl<ServerInstance, NettyClientConnection> pool = new KeyedPoolImpl<ServerInstance, NettyClientConnection>(1, 1, 300000, 1, rm, timedExecutor, service, registry);
    rm.setPool(pool);
    SegmentIdSet pg1 = new SegmentIdSet();
    pg1.addSegment(new SegmentId("0"));
    SegmentIdSet pg2 = new SegmentIdSet();
    pg2.addSegment(new SegmentId("1"));
    SegmentIdSet pg3 = new SegmentIdSet();
    pg3.addSegment(new SegmentId("2"));
    SegmentIdSet pg4 = new SegmentIdSet();
    pg4.addSegment(new SegmentId("3"));
    ServerInstance serverInstance1 = new ServerInstance("localhost", serverPort1);
    ServerInstance serverInstance2 = new ServerInstance("localhost", serverPort2);
    ServerInstance serverInstance3 = new ServerInstance("localhost", serverPort3);
    ServerInstance serverInstance4 = new ServerInstance("localhost", serverPort4);
    Map<ServerInstance, SegmentIdSet> pgMap = new HashMap<ServerInstance, SegmentIdSet>();
    pgMap.put(serverInstance1, pg1);
    pgMap.put(serverInstance2, pg2);
    pgMap.put(serverInstance3, pg3);
    pgMap.put(serverInstance4, pg4);
    String request1 = "request_0";
    String request2 = "request_1";
    String request3 = "request_2";
    String request4 = "request_3";
    Map<SegmentIdSet, String> pgMapStr = new HashMap<SegmentIdSet, String>();
    pgMapStr.put(pg1, request1);
    pgMapStr.put(pg2, request2);
    pgMapStr.put(pg3, request3);
    pgMapStr.put(pg4, request4);
    ScatterGatherRequest req = new TestScatterGatherRequest(pgMap, pgMapStr, new RoundRobinReplicaSelection(), ReplicaSelectionGranularity.SEGMENT_ID_SET, 0, 1000);
    ScatterGatherImpl scImpl = new ScatterGatherImpl(pool, service);
    final ScatterGatherStats scatterGatherStats = new ScatterGatherStats();
    BrokerMetrics brokerMetrics = new BrokerMetrics(new MetricsRegistry());
    CompositeFuture<ServerInstance, ByteBuf> fut = scImpl.scatterGather(req, scatterGatherStats, brokerMetrics);
    Map<ServerInstance, ByteBuf> v = fut.get();
    //Only 3 servers return value.
    Assert.assertEquals(v.size(), 3);
    ByteBuf b = v.get(serverInstance1);
    byte[] b2 = new byte[b.readableBytes()];
    b.readBytes(b2);
    String response = new String(b2);
    Assert.assertEquals(response, "response_0_0");
    b = v.get(serverInstance2);
    b2 = new byte[b.readableBytes()];
    b.readBytes(b2);
    response = new String(b2);
    Assert.assertEquals(response, "response_1_0");
    b = v.get(serverInstance3);
    b2 = new byte[b.readableBytes()];
    b.readBytes(b2);
    response = new String(b2);
    Assert.assertEquals(response, "response_2_0");
    //No  response from 4th server
    Assert.assertNull(v.get(serverInstance4), "No response from 4th server");
    Map<ServerInstance, Throwable> errorMap = fut.getError();
    Assert.assertEquals(errorMap.size(), 1, "One error");
    Assert.assertNotNull(errorMap.get(serverInstance4), "Server4 returned timeout");
    Thread.sleep(3000);
    pool.getStats().refresh();
    Assert.assertEquals(pool.getStats().getTotalBadDestroyed(), 1, "Total Bad destroyed");
    pool.shutdown();
    service.shutdown();
    eventLoopGroup.shutdownGracefully();
    server1.shutdownGracefully();
    server2.shutdownGracefully();
    server3.shutdownGracefully();
    server4.shutdownGracefully();
}

Example 7 with EventLoopGroup

use of org.apache.flink.shaded.netty4.io.netty.channel.EventLoopGroup in project pinot by linkedin.

the class ScatterGatherTest method testMultipleServerHappy.

@Test
public void testMultipleServerHappy() throws Exception {
    MetricsRegistry registry = new MetricsRegistry();
    // Server start
    int serverPort1 = 7071;
    int serverPort2 = 7072;
    int serverPort3 = 7073;
    int serverPort4 = 7074;
    NettyTCPServer server1 = new NettyTCPServer(serverPort1, new TestRequestHandlerFactory(0, 1), null);
    NettyTCPServer server2 = new NettyTCPServer(serverPort2, new TestRequestHandlerFactory(1, 1), null);
    NettyTCPServer server3 = new NettyTCPServer(serverPort3, new TestRequestHandlerFactory(2, 1), null);
    NettyTCPServer server4 = new NettyTCPServer(serverPort4, new TestRequestHandlerFactory(3, 1), null);
    Thread t1 = new Thread(server1);
    Thread t2 = new Thread(server2);
    Thread t3 = new Thread(server3);
    Thread t4 = new Thread(server4);
    t1.start();
    t2.start();
    t3.start();
    t4.start();
    //Client setup
    ScheduledExecutorService timedExecutor = new ScheduledThreadPoolExecutor(1);
    ExecutorService poolExecutor = MoreExecutors.sameThreadExecutor();
    ExecutorService service = new ThreadPoolExecutor(1, 1, 1, TimeUnit.DAYS, new LinkedBlockingDeque<Runnable>());
    EventLoopGroup eventLoopGroup = new NioEventLoopGroup();
    NettyClientMetrics clientMetrics = new NettyClientMetrics(registry, "client_");
    PooledNettyClientResourceManager rm = new PooledNettyClientResourceManager(eventLoopGroup, new HashedWheelTimer(), clientMetrics);
    KeyedPoolImpl<ServerInstance, NettyClientConnection> pool = new KeyedPoolImpl<ServerInstance, NettyClientConnection>(1, 1, 300000, 1, rm, timedExecutor, poolExecutor, registry);
    rm.setPool(pool);
    SegmentIdSet pg1 = new SegmentIdSet();
    pg1.addSegment(new SegmentId("0"));
    SegmentIdSet pg2 = new SegmentIdSet();
    pg2.addSegment(new SegmentId("1"));
    SegmentIdSet pg3 = new SegmentIdSet();
    pg3.addSegment(new SegmentId("2"));
    SegmentIdSet pg4 = new SegmentIdSet();
    pg4.addSegment(new SegmentId("3"));
    ServerInstance serverInstance1 = new ServerInstance("localhost", serverPort1);
    ServerInstance serverInstance2 = new ServerInstance("localhost", serverPort2);
    ServerInstance serverInstance3 = new ServerInstance("localhost", serverPort3);
    ServerInstance serverInstance4 = new ServerInstance("localhost", serverPort4);
    Map<ServerInstance, SegmentIdSet> pgMap = new HashMap<ServerInstance, SegmentIdSet>();
    pgMap.put(serverInstance1, pg1);
    pgMap.put(serverInstance2, pg2);
    pgMap.put(serverInstance3, pg3);
    pgMap.put(serverInstance4, pg4);
    String request1 = "request_0";
    String request2 = "request_1";
    String request3 = "request_2";
    String request4 = "request_3";
    Map<SegmentIdSet, String> pgMapStr = new HashMap<SegmentIdSet, String>();
    pgMapStr.put(pg1, request1);
    pgMapStr.put(pg2, request2);
    pgMapStr.put(pg3, request3);
    pgMapStr.put(pg4, request4);
    ScatterGatherRequest req = new TestScatterGatherRequest(pgMap, pgMapStr);
    ScatterGatherImpl scImpl = new ScatterGatherImpl(pool, service);
    final ScatterGatherStats scatterGatherStats = new ScatterGatherStats();
    BrokerMetrics brokerMetrics = new BrokerMetrics(new MetricsRegistry());
    CompositeFuture<ServerInstance, ByteBuf> fut = scImpl.scatterGather(req, scatterGatherStats, brokerMetrics);
    Map<ServerInstance, ByteBuf> v = fut.get();
    Assert.assertEquals(v.size(), 4);
    ByteBuf b = v.get(serverInstance1);
    byte[] b2 = new byte[b.readableBytes()];
    b.readBytes(b2);
    String response = new String(b2);
    Assert.assertEquals(response, "response_0_0");
    b = v.get(serverInstance2);
    b2 = new byte[b.readableBytes()];
    b.readBytes(b2);
    response = new String(b2);
    Assert.assertEquals(response, "response_1_0");
    b = v.get(serverInstance3);
    b2 = new byte[b.readableBytes()];
    b.readBytes(b2);
    response = new String(b2);
    Assert.assertEquals(response, "response_2_0");
    b = v.get(serverInstance4);
    b2 = new byte[b.readableBytes()];
    b.readBytes(b2);
    response = new String(b2);
    Assert.assertEquals(response, "response_3_0");
    server1.shutdownGracefully();
    server2.shutdownGracefully();
    server3.shutdownGracefully();
    server4.shutdownGracefully();
    pool.shutdown();
    service.shutdown();
    eventLoopGroup.shutdownGracefully();
}

Example 8 with EventLoopGroup

use of org.apache.flink.shaded.netty4.io.netty.channel.EventLoopGroup in project flink by apache.

the class KvStateServerTest method testSimpleRequest.

/**
	 * Tests a simple successful query via a SocketChannel.
	 */
@Test
public void testSimpleRequest() throws Exception {
    KvStateServer server = null;
    Bootstrap bootstrap = null;
    try {
        KvStateRegistry registry = new KvStateRegistry();
        KvStateRequestStats stats = new AtomicKvStateRequestStats();
        server = new KvStateServer(InetAddress.getLocalHost(), 0, 1, 1, registry, stats);
        server.start();
        KvStateServerAddress serverAddress = server.getAddress();
        int numKeyGroups = 1;
        AbstractStateBackend abstractBackend = new MemoryStateBackend();
        DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
        dummyEnv.setKvStateRegistry(registry);
        AbstractKeyedStateBackend<Integer> backend = abstractBackend.createKeyedStateBackend(dummyEnv, new JobID(), "test_op", IntSerializer.INSTANCE, numKeyGroups, new KeyGroupRange(0, 0), registry.createTaskRegistry(new JobID(), new JobVertexID()));
        final KvStateServerHandlerTest.TestRegistryListener registryListener = new KvStateServerHandlerTest.TestRegistryListener();
        registry.registerListener(registryListener);
        ValueStateDescriptor<Integer> desc = new ValueStateDescriptor<>("any", IntSerializer.INSTANCE);
        desc.setQueryable("vanilla");
        ValueState<Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
        // Update KvState
        int expectedValue = 712828289;
        int key = 99812822;
        backend.setCurrentKey(key);
        state.update(expectedValue);
        // Request
        byte[] serializedKeyAndNamespace = KvStateRequestSerializer.serializeKeyAndNamespace(key, IntSerializer.INSTANCE, VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE);
        // Connect to the server
        final BlockingQueue<ByteBuf> responses = new LinkedBlockingQueue<>();
        bootstrap = createBootstrap(new LengthFieldBasedFrameDecoder(Integer.MAX_VALUE, 0, 4, 0, 4), new ChannelInboundHandlerAdapter() {

            @Override
            public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
                responses.add((ByteBuf) msg);
            }
        });
        Channel channel = bootstrap.connect(serverAddress.getHost(), serverAddress.getPort()).sync().channel();
        long requestId = Integer.MAX_VALUE + 182828L;
        assertTrue(registryListener.registrationName.equals("vanilla"));
        ByteBuf request = KvStateRequestSerializer.serializeKvStateRequest(channel.alloc(), requestId, registryListener.kvStateId, serializedKeyAndNamespace);
        channel.writeAndFlush(request);
        ByteBuf buf = responses.poll(TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
        assertEquals(KvStateRequestType.REQUEST_RESULT, KvStateRequestSerializer.deserializeHeader(buf));
        KvStateRequestResult response = KvStateRequestSerializer.deserializeKvStateRequestResult(buf);
        assertEquals(requestId, response.getRequestId());
        int actualValue = KvStateRequestSerializer.deserializeValue(response.getSerializedResult(), IntSerializer.INSTANCE);
        assertEquals(expectedValue, actualValue);
    } finally {
        if (server != null) {
            server.shutDown();
        }
        if (bootstrap != null) {
            EventLoopGroup group = bootstrap.group();
            if (group != null) {
                group.shutdownGracefully();
            }
        }
    }
}

Example 9 with EventLoopGroup

use of org.apache.flink.shaded.netty4.io.netty.channel.EventLoopGroup in project netty by netty.

the class Http2FrameWriterBenchmark method boostrapEnvWithTransport.

private static Environment boostrapEnvWithTransport(final EnvironmentType environmentType) {
    final EnvironmentParameters params = environmentType.params();
    ServerBootstrap sb = new ServerBootstrap();
    Bootstrap cb = new Bootstrap();
    final TransportEnvironment environment = new TransportEnvironment(cb, sb);
    EventLoopGroup serverEventLoopGroup = params.newEventLoopGroup();
    sb.group(serverEventLoopGroup, serverEventLoopGroup);
    sb.channel(params.serverChannelClass());
    sb.option(ChannelOption.ALLOCATOR, params.serverAllocator());
    sb.childOption(ChannelOption.ALLOCATOR, params.serverAllocator());
    sb.childHandler(new ChannelInitializer<Channel>() {

        @Override
        protected void initChannel(Channel ch) throws Exception {
        }
    });
    cb.group(params.newEventLoopGroup());
    cb.channel(params.clientChannelClass());
    cb.option(ChannelOption.ALLOCATOR, params.clientAllocator());
    final CountDownLatch latch = new CountDownLatch(1);
    cb.handler(new ChannelInitializer<Channel>() {

        @Override
        protected void initChannel(Channel ch) throws Exception {
            ChannelPipeline p = ch.pipeline();
            Http2Connection connection = new DefaultHttp2Connection(false);
            Http2RemoteFlowController remoteFlowController = params.remoteFlowController();
            if (remoteFlowController != null) {
                connection.remote().flowController(params.remoteFlowController());
            }
            Http2LocalFlowController localFlowController = params.localFlowController();
            if (localFlowController != null) {
                connection.local().flowController(localFlowController);
            }
            environment.writer(new DefaultHttp2FrameWriter());
            Http2ConnectionEncoder encoder = new DefaultHttp2ConnectionEncoder(connection, environment.writer());
            Http2ConnectionDecoder decoder = new DefaultHttp2ConnectionDecoder(connection, encoder, new DefaultHttp2FrameReader());
            Http2ConnectionHandler connectionHandler = new Http2ConnectionHandlerBuilder().encoderEnforceMaxConcurrentStreams(false).frameListener(new Http2FrameAdapter()).codec(decoder, encoder).build();
            p.addLast(connectionHandler);
            environment.context(p.lastContext());
            // Must wait for context to be set.
            latch.countDown();
        }
    });
    environment.serverChannel(sb.bind(params.address()));
    params.address(environment.serverChannel().localAddress());
    environment.clientChannel(cb.connect(params.address()));
    try {
        if (!latch.await(5, SECONDS)) {
            throw new RuntimeException("Channel did not initialize in time");
        }
    } catch (InterruptedException ie) {
        throw new RuntimeException(ie);
    }
    return environment;
}

Example 10 with EventLoopGroup

use of org.apache.flink.shaded.netty4.io.netty.channel.EventLoopGroup in project netty by netty.

the class NioSocketChannelTest method testChannelReRegisterRead.

private static void testChannelReRegisterRead(final boolean sameEventLoop) throws Exception {
    final EventLoopGroup group = new NioEventLoopGroup(2);
    final CountDownLatch latch = new CountDownLatch(1);
    // Just some random bytes
    byte[] bytes = new byte[1024];
    PlatformDependent.threadLocalRandom().nextBytes(bytes);
    Channel sc = null;
    Channel cc = null;
    ServerBootstrap b = new ServerBootstrap();
    try {
        b.group(group).channel(NioServerSocketChannel.class).childOption(ChannelOption.SO_KEEPALIVE, true).childHandler(new ChannelInitializer<Channel>() {

            @Override
            protected void initChannel(Channel ch) throws Exception {
                ChannelPipeline pipeline = ch.pipeline();
                pipeline.addLast(new SimpleChannelInboundHandler<ByteBuf>() {

                    @Override
                    protected void channelRead0(ChannelHandlerContext ctx, ByteBuf byteBuf) {
                        // We was able to read something from the Channel after reregister.
                        latch.countDown();
                    }

                    @Override
                    public void channelActive(final ChannelHandlerContext ctx) throws Exception {
                        final EventLoop loop = group.next();
                        if (sameEventLoop) {
                            deregister(ctx, loop);
                        } else {
                            loop.execute(new Runnable() {

                                @Override
                                public void run() {
                                    deregister(ctx, loop);
                                }
                            });
                        }
                    }

                    private void deregister(ChannelHandlerContext ctx, final EventLoop loop) {
                        // As soon as the channel becomes active re-register it to another
                        // EventLoop. After this is done we should still receive the data that
                        // was written to the channel.
                        ctx.deregister().addListener(new ChannelFutureListener() {

                            @Override
                            public void operationComplete(ChannelFuture cf) {
                                Channel channel = cf.channel();
                                assertNotSame(loop, channel.eventLoop());
                                group.next().register(channel);
                            }
                        });
                    }
                });
            }
        });
        sc = b.bind(0).syncUninterruptibly().channel();
        Bootstrap bootstrap = new Bootstrap();
        bootstrap.group(group).channel(NioSocketChannel.class);
        bootstrap.handler(new ChannelInboundHandlerAdapter());
        cc = bootstrap.connect(sc.localAddress()).syncUninterruptibly().channel();
        cc.writeAndFlush(Unpooled.wrappedBuffer(bytes)).syncUninterruptibly();
        latch.await();
    } finally {
        if (cc != null) {
            cc.close();
        }
        if (sc != null) {
            sc.close();
        }
        group.shutdownGracefully();
    }
}