Examples with LengthFieldBasedFrameDecoder org.apache.flink.shaded.netty4.io.netty.handler.codec.LengthFieldBasedFrameDecoder used on opensource projects

Example 1 with LengthFieldBasedFrameDecoder

use of org.apache.flink.shaded.netty4.io.netty.handler.codec.LengthFieldBasedFrameDecoder in project flink by apache.

the class KvStateServerTest method testSimpleRequest.

/**
	 * Tests a simple successful query via a SocketChannel.
	 */
@Test
public void testSimpleRequest() throws Exception {
    KvStateServer server = null;
    Bootstrap bootstrap = null;
    try {
        KvStateRegistry registry = new KvStateRegistry();
        KvStateRequestStats stats = new AtomicKvStateRequestStats();
        server = new KvStateServer(InetAddress.getLocalHost(), 0, 1, 1, registry, stats);
        server.start();
        KvStateServerAddress serverAddress = server.getAddress();
        int numKeyGroups = 1;
        AbstractStateBackend abstractBackend = new MemoryStateBackend();
        DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
        dummyEnv.setKvStateRegistry(registry);
        AbstractKeyedStateBackend<Integer> backend = abstractBackend.createKeyedStateBackend(dummyEnv, new JobID(), "test_op", IntSerializer.INSTANCE, numKeyGroups, new KeyGroupRange(0, 0), registry.createTaskRegistry(new JobID(), new JobVertexID()));
        final KvStateServerHandlerTest.TestRegistryListener registryListener = new KvStateServerHandlerTest.TestRegistryListener();
        registry.registerListener(registryListener);
        ValueStateDescriptor<Integer> desc = new ValueStateDescriptor<>("any", IntSerializer.INSTANCE);
        desc.setQueryable("vanilla");
        ValueState<Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
        // Update KvState
        int expectedValue = 712828289;
        int key = 99812822;
        backend.setCurrentKey(key);
        state.update(expectedValue);
        // Request
        byte[] serializedKeyAndNamespace = KvStateRequestSerializer.serializeKeyAndNamespace(key, IntSerializer.INSTANCE, VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE);
        // Connect to the server
        final BlockingQueue<ByteBuf> responses = new LinkedBlockingQueue<>();
        bootstrap = createBootstrap(new LengthFieldBasedFrameDecoder(Integer.MAX_VALUE, 0, 4, 0, 4), new ChannelInboundHandlerAdapter() {

            @Override
            public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
                responses.add((ByteBuf) msg);
            }
        });
        Channel channel = bootstrap.connect(serverAddress.getHost(), serverAddress.getPort()).sync().channel();
        long requestId = Integer.MAX_VALUE + 182828L;
        assertTrue(registryListener.registrationName.equals("vanilla"));
        ByteBuf request = KvStateRequestSerializer.serializeKvStateRequest(channel.alloc(), requestId, registryListener.kvStateId, serializedKeyAndNamespace);
        channel.writeAndFlush(request);
        ByteBuf buf = responses.poll(TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
        assertEquals(KvStateRequestType.REQUEST_RESULT, KvStateRequestSerializer.deserializeHeader(buf));
        KvStateRequestResult response = KvStateRequestSerializer.deserializeKvStateRequestResult(buf);
        assertEquals(requestId, response.getRequestId());
        int actualValue = KvStateRequestSerializer.deserializeValue(response.getSerializedResult(), IntSerializer.INSTANCE);
        assertEquals(expectedValue, actualValue);
    } finally {
        if (server != null) {
            server.shutDown();
        }
        if (bootstrap != null) {
            EventLoopGroup group = bootstrap.group();
            if (group != null) {
                group.shutdownGracefully();
            }
        }
    }
}

Example 2 with LengthFieldBasedFrameDecoder

use of org.apache.flink.shaded.netty4.io.netty.handler.codec.LengthFieldBasedFrameDecoder in project pulsar by yahoo.

the class MockBrokerService method startMockBrokerService.

public void startMockBrokerService() throws Exception {
    ThreadFactory threadFactory = new ThreadFactoryBuilder().setNameFormat("mock-pulsar-%s").build();
    final int numThreads = 2;
    final int MaxMessageSize = 5 * 1024 * 1024;
    EventLoopGroup eventLoopGroup;
    try {
        if (SystemUtils.IS_OS_LINUX) {
            try {
                eventLoopGroup = new EpollEventLoopGroup(numThreads, threadFactory);
            } catch (UnsatisfiedLinkError e) {
                eventLoopGroup = new NioEventLoopGroup(numThreads, threadFactory);
            }
        } else {
            eventLoopGroup = new NioEventLoopGroup(numThreads, threadFactory);
        }
        workerGroup = eventLoopGroup;
        ServerBootstrap bootstrap = new ServerBootstrap();
        bootstrap.group(workerGroup, workerGroup);
        if (workerGroup instanceof EpollEventLoopGroup) {
            bootstrap.channel(EpollServerSocketChannel.class);
        } else {
            bootstrap.channel(NioServerSocketChannel.class);
        }
        bootstrap.childHandler(new ChannelInitializer<SocketChannel>() {

            @Override
            public void initChannel(SocketChannel ch) throws Exception {
                ch.pipeline().addLast("frameDecoder", new LengthFieldBasedFrameDecoder(MaxMessageSize, 0, 4, 0, 4));
                ch.pipeline().addLast("handler", new MockServerCnx());
            }
        });
        // Bind and start to accept incoming connections.
        bootstrap.bind(brokerServicePort).sync();
    } catch (Exception e) {
        throw e;
    }
}

Example 3 with LengthFieldBasedFrameDecoder

use of org.apache.flink.shaded.netty4.io.netty.handler.codec.LengthFieldBasedFrameDecoder in project carbondata by apache.

the class DictionaryClient method startClient.

/**
   * start dictionary client
   *
   * @param address
   * @param port
   */
public void startClient(String address, int port) {
    LOGGER.audit("Starting client on " + address + " " + port);
    long start = System.currentTimeMillis();
    // Create an Event with 1 thread.
    workerGroup = new NioEventLoopGroup(1);
    Bootstrap clientBootstrap = new Bootstrap();
    clientBootstrap.group(workerGroup).channel(NioSocketChannel.class).handler(new ChannelInitializer<SocketChannel>() {

        @Override
        public void initChannel(SocketChannel ch) throws Exception {
            ChannelPipeline pipeline = ch.pipeline();
            // Based on length provided at header, it collects all packets
            pipeline.addLast("LengthDecoder", new LengthFieldBasedFrameDecoder(1048576, 0, 2, 0, 2));
            pipeline.addLast("DictionaryClientHandler", dictionaryClientHandler);
        }
    });
    clientBootstrap.connect(new InetSocketAddress(address, port));
    LOGGER.info("Dictionary client Started, Total time spent : " + (System.currentTimeMillis() - start));
}

Example 4 with LengthFieldBasedFrameDecoder

use of org.apache.flink.shaded.netty4.io.netty.handler.codec.LengthFieldBasedFrameDecoder in project carbondata by apache.

the class DictionaryServer method bindToPort.

/**
   * Binds dictionary server to an available port.
   *
   * @param port
   */
private void bindToPort(int port) {
    long start = System.currentTimeMillis();
    // Configure the server.
    int i = 0;
    while (i < 10) {
        int newPort = port + i;
        try {
            ServerBootstrap bootstrap = new ServerBootstrap();
            bootstrap.group(boss, worker);
            bootstrap.channel(NioServerSocketChannel.class);
            bootstrap.childHandler(new ChannelInitializer<SocketChannel>() {

                @Override
                public void initChannel(SocketChannel ch) throws Exception {
                    ChannelPipeline pipeline = ch.pipeline();
                    pipeline.addLast("LengthDecoder", new LengthFieldBasedFrameDecoder(1048576, 0, 2, 0, 2));
                    pipeline.addLast("DictionaryServerHandler", dictionaryServerHandler);
                }
            });
            bootstrap.childOption(ChannelOption.SO_KEEPALIVE, true);
            bootstrap.bind(newPort).sync();
            LOGGER.audit("Dictionary Server started, Time spent " + (System.currentTimeMillis() - start) + " Listening on port " + newPort);
            this.port = newPort;
            break;
        } catch (Exception e) {
            LOGGER.error(e, "Dictionary Server Failed to bind to port:");
            if (i == 9) {
                throw new RuntimeException("Dictionary Server Could not bind to any port");
            }
        }
        i++;
    }
}

Example 5 with LengthFieldBasedFrameDecoder

use of org.apache.flink.shaded.netty4.io.netty.handler.codec.LengthFieldBasedFrameDecoder in project angel by Tencent.

the class MatrixTransportServer method start.

public void start() {
    Configuration conf = context.getConf();
    int workerNum = conf.getInt(AngelConf.ANGEL_NETTY_MATRIXTRANSFER_SERVER_EVENTGROUP_THREADNUM, AngelConf.DEFAULT_ANGEL_NETTY_MATRIXTRANSFER_SERVER_EVENTGROUP_THREADNUM);
    int sendBuffSize = conf.getInt(AngelConf.ANGEL_NETTY_MATRIXTRANSFER_SERVER_SNDBUF, AngelConf.DEFAULT_ANGEL_NETTY_MATRIXTRANSFER_SERVER_SNDBUF);
    int recvBuffSize = conf.getInt(AngelConf.ANGEL_NETTY_MATRIXTRANSFER_SERVER_RCVBUF, AngelConf.DEFAULT_ANGEL_NETTY_MATRIXTRANSFER_SERVER_RCVBUF);
    final int maxMessageSize = conf.getInt(AngelConf.ANGEL_NETTY_MATRIXTRANSFER_MAX_MESSAGE_SIZE, AngelConf.DEFAULT_ANGEL_NETTY_MATRIXTRANSFER_MAX_MESSAGE_SIZE);
    bossGroup = new NioEventLoopGroup(1);
    workerGroup = new NioEventLoopGroup(workerNum);
    ((NioEventLoopGroup) workerGroup).setIoRatio(70);
    ServerBootstrap b = new ServerBootstrap();
    b.group(bossGroup, workerGroup).channel(NioServerSocketChannel.class).option(ChannelOption.SO_SNDBUF, sendBuffSize).option(ChannelOption.SO_RCVBUF, recvBuffSize).childHandler(new ChannelInitializer<SocketChannel>() {

        @Override
        public void initChannel(SocketChannel ch) throws Exception {
            ChannelPipeline p = ch.pipeline();
            p.addLast(new LengthFieldBasedFrameDecoder(maxMessageSize, 0, 4, 0, 4));
            p.addLast(new LengthFieldPrepender(4));
            p.addLast(new MatrixTransportServerHandler(context));
        }
    });
    channelFuture = b.bind(port);
}