Examples with HttpClientCodec org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpClientCodec used on opensource projects

Example 41 with HttpClientCodec

use of org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpClientCodec in project riposte by Nike-Inc.

the class VerifyTimeoutsAndProxyConnectionPoolingWorksComponentTest method verify_incomplete_call_is_timed_out.

@Test
public void verify_incomplete_call_is_timed_out() throws InterruptedException, TimeoutException, ExecutionException, IOException {
    Bootstrap bootstrap = new Bootstrap();
    EventLoopGroup eventLoopGroup = new NioEventLoopGroup();
    try {
        CompletableFuture<Pair<String, String>> responseFromServer = new CompletableFuture<>();
        // Create a raw netty HTTP client so we can fiddle with headers and intentionally create a bad request
        // that should trigger the bad call timeout.
        bootstrap.group(eventLoopGroup).channel(NioSocketChannel.class).handler(new ChannelInitializer<SocketChannel>() {

            @Override
            protected void initChannel(SocketChannel ch) throws Exception {
                ChannelPipeline p = ch.pipeline();
                p.addLast(new HttpClientCodec());
                p.addLast(new HttpObjectAggregator(Integer.MAX_VALUE));
                p.addLast(new SimpleChannelInboundHandler<HttpObject>() {

                    @Override
                    protected void channelRead0(ChannelHandlerContext ctx, HttpObject msg) throws Exception {
                        if (msg instanceof FullHttpResponse) {
                            // Store the server response for asserting on later.
                            FullHttpResponse responseMsg = (FullHttpResponse) msg;
                            responseFromServer.complete(Pair.of(responseMsg.content().toString(CharsetUtil.UTF_8), responseMsg.headers().get(HttpHeaders.Names.CONNECTION)));
                        } else {
                            // Should never happen.
                            throw new RuntimeException("Received unexpected message type: " + msg.getClass());
                        }
                    }
                });
            }
        });
        // Connect to the server.
        Channel ch = bootstrap.connect("localhost", downstreamServerConfig.endpointsPort()).sync().channel();
        // Create a bad HTTP request. This one will be bad because it has a non-zero content-length header,
        // but we're sending no payload. The server should (correctly) sit and wait for payload bytes to
        // arrive until it hits the timeout, at which point it should return the correct error response.
        HttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, LongDelayTestEndpoint.MATCHING_PATH);
        request.headers().set(HttpHeaders.Names.HOST, "localhost");
        request.headers().set(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
        request.headers().set(HttpHeaders.Names.CONTENT_LENGTH, "100");
        long beforeCallTimeNanos = System.nanoTime();
        // Send the bad request.
        ch.writeAndFlush(request);
        // Wait for the response to be received and the connection to be closed.
        try {
            ch.closeFuture().get(incompleteCallTimeoutMillis * 10, TimeUnit.MILLISECONDS);
            responseFromServer.get(incompleteCallTimeoutMillis * 10, TimeUnit.MILLISECONDS);
        } catch (TimeoutException ex) {
            fail("The call took much longer than expected without receiving a response. " + "Cancelling this test - it's not working properly", ex);
        }
        // If we reach here then the call should be complete.
        long totalCallTimeMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - beforeCallTimeNanos);
        // Verify that we got back the correct error response.
        // It should be a MALFORMED_REQUEST with extra metadata explaining that the call was bad.
        Pair<String, String> responseInfo = responseFromServer.get();
        DefaultErrorContractDTO errorContract = objectMapper.readValue(responseInfo.getLeft(), DefaultErrorContractDTO.class);
        assertThat(errorContract).isNotNull();
        assertThat(errorContract.errors.size()).isEqualTo(1);
        DefaultErrorDTO error = errorContract.errors.get(0);
        ApiError expectedApiError = SampleCoreApiError.MALFORMED_REQUEST;
        Map<String, Object> expectedMetadata = MapBuilder.builder("cause", (Object) "Unfinished/invalid HTTP request").build();
        assertThat(error.code).isEqualTo(expectedApiError.getErrorCode());
        assertThat(error.message).isEqualTo(expectedApiError.getMessage());
        assertThat(error.metadata).isEqualTo(expectedMetadata);
        // The server should have closed the connection even though we asked for keep-alive.
        assertThat(responseInfo.getRight()).isEqualTo(HttpHeaders.Values.CLOSE);
        // Total call time should be pretty close to incompleteCallTimeoutMillis give or take a few
        // milliseconds, but due to the inability to account for slow machines running the unit tests,
        // a server that isn't warmed up, etc, we can't put a ceiling on the wiggle room we'd need, so
        // we'll just verify it took at least the minimum necessary amount of time.
        assertThat(totalCallTimeMillis).isGreaterThanOrEqualTo(incompleteCallTimeoutMillis);
    } finally {
        eventLoopGroup.shutdownGracefully();
    }
}

Example 42 with HttpClientCodec

use of org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpClientCodec in project qpid-broker-j by apache.

the class WebSocketFrameTransport method buildInputOutputPipeline.

@Override
protected void buildInputOutputPipeline(final ChannelPipeline pipeline) {
    pipeline.addLast(new HttpClientCodec());
    pipeline.addLast(new HttpObjectAggregator(65536));
    pipeline.addLast(_webSocketClientHandler);
    pipeline.addLast(_webSocketFramingOutputHandler);
    pipeline.addLast(_webSocketDeframingInputHandler);
    super.buildInputOutputPipeline(pipeline);
}

Example 43 with HttpClientCodec

use of org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpClientCodec in project ambry by linkedin.

the class NettyPerfClient method start.

/**
 * Starts the NettyPerfClient.
 * @throws InterruptedException
 */
protected void start() {
    logger.info("Starting NettyPerfClient");
    reporter.start();
    group = new NioEventLoopGroup(concurrency);
    perfClientStartTime = System.currentTimeMillis();
    for (String host : hosts) {
        logger.info("Connecting to {}:{}", host, port);
        // create a new bootstrap with a fixed remote address for each host. This is the simplest way to support
        // reconnection on failure. All bootstraps will share the same event loop group.
        Bootstrap bootstrap = new Bootstrap().group(group).channel(NioSocketChannel.class).remoteAddress(host, port);
        bootstrap.handler(new ChannelInitializer<SocketChannel>() {

            @Override
            public void initChannel(SocketChannel ch) {
                logger.info("Initializing the channel to {}:{}", host, port);
                if (sslFactory != null) {
                    ch.pipeline().addLast(new SslHandler(sslFactory.createSSLEngine(host, port, SSLFactory.Mode.CLIENT)));
                }
                ch.pipeline().addLast(new HttpClientCodec()).addLast(new ChunkedWriteHandler()).addLast(new ResponseHandler(bootstrap));
            }
        });
        for (int i = 0; i < concurrency; i++) {
            ChannelFuture future = bootstrap.connect();
            future.addListener(channelConnectListener);
        }
        hostToRequestCount.put(host, new AtomicLong(0));
        hostToSleepTime.put(host, sleepTimeInMs);
    }
    if (backgroundScheduler != null) {
        backgroundScheduler.scheduleAtFixedRate(updater, 0, 1, TimeUnit.SECONDS);
        logger.info("Background scheduler is instantiated to update sleep time.");
    }
    isRunning = true;
    logger.info("Created {} channel(s) per remote host", concurrency);
    logger.info("NettyPerfClient started");
}

Example 44 with HttpClientCodec

use of org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpClientCodec in project eat by nhnent.

the class NettyClient method startUp.

/**
 * Start up Netty client
 */
public void startUp(int actorIndex) {
    int cntOfRealThread = Config.obj().getCommon().getCountOfRealThread();
    int cntOfPort = Config.obj().getServer().getCountOfPort();
    int portNo = svrPort + (actorIndex % cntOfPort);
    int groupIndex = actorIndex % cntOfRealThread;
    logger.debug("actorIndex=>{}, portNo=>{}, groupIndex=>{}", actorIndex, portNo, groupIndex);
    try {
        int timeout = Config.obj().getCommon().getReceiveTimeoutSec();
        Bootstrap b = new Bootstrap();
        b.group(groups[groupIndex]).channel(NioSocketChannel.class).option(ChannelOption.TCP_NODELAY, true).handler(new ChannelInitializer<SocketChannel>() {

            @Override
            public void initChannel(SocketChannel ch) throws Exception {
                if (Config.obj().getServer().getSsl() != null) {
                    SslContext sslContext = SslHandler.initSSL();
                    if (sslContext != null) {
                        logger.info("use ssl");
                        ch.pipeline().addLast(sslContext.newHandler(ch.alloc()));
                    }
                }
                if (Config.obj().getServer().getSocketType().equals("STREAM")) {
                    ch.pipeline().addLast("readTimeoutHandler", new ReadTimeoutWithNoClose(timeout));
                    ch.pipeline().addLast("streamSocketPacketClientHandler", streamSocketPacketClientHandler);
                } else if (Config.obj().getServer().getSocketType().equals("WEBSOCKET")) {
                    ch.pipeline().addLast("http-codec", new HttpClientCodec());
                    ch.pipeline().addLast("aggregator", new HttpObjectAggregator(65536));
                    ch.pipeline().addLast("ws-handler", webSocketClientHandler);
                } else {
                    logger.error("Unsupported socket type");
                }
            }
        });
        Channel ch = b.connect(serverIp, portNo).sync().channel();
        logger.debug("Channel==> {}", ch.toString());
        isConnected = Boolean.TRUE;
    } catch (Exception e) {
        logger.error(ExceptionUtils.getStackTrace(e));
    }
}

Example 45 with HttpClientCodec

use of org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpClientCodec in project proxyee-down by monkeyWie.

the class HttpDownInitializer method initChannel.

@Override
protected void initChannel(Channel ch) throws Exception {
    if (bootstrap.getHttpDownInfo().getProxyConfig() != null) {
        ch.pipeline().addLast(ProxyHandleFactory.build(bootstrap.getHttpDownInfo().getProxyConfig()));
    }
    if (isSsl) {
        ch.pipeline().addLast(bootstrap.getClientSslContext().newHandler(ch.alloc()));
    }
    ch.pipeline().addLast("httpCodec", new HttpClientCodec());
    ch.pipeline().addLast(new ChannelInboundHandlerAdapter() {

        private Closeable closeable;

        private TaskInfo taskInfo = bootstrap.getHttpDownInfo().getTaskInfo();

        private HttpDownCallback callback = bootstrap.getCallback();

        @Override
        public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
            try {
                if (msg instanceof HttpContent) {
                    if (!isSucc) {
                        return;
                    }
                    HttpContent httpContent = (HttpContent) msg;
                    ByteBuf byteBuf = httpContent.content();
                    synchronized (chunkInfo) {
                        Channel nowChannel = bootstrap.getChannel(chunkInfo);
                        if (chunkInfo.getStatus() == HttpDownStatus.RUNNING && nowChannel == ctx.channel()) {
                            int readableBytes = bootstrap.doFileWriter(chunkInfo, byteBuf.nioBuffer());
                            if (readableBytes > 0) {
                                // 最后一次下载时间
                                chunkInfo.setLastDownTime(System.currentTimeMillis());
                                // 文件已下载大小
                                chunkInfo.setDownSize(chunkInfo.getDownSize() + readableBytes);
                                taskInfo.setDownSize(taskInfo.getDownSize() + readableBytes);
                                if (callback != null) {
                                    callback.onProgress(bootstrap.getHttpDownInfo(), chunkInfo);
                                }
                            }
                        } else {
                            safeClose(ctx.channel());
                            return;
                        }
                    }
                    if (isDone(chunkInfo.getDownSize(), httpContent)) {
                        LOGGER.debug("分段下载完成：channelId[" + ctx.channel().id() + "]\t" + chunkInfo);
                        bootstrap.close(chunkInfo, true);
                        // 分段下载完成回调
                        chunkInfo.setStatus(HttpDownStatus.DONE);
                        taskInfo.refresh(chunkInfo);
                        if (callback != null) {
                            callback.onChunkDone(bootstrap.getHttpDownInfo(), chunkInfo);
                        }
                        synchronized (taskInfo) {
                            if (taskInfo.getStatus() == HttpDownStatus.RUNNING && taskInfo.getChunkInfoList().stream().allMatch((chunk) -> chunk.getStatus() == HttpDownStatus.DONE)) {
                                if (!taskInfo.isSupportRange()) {
                                    // chunked编码最后更新文件大小
                                    taskInfo.setTotalSize(taskInfo.getDownSize());
                                    taskInfo.getChunkInfoList().get(0).setTotalSize(taskInfo.getDownSize());
                                }
                                // 文件下载完成回调
                                taskInfo.setStatus(HttpDownStatus.DONE);
                                LOGGER.debug("下载完成：channelId[" + ctx.channel().id() + "]\t" + chunkInfo);
                                bootstrap.close(true);
                                if (callback != null) {
                                    callback.onDone(bootstrap.getHttpDownInfo());
                                }
                            }
                        }
                    } else if (isContinue(chunkInfo.getDownSize())) {
                        // 百度响应做了手脚，会少一个字节
                        // 真实响应字节小于要下载的字节，在下载完成后要继续下载
                        LOGGER.debug("继续下载：channelId[" + ctx.channel().id() + "]\t" + chunkInfo);
                        bootstrap.retryChunkDown(chunkInfo, HttpDownStatus.CONNECTING_CONTINUE);
                    }
                } else {
                    HttpResponse httpResponse = (HttpResponse) msg;
                    Integer responseCode = httpResponse.status().code();
                    if (responseCode.toString().indexOf("20") != 0) {
                        // 应对百度近期同一时段多个连接返回400的问题
                        LOGGER.debug("响应状态码异常：" + responseCode + "\t" + chunkInfo);
                        if (responseCode == 401 || responseCode == 403 || responseCode == 404) {
                            chunkInfo.setStatus(HttpDownStatus.ERROR_WAIT_CONNECT);
                        }
                        return;
                    }
                    realContentSize = HttpDownUtil.getDownContentSize(httpResponse.headers());
                    synchronized (chunkInfo) {
                        // 判断状态是否为连接中
                        if (chunkInfo.getStatus() == HttpDownStatus.CONNECTING_NORMAL || chunkInfo.getStatus() == HttpDownStatus.CONNECTING_FAIL || chunkInfo.getStatus() == HttpDownStatus.CONNECTING_CONTINUE) {
                            LOGGER.debug("下载响应：channelId[" + ctx.channel().id() + "]\t contentSize[" + realContentSize + "]" + chunkInfo);
                            chunkInfo.setDownSize(chunkInfo.getNowStartPosition() - chunkInfo.getOriStartPosition());
                            closeable = bootstrap.initFileWriter(chunkInfo);
                            chunkInfo.setStatus(HttpDownStatus.RUNNING);
                            if (callback != null) {
                                callback.onChunkConnected(bootstrap.getHttpDownInfo(), chunkInfo);
                            }
                            isSucc = true;
                        } else {
                            safeClose(ctx.channel());
                        }
                    }
                }
            } catch (Exception e) {
                throw e;
            } finally {
                ReferenceCountUtil.release(msg);
            }
        }

        @Override
        public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
            LOGGER.error("down onChunkError:", cause);
            Channel nowChannel = bootstrap.getChannel(chunkInfo);
            safeClose(ctx.channel());
            if (nowChannel == ctx.channel()) {
                if (callback != null) {
                    callback.onChunkError(bootstrap.getHttpDownInfo(), chunkInfo, cause);
                }
                bootstrap.retryChunkDown(chunkInfo);
            }
        }

        @Override
        public void channelUnregistered(ChannelHandlerContext ctx) throws Exception {
            super.channelUnregistered(ctx);
            safeClose(ctx.channel());
        }

        private void safeClose(Channel channel) {
            try {
                HttpDownUtil.safeClose(channel, closeable);
            } catch (IOException e) {
                LOGGER.error("safeClose fail:", e);
            }
        }

        private boolean isDone(long downSize, HttpContent content) {
            return downSize == chunkInfo.getTotalSize() || (!taskInfo.isSupportRange() && content instanceof LastHttpContent);
        }

        private boolean isContinue(long downSize) {
            long downChunkSize = downSize + chunkInfo.getOriStartPosition() - chunkInfo.getNowStartPosition();
            return realContentSize == downChunkSize || (realContentSize - 1) == downChunkSize;
        }
    });
}