Examples with Future org.apache.flink.shaded.netty4.io.netty.util.concurrent.Future used on opensource projects

Example 51 with Future

use of org.apache.flink.shaded.netty4.io.netty.util.concurrent.Future in project ambry by linkedin.

the class Http2NetworkClient method sendAndPoll.

@Override
public List<ResponseInfo> sendAndPoll(List<RequestInfo> requestsToSend, Set<Integer> requestsToDrop, int pollTimeoutMs) {
    List<ResponseInfo> readyResponseInfos = new ArrayList<>();
    if (requestsToDrop.size() != 0) {
        logger.warn("Number of requestsToDrop: {}", requestsToDrop.size());
        http2ClientMetrics.http2RequestsToDropCount.inc(requestsToDrop.size());
        for (int correlationId : requestsToDrop) {
            Channel streamChannel = correlationIdInFlightToChannelMap.remove(correlationId);
            if (streamChannel != null) {
                logger.warn("Drop request on streamChannel: {}", streamChannel);
                // Drop request just generates a ResponseInfo with TimeoutError to router.
                // The stream is still transmitting, but router will ignore ResponseInfo with the same correlationId.
                // We need stream reset to cancel the stream in transmitting.
                RequestInfo requestInfo = streamChannel.attr(Http2NetworkClient.REQUEST_INFO).get();
                if (requestInfo != null) {
                    readyResponseInfos.add(new ResponseInfo(requestInfo, NetworkClientErrorCode.TimeoutError, null));
                }
            }
        }
    }
    long sendStartTime = System.currentTimeMillis();
    // Send request
    http2ClientMetrics.http2ClientSendRate.mark(requestsToSend.size());
    for (RequestInfo requestInfo : requestsToSend) {
        long streamInitiateTime = System.currentTimeMillis();
        long waitingTime = streamInitiateTime - requestInfo.getRequestCreateTime();
        http2ClientMetrics.requestToNetworkClientLatencyMs.update(waitingTime);
        this.pools.get(InetSocketAddress.createUnresolved(requestInfo.getHost(), requestInfo.getPort().getPort())).acquire().addListener((GenericFutureListener<Future<Channel>>) future -> {
            if (future.isSuccess()) {
                http2ClientMetrics.http2StreamAcquireTime.update(System.currentTimeMillis() - streamInitiateTime);
                long streamAcquiredTime = System.currentTimeMillis();
                Channel streamChannel = future.getNow();
                correlationIdInFlightToChannelMap.put(requestInfo.getRequest().getCorrelationId(), streamChannel);
                streamChannel.attr(REQUEST_INFO).set(requestInfo);
                if (!streamChannel.isWritable() || !streamChannel.parent().isWritable()) {
                    http2ClientMetrics.http2StreamNotWritableCount.inc();
                    logger.debug("Stream {} {} not writable. BytesBeforeWritable {} {}", streamChannel.hashCode(), streamChannel, streamChannel.bytesBeforeWritable(), streamChannel.parent().bytesBeforeWritable());
                }
                streamChannel.writeAndFlush(requestInfo.getRequest()).addListener(new ChannelFutureListener() {

                    @Override
                    public void operationComplete(ChannelFuture future) throws Exception {
                        if (future.isSuccess()) {
                            long writeAndFlushUsedTime = System.currentTimeMillis() - streamAcquiredTime;
                            http2ClientMetrics.http2StreamWriteAndFlushTime.update(writeAndFlushUsedTime);
                            requestInfo.setStreamSendTime(System.currentTimeMillis());
                            if (writeAndFlushUsedTime > http2ClientConfig.http2WriteAndFlushTimeoutMs) {
                                logger.debug("WriteAndFlush exceeds http2RequestTimeoutMs {}ms, used time: {}ms, stream channel {}", http2ClientConfig.http2WriteAndFlushTimeoutMs, writeAndFlushUsedTime, streamChannel);
                                if (http2ClientConfig.http2DropRequestOnWriteAndFlushTimeout) {
                                    RequestInfo requestInfoFromChannelAttr = releaseAndCloseStreamChannel(streamChannel);
                                    if (requestInfoFromChannelAttr != null) {
                                        http2ClientResponseHandler.getResponseInfoQueue().put(new ResponseInfo(requestInfo, NetworkClientErrorCode.NetworkError, null));
                                    }
                                }
                            }
                        } else {
                            http2ClientMetrics.http2StreamWriteAndFlushErrorCount.inc();
                            logger.warn("Stream {} {} writeAndFlush fail. Cause: {}", streamChannel.hashCode(), streamChannel, future.cause().toString());
                            RequestInfo requestInfoFromChannelAttr = releaseAndCloseStreamChannel(streamChannel);
                            if (requestInfoFromChannelAttr != null) {
                                http2ClientResponseHandler.getResponseInfoQueue().put(new ResponseInfo(requestInfoFromChannelAttr, NetworkClientErrorCode.NetworkError, null));
                            }
                        }
                    }
                });
            } else {
                logger.error("Couldn't acquire stream channel to {}:{} . Cause:", requestInfo.getHost(), requestInfo.getPort().getPort(), future.cause());
                requestInfo.getRequest().release();
                http2ClientResponseHandler.getResponseInfoQueue().put(new ResponseInfo(requestInfo, NetworkClientErrorCode.NetworkError, null));
            }
        });
    }
    http2ClientMetrics.http2ClientSendTime.update(System.currentTimeMillis() - sendStartTime);
    http2ClientResponseHandler.getResponseInfoQueue().poll(readyResponseInfos, pollTimeoutMs);
    for (ResponseInfo responseInfo : readyResponseInfos) {
        correlationIdInFlightToChannelMap.remove(responseInfo.getRequestInfo().getRequest().getCorrelationId());
    }
    http2ClientMetrics.http2ClientSendRate.mark(readyResponseInfos.size());
    http2ClientMetrics.http2ClientSendAndPollTime.update(System.currentTimeMillis() - sendStartTime);
    return readyResponseInfos;
}

Example 52 with Future

use of org.apache.flink.shaded.netty4.io.netty.util.concurrent.Future in project ambry by linkedin.

the class Http2NetworkClient method warmUpConnections.

@Override
public int warmUpConnections(List<DataNodeId> dataNodeIds, int connectionWarmUpPercentagePerDataNode, long timeForWarmUp, List<ResponseInfo> responseInfoList) {
    long startTime = System.currentTimeMillis();
    AtomicInteger successCount = new AtomicInteger();
    AtomicInteger failCount = new AtomicInteger();
    int warmUpConnectionPerPort = http2ClientConfig.http2MinConnectionPerPort * connectionWarmUpPercentagePerDataNode / 100;
    int expectedConnections = dataNodeIds.size() * warmUpConnectionPerPort;
    for (DataNodeId dataNodeId : dataNodeIds) {
        for (int i = 0; i < warmUpConnectionPerPort; i++) {
            this.pools.get(InetSocketAddress.createUnresolved(dataNodeId.getHostname(), dataNodeId.getHttp2Port())).acquire().addListener((GenericFutureListener<Future<Channel>>) future -> {
                if (future.isSuccess()) {
                    Channel streamChannel = future.getNow();
                    releaseAndCloseStreamChannel(streamChannel);
                    successCount.incrementAndGet();
                } else {
                    failCount.incrementAndGet();
                    responseInfoList.add(new ResponseInfo(null, NetworkClientErrorCode.NetworkError, null, dataNodeId));
                    logger.error("Couldn't acquire stream channel to {}:{} . Cause: {}.", dataNodeId.getHostname(), dataNodeId.getHttp2Port(), future.cause());
                }
            });
        }
    }
    while (System.currentTimeMillis() - startTime < timeForWarmUp) {
        if (successCount.get() + failCount.get() == expectedConnections) {
            break;
        } else {
            try {
                Thread.sleep(300);
            } catch (InterruptedException e) {
                break;
            }
        }
    }
    logger.info("HTTP2 connection warm up done. Tried: {}, Succeeded: {}, Failed: {}, Time elapsed: {} ms", expectedConnections, successCount, failCount, System.currentTimeMillis() - startTime);
    return successCount.get();
}

Example 53 with Future

use of org.apache.flink.shaded.netty4.io.netty.util.concurrent.Future in project zeebe by zeebe-io.

the class NettyMessagingService method bind.

/**
 * Recursively binds the given bootstrap to the given interfaces.
 *
 * @param bootstrap the bootstrap to bind
 * @param addressIterator an iterator of Addresses to which to bind
 * @param future the future to completed once the bootstrap has been bound to all provided
 *     interfaces
 */
private void bind(final ServerBootstrap bootstrap, final Iterator<Address> addressIterator, final CompletableFuture<Void> future) {
    if (addressIterator.hasNext()) {
        final Address address = addressIterator.next();
        bootstrap.bind(address.host(), address.port()).addListener((ChannelFutureListener) f -> {
            if (f.isSuccess()) {
                log.info("TCP server listening for connections on {}", address);
                serverChannel = f.channel();
                bind(bootstrap, addressIterator, future);
            } else {
                log.warn("Failed to bind TCP server to port {} due to {}", address, f.cause());
                future.completeExceptionally(f.cause());
            }
        });
    } else {
        future.complete(null);
    }
}

Example 54 with Future

use of org.apache.flink.shaded.netty4.io.netty.util.concurrent.Future in project RemoteShuffleService by alibaba.

the class ShuffleClientSuiteJ method setupEnv.

private void setupEnv() throws IOException, InterruptedException {
    RssConf conf = new RssConf();
    conf.set("rss.pushdata.retry.thread.num", "1");
    conf.set("rss.push.data.buffer.size", "1K");
    shuffleClient = new ShuffleClientImpl(conf);
    masterLocation.setPeer(slaveLocation);
    when(endpointRef.askSync(new ControlMessages.RegisterShuffle(TEST_APPLICATION_ID, TEST_SHUFFLE_ID, 1, 1), ClassTag$.MODULE$.apply(ControlMessages.RegisterShuffleResponse.class))).thenAnswer(t -> new ControlMessages.RegisterShuffleResponse(StatusCode.Success, new ArrayList<PartitionLocation>() {

        {
            add(masterLocation);
        }
    }));
    shuffleClient.setupMetaServiceRef(endpointRef);
    ChannelFuture mockedFuture = new ChannelFuture() {

        @Override
        public Channel channel() {
            return null;
        }

        @Override
        public ChannelFuture addListener(GenericFutureListener<? extends Future<? super Void>> listener) {
            return null;
        }

        @Override
        public ChannelFuture addListeners(GenericFutureListener<? extends Future<? super Void>>... listeners) {
            return null;
        }

        @Override
        public ChannelFuture removeListener(GenericFutureListener<? extends Future<? super Void>> listener) {
            return null;
        }

        @Override
        public ChannelFuture removeListeners(GenericFutureListener<? extends Future<? super Void>>... listeners) {
            return null;
        }

        @Override
        public ChannelFuture sync() throws InterruptedException {
            return null;
        }

        @Override
        public ChannelFuture syncUninterruptibly() {
            return null;
        }

        @Override
        public ChannelFuture await() throws InterruptedException {
            return null;
        }

        @Override
        public ChannelFuture awaitUninterruptibly() {
            return null;
        }

        @Override
        public boolean isVoid() {
            return false;
        }

        @Override
        public boolean isSuccess() {
            return true;
        }

        @Override
        public boolean isCancellable() {
            return false;
        }

        @Override
        public Throwable cause() {
            return null;
        }

        @Override
        public boolean await(long timeout, TimeUnit unit) throws InterruptedException {
            return true;
        }

        @Override
        public boolean await(long timeoutMillis) throws InterruptedException {
            return true;
        }

        @Override
        public boolean awaitUninterruptibly(long timeout, TimeUnit unit) {
            return true;
        }

        @Override
        public boolean awaitUninterruptibly(long timeoutMillis) {
            return true;
        }

        @Override
        public Void getNow() {
            return null;
        }

        @Override
        public boolean cancel(boolean mayInterruptIfRunning) {
            return false;
        }

        @Override
        public boolean isCancelled() {
            return false;
        }

        @Override
        public boolean isDone() {
            return true;
        }

        @Override
        public Void get() throws InterruptedException, ExecutionException {
            return null;
        }

        @Override
        public Void get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
            return null;
        }
    };
    when(client.pushData(any(), any())).thenAnswer(t -> mockedFuture);
    when(clientFactory.createClient(masterLocation.getHost(), masterLocation.getPushPort(), TEST_REDUCRE_ID)).thenAnswer(t -> client);
    when(client.pushMergedData(any(), any())).thenAnswer(t -> mockedFuture);
    when(clientFactory.createClient(masterLocation.getHost(), masterLocation.getPushPort())).thenAnswer(t -> client);
    shuffleClient.dataClientFactory = clientFactory;
}

Example 55 with Future

use of org.apache.flink.shaded.netty4.io.netty.util.concurrent.Future in project tinyradius-netty by globalreachtech.

the class RadiusClientTest method communicateEndpointListFirstSuccess.

@Test
void communicateEndpointListFirstSuccess() throws RadiusPacketException {
    final byte id = (byte) random.nextInt(256);
    final RadiusResponse response = RadiusResponse.create(DefaultDictionary.INSTANCE, (byte) 2, id, null, Collections.emptyList());
    final CapturingOutboundHandler capturingOutboundHandler = spy(new CapturingOutboundHandler(p -> p.trySuccess(response)));
    final RadiusClient radiusClient = new RadiusClient(bootstrap, address, timeoutHandler, capturingOutboundHandler);
    final InetSocketAddress address2 = new InetSocketAddress(1);
    // never used
    final RadiusEndpoint stubEndpoint2 = new RadiusEndpoint(address2, "secret2");
    final InetSocketAddress address3 = new InetSocketAddress(2);
    // never used
    final RadiusEndpoint stubEndpoint3 = new RadiusEndpoint(address3, "secret3");
    final List<RadiusEndpoint> endpoints = Arrays.asList(stubEndpoint, stubEndpoint2, stubEndpoint3);
    final RadiusRequest request = RadiusRequest.create(dictionary, (byte) 1, (byte) 1, null, Collections.emptyList());
    final Future<RadiusResponse> future = radiusClient.communicate(request, endpoints);
    await().until(future::isDone);
    assertTrue(future.isSuccess());
    assertEquals(response, future.getNow());
    assertEquals(1, capturingOutboundHandler.requests.size());
    assertEquals("secret", capturingOutboundHandler.requests.get(0).getEndpoint().getSecret());
}