Examples with AppendBatchSizeTracker io.pravega.shared.protocol.netty.AppendBatchSizeTracker used on opensource projects

Example 1 with AppendBatchSizeTracker

use of io.pravega.shared.protocol.netty.AppendBatchSizeTracker in project pravega by pravega.

the class ConnectionFactoryImpl method establishConnection.

@Override
public CompletableFuture<ClientConnection> establishConnection(PravegaNodeUri location, ReplyProcessor rp) {
    Preconditions.checkNotNull(location);
    Exceptions.checkNotClosed(closed.get(), this);
    final SslContext sslCtx;
    if (clientConfig.isEnableTls()) {
        try {
            SslContextBuilder sslCtxFactory = SslContextBuilder.forClient();
            if (Strings.isNullOrEmpty(clientConfig.getTrustStore())) {
                sslCtxFactory = sslCtxFactory.trustManager(FingerprintTrustManagerFactory.getInstance(FingerprintTrustManagerFactory.getDefaultAlgorithm()));
            } else {
                sslCtxFactory = SslContextBuilder.forClient().trustManager(new File(clientConfig.getTrustStore()));
            }
            sslCtx = sslCtxFactory.build();
        } catch (SSLException | NoSuchAlgorithmException e) {
            throw new RuntimeException(e);
        }
    } else {
        sslCtx = null;
    }
    AppendBatchSizeTracker batchSizeTracker = new AppendBatchSizeTrackerImpl();
    ClientConnectionInboundHandler handler = new ClientConnectionInboundHandler(location.getEndpoint(), rp, batchSizeTracker);
    Bootstrap b = new Bootstrap();
    b.group(group).channel(nio ? NioSocketChannel.class : EpollSocketChannel.class).option(ChannelOption.TCP_NODELAY, true).handler(new ChannelInitializer<SocketChannel>() {

        @Override
        public void initChannel(SocketChannel ch) throws Exception {
            ChannelPipeline p = ch.pipeline();
            if (sslCtx != null) {
                SslHandler sslHandler = sslCtx.newHandler(ch.alloc(), location.getEndpoint(), location.getPort());
                if (clientConfig.isValidateHostName()) {
                    SSLEngine sslEngine = sslHandler.engine();
                    SSLParameters sslParameters = sslEngine.getSSLParameters();
                    sslParameters.setEndpointIdentificationAlgorithm("HTTPS");
                    sslEngine.setSSLParameters(sslParameters);
                }
                p.addLast(sslHandler);
            }
            // p.addLast(new LoggingHandler(LogLevel.INFO));
            p.addLast(new ExceptionLoggingHandler(location.getEndpoint()), new CommandEncoder(batchSizeTracker), new LengthFieldBasedFrameDecoder(WireCommands.MAX_WIRECOMMAND_SIZE, 4, 4), new CommandDecoder(), handler);
        }
    });
    // Start the client.
    CompletableFuture<ClientConnection> connectionComplete = new CompletableFuture<>();
    try {
        b.connect(location.getEndpoint(), location.getPort()).addListener(new ChannelFutureListener() {

            @Override
            public void operationComplete(ChannelFuture future) {
                if (future.isSuccess()) {
                    // since ChannelFuture is complete future.channel() is not a blocking call.
                    Channel ch = future.channel();
                    log.debug("Connect operation completed for channel:{}, local address:{}, remote address:{}", ch.id(), ch.localAddress(), ch.remoteAddress());
                    // Once a channel is closed the channel group implementation removes it.
                    allChannels.add(ch);
                    connectionComplete.complete(handler);
                } else {
                    connectionComplete.completeExceptionally(new ConnectionFailedException(future.cause()));
                }
            }
        });
    } catch (Exception e) {
        connectionComplete.completeExceptionally(new ConnectionFailedException(e));
    }
    // check if channel is registered.
    CompletableFuture<Void> channelRegisteredFuture = new CompletableFuture<>();
    handler.completeWhenRegistered(channelRegisteredFuture);
    return connectionComplete.thenCombine(channelRegisteredFuture, (clientConnection, v) -> clientConnection);
}

Example 2 with AppendBatchSizeTracker

use of io.pravega.shared.protocol.netty.AppendBatchSizeTracker in project pravega by pravega.

the class CommandEncoder method write.

@Synchronized
public void write(Append append) throws IOException {
    Exceptions.checkNotClosed(this.closed.get(), this);
    Session session = setupSegments.get(new SimpleImmutableEntry<>(append.getSegment(), append.getWriterId()));
    validateAppend(append, session);
    final ByteBuf data = append.getData().slice();
    final AppendBatchSizeTracker blockSizeSupplier = (appendTracker == null) ? null : appendTracker.apply(append.getFlowId());
    if (blockSizeSupplier != null) {
        blockSizeSupplier.recordAppend(append.getEventNumber(), data.readableBytes());
    }
    if (isChannelFree()) {
        if (session.isFree()) {
            session.record(append);
            startAppend(blockSizeSupplier, append);
            continueAppend(data);
            if (bytesLeftInBlock == 0) {
                completeAppend(null);
                flushBuffer();
            }
        } else {
            session.record(append);
            session.write(data);
            session.flushToBuffer();
        }
    } else {
        session.record(append);
        if (isChannelOwner(append.getWriterId(), append.getSegment())) {
            if (bytesLeftInBlock > data.readableBytes()) {
                continueAppend(data);
            } else {
                ByteBuf dataInsideBlock = data.readSlice(bytesLeftInBlock);
                completeAppend(dataInsideBlock, data);
                flushAllToBuffer();
                flushBuffer();
            }
        } else {
            session.write(data);
        }
    }
}

Example 3 with AppendBatchSizeTracker

use of io.pravega.shared.protocol.netty.AppendBatchSizeTracker in project pravega by pravega.

the class CommandEncoderTest method testAppendSizeQueuedBreak.

@Test
public void testAppendSizeQueuedBreak() throws IOException {
    AppendBatchSizeTracker batchSizeTracker = new FixedBatchSizeTracker(100);
    DecodingOutputStream output = new DecodingOutputStream();
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
    CommandEncoder commandEncoder = new CommandEncoder(x -> batchSizeTracker, null, output, null, endpoint);
    UUID writerId1 = UUID.randomUUID();
    WireCommand setupAppend = new WireCommands.SetupAppend(0, writerId1, "seg", "");
    commandEncoder.write(setupAppend);
    assertEquals(output.decoded.remove(0), setupAppend);
    UUID writerId2 = UUID.randomUUID();
    setupAppend = new WireCommands.SetupAppend(0, writerId2, "seg", "");
    commandEncoder.write(setupAppend);
    assertEquals(output.decoded.remove(0), setupAppend);
    ByteBuf data = Unpooled.wrappedBuffer(new byte[40]);
    WireCommands.Event event = new WireCommands.Event(data);
    Append append1 = new Append("seg", writerId1, 1, event, 0);
    commandEncoder.write(append1);
    Append appendOther1 = new Append("seg", writerId2, 101, event, 0);
    WireCommands.Event largeEvent = new WireCommands.Event(Unpooled.wrappedBuffer(new byte[CommandEncoder.MAX_QUEUED_SIZE]));
    Append appendOther2 = new Append("seg", writerId2, 102, largeEvent, 0);
    commandEncoder.write(appendOther1);
    commandEncoder.write(appendOther2);
    AppendBlock block = (AppendBlock) output.decoded.remove(0);
    assertEquals(108, block.getData().readableBytes());
    AppendBlockEnd blockEnd = (AppendBlockEnd) output.decoded.remove(0);
    assertEquals(writerId1, blockEnd.getWriterId());
    assertEquals(48, blockEnd.getSizeOfWholeEvents());
    assertEquals(0, blockEnd.getData().readableBytes());
    assertEquals(1, blockEnd.getNumEvents());
    block = (AppendBlock) output.decoded.remove(0);
    assertEquals(CommandEncoder.MAX_QUEUED_SIZE + 48 + 8, block.getData().readableBytes());
    blockEnd = (AppendBlockEnd) output.decoded.remove(0);
    assertEquals(writerId2, blockEnd.getWriterId());
    assertEquals(CommandEncoder.MAX_QUEUED_SIZE + 48 + 8, blockEnd.getSizeOfWholeEvents());
    assertEquals(0, blockEnd.getData().readableBytes());
    assertEquals(2, blockEnd.getNumEvents());
}

Example 4 with AppendBatchSizeTracker

use of io.pravega.shared.protocol.netty.AppendBatchSizeTracker in project pravega by pravega.

the class CommandEncoderTest method testAppendsAreBatched.

@Test
public void testAppendsAreBatched() throws IOException {
    AppendBatchSizeTracker batchSizeTracker = new FixedBatchSizeTracker(100);
    DecodingOutputStream output = new DecodingOutputStream();
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
    CommandEncoder commandEncoder = new CommandEncoder(x -> batchSizeTracker, null, output, null, endpoint);
    UUID writerId = UUID.randomUUID();
    WireCommand setupAppend = new WireCommands.SetupAppend(0, writerId, "seg", "");
    commandEncoder.write(setupAppend);
    assertEquals(output.decoded.remove(0), setupAppend);
    ByteBuf data = Unpooled.wrappedBuffer(new byte[40]);
    WireCommands.Event event = new WireCommands.Event(data);
    Append append1 = new Append("seg", writerId, 1, event, 0);
    Append append2 = new Append("seg", writerId, 2, event, 0);
    Append append3 = new Append("seg", writerId, 3, event, 0);
    commandEncoder.write(append1);
    commandEncoder.write(append2);
    commandEncoder.write(append3);
    AppendBlock block = (AppendBlock) output.decoded.remove(0);
    assertEquals(108, block.getData().readableBytes());
    AppendBlockEnd blockEnd = (AppendBlockEnd) output.decoded.remove(0);
    assertEquals(writerId, blockEnd.getWriterId());
    assertEquals(80 + 16, blockEnd.getSizeOfWholeEvents());
    assertEquals(40 + 4, blockEnd.getData().readableBytes());
    assertEquals(3, blockEnd.getNumEvents());
}

Example 5 with AppendBatchSizeTracker

use of io.pravega.shared.protocol.netty.AppendBatchSizeTracker in project pravega by pravega.

the class CommandEncoderTest method testBatchInterupted.

@Test
public void testBatchInterupted() throws IOException {
    AppendBatchSizeTracker batchSizeTracker = new FixedBatchSizeTracker(100);
    DecodingOutputStream output = new DecodingOutputStream();
    PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT);
    CommandEncoder commandEncoder = new CommandEncoder(x -> batchSizeTracker, null, output, null, endpoint);
    UUID writerId = UUID.randomUUID();
    WireCommand setupAppend = new WireCommands.SetupAppend(0, writerId, "seg", "");
    commandEncoder.write(setupAppend);
    assertEquals(output.decoded.remove(0), setupAppend);
    ByteBuf data = Unpooled.wrappedBuffer(new byte[40]);
    WireCommands.Event event = new WireCommands.Event(data);
    Append append1 = new Append("seg", writerId, 1, event, 0);
    commandEncoder.write(append1);
    commandEncoder.write(new WireCommands.KeepAlive());
    AppendBlock block = (AppendBlock) output.decoded.remove(0);
    assertEquals(108, block.getData().readableBytes());
    AppendBlockEnd blockEnd = (AppendBlockEnd) output.decoded.remove(0);
    assertEquals(writerId, blockEnd.getWriterId());
    assertEquals(48, blockEnd.getSizeOfWholeEvents());
    assertEquals(0, blockEnd.getData().readableBytes());
    assertEquals(1, blockEnd.getNumEvents());
    WireCommands.KeepAlive breakingCommand = (WireCommands.KeepAlive) output.decoded.remove(0);
    assertNotNull(breakingCommand);
}