Examples with PooledByteBufAllocator io.netty.buffer.PooledByteBufAllocator used on opensource projects

Example 21 with PooledByteBufAllocator

use of io.netty.buffer.PooledByteBufAllocator in project bookkeeper by apache.

the class ByteBufAllocatorBuilderTest method testPooled.

@Test
public void testPooled() {
    PooledByteBufAllocator pooledAlloc = new PooledByteBufAllocator(true);
    ByteBufAllocator alloc = ByteBufAllocatorBuilder.create().poolingPolicy(PoolingPolicy.PooledDirect).pooledAllocator(pooledAlloc).build();
    assertTrue(alloc.isDirectBufferPooled());
    ByteBuf buf1 = alloc.buffer();
    assertEquals(pooledAlloc, buf1.alloc());
    assertFalse(buf1.hasArray());
    buf1.release();
    ByteBuf buf2 = alloc.directBuffer();
    assertEquals(pooledAlloc, buf2.alloc());
    assertFalse(buf2.hasArray());
    buf2.release();
    ByteBuf buf3 = alloc.heapBuffer();
    assertEquals(pooledAlloc, buf3.alloc());
    assertTrue(buf3.hasArray());
    buf3.release();
}

Example 22 with PooledByteBufAllocator

use of io.netty.buffer.PooledByteBufAllocator in project bookkeeper by apache.

the class ByteBufAllocatorBuilderTest method testPooledWithDefaultAllocator.

@Test
public void testPooledWithDefaultAllocator() {
    ByteBufAllocator alloc = ByteBufAllocatorBuilder.create().poolingPolicy(PoolingPolicy.PooledDirect).poolingConcurrency(3).build();
    assertTrue(alloc.isDirectBufferPooled());
    ByteBuf buf1 = alloc.buffer();
    assertEquals(PooledByteBufAllocator.class, buf1.alloc().getClass());
    assertEquals(3, ((PooledByteBufAllocator) buf1.alloc()).metric().numDirectArenas());
    assertFalse(buf1.hasArray());
    buf1.release();
    ByteBuf buf2 = alloc.directBuffer();
    assertFalse(buf2.hasArray());
    buf2.release();
    ByteBuf buf3 = alloc.heapBuffer();
    assertTrue(buf3.hasArray());
    buf3.release();
}

Example 23 with PooledByteBufAllocator

use of io.netty.buffer.PooledByteBufAllocator in project bookkeeper by apache.

the class BookieWriteLedgerTest method testLedgerCreateByteBufRefCnt.

@Test
@SuppressWarnings("unchecked")
public void testLedgerCreateByteBufRefCnt() throws Exception {
    final LedgerHandle lh = bkc.createLedger(5, 3, 2, digestType, ledgerPassword, null);
    final List<AbstractByteBufAllocator> allocs = Lists.newArrayList(new PooledByteBufAllocator(true), new PooledByteBufAllocator(false), new UnpooledByteBufAllocator(true), new UnpooledByteBufAllocator(false));
    int entryId = 0;
    for (AbstractByteBufAllocator alloc : allocs) {
        final ByteBuf data = alloc.buffer(10);
        data.writeBytes(("fragment0" + entryId).getBytes());
        assertEquals("ref count on ByteBuf should be 1", 1, data.refCnt());
        CompletableFuture<Integer> cf = new CompletableFuture<>();
        lh.asyncAddEntry(data, (rc, handle, eId, ctx) -> {
            CompletableFuture<Integer> future = (CompletableFuture<Integer>) ctx;
            future.complete(rc);
        }, cf);
        int rc = cf.get();
        assertEquals("rc code is OK", BKException.Code.OK, rc);
        for (int i = 0; i < 10; i++) {
            if (data.refCnt() == 0) {
                break;
            }
            // recycler runs asynchronously
            TimeUnit.MILLISECONDS.sleep(250);
        }
        assertEquals("writing entry with id " + entryId + ", ref count on ByteBuf should be 0 ", 0, data.refCnt());
        org.apache.bookkeeper.client.api.LedgerEntry e = lh.read(entryId, entryId).getEntry(entryId);
        assertEquals("entry data is correct", "fragment0" + entryId, new String(e.getEntryBytes()));
        entryId++;
    }
    bkc.deleteLedger(lh.ledgerId);
}

Example 24 with PooledByteBufAllocator

use of io.netty.buffer.PooledByteBufAllocator in project bookkeeper by apache.

the class BookieWriteLedgerTest method testLedgerCreateAdvByteBufRefCnt.

@Test
@SuppressWarnings("unchecked")
public void testLedgerCreateAdvByteBufRefCnt() throws Exception {
    long ledgerId = rng.nextLong();
    ledgerId &= Long.MAX_VALUE;
    if (!baseConf.getLedgerManagerFactoryClass().equals(LongHierarchicalLedgerManagerFactory.class)) {
        // since LongHierarchicalLedgerManager supports ledgerIds of
        // decimal length upto 19 digits but other
        // LedgerManagers only upto 10 decimals
        ledgerId %= 9999999999L;
    }
    final LedgerHandle lh = bkc.createLedgerAdv(ledgerId, 5, 3, 2, digestType, ledgerPassword, null);
    final List<AbstractByteBufAllocator> allocs = Lists.newArrayList(new PooledByteBufAllocator(true), new PooledByteBufAllocator(false), new UnpooledByteBufAllocator(true), new UnpooledByteBufAllocator(false));
    long entryId = 0;
    for (AbstractByteBufAllocator alloc : allocs) {
        final ByteBuf data = alloc.buffer(10);
        data.writeBytes(("fragment0" + entryId).getBytes());
        assertEquals("ref count on ByteBuf should be 1", 1, data.refCnt());
        CompletableFuture<Integer> cf = new CompletableFuture<>();
        lh.asyncAddEntry(entryId, data, (rc, handle, eId, qwcLatency, ctx) -> {
            CompletableFuture<Integer> future = (CompletableFuture<Integer>) ctx;
            future.complete(rc);
        }, cf);
        int rc = cf.get();
        assertEquals("rc code is OK", BKException.Code.OK, rc);
        for (int i = 0; i < 10; i++) {
            if (data.refCnt() == 0) {
                break;
            }
            // recycler runs asynchronously
            TimeUnit.MILLISECONDS.sleep(250);
        }
        assertEquals("writing entry with id " + entryId + ", ref count on ByteBuf should be 0 ", 0, data.refCnt());
        org.apache.bookkeeper.client.api.LedgerEntry e = lh.read(entryId, entryId).getEntry(entryId);
        assertEquals("entry data is correct", "fragment0" + entryId, new String(e.getEntryBytes()));
        entryId++;
    }
    bkc.deleteLedger(lh.ledgerId);
}

Example 25 with PooledByteBufAllocator

use of io.netty.buffer.PooledByteBufAllocator in project bookkeeper by apache.

the class BookieNettyServer method listenOn.

private void listenOn(InetSocketAddress address, BookieSocketAddress bookieAddress) throws InterruptedException {
    if (!conf.isDisableServerSocketBind()) {
        ServerBootstrap bootstrap = new ServerBootstrap();
        bootstrap.option(ChannelOption.ALLOCATOR, allocator);
        bootstrap.childOption(ChannelOption.ALLOCATOR, allocator);
        bootstrap.group(eventLoopGroup, eventLoopGroup);
        bootstrap.childOption(ChannelOption.TCP_NODELAY, conf.getServerTcpNoDelay());
        bootstrap.childOption(ChannelOption.SO_LINGER, conf.getServerSockLinger());
        bootstrap.childOption(ChannelOption.RCVBUF_ALLOCATOR, new AdaptiveRecvByteBufAllocator(conf.getRecvByteBufAllocatorSizeMin(), conf.getRecvByteBufAllocatorSizeInitial(), conf.getRecvByteBufAllocatorSizeMax()));
        bootstrap.option(ChannelOption.WRITE_BUFFER_WATER_MARK, new WriteBufferWaterMark(conf.getServerWriteBufferLowWaterMark(), conf.getServerWriteBufferHighWaterMark()));
        if (eventLoopGroup instanceof EpollEventLoopGroup) {
            bootstrap.channel(EpollServerSocketChannel.class);
        } else {
            bootstrap.channel(NioServerSocketChannel.class);
        }
        bootstrap.childHandler(new ChannelInitializer<SocketChannel>() {

            @Override
            protected void initChannel(SocketChannel ch) throws Exception {
                synchronized (suspensionLock) {
                    while (suspended) {
                        suspensionLock.wait();
                    }
                }
                BookieSideConnectionPeerContextHandler contextHandler = new BookieSideConnectionPeerContextHandler();
                ChannelPipeline pipeline = ch.pipeline();
                // For ByteBufList, skip the usual LengthFieldPrepender and have the encoder itself to add it
                pipeline.addLast("bytebufList", ByteBufList.ENCODER_WITH_SIZE);
                pipeline.addLast("lengthbaseddecoder", new LengthFieldBasedFrameDecoder(maxFrameSize, 0, 4, 0, 4));
                pipeline.addLast("lengthprepender", new LengthFieldPrepender(4));
                pipeline.addLast("bookieProtoDecoder", new BookieProtoEncoding.RequestDecoder(registry));
                pipeline.addLast("bookieProtoEncoder", new BookieProtoEncoding.ResponseEncoder(registry));
                pipeline.addLast("bookieAuthHandler", new AuthHandler.ServerSideHandler(contextHandler.getConnectionPeer(), authProviderFactory));
                ChannelInboundHandler requestHandler = isRunning.get() ? new BookieRequestHandler(conf, requestProcessor, allChannels) : new RejectRequestHandler();
                pipeline.addLast("bookieRequestHandler", requestHandler);
                pipeline.addLast("contextHandler", contextHandler);
            }
        });
        // Bind and start to accept incoming connections
        LOG.info("Binding bookie-rpc endpoint to {}", address);
        Channel listen = bootstrap.bind(address.getAddress(), address.getPort()).sync().channel();
        if (listen.localAddress() instanceof InetSocketAddress) {
            if (conf.getBookiePort() == 0) {
                // this is really really nasty. It's using the configuration object as a notification
                // bus. We should get rid of this at some point
                conf.setBookiePort(((InetSocketAddress) listen.localAddress()).getPort());
            }
        }
    }
    if (conf.isEnableLocalTransport()) {
        ServerBootstrap jvmBootstrap = new ServerBootstrap();
        jvmBootstrap.childOption(ChannelOption.ALLOCATOR, new PooledByteBufAllocator(true));
        jvmBootstrap.group(jvmEventLoopGroup, jvmEventLoopGroup);
        jvmBootstrap.childOption(ChannelOption.TCP_NODELAY, conf.getServerTcpNoDelay());
        jvmBootstrap.childOption(ChannelOption.SO_KEEPALIVE, conf.getServerSockKeepalive());
        jvmBootstrap.childOption(ChannelOption.SO_LINGER, conf.getServerSockLinger());
        jvmBootstrap.childOption(ChannelOption.RCVBUF_ALLOCATOR, new AdaptiveRecvByteBufAllocator(conf.getRecvByteBufAllocatorSizeMin(), conf.getRecvByteBufAllocatorSizeInitial(), conf.getRecvByteBufAllocatorSizeMax()));
        jvmBootstrap.option(ChannelOption.WRITE_BUFFER_WATER_MARK, new WriteBufferWaterMark(conf.getServerWriteBufferLowWaterMark(), conf.getServerWriteBufferHighWaterMark()));
        if (jvmEventLoopGroup instanceof DefaultEventLoopGroup) {
            jvmBootstrap.channel(LocalServerChannel.class);
        } else if (jvmEventLoopGroup instanceof EpollEventLoopGroup) {
            jvmBootstrap.channel(EpollServerSocketChannel.class);
        } else {
            jvmBootstrap.channel(NioServerSocketChannel.class);
        }
        jvmBootstrap.childHandler(new ChannelInitializer<LocalChannel>() {

            @Override
            protected void initChannel(LocalChannel ch) throws Exception {
                synchronized (suspensionLock) {
                    while (suspended) {
                        suspensionLock.wait();
                    }
                }
                BookieSideConnectionPeerContextHandler contextHandler = new BookieSideConnectionPeerContextHandler();
                ChannelPipeline pipeline = ch.pipeline();
                pipeline.addLast("lengthbaseddecoder", new LengthFieldBasedFrameDecoder(maxFrameSize, 0, 4, 0, 4));
                pipeline.addLast("lengthprepender", new LengthFieldPrepender(4));
                pipeline.addLast("bookieProtoDecoder", new BookieProtoEncoding.RequestDecoder(registry));
                pipeline.addLast("bookieProtoEncoder", new BookieProtoEncoding.ResponseEncoder(registry));
                pipeline.addLast("bookieAuthHandler", new AuthHandler.ServerSideHandler(contextHandler.getConnectionPeer(), authProviderFactory));
                ChannelInboundHandler requestHandler = isRunning.get() ? new BookieRequestHandler(conf, requestProcessor, allChannels) : new RejectRequestHandler();
                pipeline.addLast("bookieRequestHandler", requestHandler);
                pipeline.addLast("contextHandler", contextHandler);
            }
        });
        LOG.info("Binding jvm bookie-rpc endpoint to {}", bookieId.toString());
        // use the same address 'name', so clients can find local Bookie still discovering them using ZK
        jvmBootstrap.bind(new LocalAddress(bookieId.toString())).sync();
        LocalBookiesRegistry.registerLocalBookieAddress(bookieId);
    }
}