Examples with ByteBufList org.apache.bookkeeper.util.ByteBufList used on opensource projects

Example 1 with ByteBufList

use of org.apache.bookkeeper.util.ByteBufList in project bookkeeper by apache.

the class ProtocolBenchmark method testAddEntryV2.

@Benchmark
public void testAddEntryV2() throws Exception {
    ByteBufList list = ByteBufList.get(entry.slice());
    BookieProtocol.AddRequest req = BookieProtocol.AddRequest.create(BookieProtocol.CURRENT_PROTOCOL_VERSION, ledgerId, entryId, flags, masterKey, list);
    Object res = this.reqEnDeV2.encode(req, ByteBufAllocator.DEFAULT);
    ReferenceCountUtil.release(res);
    ReferenceCountUtil.release(list);
}

Example 2 with ByteBufList

use of org.apache.bookkeeper.util.ByteBufList in project bookkeeper by apache.

the class BookieClientTest method testWriteGaps.

@Test
public void testWriteGaps() throws Exception {
    final Object notifyObject = new Object();
    byte[] passwd = new byte[20];
    Arrays.fill(passwd, (byte) 'a');
    BookieSocketAddress addr = bs.getLocalAddress();
    ResultStruct arc = new ResultStruct();
    BookieClient bc = new BookieClient(new ClientConfiguration(), eventLoopGroup, executor, scheduler, NullStatsLogger.INSTANCE);
    ByteBufList bb = createByteBuffer(1, 1, 1);
    bc.addEntry(addr, 1, passwd, 1, bb, wrcb, arc, BookieProtocol.FLAG_NONE);
    synchronized (arc) {
        arc.wait(1000);
        assertEquals(0, arc.rc);
        bc.readEntry(addr, 1, 1, recb, arc, BookieProtocol.FLAG_NONE);
        arc.wait(1000);
        assertEquals(0, arc.rc);
        assertEquals(1, arc.entry.getInt());
    }
    bb = createByteBuffer(2, 1, 2);
    bc.addEntry(addr, 1, passwd, 2, bb, wrcb, null, BookieProtocol.FLAG_NONE);
    bb = createByteBuffer(3, 1, 3);
    bc.addEntry(addr, 1, passwd, 3, bb, wrcb, null, BookieProtocol.FLAG_NONE);
    bb = createByteBuffer(5, 1, 5);
    bc.addEntry(addr, 1, passwd, 5, bb, wrcb, null, BookieProtocol.FLAG_NONE);
    bb = createByteBuffer(7, 1, 7);
    bc.addEntry(addr, 1, passwd, 7, bb, wrcb, null, BookieProtocol.FLAG_NONE);
    synchronized (notifyObject) {
        bb = createByteBuffer(11, 1, 11);
        bc.addEntry(addr, 1, passwd, 11, bb, wrcb, notifyObject, BookieProtocol.FLAG_NONE);
        notifyObject.wait();
    }
    synchronized (arc) {
        bc.readEntry(addr, 1, 6, recb, arc, BookieProtocol.FLAG_NONE);
        arc.wait(1000);
        assertEquals(BKException.Code.NoSuchEntryException, arc.rc);
    }
    synchronized (arc) {
        bc.readEntry(addr, 1, 7, recb, arc, BookieProtocol.FLAG_NONE);
        arc.wait(1000);
        assertEquals(0, arc.rc);
        assertEquals(7, arc.entry.getInt(), BookieProtocol.FLAG_NONE);
    }
    synchronized (arc) {
        bc.readEntry(addr, 1, 1, recb, arc, BookieProtocol.FLAG_NONE);
        arc.wait(1000);
        assertEquals(0, arc.rc);
        assertEquals(1, arc.entry.getInt());
    }
    synchronized (arc) {
        bc.readEntry(addr, 1, 2, recb, arc, BookieProtocol.FLAG_NONE);
        arc.wait(1000);
        assertEquals(0, arc.rc);
        assertEquals(2, arc.entry.getInt());
    }
    synchronized (arc) {
        bc.readEntry(addr, 1, 3, recb, arc, BookieProtocol.FLAG_NONE);
        arc.wait(1000);
        assertEquals(0, arc.rc);
        assertEquals(3, arc.entry.getInt());
    }
    synchronized (arc) {
        bc.readEntry(addr, 1, 4, recb, arc, BookieProtocol.FLAG_NONE);
        arc.wait(1000);
        assertEquals(BKException.Code.NoSuchEntryException, arc.rc);
    }
    synchronized (arc) {
        bc.readEntry(addr, 1, 11, recb, arc, BookieProtocol.FLAG_NONE);
        arc.wait(1000);
        assertEquals(0, arc.rc);
        assertEquals(11, arc.entry.getInt());
    }
    synchronized (arc) {
        bc.readEntry(addr, 1, 5, recb, arc, BookieProtocol.FLAG_NONE);
        arc.wait(1000);
        assertEquals(0, arc.rc);
        assertEquals(5, arc.entry.getInt());
    }
    synchronized (arc) {
        bc.readEntry(addr, 1, 10, recb, arc, BookieProtocol.FLAG_NONE);
        arc.wait(1000);
        assertEquals(BKException.Code.NoSuchEntryException, arc.rc);
    }
    synchronized (arc) {
        bc.readEntry(addr, 1, 12, recb, arc, BookieProtocol.FLAG_NONE);
        arc.wait(1000);
        assertEquals(BKException.Code.NoSuchEntryException, arc.rc);
    }
    synchronized (arc) {
        bc.readEntry(addr, 1, 13, recb, arc, BookieProtocol.FLAG_NONE);
        arc.wait(1000);
        assertEquals(BKException.Code.NoSuchEntryException, arc.rc);
    }
}

Example 3 with ByteBufList

use of org.apache.bookkeeper.util.ByteBufList in project bookkeeper by apache.

the class TestPendingReadLacOp method testPendingReadLacOpMissingLAC.

@Test
public void testPendingReadLacOpMissingLAC() throws Exception {
    LedgerHandle lh = bkc.createLedger(3, 3, 2, BookKeeper.DigestType.MAC, pwd);
    lh.append(data);
    lh.append(data);
    lh.append(data);
    final CompletableFuture<Long> result = new CompletableFuture<>();
    PendingReadLacOp pro = new PendingReadLacOp(lh, (rc, lac) -> result.complete(lac)) {

        @Override
        public void initiate() {
            for (int i = 0; i < lh.metadata.currentEnsemble.size(); i++) {
                final int index = i;
                ByteBufList buffer = lh.getDigestManager().computeDigestAndPackageForSendingLac(1);
                bkc.scheduler.schedule(() -> {
                    readLacComplete(0, lh.getId(), buffer.getBuffer(0), null, index);
                }, 0, TimeUnit.SECONDS);
                lh.bk.getBookieClient().readLac(lh.metadata.currentEnsemble.get(i), lh.ledgerId, this, i);
            }
        }
    };
    pro.initiate();
    assertEquals(result.get().longValue(), 1);
}

Example 4 with ByteBufList

use of org.apache.bookkeeper.util.ByteBufList in project bookkeeper by apache.

the class LedgerFragmentReplicator method recoverLedgerFragmentEntry.

/**
 * This method asynchronously recovers a specific ledger entry by reading
 * the values via the BookKeeper Client (which would read it from the other
 * replicas) and then writing it to the chosen new bookie.
 *
 * @param entryId
 *            Ledger Entry ID to recover.
 * @param lh
 *            LedgerHandle for the ledger
 * @param ledgerFragmentEntryMcb
 *            MultiCallback to invoke once we've recovered the current
 *            ledger entry.
 * @param newBookies
 *            New bookies we want to use to recover and replicate the ledger
 *            entries that were stored on the failed bookie.
 */
private void recoverLedgerFragmentEntry(final Long entryId, final LedgerHandle lh, final AsyncCallback.VoidCallback ledgerFragmentEntryMcb, final Set<BookieSocketAddress> newBookies) throws InterruptedException {
    final AtomicInteger numCompleted = new AtomicInteger(0);
    final AtomicBoolean completed = new AtomicBoolean(false);
    final WriteCallback multiWriteCallback = new WriteCallback() {

        @Override
        public void writeComplete(int rc, long ledgerId, long entryId, BookieSocketAddress addr, Object ctx) {
            if (rc != BKException.Code.OK) {
                LOG.error("BK error writing entry for ledgerId: {}, entryId: {}, bookie: {}", ledgerId, entryId, addr, BKException.create(rc));
                if (completed.compareAndSet(false, true)) {
                    ledgerFragmentEntryMcb.processResult(rc, null, null);
                }
            } else {
                numEntriesWritten.inc();
                if (ctx instanceof Long) {
                    numBytesWritten.registerSuccessfulValue((Long) ctx);
                }
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Success writing ledger id {}, entry id {} to a new bookie {}!", ledgerId, entryId, addr);
                }
                if (numCompleted.incrementAndGet() == newBookies.size() && completed.compareAndSet(false, true)) {
                    ledgerFragmentEntryMcb.processResult(rc, null, null);
                }
            }
        }
    };
    /*
         * Read the ledger entry using the LedgerHandle. This will allow us to
         * read the entry from one of the other replicated bookies other than
         * the dead one.
         */
    lh.asyncReadEntries(entryId, entryId, new ReadCallback() {

        @Override
        public void readComplete(int rc, LedgerHandle lh, Enumeration<LedgerEntry> seq, Object ctx) {
            if (rc != BKException.Code.OK) {
                LOG.error("BK error reading ledger entry: " + entryId, BKException.create(rc));
                ledgerFragmentEntryMcb.processResult(rc, null, null);
                return;
            }
            /*
                 * Now that we've read the ledger entry, write it to the new
                 * bookie we've selected.
                 */
            LedgerEntry entry = seq.nextElement();
            byte[] data = entry.getEntry();
            final long dataLength = data.length;
            numEntriesRead.inc();
            numBytesRead.registerSuccessfulValue(dataLength);
            ByteBufList toSend = lh.getDigestManager().computeDigestAndPackageForSending(entryId, lh.getLastAddConfirmed(), entry.getLength(), Unpooled.wrappedBuffer(data, 0, data.length));
            for (BookieSocketAddress newBookie : newBookies) {
                bkc.getBookieClient().addEntry(newBookie, lh.getId(), lh.getLedgerKey(), entryId, ByteBufList.clone(toSend), multiWriteCallback, dataLength, BookieProtocol.FLAG_RECOVERY_ADD);
            }
            toSend.release();
        }
    }, null);
}

Example 5 with ByteBufList

use of org.apache.bookkeeper.util.ByteBufList in project bookkeeper by apache.

the class MockBookKeeperTestCase method setupBookieClientAddEntry.

@SuppressWarnings("unchecked")
protected void setupBookieClientAddEntry() {
    doAnswer(invokation -> {
        Object[] args = invokation.getArguments();
        BookkeeperInternalCallbacks.WriteCallback callback = (BookkeeperInternalCallbacks.WriteCallback) args[5];
        BookieSocketAddress bookieSocketAddress = (BookieSocketAddress) args[0];
        long ledgerId = (Long) args[1];
        long entryId = (Long) args[3];
        ByteBufList toSend = (ByteBufList) args[4];
        Object ctx = args[6];
        int options = (int) args[7];
        boolean isRecoveryAdd = ((short) options & BookieProtocol.FLAG_RECOVERY_ADD) == BookieProtocol.FLAG_RECOVERY_ADD;
        executor.executeOrdered(ledgerId, () -> {
            byte[] entry;
            try {
                entry = extractEntryPayload(ledgerId, entryId, toSend);
            } catch (BKDigestMatchException e) {
                callback.writeComplete(Code.DigestMatchException, ledgerId, entryId, bookieSocketAddress, ctx);
                return;
            }
            boolean fenced = fencedLedgers.contains(ledgerId);
            if (fenced && !isRecoveryAdd) {
                callback.writeComplete(BKException.Code.LedgerFencedException, ledgerId, entryId, bookieSocketAddress, ctx);
            } else {
                if (failedBookies.contains(bookieSocketAddress)) {
                    callback.writeComplete(NoBookieAvailableException, ledgerId, entryId, bookieSocketAddress, ctx);
                    return;
                }
                if (getMockLedgerContentsInBookie(ledgerId, bookieSocketAddress).isEmpty()) {
                    registerMockEntryForRead(ledgerId, BookieProtocol.LAST_ADD_CONFIRMED, bookieSocketAddress, new byte[0], BookieProtocol.INVALID_ENTRY_ID);
                }
                registerMockEntryForRead(ledgerId, entryId, bookieSocketAddress, entry, ledgerId);
                callback.writeComplete(BKException.Code.OK, ledgerId, entryId, bookieSocketAddress, ctx);
            }
        });
        return null;
    }).when(bookieClient).addEntry(any(BookieSocketAddress.class), anyLong(), any(byte[].class), anyLong(), any(ByteBufList.class), any(BookkeeperInternalCallbacks.WriteCallback.class), any(), anyInt());
}