Examples with WriteCallback org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.WriteCallback used on opensource projects

Example 6 with WriteCallback

use of org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.WriteCallback in project bookkeeper by apache.

the class LedgerFragmentReplicator method recoverLedgerFragmentEntry.

/**
 * This method asynchronously recovers a specific ledger entry by reading
 * the values via the BookKeeper Client (which would read it from the other
 * replicas) and then writing it to the chosen new bookie.
 *
 * @param entryId
 *            Ledger Entry ID to recover.
 * @param lh
 *            LedgerHandle for the ledger
 * @param ledgerFragmentEntryMcb
 *            MultiCallback to invoke once we've recovered the current
 *            ledger entry.
 * @param newBookies
 *            New bookies we want to use to recover and replicate the ledger
 *            entries that were stored on the failed bookie.
 */
private void recoverLedgerFragmentEntry(final Long entryId, final LedgerHandle lh, final AsyncCallback.VoidCallback ledgerFragmentEntryMcb, final Set<BookieSocketAddress> newBookies) throws InterruptedException {
    final AtomicInteger numCompleted = new AtomicInteger(0);
    final AtomicBoolean completed = new AtomicBoolean(false);
    final WriteCallback multiWriteCallback = new WriteCallback() {

        @Override
        public void writeComplete(int rc, long ledgerId, long entryId, BookieSocketAddress addr, Object ctx) {
            if (rc != BKException.Code.OK) {
                LOG.error("BK error writing entry for ledgerId: {}, entryId: {}, bookie: {}", ledgerId, entryId, addr, BKException.create(rc));
                if (completed.compareAndSet(false, true)) {
                    ledgerFragmentEntryMcb.processResult(rc, null, null);
                }
            } else {
                numEntriesWritten.inc();
                if (ctx instanceof Long) {
                    numBytesWritten.registerSuccessfulValue((Long) ctx);
                }
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Success writing ledger id {}, entry id {} to a new bookie {}!", ledgerId, entryId, addr);
                }
                if (numCompleted.incrementAndGet() == newBookies.size() && completed.compareAndSet(false, true)) {
                    ledgerFragmentEntryMcb.processResult(rc, null, null);
                }
            }
        }
    };
    /*
         * Read the ledger entry using the LedgerHandle. This will allow us to
         * read the entry from one of the other replicated bookies other than
         * the dead one.
         */
    lh.asyncReadEntries(entryId, entryId, new ReadCallback() {

        @Override
        public void readComplete(int rc, LedgerHandle lh, Enumeration<LedgerEntry> seq, Object ctx) {
            if (rc != BKException.Code.OK) {
                LOG.error("BK error reading ledger entry: " + entryId, BKException.create(rc));
                ledgerFragmentEntryMcb.processResult(rc, null, null);
                return;
            }
            /*
                 * Now that we've read the ledger entry, write it to the new
                 * bookie we've selected.
                 */
            LedgerEntry entry = seq.nextElement();
            byte[] data = entry.getEntry();
            final long dataLength = data.length;
            numEntriesRead.inc();
            numBytesRead.registerSuccessfulValue(dataLength);
            ByteBufList toSend = lh.getDigestManager().computeDigestAndPackageForSending(entryId, lh.getLastAddConfirmed(), entry.getLength(), Unpooled.wrappedBuffer(data, 0, data.length));
            for (BookieSocketAddress newBookie : newBookies) {
                bkc.getBookieClient().addEntry(newBookie, lh.getId(), lh.getLedgerKey(), entryId, ByteBufList.clone(toSend), multiWriteCallback, dataLength, BookieProtocol.FLAG_RECOVERY_ADD);
            }
            toSend.release();
        }
    }, null);
}

Example 7 with WriteCallback

use of org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.WriteCallback in project bookkeeper by apache.

the class BookieJournalForceTest method testAckAfterSync.

@Test
public void testAckAfterSync() throws Exception {
    File journalDir = tempDir.newFolder();
    Bookie.checkDirectoryStructure(Bookie.getCurrentDirectory(journalDir));
    ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
    conf.setJournalDirName(journalDir.getPath()).setZkServers(null);
    JournalChannel jc = spy(new JournalChannel(journalDir, 1));
    whenNew(JournalChannel.class).withAnyArguments().thenReturn(jc);
    LedgerDirsManager ledgerDirsManager = mock(LedgerDirsManager.class);
    Journal journal = new Journal(0, journalDir, conf, ledgerDirsManager);
    // machinery to suspend ForceWriteThread
    CountDownLatch forceWriteThreadSuspendedLatch = new CountDownLatch(1);
    LinkedBlockingQueue<ForceWriteRequest> supportQueue = enableForceWriteThreadSuspension(forceWriteThreadSuspendedLatch, journal);
    journal.start();
    LogMark lastLogMarkBeforeWrite = journal.getLastLogMark().markLog().getCurMark();
    CountDownLatch latch = new CountDownLatch(1);
    long ledgerId = 1;
    long entryId = 0;
    journal.logAddEntry(ledgerId, entryId, DATA, false, /* ackBeforeSync */
    new WriteCallback() {

        @Override
        public void writeComplete(int rc, long ledgerId, long entryId, BookieSocketAddress addr, Object ctx) {
            latch.countDown();
        }
    }, null);
    // wait that an entry is written to the ForceWriteThread queue
    while (supportQueue.isEmpty()) {
        Thread.sleep(100);
    }
    assertEquals(1, latch.getCount());
    assertEquals(1, supportQueue.size());
    // in constructor of JournalChannel we are calling forceWrite(true) but it is not tracked by PowerMock
    // because the 'spy' is applied only on return from the constructor
    verify(jc, times(0)).forceWrite(true);
    // let ForceWriteThread work
    forceWriteThreadSuspendedLatch.countDown();
    // callback should complete now
    assertTrue(latch.await(20, TimeUnit.SECONDS));
    verify(jc, atLeast(1)).forceWrite(false);
    assertEquals(0, supportQueue.size());
    // verify that log marker advanced
    LastLogMark lastLogMarkAfterForceWrite = journal.getLastLogMark();
    assertTrue(lastLogMarkAfterForceWrite.getCurMark().compare(lastLogMarkBeforeWrite) > 0);
    journal.shutdown();
}

Example 8 with WriteCallback

use of org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.WriteCallback in project bookkeeper by apache.

the class BookieWriteToJournalTest method testJournalLogAddEntryCalledCorrectly.

/**
 * test that Bookie calls correctly Journal.logAddEntry about "ackBeforeSync" parameter.
 */
@Test
public void testJournalLogAddEntryCalledCorrectly() throws Exception {
    File journalDir = tempDir.newFolder();
    Bookie.checkDirectoryStructure(Bookie.getCurrentDirectory(journalDir));
    File ledgerDir = tempDir.newFolder();
    Bookie.checkDirectoryStructure(Bookie.getCurrentDirectory(ledgerDir));
    ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
    conf.setJournalDirName(journalDir.getPath()).setLedgerDirNames(new String[] { ledgerDir.getPath() }).setZkServers(null);
    BookieSocketAddress bookieAddress = Bookie.getBookieAddress(conf);
    CountDownLatch journalJoinLatch = new CountDownLatch(1);
    Journal journal = mock(Journal.class);
    MutableBoolean effectiveAckBeforeSync = new MutableBoolean(false);
    doAnswer((Answer) (InvocationOnMock iom) -> {
        ByteBuf entry = iom.getArgument(0);
        long ledgerId = entry.getLong(entry.readerIndex() + 0);
        long entryId = entry.getLong(entry.readerIndex() + 8);
        boolean ackBeforeSync = iom.getArgument(1);
        WriteCallback callback = iom.getArgument(2);
        Object ctx = iom.getArgument(3);
        effectiveAckBeforeSync.setValue(ackBeforeSync);
        callback.writeComplete(BKException.Code.OK, ledgerId, entryId, bookieAddress, ctx);
        return null;
    }).when(journal).logAddEntry(any(ByteBuf.class), anyBoolean(), any(WriteCallback.class), any());
    // bookie will continue to work as soon as the journal thread is alive
    doAnswer((Answer) (InvocationOnMock iom) -> {
        journalJoinLatch.await();
        return null;
    }).when(journal).joinThread();
    whenNew(Journal.class).withAnyArguments().thenReturn(journal);
    Bookie b = new Bookie(conf);
    b.start();
    long ledgerId = 1;
    long entryId = 0;
    Object expectedCtx = "foo";
    byte[] masterKey = new byte[64];
    for (boolean ackBeforeSync : new boolean[] { true, false }) {
        CountDownLatch latch = new CountDownLatch(1);
        final ByteBuf data = Unpooled.buffer();
        data.writeLong(ledgerId);
        data.writeLong(entryId);
        final long expectedEntryId = entryId;
        b.addEntry(data, ackBeforeSync, (int rc, long ledgerId1, long entryId1, BookieSocketAddress addr, Object ctx) -> {
            assertSame(expectedCtx, ctx);
            assertEquals(ledgerId, ledgerId1);
            assertEquals(expectedEntryId, entryId1);
            latch.countDown();
        }, expectedCtx, masterKey);
        latch.await(30, TimeUnit.SECONDS);
        assertEquals(ackBeforeSync, effectiveAckBeforeSync.booleanValue());
        entryId++;
    }
    // let bookie exit main thread
    journalJoinLatch.countDown();
    b.shutdown();
}

Example 9 with WriteCallback

use of org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.WriteCallback in project bookkeeper by apache.

the class ConcurrentLedgerTest method doWrites.

private long doWrites(int ledgers, int size, int totalwrites) throws IOException, InterruptedException, BookieException {
    throttle = new Semaphore(10000);
    WriteCallback cb = new WriteCallback() {

        @Override
        public void writeComplete(int rc, long ledgerId, long entryId, BookieSocketAddress addr, Object ctx) {
            AtomicInteger counter = (AtomicInteger) ctx;
            counter.getAndIncrement();
            throttle.release();
        }
    };
    AtomicInteger counter = new AtomicInteger();
    long start = System.currentTimeMillis();
    for (int i = 1; i <= totalwrites / ledgers; i++) {
        for (int j = 1; j <= ledgers; j++) {
            ByteBuffer bytes = ByteBuffer.allocate(size);
            bytes.putLong(j);
            bytes.putLong(i);
            bytes.putLong(j + 2);
            bytes.putLong(i + 3);
            bytes.put(("This is ledger " + j + " entry " + i).getBytes());
            bytes.position(0);
            bytes.limit(bytes.capacity());
            throttle.acquire();
            bookie.addEntry(Unpooled.wrappedBuffer(bytes), false, cb, counter, zeros);
        }
    }
    long finish = System.currentTimeMillis();
    return finish - start;
}

Example 10 with WriteCallback

use of org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.WriteCallback in project bookkeeper by apache.

the class BookieClient method main.

/**
 * @param args
 * @throws IOException
 * @throws NumberFormatException
 * @throws InterruptedException
 */
public static void main(String[] args) throws NumberFormatException, IOException, InterruptedException {
    if (args.length != 3) {
        System.err.println("USAGE: BookieClient bookieHost port ledger#");
        return;
    }
    WriteCallback cb = new WriteCallback() {

        public void writeComplete(int rc, long ledger, long entry, BookieSocketAddress addr, Object ctx) {
            Counter counter = (Counter) ctx;
            counter.dec();
            if (rc != 0) {
                System.out.println("rc = " + rc + " for " + entry + "@" + ledger);
            }
        }
    };
    Counter counter = new Counter();
    byte[] hello = "hello".getBytes(UTF_8);
    long ledger = Long.parseLong(args[2]);
    EventLoopGroup eventLoopGroup = new NioEventLoopGroup(1);
    OrderedExecutor executor = OrderedExecutor.newBuilder().name("BookieClientWorker").numThreads(1).build();
    ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory("BookKeeperClientScheduler"));
    BookieClient bc = new BookieClient(new ClientConfiguration(), eventLoopGroup, executor, scheduler, NullStatsLogger.INSTANCE);
    BookieSocketAddress addr = new BookieSocketAddress(args[0], Integer.parseInt(args[1]));
    for (int i = 0; i < 100000; i++) {
        counter.inc();
        bc.addEntry(addr, ledger, new byte[0], i, ByteBufList.get(Unpooled.wrappedBuffer(hello)), cb, counter, 0);
    }
    counter.wait(0);
    System.out.println("Total = " + counter.total());
    scheduler.shutdown();
    eventLoopGroup.shutdownGracefully();
    executor.shutdown();
}