Examples with Checkpoint org.apache.bookkeeper.bookie.CheckpointSource.Checkpoint used on opensource projects

Example 11 with Checkpoint

use of org.apache.bookkeeper.bookie.CheckpointSource.Checkpoint in project bookkeeper by apache.

the class TestSyncThread method testSyncThreadSuspension.

/**
 * Test that sync thread suspension works.
 * i.e. when we suspend the syncthread, nothing
 * will be synced.
 */
@Test
public void testSyncThreadSuspension() throws Exception {
    int flushInterval = 100;
    ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
    conf.setFlushInterval(flushInterval);
    CheckpointSource checkpointSource = new DummyCheckpointSource();
    LedgerDirsListener listener = new DummyLedgerDirsListener();
    final AtomicInteger checkpointCount = new AtomicInteger(0);
    LedgerStorage storage = new DummyLedgerStorage() {

        @Override
        public void checkpoint(Checkpoint checkpoint) throws IOException {
            checkpointCount.incrementAndGet();
        }
    };
    final SyncThread t = new SyncThread(conf, listener, storage, checkpointSource);
    t.startCheckpoint(Checkpoint.MAX);
    while (checkpointCount.get() == 0) {
        Thread.sleep(flushInterval);
    }
    t.suspendSync();
    Thread.sleep(flushInterval);
    int count = checkpointCount.get();
    for (int i = 0; i < 10; i++) {
        t.startCheckpoint(Checkpoint.MAX);
        assertEquals("Checkpoint count shouldn't change", count, checkpointCount.get());
    }
    t.resumeSync();
    int i = 0;
    while (checkpointCount.get() == count) {
        Thread.sleep(flushInterval);
        i++;
        if (i > 100) {
            fail("Checkpointing never resumed");
        }
    }
    t.shutdown();
}

Example 12 with Checkpoint

use of org.apache.bookkeeper.bookie.CheckpointSource.Checkpoint in project bookkeeper by apache.

the class DbLedgerStorage method checkpoint.

@Override
public void checkpoint(Checkpoint checkpoint) throws IOException {
    Checkpoint thisCheckpoint = checkpointSource.newCheckpoint();
    if (lastCheckpoint.compareTo(checkpoint) > 0) {
        return;
    }
    long startTime = MathUtils.nowInNano();
    // Only a single flush operation can happen at a time
    flushMutex.lock();
    try {
        // Swap the write cache so that writes can continue to happen while the flush is
        // ongoing
        swapWriteCache();
        long sizeToFlush = writeCacheBeingFlushed.size();
        if (log.isDebugEnabled()) {
            log.debug("Flushing entries. count: {} -- size {} Mb", writeCacheBeingFlushed.count(), sizeToFlush / 1024.0 / 1024);
        }
        // Write all the pending entries into the entry logger and collect the offset
        // position for each entry
        Batch batch = entryLocationIndex.newBatch();
        writeCacheBeingFlushed.forEach((ledgerId, entryId, entry) -> {
            try {
                long location = entryLogger.addEntry(ledgerId, entry, true);
                entryLocationIndex.addLocation(batch, ledgerId, entryId, location);
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        });
        entryLogger.flush();
        long batchFlushStarTime = System.nanoTime();
        batch.flush();
        batch.close();
        if (log.isDebugEnabled()) {
            log.debug("DB batch flushed time : {} s", MathUtils.elapsedNanos(batchFlushStarTime) / (double) TimeUnit.SECONDS.toNanos(1));
        }
        ledgerIndex.flush();
        cleanupExecutor.execute(() -> {
            // is single-threaded
            try {
                if (log.isDebugEnabled()) {
                    log.debug("Removing deleted ledgers from db indexes");
                }
                entryLocationIndex.removeOffsetFromDeletedLedgers();
                ledgerIndex.removeDeletedLedgers();
            } catch (Throwable t) {
                log.warn("Failed to cleanup db indexes", t);
            }
        });
        lastCheckpoint = thisCheckpoint;
        // Discard all the entry from the write cache, since they're now persisted
        writeCacheBeingFlushed.clear();
        double flushTimeSeconds = MathUtils.elapsedNanos(startTime) / (double) TimeUnit.SECONDS.toNanos(1);
        double flushThroughput = sizeToFlush / 1024.0 / 1024.0 / flushTimeSeconds;
        if (log.isDebugEnabled()) {
            log.debug("Flushing done time {} s -- Written {} MB/s", flushTimeSeconds, flushThroughput);
        }
        recordSuccessfulEvent(flushStats, startTime);
        flushSizeStats.registerSuccessfulValue(sizeToFlush);
    } catch (IOException e) {
        // Leave IOExecption as it is
        throw e;
    } catch (RuntimeException e) {
        // Wrap unchecked exceptions
        throw new IOException(e);
    } finally {
        try {
            isFlushOngoing.set(false);
        } finally {
            flushMutex.unlock();
        }
    }
}

Example 13 with Checkpoint

use of org.apache.bookkeeper.bookie.CheckpointSource.Checkpoint in project bookkeeper by apache.

the class EntryMemTable method snapshot.

/**
 * Snapshot current EntryMemTable. if given <i>oldCp</i> is older than current checkpoint,
 * we don't do any snapshot. If snapshot happened, we return the checkpoint of the snapshot.
 *
 * @param oldCp
 *          checkpoint
 * @return checkpoint of the snapshot, null means no snapshot
 * @throws IOException
 */
Checkpoint snapshot(Checkpoint oldCp) throws IOException {
    Checkpoint cp = null;
    // No-op if snapshot currently has entries
    if (this.snapshot.isEmpty() && this.kvmap.compareTo(oldCp) < 0) {
        final long startTimeNanos = MathUtils.nowInNano();
        this.lock.writeLock().lock();
        try {
            if (this.snapshot.isEmpty() && !this.kvmap.isEmpty() && this.kvmap.compareTo(oldCp) < 0) {
                this.snapshot = this.kvmap;
                this.kvmap = newSkipList();
                // get the checkpoint of the memtable.
                cp = this.kvmap.cp;
                // Reset heap to not include any keys
                this.size.set(0);
                // Reset allocator so we get a fresh buffer for the new EntryMemTable
                this.allocator = new SkipListArena(conf);
            }
        } finally {
            this.lock.writeLock().unlock();
        }
        if (null != cp) {
            snapshotStats.registerSuccessfulEvent(MathUtils.elapsedNanos(startTimeNanos), TimeUnit.NANOSECONDS);
        } else {
            snapshotStats.registerFailedEvent(MathUtils.elapsedNanos(startTimeNanos), TimeUnit.NANOSECONDS);
        }
    }
    return cp;
}