Examples with DistributedLogManager org.apache.distributedlog.api.DistributedLogManager used on opensource projects

Example 61 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestDLCK method testCheckAndRepairDLNamespace.

@Test(timeout = 60000)
@SuppressWarnings("deprecation")
public void testCheckAndRepairDLNamespace() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setOutputBufferSize(0);
    confLocal.setLogSegmentSequenceNumberValidationEnabled(false);
    confLocal.setLogSegmentCacheEnabled(false);
    URI uri = createDLMURI("/check-and-repair-dl-namespace");
    zkc.get().create(uri.getPath(), new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    Namespace namespace = NamespaceBuilder.newBuilder().conf(confLocal).uri(uri).build();
    OrderedScheduler scheduler = OrderedScheduler.newSchedulerBuilder().name("dlck-tool").numThreads(1).build();
    ExecutorService executorService = Executors.newCachedThreadPool();
    String streamName = "check-and-repair-dl-namespace";
    // Create completed log segments
    DistributedLogManager dlm = namespace.openLog(streamName);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 1L, 1L, 10, false);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 2L, 11L, 10, true);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 3L, 21L, 10, false);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 4L, 31L, 10, true);
    // dryrun
    DistributedLogAdmin.checkAndRepairDLNamespace(uri, namespace, new DryrunLogSegmentMetadataStoreUpdater(confLocal, getLogSegmentMetadataStore(namespace)), scheduler, false, false);
    Map<Long, LogSegmentMetadata> segments = getLogSegments(dlm);
    LOG.info("segments after drynrun {}", segments);
    verifyLogSegment(segments, new DLSN(1L, 18L, 0L), 1L, 10, 10L);
    verifyLogSegment(segments, new DLSN(2L, 16L, 0L), 2L, 9, 19L);
    verifyLogSegment(segments, new DLSN(3L, 18L, 0L), 3L, 10, 30L);
    verifyLogSegment(segments, new DLSN(4L, 16L, 0L), 4L, 9, 39L);
    // check and repair
    DistributedLogAdmin.checkAndRepairDLNamespace(uri, namespace, LogSegmentMetadataStoreUpdater.createMetadataUpdater(confLocal, getLogSegmentMetadataStore(namespace)), scheduler, false, false);
    segments = getLogSegments(dlm);
    LOG.info("segments after repair {}", segments);
    verifyLogSegment(segments, new DLSN(1L, 18L, 0L), 1L, 10, 10L);
    verifyLogSegment(segments, new DLSN(2L, 18L, 0L), 2L, 10, 20L);
    verifyLogSegment(segments, new DLSN(3L, 18L, 0L), 3L, 10, 30L);
    verifyLogSegment(segments, new DLSN(4L, 18L, 0L), 4L, 10, 40L);
    dlm.close();
    SchedulerUtils.shutdownScheduler(executorService, 5, TimeUnit.MINUTES);
    namespace.close();
}

Example 62 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestBKSyncLogReader method testDeletingLogWhileReading.

@Test(timeout = 60000)
public void testDeletingLogWhileReading() throws Exception {
    String name = testName.getMethodName();
    DistributedLogManager dlm = createNewDLM(conf, name);
    BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
    for (long i = 1; i < 10; i++) {
        LogRecord op = DLMTestUtil.getLogRecordInstance(i);
        out.write(op);
    }
    out.closeAndComplete();
    LogReader reader = dlm.getInputStream(1L);
    for (int i = 1; i < 10; i++) {
        LogRecord record = waitForNextRecord(reader);
        assertEquals((long) i, record.getTransactionId());
    }
    DistributedLogManager deleteDLM = createNewDLM(conf, name);
    deleteDLM.delete();
    LogRecord record;
    try {
        record = reader.readNext(false);
        while (null == record) {
            record = reader.readNext(false);
        }
        fail("Should fail reading next with LogNotFound");
    } catch (LogNotFoundException lnfe) {
    // expected
    }
}

Example 63 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestBKSyncLogReader method testReadRecordsWhenReadAheadCatchingUp2.

@Test(timeout = 60000)
public void testReadRecordsWhenReadAheadCatchingUp2() throws Exception {
    String name = testName.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(Integer.MAX_VALUE);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    final BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
    for (long i = 1L; i <= 10L; i++) {
        LogRecord record = DLMTestUtil.getLogRecordInstance(i);
        out.write(record);
    }
    out.flush();
    out.commit();
    final AtomicLong nextTxId = new AtomicLong(11L);
    logger.info("Write first 10 records");
    ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
    executorService.scheduleAtFixedRate(new Runnable() {

        @Override
        public void run() {
            long txid = nextTxId.getAndIncrement();
            LogRecord record = DLMTestUtil.getLogRecordInstance(txid);
            try {
                out.write(record);
            } catch (IOException e) {
            // ignore the ioe
            }
        }
    }, 0, 400, TimeUnit.MILLISECONDS);
    // open a reader to read
    BKSyncLogReader reader = (BKSyncLogReader) dlm.getInputStream(1L);
    // resume reading from sync reader. so it should be able to read all 10 records
    // and return null to claim it as caughtup
    LogRecord record = reader.readNext(false);
    int numReads = 0;
    long expectedTxId = 1L;
    while (null != record) {
        ++numReads;
        assertEquals(expectedTxId, record.getTransactionId());
        DLMTestUtil.verifyLogRecord(record);
        ++expectedTxId;
        record = reader.readNext(false);
    }
    assertTrue(numReads >= 10);
    executorService.shutdown();
    out.close();
    reader.close();
    dlm.close();
}

Example 64 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestNonBlockingReads method testNonBlockingReadRecovery.

@Test(timeout = 100000)
public void testNonBlockingReadRecovery() throws Exception {
    String name = "distrlog-non-blocking-reader-recovery";
    final DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setReadAheadBatchSize(10);
    confLocal.setReadAheadMaxRecords(10);
    final DistributedLogManager dlm = createNewDLM(confLocal, name);
    ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1);
    ScheduledFuture writerClosedFuture = null;
    try {
        final Thread currentThread = Thread.currentThread();
        writerClosedFuture = executor.schedule(new Runnable() {

            @Override
            public void run() {
                try {
                    writeRecordsForNonBlockingReads(confLocal, dlm, true);
                } catch (Exception exc) {
                    currentThread.interrupt();
                }
            }
        }, 100, TimeUnit.MILLISECONDS);
        readNonBlocking(dlm, false);
        assertFalse(currentThread.isInterrupted());
    } finally {
        if (writerClosedFuture != null) {
            // ensure writer.closeAndComplete is done before we close dlm
            writerClosedFuture.get();
        }
        executor.shutdown();
        dlm.close();
    }
}

Example 65 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestNonBlockingReads method testHandleInconsistentMetadata.

@Test(timeout = 60000)
public void testHandleInconsistentMetadata() throws Exception {
    String name = "distrlog-inconsistent-metadata-blocking-read";
    long numRecordsWritten = createStreamWithInconsistentMetadata(name);
    DistributedLogManager dlm = createNewDLM(conf, name);
    try {
        LogReader reader = dlm.getInputStream(45);
        long numRecordsRead = 0;
        LogRecord record = reader.readNext(false);
        long lastTxId = -1;
        while (numRecordsRead < numRecordsWritten / 2) {
            if (null != record) {
                DLMTestUtil.verifyLogRecord(record);
                Assert.assertTrue(lastTxId < record.getTransactionId());
                lastTxId = record.getTransactionId();
                numRecordsRead++;
            } else {
                Thread.sleep(1);
            }
            record = reader.readNext(false);
        }
        reader.close();
        assertEquals(numRecordsWritten / 2, numRecordsRead);
    } finally {
        dlm.close();
    }
}