Examples with DistributedLogManager org.apache.distributedlog.api.DistributedLogManager used on opensource projects

Example 66 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestNonBlockingReads method testHandleInconsistentMetadataNonBlocking.

@Test(timeout = 15000)
public void testHandleInconsistentMetadataNonBlocking() throws Exception {
    String name = "distrlog-inconsistent-metadata-nonblocking-read";
    long numRecordsWritten = createStreamWithInconsistentMetadata(name);
    DistributedLogManager dlm = createNewDLM(conf, name);
    try {
        LogReader reader = dlm.getInputStream(45);
        long numRecordsRead = 0;
        long lastTxId = -1;
        while (numRecordsRead < (numRecordsWritten / 2)) {
            LogRecord record = reader.readNext(false);
            if (record != null) {
                DLMTestUtil.verifyLogRecord(record);
                Assert.assertTrue(lastTxId < record.getTransactionId());
                lastTxId = record.getTransactionId();
                numRecordsRead++;
            } else {
                Thread.sleep(1);
            }
        }
        reader.close();
    } finally {
        dlm.close();
    }
}

Example 67 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestNonBlockingReads method createStreamWithInconsistentMetadata.

private long createStreamWithInconsistentMetadata(String name) throws Exception {
    DistributedLogManager dlm = createNewDLM(conf, name);
    ZooKeeperClient zkClient = TestZooKeeperClientBuilder.newBuilder().uri(createDLMURI("/")).build();
    long txid = 1;
    long numRecordsWritten = 0;
    int segmentSize = 10;
    for (long i = 0; i < 3; i++) {
        BKAsyncLogWriter out = (BKAsyncLogWriter) dlm.startAsyncLogSegmentNonPartitioned();
        for (long j = 1; j <= segmentSize; j++) {
            LogRecord op = DLMTestUtil.getLogRecordInstance(txid++);
            Utils.ioResult(out.write(op));
            numRecordsWritten++;
        }
        out.closeAndComplete();
    }
    BKLogWriteHandler blplm = ((BKDistributedLogManager) (dlm)).createWriteHandler(true);
    String completedZNode = blplm.completedLedgerZNode(txid - segmentSize, txid - 1, 3);
    LogSegmentMetadata metadata = Utils.ioResult(LogSegmentMetadata.read(zkClient, completedZNode));
    zkClient.get().delete(completedZNode, -1);
    LogSegmentMetadata metadataToChange = metadata.mutator().setLastEntryId(metadata.getLastEntryId() + 100).setLastTxId(metadata.getLastTxId() + 100).build();
    metadataToChange.write(zkClient);
    txid += 100;
    for (long i = 0; i < 3; i++) {
        BKAsyncLogWriter out = (BKAsyncLogWriter) dlm.startAsyncLogSegmentNonPartitioned();
        for (long j = 1; j <= segmentSize; j++) {
            LogRecord op = DLMTestUtil.getLogRecordInstance(txid++);
            Utils.ioResult(out.write(op));
            numRecordsWritten++;
        }
        out.closeAndComplete();
    }
    dlm.close();
    return numRecordsWritten;
}

Example 68 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestNonBlockingReads method testNonBlockingReadAheadStall.

@Test(timeout = 60000)
public void testNonBlockingReadAheadStall() throws Exception {
    String name = "distrlog-non-blocking-reader-stall";
    final DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setReadAheadBatchSize(1);
    confLocal.setReadAheadMaxRecords(3);
    confLocal.setReadLACLongPollTimeout(249);
    confLocal.setReaderIdleWarnThresholdMillis(500);
    confLocal.setReaderIdleErrorThresholdMillis(30000);
    final DistributedLogManager dlm = createNewDLM(confLocal, name);
    ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1);
    ScheduledFuture writerClosedFuture = null;
    try {
        final Thread currentThread = Thread.currentThread();
        writerClosedFuture = executor.schedule(new Runnable() {

            @Override
            public void run() {
                try {
                    writeRecordsForNonBlockingReads(confLocal, dlm, false, 3);
                } catch (Exception exc) {
                    currentThread.interrupt();
                }
            }
        }, 10, TimeUnit.MILLISECONDS);
        boolean exceptionEncountered = false;
        try {
            readNonBlocking(dlm, false, 3, false);
        } catch (IdleReaderException exc) {
            LOG.info("Exception encountered", exc);
            exceptionEncountered = true;
        }
        assertFalse(exceptionEncountered);
        assertFalse(currentThread.isInterrupted());
    } finally {
        if (writerClosedFuture != null) {
            // ensure writer.closeAndComplete is done before we close dlm
            writerClosedFuture.get();
        }
        executor.shutdown();
        dlm.close();
    }
}

Example 69 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestNonBlockingReads method testHandleInconsistentMetadataDLSNNonBlocking.

@Test(timeout = 15000)
public void testHandleInconsistentMetadataDLSNNonBlocking() throws Exception {
    String name = "distrlog-inconsistent-metadata-nonblocking-read-dlsn";
    long numRecordsWritten = createStreamWithInconsistentMetadata(name);
    DistributedLogManager dlm = createNewDLM(conf, name);
    try {
        LogReader reader = dlm.getInputStream(DLSN.InitialDLSN);
        long numRecordsRead = 0;
        long lastTxId = -1;
        while (numRecordsRead < numRecordsWritten) {
            LogRecord record = reader.readNext(false);
            if (record != null) {
                DLMTestUtil.verifyLogRecord(record);
                Assert.assertTrue(lastTxId < record.getTransactionId());
                lastTxId = record.getTransactionId();
                numRecordsRead++;
            } else {
                Thread.sleep(1);
            }
        }
        reader.close();
    } finally {
        dlm.close();
    }
}

Example 70 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestNonBlockingReads method testNonBlockingReadIdleError.

@Test(timeout = 100000)
public void testNonBlockingReadIdleError() throws Exception {
    String name = "distrlog-non-blocking-reader-error";
    final DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setReadAheadBatchSize(1);
    confLocal.setReadAheadMaxRecords(1);
    confLocal.setReadLACLongPollTimeout(24);
    confLocal.setReaderIdleWarnThresholdMillis(50);
    confLocal.setReaderIdleErrorThresholdMillis(100);
    final DistributedLogManager dlm = createNewDLM(confLocal, name);
    ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1);
    ScheduledFuture writerClosedFuture = null;
    try {
        final Thread currentThread = Thread.currentThread();
        writerClosedFuture = executor.schedule(new Runnable() {

            @Override
            public void run() {
                try {
                    writeRecordsForNonBlockingReads(confLocal, dlm, false);
                } catch (Exception exc) {
                    currentThread.interrupt();
                }
            }
        }, 100, TimeUnit.MILLISECONDS);
        boolean exceptionEncountered = false;
        try {
            readNonBlocking(dlm, false, DEFAULT_SEGMENT_SIZE, true);
        } catch (IdleReaderException exc) {
            exceptionEncountered = true;
        }
        assertTrue(exceptionEncountered);
        assertFalse(currentThread.isInterrupted());
    } finally {
        if (writerClosedFuture != null) {
            // ensure writer.closeAndComplete is done before we close dlm
            writerClosedFuture.get();
        }
        executor.shutdown();
        dlm.close();
    }
}