Examples with DistributedLogManager org.apache.distributedlog.api.DistributedLogManager used on opensource projects

Example 46 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestAsyncReaderWriter method writeRecordsWithOutstandingWriteLimit.

public void writeRecordsWithOutstandingWriteLimit(int stream, int global, boolean shouldFail) throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setPerWriterOutstandingWriteLimit(stream);
    confLocal.setOutstandingWriteLimitDarkmode(false);
    DistributedLogManager dlm;
    if (global > -1) {
        dlm = createNewDLM(confLocal, runtime.getMethodName(), new SimplePermitLimiter(false, global, new NullStatsLogger(), true, new FixedValueFeature("", 0)));
    } else {
        dlm = createNewDLM(confLocal, runtime.getMethodName());
    }
    BKAsyncLogWriter writer = (BKAsyncLogWriter) (dlm.startAsyncLogSegmentNonPartitioned());
    ArrayList<CompletableFuture<DLSN>> results = new ArrayList<CompletableFuture<DLSN>>(1000);
    for (int i = 0; i < 1000; i++) {
        results.add(writer.write(DLMTestUtil.getLogRecordInstance(1L)));
    }
    for (CompletableFuture<DLSN> result : results) {
        try {
            Utils.ioResult(result);
            if (shouldFail) {
                fail("should fail due to no outstanding writes permitted");
            }
        } catch (OverCapacityException ex) {
            assertTrue(shouldFail);
        }
    }
    writer.closeAndComplete();
    dlm.close();
}

Example 47 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestAsyncReaderWriter method testAsyncReadMissingLogSegmentsNotification.

@DistributedLogAnnotations.FlakyTest
@Test(timeout = 60000)
public void testAsyncReadMissingLogSegmentsNotification() throws Exception {
    String name = "distrlog-async-reader-missing-zk-notification";
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setReadAheadBatchSize(1);
    confLocal.setReadAheadMaxRecords(1);
    confLocal.setReadLACLongPollTimeout(49);
    confLocal.setReaderIdleWarnThresholdMillis(100);
    confLocal.setReaderIdleErrorThresholdMillis(20000);
    final DistributedLogManager dlm = createNewDLM(confLocal, name);
    final Thread currentThread = Thread.currentThread();
    final int segmentSize = 10;
    final int numSegments = 3;
    final CountDownLatch latch = new CountDownLatch(1);
    final CountDownLatch readLatch = new CountDownLatch(1);
    final ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1);
    executor.schedule(new Runnable() {

        @Override
        public void run() {
            try {
                int txid = 1;
                for (long i = 0; i < numSegments; i++) {
                    BKSyncLogWriter writer = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
                    for (long j = 1; j <= segmentSize; j++) {
                        writer.write(DLMTestUtil.getLargeLogRecordInstance(txid++));
                        if ((i == 0) && (j == 1)) {
                            latch.countDown();
                        } else {
                            // wait for reader to start
                            readLatch.await();
                        }
                    }
                    writer.closeAndComplete();
                    Thread.sleep(100);
                }
            } catch (Exception exc) {
                if (!executor.isShutdown()) {
                    currentThread.interrupt();
                }
            }
        }
    }, 0, TimeUnit.MILLISECONDS);
    latch.await();
    BKAsyncLogReader reader = (BKAsyncLogReader) dlm.getAsyncLogReader(DLSN.InitialDLSN);
    reader.disableReadAheadLogSegmentsNotification();
    boolean exceptionEncountered = false;
    int recordCount = 0;
    try {
        while (true) {
            CompletableFuture<LogRecordWithDLSN> record = reader.readNext();
            Utils.ioResult(record);
            if (recordCount == 0) {
                readLatch.countDown();
            }
            recordCount++;
            if (recordCount >= segmentSize * numSegments) {
                break;
            }
        }
    } catch (IdleReaderException exc) {
        exceptionEncountered = true;
    }
    assertTrue(!exceptionEncountered);
    Assert.assertEquals(recordCount, segmentSize * numSegments);
    assertTrue(!currentThread.isInterrupted());
    Utils.close(reader);
    executor.shutdown();
}

Example 48 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestAsyncReaderWriter method testIdleReaderExceptionWhenKeepAliveIsDisabled.

@Test(timeout = 60000)
public void testIdleReaderExceptionWhenKeepAliveIsDisabled() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setPeriodicKeepAliveMilliSeconds(0);
    confLocal.setReadLACLongPollTimeout(9);
    confLocal.setReaderIdleWarnThresholdMillis(20);
    confLocal.setReaderIdleErrorThresholdMillis(40);
    URI uri = createDLMURI("/" + name);
    ensureURICreated(uri);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    BKAsyncLogWriter writer = (BKAsyncLogWriter) Utils.ioResult(dlm.openAsyncLogWriter());
    writer.write(DLMTestUtil.getLogRecordInstance(1L));
    AsyncLogReader reader = Utils.ioResult(dlm.openAsyncLogReader(DLSN.InitialDLSN));
    try {
        Utils.ioResult(reader.readNext());
        fail("Should fail when stream is idle");
    } catch (IdleReaderException ire) {
    // expected
    }
    Utils.close(reader);
    writer.close();
    dlm.close();
}

Example 49 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestAsyncReaderWriter method testSimpleAsyncReadPosition.

/**
 * Test Async Read by positioning to a given position in the log.
 * @throws Exception
 */
@Test(timeout = 60000)
public void testSimpleAsyncReadPosition() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(testConf);
    confLocal.setOutputBufferSize(1024);
    confLocal.setReadAheadWaitTime(10);
    confLocal.setReadAheadBatchSize(10);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    int numLogSegments = 3;
    int numRecordsPerLogSegment = 10;
    long txid = 1L;
    // write 3 log segments, 10 records per log segment
    txid = writeRecords(dlm, numLogSegments, numRecordsPerLogSegment, txid, false);
    // write another log segment with 5 records
    txid = writeLogSegment(dlm, 5, txid, Integer.MAX_VALUE, false);
    final CountDownLatch syncLatch = new CountDownLatch((int) (txid - 14));
    final CountDownLatch doneLatch = new CountDownLatch(1);
    final AtomicBoolean errorsFound = new AtomicBoolean(false);
    final AsyncLogReader reader = dlm.getAsyncLogReader(new DLSN(2, 2, 4));
    assertEquals(name, reader.getStreamName());
    boolean monotonic = LogSegmentMetadata.supportsSequenceId(confLocal.getDLLedgerMetadataLayoutVersion());
    TestAsyncReaderWriter.readNext(reader, new DLSN(2, 3, 0), monotonic ? 13L : Long.MIN_VALUE, monotonic, syncLatch, doneLatch, errorsFound);
    doneLatch.await();
    assertFalse("Errors found on reading records", errorsFound.get());
    syncLatch.await();
    Utils.close(reader);
    dlm.close();
}

Example 50 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestAsyncReaderWriter method testGetLastTxId.

@Test(timeout = 60000)
public void testGetLastTxId() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setImmediateFlushEnabled(true);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    AsyncLogWriter writer = dlm.startAsyncLogSegmentNonPartitioned();
    int numRecords = 10;
    for (int i = 0; i < numRecords; i++) {
        Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(i)));
        assertEquals("last tx id should become " + i, i, writer.getLastTxId());
    }
    // open a writer to recover the inprogress log segment
    AsyncLogWriter recoverWriter = dlm.startAsyncLogSegmentNonPartitioned();
    assertEquals("recovered last tx id should be " + (numRecords - 1), numRecords - 1, recoverWriter.getLastTxId());
}