Examples with AsyncLogReader org.apache.distributedlog.api.AsyncLogReader used on opensource projects

Example 21 with AsyncLogReader

use of org.apache.distributedlog.api.AsyncLogReader in project bookkeeper by apache.

the class TestAsyncReaderWriter method testSimpleAsyncReadWritePiggyBack.

@Test(timeout = 60000)
public void testSimpleAsyncReadWritePiggyBack() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(testConf);
    confLocal.setEnableReadAhead(true);
    confLocal.setReadAheadWaitTime(500);
    confLocal.setReadAheadBatchSize(10);
    confLocal.setReadAheadMaxRecords(100);
    confLocal.setOutputBufferSize(1024);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(100);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    final AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InvalidDLSN);
    int numLogSegments = 3;
    int numRecordsPerLogSegment = 10;
    final CountDownLatch readLatch = new CountDownLatch(30);
    final CountDownLatch readDoneLatch = new CountDownLatch(1);
    final AtomicBoolean readErrors = new AtomicBoolean(false);
    final CountDownLatch writeLatch = new CountDownLatch(30);
    final AtomicBoolean writeErrors = new AtomicBoolean(false);
    int txid = 1;
    for (long i = 0; i < numLogSegments; i++) {
        final long currentLogSegmentSeqNo = i + 1;
        BKAsyncLogWriter writer = (BKAsyncLogWriter) (dlm.startAsyncLogSegmentNonPartitioned());
        for (long j = 0; j < numRecordsPerLogSegment; j++) {
            Thread.sleep(50);
            final LogRecord record = DLMTestUtil.getLargeLogRecordInstance(txid++);
            CompletableFuture<DLSN> dlsnFuture = writer.write(record);
            dlsnFuture.whenComplete(new WriteFutureEventListener(record, currentLogSegmentSeqNo, j, writeLatch, writeErrors, false));
            if (i == 0 && j == 0) {
                boolean monotonic = LogSegmentMetadata.supportsSequenceId(confLocal.getDLLedgerMetadataLayoutVersion());
                TestAsyncReaderWriter.readNext(reader, DLSN.InvalidDLSN, monotonic ? 0L : Long.MIN_VALUE, monotonic, readLatch, readDoneLatch, readErrors);
            }
        }
        writer.closeAndComplete();
    }
    writeLatch.await();
    assertFalse("All writes should succeed", writeErrors.get());
    readDoneLatch.await();
    assertFalse("All reads should succeed", readErrors.get());
    readLatch.await();
    Utils.close(reader);
    dlm.close();
}

Example 22 with AsyncLogReader

use of org.apache.distributedlog.api.AsyncLogReader in project bookkeeper by apache.

the class TestAsyncReaderWriter method testSimpleAsyncReadWriteInternal.

void testSimpleAsyncReadWriteInternal(String name, boolean immediateFlush, int logSegmentVersion) throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(testConf);
    confLocal.setReadAheadWaitTime(10);
    confLocal.setReadAheadBatchSize(10);
    confLocal.setOutputBufferSize(1024);
    confLocal.setDLLedgerMetadataLayoutVersion(logSegmentVersion);
    confLocal.setImmediateFlushEnabled(immediateFlush);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    int numLogSegments = 3;
    int numRecordsPerLogSegment = 10;
    final CountDownLatch readLatch = new CountDownLatch(numLogSegments * numRecordsPerLogSegment);
    final CountDownLatch readDoneLatch = new CountDownLatch(1);
    final AtomicBoolean readErrors = new AtomicBoolean(false);
    final CountDownLatch writeLatch = new CountDownLatch(numLogSegments * numRecordsPerLogSegment);
    final AtomicBoolean writeErrors = new AtomicBoolean(false);
    final AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InvalidDLSN);
    assertEquals(name, reader.getStreamName());
    int txid = 1;
    for (long i = 0; i < 3; i++) {
        final long currentLogSegmentSeqNo = i + 1;
        BKAsyncLogWriter writer = (BKAsyncLogWriter) (dlm.startAsyncLogSegmentNonPartitioned());
        for (long j = 0; j < 10; j++) {
            final long currentEntryId = j;
            final LogRecord record = DLMTestUtil.getLargeLogRecordInstance(txid++);
            CompletableFuture<DLSN> dlsnFuture = writer.write(record);
            dlsnFuture.whenComplete(new WriteFutureEventListener(record, currentLogSegmentSeqNo, currentEntryId, writeLatch, writeErrors, true));
            if (i == 0 && j == 0) {
                boolean monotonic = LogSegmentMetadata.supportsSequenceId(logSegmentVersion);
                TestAsyncReaderWriter.readNext(reader, DLSN.InvalidDLSN, monotonic ? 0L : Long.MIN_VALUE, monotonic, readLatch, readDoneLatch, readErrors);
            }
        }
        writer.closeAndComplete();
    }
    writeLatch.await();
    assertFalse("All writes should succeed", writeErrors.get());
    readDoneLatch.await();
    assertFalse("All reads should succeed", readErrors.get());
    readLatch.await();
    Utils.close(reader);
    dlm.close();
}

Example 23 with AsyncLogReader

use of org.apache.distributedlog.api.AsyncLogReader in project bookkeeper by apache.

the class TestAsyncReaderWriter method testBulkAsyncRead.

@Test(timeout = 60000)
public void testBulkAsyncRead() throws Exception {
    String name = "distrlog-bulkasyncread";
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setOutputBufferSize(0);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setReadAheadWaitTime(10);
    confLocal.setReadAheadMaxRecords(10000);
    confLocal.setReadAheadBatchSize(10);
    int numLogSegments = 3;
    int numRecordsPerLogSegment = 20;
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    writeRecords(dlm, numLogSegments, numRecordsPerLogSegment, 1L, false);
    final AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InitialDLSN);
    int expectedTxID = 1;
    int numReads = 0;
    while (expectedTxID <= numLogSegments * numRecordsPerLogSegment) {
        if (expectedTxID == numLogSegments * numRecordsPerLogSegment) {
            break;
        }
        List<LogRecordWithDLSN> records = Utils.ioResult(reader.readBulk(20));
        LOG.info("Bulk read {} entries.", records.size());
        assertTrue(records.size() >= 1);
        for (LogRecordWithDLSN record : records) {
            assertEquals(expectedTxID, record.getTransactionId());
            ++expectedTxID;
        }
        ++numReads;
    }
    // we expect bulk read works
    assertTrue(numReads < 60);
    Utils.close(reader);
    dlm.close();
}

Example 24 with AsyncLogReader

use of org.apache.distributedlog.api.AsyncLogReader in project bookkeeper by apache.

the class TestAsyncReaderWriter method simpleAsyncReadTest.

void simpleAsyncReadTest(String name, DistributedLogConfiguration confLocal) throws Exception {
    confLocal.setOutputBufferSize(1024);
    confLocal.setReadAheadWaitTime(10);
    confLocal.setReadAheadBatchSize(10);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    int numLogSegments = 3;
    int numRecordsPerLogSegment = 10;
    // Write 30 records: 3 log segments, 10 records per log segment
    long txid = 1L;
    txid = writeRecords(dlm, numLogSegments, numRecordsPerLogSegment, txid, false);
    // Write another log segment with 5 records and flush every 2 records
    txid = writeLogSegment(dlm, 5, txid, 2, false);
    final AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InvalidDLSN);
    final CountDownLatch syncLatch = new CountDownLatch((int) (txid - 1));
    final CountDownLatch completionLatch = new CountDownLatch(1);
    final AtomicBoolean errorsFound = new AtomicBoolean(false);
    boolean monotonic = LogSegmentMetadata.supportsSequenceId(confLocal.getDLLedgerMetadataLayoutVersion());
    TestAsyncReaderWriter.readNext(reader, DLSN.InvalidDLSN, monotonic ? 0L : Long.MIN_VALUE, monotonic, syncLatch, completionLatch, errorsFound);
    completionLatch.await();
    assertFalse("Errors encountered on reading records", errorsFound.get());
    syncLatch.await();
    Utils.close(reader);
    dlm.close();
}

Example 25 with AsyncLogReader

use of org.apache.distributedlog.api.AsyncLogReader in project bookkeeper by apache.

the class TestAsyncReaderWriter method testAsyncReadEmptyRecords.

@Test(timeout = 60000)
public void testAsyncReadEmptyRecords() throws Exception {
    String name = "distrlog-simpleasyncreadempty";
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setReadAheadWaitTime(10);
    confLocal.setReadAheadBatchSize(10);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    int numLogSegments = 3;
    int numRecordsPerLogSegment = 10;
    long txid = 1L;
    // write 3 log segments, 10 records per log segment
    txid = writeRecords(dlm, numLogSegments, numRecordsPerLogSegment, txid, true);
    // write another log segment with 5 records and flush every 2 records
    txid = writeLogSegment(dlm, 5, txid, 2, true);
    AsyncLogReader asyncReader = dlm.getAsyncLogReader(DLSN.InvalidDLSN);
    assertEquals("Expected stream name = " + name + " but " + asyncReader.getStreamName() + " found", name, asyncReader.getStreamName());
    long numTrans = 0;
    DLSN lastDLSN = DLSN.InvalidDLSN;
    LogRecordWithDLSN record = Utils.ioResult(asyncReader.readNext());
    while (null != record) {
        DLMTestUtil.verifyEmptyLogRecord(record);
        assertEquals(0, record.getDlsn().getSlotId());
        assertTrue(record.getDlsn().compareTo(lastDLSN) > 0);
        lastDLSN = record.getDlsn();
        numTrans++;
        if (numTrans >= (txid - 1)) {
            break;
        }
        record = Utils.ioResult(asyncReader.readNext());
    }
    assertEquals((txid - 1), numTrans);
    Utils.close(asyncReader);
    dlm.close();
}