Examples with DistributedLogManager org.apache.distributedlog.api.DistributedLogManager used on opensource projects

Example 36 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestAsyncBulkWrite method testSimpleAsyncBulkWriteSpanningEntryAndLedger.

/**
 * Test Case: A large write batch will span records into multiple entries and ledgers.
 * @throws Exception
 */
@Test(timeout = 60000)
public void testSimpleAsyncBulkWriteSpanningEntryAndLedger() throws Exception {
    String name = "distrlog-testSimpleAsyncBulkWriteSpanningEntryAndLedger";
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(testConf);
    confLocal.setOutputBufferSize(1024);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    BKAsyncLogWriter writer = (BKAsyncLogWriter) (dlm.startAsyncLogSegmentNonPartitioned());
    int batchSize = 100;
    int recSize = 1024;
    // First entry.
    long ledgerIndex = 1;
    long entryIndex = 0;
    long slotIndex = 0;
    long txIndex = 1;
    checkAllSucceeded(writer, batchSize, recSize, ledgerIndex, entryIndex, slotIndex, txIndex);
    // New entry.
    entryIndex++;
    slotIndex = 0;
    txIndex += batchSize;
    checkAllSucceeded(writer, batchSize, recSize, ledgerIndex, entryIndex, slotIndex, txIndex);
    // Roll ledger.
    ledgerIndex++;
    entryIndex = 0;
    slotIndex = 0;
    txIndex += batchSize;
    writer.closeAndComplete();
    writer = (BKAsyncLogWriter) (dlm.startAsyncLogSegmentNonPartitioned());
    checkAllSucceeded(writer, batchSize, recSize, ledgerIndex, entryIndex, slotIndex, txIndex);
    writer.closeAndComplete();
    dlm.close();
}

Example 37 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestAsyncReaderWriter method testReadBrokenEntriesAndLargeBatchSizeCrossSegment.

@Test(timeout = 60000)
public void testReadBrokenEntriesAndLargeBatchSizeCrossSegment() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setReadAheadWaitTime(10);
    confLocal.setReadAheadBatchSize(8);
    confLocal.setPositionGapDetectionEnabled(false);
    confLocal.setReadAheadSkipBrokenEntries(true);
    confLocal.setEIInjectReadAheadBrokenEntries(true);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    int numLogSegments = 3;
    int numRecordsPerLogSegment = 5;
    long txid = 1L;
    txid = writeRecords(dlm, numLogSegments, numRecordsPerLogSegment, txid, false);
    AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InvalidDLSN);
    // And so on for the next segment, so 4 records in each segment, for 12 good records
    for (int i = 0; i < 12; i++) {
        LogRecordWithDLSN record = Utils.ioResult(reader.readNext());
        assertFalse(record.getDlsn().getEntryId() % 10 == 0);
    }
    Utils.close(reader);
    dlm.close();
}

Example 38 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestAsyncReaderWriter method testReadBrokenEntriesWithGapDetection.

@Test(timeout = 60000)
public void testReadBrokenEntriesWithGapDetection() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setReadAheadWaitTime(10);
    confLocal.setReadAheadBatchSize(1);
    confLocal.setPositionGapDetectionEnabled(true);
    confLocal.setReadAheadSkipBrokenEntries(true);
    confLocal.setEIInjectReadAheadBrokenEntries(true);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    int numLogSegments = 1;
    int numRecordsPerLogSegment = 100;
    long txid = 1L;
    txid = writeRecords(dlm, numLogSegments, numRecordsPerLogSegment, txid, false);
    AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InvalidDLSN);
    try {
        // record in each ledger is discarded, for 30 - 3 = 27 records.
        for (int i = 0; i < 30; i++) {
            LogRecordWithDLSN record = Utils.ioResult(reader.readNext());
            assertFalse(record.getDlsn().getEntryId() % 10 == 0);
        }
        fail("should have thrown");
    } catch (DLIllegalStateException e) {
    }
    Utils.close(reader);
    dlm.close();
}

Example 39 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestAsyncReaderWriter method testBulkAsyncReadWithWriteBatch.

@Test(timeout = 60000)
public void testBulkAsyncReadWithWriteBatch() throws Exception {
    String name = "distrlog-bulkasyncread-with-writebatch";
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setOutputBufferSize(1024000);
    confLocal.setReadAheadWaitTime(10);
    confLocal.setReadAheadMaxRecords(10000);
    confLocal.setReadAheadBatchSize(10);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    int numLogSegments = 3;
    int numRecordsPerLogSegment = 20;
    writeRecords(dlm, numLogSegments, numRecordsPerLogSegment, 1L, false);
    final AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InitialDLSN);
    int expectedTxID = 1;
    for (long i = 0; i < 3; i++) {
        // since we batched 20 entries into single bookkeeper entry
        // we should be able to read 20 entries as a batch.
        List<LogRecordWithDLSN> records = Utils.ioResult(reader.readBulk(20));
        assertEquals(20, records.size());
        for (LogRecordWithDLSN record : records) {
            assertEquals(expectedTxID, record.getTransactionId());
            ++expectedTxID;
        }
    }
    Utils.close(reader);
    dlm.close();
}

Example 40 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestAsyncReaderWriter method testMaxReadAheadRecords.

@Test(timeout = 60000)
public void testMaxReadAheadRecords() throws Exception {
    int maxRecords = 1;
    int batchSize = 8;
    int maxAllowedCachedRecords = maxRecords + batchSize - 1;
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(Integer.MAX_VALUE);
    confLocal.setReadAheadMaxRecords(maxRecords);
    confLocal.setReadAheadBatchSize(batchSize);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    AsyncLogWriter writer = dlm.startAsyncLogSegmentNonPartitioned();
    int numRecords = 40;
    for (int i = 1; i <= numRecords; i++) {
        Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(i)));
        assertEquals("last tx id should become " + i, i, writer.getLastTxId());
    }
    LogRecord record = DLMTestUtil.getLogRecordInstance(numRecords);
    record.setControl();
    Utils.ioResult(writer.write(record));
    BKAsyncLogReader reader = (BKAsyncLogReader) dlm.getAsyncLogReader(DLSN.InitialDLSN);
    record = Utils.ioResult(reader.readNext());
    LOG.info("Read record {}", record);
    assertEquals(1L, record.getTransactionId());
    assertNotNull(reader.getReadAheadReader());
    assertTrue(reader.getReadAheadReader().getNumCachedEntries() <= maxAllowedCachedRecords);
    for (int i = 2; i <= numRecords; i++) {
        record = Utils.ioResult(reader.readNext());
        LOG.info("Read record {}", record);
        assertEquals((long) i, record.getTransactionId());
        TimeUnit.MILLISECONDS.sleep(20);
        int numCachedEntries = reader.getReadAheadReader().getNumCachedEntries();
        assertTrue("Should cache less than " + batchSize + " records but already found " + numCachedEntries + " records when reading " + i + "th record", numCachedEntries <= maxAllowedCachedRecords);
    }
    Utils.close(reader);
}