Examples with AsyncLogReader org.apache.distributedlog.api.AsyncLogReader used on opensource projects

Example 6 with AsyncLogReader

use of org.apache.distributedlog.api.AsyncLogReader in project bookkeeper by apache.

the class TestAsyncReaderLock method testReaderLockSharedDlmDoesNotConflict.

@Test(timeout = 60000)
public void testReaderLockSharedDlmDoesNotConflict() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogManager dlm0 = createNewDLM(conf, name);
    BKAsyncLogWriter writer = (BKAsyncLogWriter) (dlm0.startAsyncLogSegmentNonPartitioned());
    writer.write(DLMTestUtil.getLogRecordInstance(1L));
    writer.write(DLMTestUtil.getLogRecordInstance(2L));
    writer.closeAndComplete();
    DistributedLogManager dlm1 = createNewDLM(conf, name);
    CompletableFuture<AsyncLogReader> futureReader1 = dlm1.getAsyncLogReaderWithLock(DLSN.InitialDLSN);
    CompletableFuture<AsyncLogReader> futureReader2 = dlm1.getAsyncLogReaderWithLock(DLSN.InitialDLSN);
    // Both use the same client id, so there's no lock conflict. Not necessarily ideal, but how the
    // system currently works.
    Utils.ioResult(futureReader1);
    Utils.ioResult(futureReader2);
    dlm0.close();
    dlm1.close();
}

Example 7 with AsyncLogReader

use of org.apache.distributedlog.api.AsyncLogReader in project bookkeeper by apache.

the class TestBKDistributedLogManager method testContinuousReadersWithEmptyLedgers.

@Test(timeout = 60000)
public void testContinuousReadersWithEmptyLedgers() throws Exception {
    String name = "distrlog-continuous-emptyledgers";
    DistributedLogManager dlm = createNewDLM(conf, name);
    long txid = 1;
    for (long i = 0; i < 3; i++) {
        long start = txid;
        BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
        for (long j = 1; j <= DEFAULT_SEGMENT_SIZE; j++) {
            LogRecord op = DLMTestUtil.getLogRecordInstance(txid++);
            out.write(op);
        }
        BKLogSegmentWriter writer = out.getCachedLogWriter();
        out.closeAndComplete();
        BKLogWriteHandler blplm = ((BKDistributedLogManager) (dlm)).createWriteHandler(true);
        assertNotNull(zkc.exists(blplm.completedLedgerZNode(start, txid - 1, writer.getLogSegmentSequenceNumber()), false));
        BKLogSegmentWriter perStreamLogWriter = blplm.startLogSegment(txid - 1);
        blplm.completeAndCloseLogSegment(perStreamLogWriter.getLogSegmentSequenceNumber(), perStreamLogWriter.getLogSegmentId(), txid - 1, txid - 1, 0);
        assertNotNull(zkc.exists(blplm.completedLedgerZNode(txid - 1, txid - 1, perStreamLogWriter.getLogSegmentSequenceNumber()), false));
        Utils.ioResult(blplm.asyncClose());
    }
    BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
    for (long j = 1; j <= DEFAULT_SEGMENT_SIZE / 2; j++) {
        LogRecord op = DLMTestUtil.getLogRecordInstance(txid++);
        out.write(op);
    }
    out.flush();
    out.commit();
    out.close();
    dlm.close();
    dlm = createNewDLM(conf, name);
    AsyncLogReader asyncreader = dlm.getAsyncLogReader(DLSN.InvalidDLSN);
    long numTrans = 0;
    LogRecordWithDLSN record = Utils.ioResult(asyncreader.readNext());
    while (null != record) {
        DLMTestUtil.verifyLogRecord(record);
        numTrans++;
        if (numTrans >= (txid - 1)) {
            break;
        }
        record = Utils.ioResult(asyncreader.readNext());
    }
    assertEquals((txid - 1), numTrans);
    Utils.close(asyncreader);
    LogReader reader = dlm.getInputStream(1);
    numTrans = 0;
    record = reader.readNext(false);
    while (null != record) {
        DLMTestUtil.verifyLogRecord(record);
        numTrans++;
        record = reader.readNext(false);
    }
    assertEquals((txid - 1), numTrans);
    reader.close();
    assertEquals(txid - 1, dlm.getLogRecordCount());
    dlm.close();
}

Example 8 with AsyncLogReader

use of org.apache.distributedlog.api.AsyncLogReader in project bookkeeper by apache.

the class TestAsyncReaderWriter method testReadBrokenEntriesAndLargeBatchSizeCrossSegment.

@Test(timeout = 60000)
public void testReadBrokenEntriesAndLargeBatchSizeCrossSegment() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setReadAheadWaitTime(10);
    confLocal.setReadAheadBatchSize(8);
    confLocal.setPositionGapDetectionEnabled(false);
    confLocal.setReadAheadSkipBrokenEntries(true);
    confLocal.setEIInjectReadAheadBrokenEntries(true);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    int numLogSegments = 3;
    int numRecordsPerLogSegment = 5;
    long txid = 1L;
    txid = writeRecords(dlm, numLogSegments, numRecordsPerLogSegment, txid, false);
    AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InvalidDLSN);
    // And so on for the next segment, so 4 records in each segment, for 12 good records
    for (int i = 0; i < 12; i++) {
        LogRecordWithDLSN record = Utils.ioResult(reader.readNext());
        assertFalse(record.getDlsn().getEntryId() % 10 == 0);
    }
    Utils.close(reader);
    dlm.close();
}

Example 9 with AsyncLogReader

use of org.apache.distributedlog.api.AsyncLogReader in project bookkeeper by apache.

the class TestAsyncReaderWriter method testReadBrokenEntriesWithGapDetection.

@Test(timeout = 60000)
public void testReadBrokenEntriesWithGapDetection() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setReadAheadWaitTime(10);
    confLocal.setReadAheadBatchSize(1);
    confLocal.setPositionGapDetectionEnabled(true);
    confLocal.setReadAheadSkipBrokenEntries(true);
    confLocal.setEIInjectReadAheadBrokenEntries(true);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    int numLogSegments = 1;
    int numRecordsPerLogSegment = 100;
    long txid = 1L;
    txid = writeRecords(dlm, numLogSegments, numRecordsPerLogSegment, txid, false);
    AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InvalidDLSN);
    try {
        // record in each ledger is discarded, for 30 - 3 = 27 records.
        for (int i = 0; i < 30; i++) {
            LogRecordWithDLSN record = Utils.ioResult(reader.readNext());
            assertFalse(record.getDlsn().getEntryId() % 10 == 0);
        }
        fail("should have thrown");
    } catch (DLIllegalStateException e) {
    }
    Utils.close(reader);
    dlm.close();
}

Example 10 with AsyncLogReader

use of org.apache.distributedlog.api.AsyncLogReader in project bookkeeper by apache.

the class TestAsyncReaderWriter method testBulkAsyncReadWithWriteBatch.

@Test(timeout = 60000)
public void testBulkAsyncReadWithWriteBatch() throws Exception {
    String name = "distrlog-bulkasyncread-with-writebatch";
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setOutputBufferSize(1024000);
    confLocal.setReadAheadWaitTime(10);
    confLocal.setReadAheadMaxRecords(10000);
    confLocal.setReadAheadBatchSize(10);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    int numLogSegments = 3;
    int numRecordsPerLogSegment = 20;
    writeRecords(dlm, numLogSegments, numRecordsPerLogSegment, 1L, false);
    final AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InitialDLSN);
    int expectedTxID = 1;
    for (long i = 0; i < 3; i++) {
        // since we batched 20 entries into single bookkeeper entry
        // we should be able to read 20 entries as a batch.
        List<LogRecordWithDLSN> records = Utils.ioResult(reader.readBulk(20));
        assertEquals(20, records.size());
        for (LogRecordWithDLSN record : records) {
            assertEquals(expectedTxID, record.getTransactionId());
            ++expectedTxID;
        }
    }
    Utils.close(reader);
    dlm.close();
}