Examples with AsyncLogWriter org.apache.distributedlog.api.AsyncLogWriter used on opensource projects

Example 11 with AsyncLogWriter

use of org.apache.distributedlog.api.AsyncLogWriter in project bookkeeper by apache.

the class TestBKLogSegmentEntryReader method testReadEntriesFromInprogressSegment.

@Test(timeout = 60000)
public void testReadEntriesFromInprogressSegment() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setNumPrefetchEntriesPerLogSegment(20);
    confLocal.setMaxPrefetchEntriesPerLogSegment(20);
    DistributedLogManager dlm = createNewDLM(confLocal, runtime.getMethodName());
    AsyncLogWriter writer = createInprogressLogSegment(dlm, confLocal, 5);
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(segments.size() + " log segments found, expected to be only one", 1, segments.size());
    BKLogSegmentEntryReader reader = createEntryReader(segments.get(0), 0, confLocal);
    reader.start();
    long expectedLastAddConfirmed = 8L;
    // wait until sending out all prefetch requests
    while (reader.readAheadEntries.size() < expectedLastAddConfirmed + 2) {
        TimeUnit.MILLISECONDS.sleep(10);
    }
    assertEquals(expectedLastAddConfirmed + 2, reader.getNextEntryId());
    long txId = 1L;
    long entryId = 0L;
    while (true) {
        Entry.Reader entryReader = Utils.ioResult(reader.readNext(1)).get(0);
        LogRecordWithDLSN record = entryReader.nextRecord();
        while (null != record) {
            if (!record.isControl()) {
                DLMTestUtil.verifyLogRecord(record);
                assertEquals(txId, record.getTransactionId());
                ++txId;
            }
            DLSN dlsn = record.getDlsn();
            assertEquals(1L, dlsn.getLogSegmentSequenceNo());
            assertEquals(entryId, dlsn.getEntryId());
            record = entryReader.nextRecord();
        }
        ++entryId;
        if (entryId == expectedLastAddConfirmed + 1) {
            break;
        }
    }
    assertEquals(6L, txId);
    CompletableFuture<List<Entry.Reader>> nextReadFuture = reader.readNext(1);
    // write another record to commit previous writes
    Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(txId)));
    // the long poll will be satisfied
    List<Entry.Reader> nextReadEntries = Utils.ioResult(nextReadFuture);
    assertEquals(1, nextReadEntries.size());
    assertTrue(reader.hasCaughtUpOnInprogress());
    Entry.Reader entryReader = nextReadEntries.get(0);
    LogRecordWithDLSN record = entryReader.nextRecord();
    assertNotNull(record);
    assertTrue(record.isControl());
    assertNull(entryReader.nextRecord());
    // once the read is advanced, we will prefetch next record
    while (reader.getNextEntryId() <= entryId) {
        TimeUnit.MILLISECONDS.sleep(10);
    }
    assertEquals(entryId + 2, reader.getNextEntryId());
    assertEquals(1, reader.readAheadEntries.size());
    Utils.close(reader);
    Utils.close(writer);
}

Example 12 with AsyncLogWriter

use of org.apache.distributedlog.api.AsyncLogWriter in project bookkeeper by apache.

the class TestReadAheadEntryReader method generateCompletedLogSegments.

void generateCompletedLogSegments(DistributedLogManager dlm, long numCompletedSegments, long segmentSize, long startTxId) throws Exception {
    long txid = startTxId;
    for (long i = 0; i < numCompletedSegments; i++) {
        AsyncLogWriter writer = Utils.ioResult(dlm.openAsyncLogWriter());
        for (long j = 1; j <= segmentSize; j++) {
            Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(txid++)));
            LogRecord ctrlRecord = DLMTestUtil.getLogRecordInstance(txid);
            ctrlRecord.setControl();
            Utils.ioResult(writer.write(ctrlRecord));
        }
        Utils.close(writer);
    }
}

Example 13 with AsyncLogWriter

use of org.apache.distributedlog.api.AsyncLogWriter in project bookkeeper by apache.

the class TestReadAheadEntryReader method testPositioningIgnoreTruncationStatus.

@Test(timeout = 60000)
public void testPositioningIgnoreTruncationStatus() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(baseConf);
    confLocal.setIgnoreTruncationStatus(true);
    String streamName = runtime.getMethodName();
    BKDistributedLogManager dlm = createNewDLM(confLocal, streamName);
    // generate list of log segments
    generateCompletedLogSegments(dlm, 3, 2);
    AsyncLogWriter writer = Utils.ioResult(dlm.openAsyncLogWriter());
    Utils.ioResult(writer.truncate(new DLSN(2L, 1L, 0L)));
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    // positioning on a truncated log segment (segment 1)
    ReadAheadEntryReader readAheadEntryReader = createEntryReader(streamName, DLSN.InitialDLSN, dlm, confLocal);
    readAheadEntryReader.start(segments);
    // ensure initialization to complete
    ensureOrderSchedulerEmpty(streamName);
    expectNoException(readAheadEntryReader);
    Utils.close(readAheadEntryReader);
    // positioning on a partially truncated log segment (segment 2) before min active dlsn
    readAheadEntryReader = createEntryReader(streamName, new DLSN(2L, 0L, 0L), dlm, confLocal);
    readAheadEntryReader.start(segments);
    // ensure initialization to complete
    ensureOrderSchedulerEmpty(streamName);
    expectNoException(readAheadEntryReader);
    Utils.close(readAheadEntryReader);
    // positioning on a partially truncated log segment (segment 2) after min active dlsn
    readAheadEntryReader = createEntryReader(streamName, new DLSN(2L, 1L, 0L), dlm, confLocal);
    readAheadEntryReader.start(segments);
    // ensure initialization to complete
    ensureOrderSchedulerEmpty(streamName);
    expectNoException(readAheadEntryReader);
    Utils.close(readAheadEntryReader);
    Utils.close(writer);
    dlm.close();
}

Example 14 with AsyncLogWriter

use of org.apache.distributedlog.api.AsyncLogWriter in project bookkeeper by apache.

the class TestReadAheadEntryReader method testPositioningAtInvalidLogSegment.

@Test(timeout = 60000)
public void testPositioningAtInvalidLogSegment() throws Exception {
    String streamName = runtime.getMethodName();
    BKDistributedLogManager dlm = createNewDLM(baseConf, streamName);
    // generate list of log segments
    generateCompletedLogSegments(dlm, 3, 3);
    AsyncLogWriter writer = Utils.ioResult(dlm.openAsyncLogWriter());
    Utils.ioResult(writer.truncate(new DLSN(2L, 1L, 0L)));
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    // positioning on a truncated log segment (segment 1)
    ReadAheadEntryReader readAheadEntryReader = createEntryReader(streamName, DLSN.InitialDLSN, dlm, baseConf);
    readAheadEntryReader.start(segments);
    // ensure initialization to complete
    ensureOrderSchedulerEmpty(streamName);
    expectNoException(readAheadEntryReader);
    Entry.Reader entryReader = readAheadEntryReader.getNextReadAheadEntry(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
    assertEquals(2L, entryReader.getLSSN());
    assertEquals(1L, entryReader.getEntryId());
    entryReader.release();
    Utils.close(readAheadEntryReader);
    // positioning on a partially truncated log segment (segment 2) before min active dlsn
    readAheadEntryReader = createEntryReader(streamName, new DLSN(2L, 0L, 0L), dlm, baseConf);
    readAheadEntryReader.start(segments);
    // ensure initialization to complete
    ensureOrderSchedulerEmpty(streamName);
    expectNoException(readAheadEntryReader);
    entryReader = readAheadEntryReader.getNextReadAheadEntry(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
    assertEquals(2L, entryReader.getLSSN());
    assertEquals(1L, entryReader.getEntryId());
    entryReader.release();
    Utils.close(readAheadEntryReader);
    // positioning on a partially truncated log segment (segment 2) after min active dlsn
    readAheadEntryReader = createEntryReader(streamName, new DLSN(2L, 2L, 0L), dlm, baseConf);
    readAheadEntryReader.start(segments);
    // ensure initialization to complete
    ensureOrderSchedulerEmpty(streamName);
    expectNoException(readAheadEntryReader);
    entryReader = readAheadEntryReader.getNextReadAheadEntry(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
    assertEquals(2L, entryReader.getLSSN());
    assertEquals(2L, entryReader.getEntryId());
    entryReader.release();
    Utils.close(readAheadEntryReader);
    Utils.close(writer);
    dlm.close();
}

Example 15 with AsyncLogWriter

use of org.apache.distributedlog.api.AsyncLogWriter in project bookkeeper by apache.

the class TestReadAheadEntryReader method createInprogressLogSegment.

AsyncLogWriter createInprogressLogSegment(DistributedLogManager dlm, DistributedLogConfiguration conf, long segmentSize) throws Exception {
    AsyncLogWriter writer = Utils.ioResult(dlm.openAsyncLogWriter());
    for (long i = 1L; i <= segmentSize; i++) {
        Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(i)));
        LogRecord ctrlRecord = DLMTestUtil.getLogRecordInstance(i);
        ctrlRecord.setControl();
        Utils.ioResult(writer.write(ctrlRecord));
    }
    return writer;
}