Examples with DistributedLogManager org.apache.distributedlog.api.DistributedLogManager used on opensource projects

Example 21 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestBKLogReadHandler method testGetFirstDLSNAfterCleanTruncation.

@Test(timeout = 60000)
public void testGetFirstDLSNAfterCleanTruncation() throws Exception {
    String dlName = runtime.getMethodName();
    prepareLogSegmentsNonPartitioned(dlName, 3, 10);
    DistributedLogManager dlm = createNewDLM(conf, dlName);
    BKLogReadHandler readHandler = ((BKDistributedLogManager) dlm).createReadHandler();
    AsyncLogWriter writer = dlm.startAsyncLogSegmentNonPartitioned();
    CompletableFuture<Boolean> futureSuccess = writer.truncate(new DLSN(2, 0, 0));
    Boolean success = Utils.ioResult(futureSuccess);
    assertTrue(success);
    CompletableFuture<LogRecordWithDLSN> futureRecord = readHandler.asyncGetFirstLogRecord();
    LogRecordWithDLSN record = Utils.ioResult(futureRecord);
    assertEquals(new DLSN(2, 0, 0), record.getDlsn());
}

Example 22 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestBKLogReadHandler method testGetLogRecordCountLastRecord.

@Test(timeout = 60000)
public void testGetLogRecordCountLastRecord() throws Exception {
    String dlName = runtime.getMethodName();
    prepareLogSegmentsNonPartitioned(dlName, 11, 3);
    DistributedLogManager dlm = createNewDLM(conf, dlName);
    BKLogReadHandler readHandler = ((BKDistributedLogManager) dlm).createReadHandler();
    CompletableFuture<Long> count = null;
    count = readHandler.asyncGetLogRecordCount(new DLSN(11, 2, 0));
    assertEquals(1, Utils.ioResult(count).longValue());
}

Example 23 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestBKLogReadHandler method testGetFirstDLSNWithOpenLedger.

@Test(timeout = 60000)
public void testGetFirstDLSNWithOpenLedger() throws Exception {
    String dlName = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setOutputBufferSize(0);
    int numEntriesPerSegment = 10;
    DistributedLogManager dlm1 = createNewDLM(confLocal, dlName);
    long txid = 1;
    ArrayList<CompletableFuture<DLSN>> futures = new ArrayList<CompletableFuture<DLSN>>(numEntriesPerSegment);
    AsyncLogWriter out = dlm1.startAsyncLogSegmentNonPartitioned();
    for (int eid = 0; eid < numEntriesPerSegment; ++eid) {
        futures.add(out.write(DLMTestUtil.getLogRecordInstance(txid)));
        ++txid;
    }
    Utils.ioResult(FutureUtils.collect(futures));
    // commit
    LogRecord controlRecord = new LogRecord(txid, DistributedLogConstants.CONTROL_RECORD_CONTENT);
    controlRecord.setControl();
    Utils.ioResult(out.write(controlRecord));
    DLSN last = dlm1.getLastDLSN();
    assertEquals(new DLSN(1, 9, 0), last);
    DLSN first = Utils.ioResult(dlm1.getFirstDLSNAsync());
    assertEquals(new DLSN(1, 0, 0), first);
    Utils.close(out);
}

Example 24 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestRollLogSegments method testUnableToRollLogSegments.

@Test(timeout = 60000)
public void testUnableToRollLogSegments() throws Exception {
    String name = "distrlog-unable-to-roll-log-segments";
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setOutputBufferSize(0);
    confLocal.setLogSegmentRollingIntervalMinutes(0);
    confLocal.setMaxLogSegmentBytes(1);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    BKAsyncLogWriter writer = (BKAsyncLogWriter) dlm.startAsyncLogSegmentNonPartitioned();
    long txId = 1L;
    // Create Log Segments
    Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(txId)));
    FailpointUtils.setFailpoint(FailpointUtils.FailPointName.FP_StartLogSegmentBeforeLedgerCreate, FailpointUtils.FailPointActions.FailPointAction_Throw);
    try {
        // If we couldn't open new log segment, we should keep using the old one
        final int numRecords = 10;
        final CountDownLatch latch = new CountDownLatch(numRecords);
        for (int i = 0; i < numRecords; i++) {
            writer.write(DLMTestUtil.getLogRecordInstance(++txId)).whenComplete(new FutureEventListener<DLSN>() {

                @Override
                public void onSuccess(DLSN value) {
                    logger.info("Completed entry : {}.", value);
                    latch.countDown();
                }

                @Override
                public void onFailure(Throwable cause) {
                    logger.error("Failed to write entries : ", cause);
                }
            });
        }
        latch.await();
        writer.close();
        List<LogSegmentMetadata> segments = dlm.getLogSegments();
        logger.info("LogSegments: {}", segments);
        assertEquals(1, segments.size());
        long expectedTxID = 1L;
        LogReader reader = dlm.getInputStream(DLSN.InitialDLSN);
        LogRecordWithDLSN record = reader.readNext(false);
        while (null != record) {
            DLMTestUtil.verifyLogRecord(record);
            assertEquals(expectedTxID++, record.getTransactionId());
            assertEquals(record.getTransactionId() - 1, record.getSequenceId());
            record = reader.readNext(false);
        }
        assertEquals(12L, expectedTxID);
        reader.close();
        dlm.close();
    } finally {
        FailpointUtils.removeFailpoint(FailpointUtils.FailPointName.FP_StartLogSegmentBeforeLedgerCreate);
    }
}

Example 25 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestRollLogSegments method testCaughtUpReaderOnLogSegmentRolling.

@FlakyTest
@Test(timeout = 60000)
@SuppressWarnings("deprecation")
public void testCaughtUpReaderOnLogSegmentRolling() throws Exception {
    String name = "distrlog-caughtup-reader-on-logsegment-rolling";
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setOutputBufferSize(4 * 1024 * 1024);
    confLocal.setTraceReadAheadMetadataChanges(true);
    confLocal.setEnsembleSize(1);
    confLocal.setWriteQuorumSize(1);
    confLocal.setAckQuorumSize(1);
    confLocal.setReadLACLongPollTimeout(99999999);
    confLocal.setReaderIdleWarnThresholdMillis(2 * 99999999 + 1);
    confLocal.setBKClientReadTimeout(99999999 + 1);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    BKSyncLogWriter writer = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
    // 1) writer added 5 entries.
    final int numEntries = 5;
    for (int i = 1; i <= numEntries; i++) {
        writer.write(DLMTestUtil.getLogRecordInstance(i));
        writer.flush();
        writer.commit();
    }
    BKDistributedLogManager readDLM = (BKDistributedLogManager) createNewDLM(confLocal, name);
    final BKAsyncLogReader reader = (BKAsyncLogReader) readDLM.getAsyncLogReader(DLSN.InitialDLSN);
    // 2) reader should be able to read 5 entries.
    for (long i = 1; i <= numEntries; i++) {
        LogRecordWithDLSN record = Utils.ioResult(reader.readNext());
        DLMTestUtil.verifyLogRecord(record);
        assertEquals(i, record.getTransactionId());
        assertEquals(record.getTransactionId() - 1, record.getSequenceId());
    }
    BKLogSegmentWriter perStreamWriter = writer.segmentWriter;
    BookKeeperClient bkc = DLMTestUtil.getBookKeeperClient(readDLM);
    LedgerHandle readLh = bkc.get().openLedgerNoRecovery(getLedgerHandle(perStreamWriter).getId(), BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes(UTF_8));
    // Writer moved to lac = 9, while reader knows lac = 8 and moving to wait on 9
    checkAndWaitWriterReaderPosition(perStreamWriter, 9, reader, 9, readLh, 8);
    // write 6th record
    writer.write(DLMTestUtil.getLogRecordInstance(numEntries + 1));
    writer.flush();
    // Writer moved to lac = 10, while reader knows lac = 9 and moving to wait on 10
    checkAndWaitWriterReaderPosition(perStreamWriter, 10, reader, 10, readLh, 9);
    // write records without commit to simulate similar failure cases
    writer.write(DLMTestUtil.getLogRecordInstance(numEntries + 2));
    writer.flush();
    // Writer moved to lac = 11, while reader knows lac = 10 and moving to wait on 11
    checkAndWaitWriterReaderPosition(perStreamWriter, 11, reader, 11, readLh, 10);
    while (true) {
        BKLogSegmentEntryReader entryReader = (BKLogSegmentEntryReader) reader.getReadAheadReader().getCurrentSegmentReader().getEntryReader();
        if (null != entryReader && null != entryReader.getOutstandingLongPoll()) {
            break;
        }
        Thread.sleep(1000);
    }
    logger.info("Waiting for long poll getting interrupted with metadata changed");
    // simulate a recovery without closing ledger causing recording wrong last dlsn
    BKLogWriteHandler writeHandler = writer.getCachedWriteHandler();
    writeHandler.completeAndCloseLogSegment(writeHandler.inprogressZNodeName(perStreamWriter.getLogSegmentId(), perStreamWriter.getStartTxId(), perStreamWriter.getLogSegmentSequenceNumber()), perStreamWriter.getLogSegmentSequenceNumber(), perStreamWriter.getLogSegmentId(), perStreamWriter.getStartTxId(), perStreamWriter.getLastTxId(), perStreamWriter.getPositionWithinLogSegment() - 1, 9, 0);
    BKSyncLogWriter anotherWriter = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
    anotherWriter.write(DLMTestUtil.getLogRecordInstance(numEntries + 3));
    anotherWriter.flush();
    anotherWriter.commit();
    anotherWriter.closeAndComplete();
    for (long i = numEntries + 1; i <= numEntries + 3; i++) {
        LogRecordWithDLSN record = Utils.ioResult(reader.readNext());
        DLMTestUtil.verifyLogRecord(record);
        assertEquals(i, record.getTransactionId());
    }
    Utils.close(reader);
    readDLM.close();
}