Examples with DistributedLogManager org.apache.distributedlog.api.DistributedLogManager used on opensource projects

Example 41 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestAsyncReaderWriter method testWriteControlRecord.

/**
 * Test writing control records to writers: writers should be able to write control records, and
 * the readers should skip control records while reading.
 */
@Test(timeout = 60000)
public void testWriteControlRecord() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(testConf);
    confLocal.setOutputBufferSize(1024);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    // Write 3 log segments. For each log segments, write one control record and nine user records.
    int txid = 1;
    for (long i = 0; i < 3; i++) {
        final long currentLogSegmentSeqNo = i + 1;
        BKAsyncLogWriter writer = (BKAsyncLogWriter) (dlm.startAsyncLogSegmentNonPartitioned());
        DLSN dlsn = Utils.ioResult(writer.writeControlRecord(new LogRecord(txid++, "control".getBytes(UTF_8))));
        assertEquals(currentLogSegmentSeqNo, dlsn.getLogSegmentSequenceNo());
        assertEquals(0, dlsn.getEntryId());
        assertEquals(0, dlsn.getSlotId());
        for (long j = 1; j < 10; j++) {
            final LogRecord record = DLMTestUtil.getLargeLogRecordInstance(txid++);
            Utils.ioResult(writer.write(record));
        }
        writer.closeAndComplete();
    }
    dlm.close();
    // Read all the written data: It should skip control records and only return user records.
    DistributedLogManager readDlm = createNewDLM(confLocal, name);
    LogReader reader = readDlm.getInputStream(1);
    long numTrans = 0;
    long expectedTxId = 2;
    LogRecord record = reader.readNext(false);
    while (null != record) {
        DLMTestUtil.verifyLargeLogRecord(record);
        numTrans++;
        assertEquals(expectedTxId, record.getTransactionId());
        if (expectedTxId % 10 == 0) {
            expectedTxId += 2;
        } else {
            ++expectedTxId;
        }
        record = reader.readNext(false);
    }
    reader.close();
    assertEquals(3 * 9, numTrans);
    assertEquals(3 * 9, readDlm.getLogRecordCount());
    readDlm.close();
}

Example 42 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestAsyncReaderWriter method testReleaseLockAfterFailedToRecover.

@Test(timeout = 60000)
public void testReleaseLockAfterFailedToRecover() throws Exception {
    String name = "release-lock-after-failed-to-recover";
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(testConf);
    confLocal.setLockTimeout(0);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setOutputBufferSize(0);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    BKAsyncLogWriter writer = (BKAsyncLogWriter) (dlm.startAsyncLogSegmentNonPartitioned());
    Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(1L)));
    writer.abort();
    for (int i = 0; i < 2; i++) {
        FailpointUtils.setFailpoint(FailpointUtils.FailPointName.FP_RecoverIncompleteLogSegments, FailpointUtils.FailPointActions.FailPointAction_Throw);
        try {
            dlm.startAsyncLogSegmentNonPartitioned();
            fail("Should fail during recovering incomplete log segments");
        } catch (IOException ioe) {
        // expected;
        } finally {
            FailpointUtils.removeFailpoint(FailpointUtils.FailPointName.FP_RecoverIncompleteLogSegments);
        }
    }
    writer = (BKAsyncLogWriter) (dlm.startAsyncLogSegmentNonPartitioned());
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(1, segments.size());
    assertFalse(segments.get(0).isInProgress());
    writer.close();
    dlm.close();
}

Example 43 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestAsyncReaderWriter method testIdleReaderExceptionWhenKeepAliveIsEnabled.

@Test(timeout = 60000)
public void testIdleReaderExceptionWhenKeepAliveIsEnabled() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setPeriodicKeepAliveMilliSeconds(1000);
    confLocal.setReadLACLongPollTimeout(999);
    confLocal.setReaderIdleWarnThresholdMillis(2000);
    confLocal.setReaderIdleErrorThresholdMillis(4000);
    URI uri = createDLMURI("/" + name);
    ensureURICreated(uri);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    BKAsyncLogWriter writer = (BKAsyncLogWriter) Utils.ioResult(dlm.openAsyncLogWriter());
    writer.write(DLMTestUtil.getLogRecordInstance(1L));
    AsyncLogReader reader = Utils.ioResult(dlm.openAsyncLogReader(DLSN.InitialDLSN));
    LogRecordWithDLSN record = Utils.ioResult(reader.readNext());
    assertEquals(1L, record.getTransactionId());
    DLMTestUtil.verifyLogRecord(record);
    Utils.close(reader);
    writer.close();
    dlm.close();
}

Example 44 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestAsyncReaderWriter method testCreateLogStreamWithDifferentReplicationFactor.

@Test(timeout = 60000)
public void testCreateLogStreamWithDifferentReplicationFactor() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    ConcurrentBaseConfiguration baseConf = new ConcurrentConstConfiguration(confLocal);
    DynamicDistributedLogConfiguration dynConf = new DynamicDistributedLogConfiguration(baseConf);
    dynConf.setProperty(DistributedLogConfiguration.BKDL_BOOKKEEPER_ENSEMBLE_SIZE, DistributedLogConfiguration.BKDL_BOOKKEEPER_ENSEMBLE_SIZE_DEFAULT - 1);
    URI uri = createDLMURI("/" + name);
    ensureURICreated(uri);
    Namespace namespace = NamespaceBuilder.newBuilder().conf(confLocal).uri(uri).build();
    // use the pool
    DistributedLogManager dlm = namespace.openLog(name + "-pool");
    AsyncLogWriter writer = dlm.startAsyncLogSegmentNonPartitioned();
    Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(1L)));
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(1, segments.size());
    long ledgerId = segments.get(0).getLogSegmentId();
    LedgerHandle lh = ((BKNamespaceDriver) namespace.getNamespaceDriver()).getReaderBKC().get().openLedgerNoRecovery(ledgerId, BookKeeper.DigestType.CRC32, confLocal.getBKDigestPW().getBytes(UTF_8));
    LedgerMetadata metadata = BookKeeperAccessor.getLedgerMetadata(lh);
    assertEquals(DistributedLogConfiguration.BKDL_BOOKKEEPER_ENSEMBLE_SIZE_DEFAULT, metadata.getEnsembleSize());
    lh.close();
    Utils.close(writer);
    dlm.close();
    // use customized configuration
    dlm = namespace.openLog(name + "-custom", java.util.Optional.empty(), java.util.Optional.of(dynConf), java.util.Optional.empty());
    writer = dlm.startAsyncLogSegmentNonPartitioned();
    Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(1L)));
    segments = dlm.getLogSegments();
    assertEquals(1, segments.size());
    ledgerId = segments.get(0).getLogSegmentId();
    lh = ((BKNamespaceDriver) namespace.getNamespaceDriver()).getReaderBKC().get().openLedgerNoRecovery(ledgerId, BookKeeper.DigestType.CRC32, confLocal.getBKDigestPW().getBytes(UTF_8));
    metadata = BookKeeperAccessor.getLedgerMetadata(lh);
    assertEquals(DistributedLogConfiguration.BKDL_BOOKKEEPER_ENSEMBLE_SIZE_DEFAULT - 1, metadata.getEnsembleSize());
    lh.close();
    Utils.close(writer);
    dlm.close();
    namespace.close();
}

Example 45 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestAsyncReaderWriter method testMarkEndOfStreamAtBeginningOfSegment.

@Test(timeout = 60000)
public void testMarkEndOfStreamAtBeginningOfSegment() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    BKAsyncLogWriter writer = (BKAsyncLogWriter) dlm.startAsyncLogSegmentNonPartitioned();
    Utils.ioResult(writer.markEndOfStream());
    try {
        Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(1)));
        fail("Should have thrown");
    } catch (EndOfStreamException ex) {
    }
    writer.close();
    BKAsyncLogReader reader = (BKAsyncLogReader) dlm.getAsyncLogReader(DLSN.InitialDLSN);
    try {
        LogRecord record = Utils.ioResult(reader.readNext());
        fail("Should have thrown");
    } catch (EndOfStreamException ex) {
    }
    Utils.close(reader);
}