Examples with DistributedLogManager org.apache.distributedlog.api.DistributedLogManager used on opensource projects

Example 11 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestBKDistributedLogManager method testMarkEndOfStream.

@Test(timeout = 60000)
public void testMarkEndOfStream() throws Exception {
    String name = "distrlog-mark-end-of-stream";
    DistributedLogManager dlm = createNewDLM(conf, name);
    long txid = 1;
    txid = writeAndMarkEndOfStream(dlm, txid);
    LogReader reader = dlm.getInputStream(1);
    long numTrans = 0;
    boolean exceptionEncountered = false;
    LogRecord record = null;
    try {
        record = reader.readNext(false);
        long expectedTxId = 1;
        while (null != record) {
            DLMTestUtil.verifyLogRecord(record);
            assertEquals(expectedTxId, record.getTransactionId());
            expectedTxId++;
            numTrans++;
            record = reader.readNext(false);
        }
    } catch (EndOfStreamException exc) {
        LOG.info("Encountered EndOfStream on reading records after {}", record);
        exceptionEncountered = true;
    }
    assertEquals((txid - 1), numTrans);
    assertTrue(exceptionEncountered);
    exceptionEncountered = false;
    try {
        reader.readNext(false);
    } catch (EndOfStreamException exc) {
        exceptionEncountered = true;
    }
    assertTrue(exceptionEncountered);
    reader.close();
}

Example 12 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestBKDistributedLogManager method testContinuousReaderBulk.

@Test(timeout = 60000)
public void testContinuousReaderBulk() throws Exception {
    String name = "distrlog-continuous-bulk";
    DistributedLogManager dlm = createNewDLM(conf, name);
    long txid = 1;
    for (long i = 0; i < 3; i++) {
        BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
        for (long j = 1; j <= DEFAULT_SEGMENT_SIZE; j++) {
            LogRecord op = DLMTestUtil.getLogRecordInstance(txid++);
            out.write(op);
        }
        out.closeAndComplete();
    }
    BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
    for (long j = 1; j <= DEFAULT_SEGMENT_SIZE / 2; j++) {
        LogRecord op = DLMTestUtil.getLogRecordInstance(txid++);
        out.write(op);
    }
    out.flush();
    out.commit();
    out.close();
    dlm.close();
    dlm = createNewDLM(conf, name);
    LogReader reader = dlm.getInputStream(1);
    long numTrans = 0;
    List<LogRecordWithDLSN> recordList = reader.readBulk(false, 13);
    long lastTxId = -1;
    while (!recordList.isEmpty()) {
        for (LogRecord record : recordList) {
            assert (lastTxId < record.getTransactionId());
            lastTxId = record.getTransactionId();
            DLMTestUtil.verifyLogRecord(record);
            numTrans++;
        }
        recordList = reader.readBulk(false, 13);
    }
    reader.close();
    assertEquals((txid - 1), numTrans);
}

Example 13 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestBKDistributedLogManager method markEndOfStreamOnEmptyLogSegment.

private void markEndOfStreamOnEmptyLogSegment(int numCompletedSegments) throws Exception {
    String name = "distrlog-mark-end-empty-" + numCompletedSegments;
    DistributedLogManager dlm = createNewDLM(conf, name);
    DLMTestUtil.generateCompletedLogSegments(dlm, conf, numCompletedSegments, DEFAULT_SEGMENT_SIZE);
    BKSyncLogWriter writer = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
    writer.markEndOfStream();
    LogReader reader = dlm.getInputStream(1);
    long numTrans = 0;
    boolean exceptionEncountered = false;
    try {
        LogRecord record = reader.readNext(false);
        long lastTxId = -1;
        while (null != record) {
            DLMTestUtil.verifyLogRecord(record);
            assert (lastTxId < record.getTransactionId());
            lastTxId = record.getTransactionId();
            numTrans++;
            record = reader.readNext(false);
        }
    } catch (EndOfStreamException exc) {
        exceptionEncountered = true;
    }
    assertEquals(numCompletedSegments * DEFAULT_SEGMENT_SIZE, numTrans);
    assertTrue(exceptionEncountered);
    exceptionEncountered = false;
    try {
        reader.readNext(false);
    } catch (EndOfStreamException exc) {
        exceptionEncountered = true;
    }
    assertTrue(exceptionEncountered);
    reader.close();
}

Example 14 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestBKDistributedLogManager method testSubscriptionsStore.

@Test(timeout = 60000)
public void testSubscriptionsStore() throws Exception {
    String name = "distrlog-subscriptions-store";
    String subscriber0 = "subscriber-0";
    String subscriber1 = "subscriber-1";
    String subscriber2 = "subscriber-2";
    DLSN commitPosition0 = new DLSN(4, 33, 5);
    DLSN commitPosition1 = new DLSN(4, 34, 5);
    DLSN commitPosition2 = new DLSN(5, 34, 5);
    DLSN commitPosition3 = new DLSN(6, 35, 6);
    DistributedLogManager dlm = createNewDLM(conf, name);
    SubscriptionsStore store = dlm.getSubscriptionsStore();
    // no data
    assertEquals(Utils.ioResult(store.getLastCommitPosition(subscriber0)), DLSN.NonInclusiveLowerBound);
    assertEquals(Utils.ioResult(store.getLastCommitPosition(subscriber1)), DLSN.NonInclusiveLowerBound);
    assertEquals(Utils.ioResult(store.getLastCommitPosition(subscriber2)), DLSN.NonInclusiveLowerBound);
    // empty
    assertTrue(Utils.ioResult(store.getLastCommitPositions()).isEmpty());
    // subscriber 0 advance
    Utils.ioResult(store.advanceCommitPosition(subscriber0, commitPosition0));
    assertEquals(commitPosition0, Utils.ioResult(store.getLastCommitPosition(subscriber0)));
    Map<String, DLSN> committedPositions = Utils.ioResult(store.getLastCommitPositions());
    assertEquals(1, committedPositions.size());
    assertEquals(commitPosition0, committedPositions.get(subscriber0));
    // subscriber 1 advance
    Utils.ioResult(store.advanceCommitPosition(subscriber1, commitPosition1));
    assertEquals(commitPosition1, Utils.ioResult(store.getLastCommitPosition(subscriber1)));
    committedPositions = Utils.ioResult(store.getLastCommitPositions());
    assertEquals(2, committedPositions.size());
    assertEquals(commitPosition0, committedPositions.get(subscriber0));
    assertEquals(commitPosition1, committedPositions.get(subscriber1));
    // subscriber 2 advance
    Utils.ioResult(store.advanceCommitPosition(subscriber2, commitPosition2));
    assertEquals(commitPosition2, Utils.ioResult(store.getLastCommitPosition(subscriber2)));
    committedPositions = Utils.ioResult(store.getLastCommitPositions());
    assertEquals(3, committedPositions.size());
    assertEquals(commitPosition0, committedPositions.get(subscriber0));
    assertEquals(commitPosition1, committedPositions.get(subscriber1));
    assertEquals(commitPosition2, committedPositions.get(subscriber2));
    // subscriber 2 advance again
    DistributedLogManager newDLM = createNewDLM(conf, name);
    SubscriptionsStore newStore = newDLM.getSubscriptionsStore();
    Utils.ioResult(newStore.advanceCommitPosition(subscriber2, commitPosition3));
    newStore.close();
    newDLM.close();
    committedPositions = Utils.ioResult(store.getLastCommitPositions());
    assertEquals(3, committedPositions.size());
    assertEquals(commitPosition0, committedPositions.get(subscriber0));
    assertEquals(commitPosition1, committedPositions.get(subscriber1));
    assertEquals(commitPosition3, committedPositions.get(subscriber2));
    dlm.close();
}

Example 15 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestBKDistributedLogManager method testContinuousReadersWithEmptyLedgers.

@Test(timeout = 60000)
public void testContinuousReadersWithEmptyLedgers() throws Exception {
    String name = "distrlog-continuous-emptyledgers";
    DistributedLogManager dlm = createNewDLM(conf, name);
    long txid = 1;
    for (long i = 0; i < 3; i++) {
        long start = txid;
        BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
        for (long j = 1; j <= DEFAULT_SEGMENT_SIZE; j++) {
            LogRecord op = DLMTestUtil.getLogRecordInstance(txid++);
            out.write(op);
        }
        BKLogSegmentWriter writer = out.getCachedLogWriter();
        out.closeAndComplete();
        BKLogWriteHandler blplm = ((BKDistributedLogManager) (dlm)).createWriteHandler(true);
        assertNotNull(zkc.exists(blplm.completedLedgerZNode(start, txid - 1, writer.getLogSegmentSequenceNumber()), false));
        BKLogSegmentWriter perStreamLogWriter = blplm.startLogSegment(txid - 1);
        blplm.completeAndCloseLogSegment(perStreamLogWriter.getLogSegmentSequenceNumber(), perStreamLogWriter.getLogSegmentId(), txid - 1, txid - 1, 0);
        assertNotNull(zkc.exists(blplm.completedLedgerZNode(txid - 1, txid - 1, perStreamLogWriter.getLogSegmentSequenceNumber()), false));
        Utils.ioResult(blplm.asyncClose());
    }
    BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
    for (long j = 1; j <= DEFAULT_SEGMENT_SIZE / 2; j++) {
        LogRecord op = DLMTestUtil.getLogRecordInstance(txid++);
        out.write(op);
    }
    out.flush();
    out.commit();
    out.close();
    dlm.close();
    dlm = createNewDLM(conf, name);
    AsyncLogReader asyncreader = dlm.getAsyncLogReader(DLSN.InvalidDLSN);
    long numTrans = 0;
    LogRecordWithDLSN record = Utils.ioResult(asyncreader.readNext());
    while (null != record) {
        DLMTestUtil.verifyLogRecord(record);
        numTrans++;
        if (numTrans >= (txid - 1)) {
            break;
        }
        record = Utils.ioResult(asyncreader.readNext());
    }
    assertEquals((txid - 1), numTrans);
    Utils.close(asyncreader);
    LogReader reader = dlm.getInputStream(1);
    numTrans = 0;
    record = reader.readNext(false);
    while (null != record) {
        DLMTestUtil.verifyLogRecord(record);
        numTrans++;
        record = reader.readNext(false);
    }
    assertEquals((txid - 1), numTrans);
    reader.close();
    assertEquals(txid - 1, dlm.getLogRecordCount());
    dlm.close();
}