Examples with LogReader org.apache.distributedlog.api.LogReader used on opensource projects

Example 6 with LogReader

use of org.apache.distributedlog.api.LogReader in project bookkeeper by apache.

the class TestBKDistributedLogManager method markEndOfStreamOnEmptyLogSegment.

private void markEndOfStreamOnEmptyLogSegment(int numCompletedSegments) throws Exception {
    String name = "distrlog-mark-end-empty-" + numCompletedSegments;
    DistributedLogManager dlm = createNewDLM(conf, name);
    DLMTestUtil.generateCompletedLogSegments(dlm, conf, numCompletedSegments, DEFAULT_SEGMENT_SIZE);
    BKSyncLogWriter writer = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
    writer.markEndOfStream();
    LogReader reader = dlm.getInputStream(1);
    long numTrans = 0;
    boolean exceptionEncountered = false;
    try {
        LogRecord record = reader.readNext(false);
        long lastTxId = -1;
        while (null != record) {
            DLMTestUtil.verifyLogRecord(record);
            assert (lastTxId < record.getTransactionId());
            lastTxId = record.getTransactionId();
            numTrans++;
            record = reader.readNext(false);
        }
    } catch (EndOfStreamException exc) {
        exceptionEncountered = true;
    }
    assertEquals(numCompletedSegments * DEFAULT_SEGMENT_SIZE, numTrans);
    assertTrue(exceptionEncountered);
    exceptionEncountered = false;
    try {
        reader.readNext(false);
    } catch (EndOfStreamException exc) {
        exceptionEncountered = true;
    }
    assertTrue(exceptionEncountered);
    reader.close();
}

Example 7 with LogReader

use of org.apache.distributedlog.api.LogReader in project bookkeeper by apache.

the class TestBKDistributedLogManager method testContinuousReadersWithEmptyLedgers.

@Test(timeout = 60000)
public void testContinuousReadersWithEmptyLedgers() throws Exception {
    String name = "distrlog-continuous-emptyledgers";
    DistributedLogManager dlm = createNewDLM(conf, name);
    long txid = 1;
    for (long i = 0; i < 3; i++) {
        long start = txid;
        BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
        for (long j = 1; j <= DEFAULT_SEGMENT_SIZE; j++) {
            LogRecord op = DLMTestUtil.getLogRecordInstance(txid++);
            out.write(op);
        }
        BKLogSegmentWriter writer = out.getCachedLogWriter();
        out.closeAndComplete();
        BKLogWriteHandler blplm = ((BKDistributedLogManager) (dlm)).createWriteHandler(true);
        assertNotNull(zkc.exists(blplm.completedLedgerZNode(start, txid - 1, writer.getLogSegmentSequenceNumber()), false));
        BKLogSegmentWriter perStreamLogWriter = blplm.startLogSegment(txid - 1);
        blplm.completeAndCloseLogSegment(perStreamLogWriter.getLogSegmentSequenceNumber(), perStreamLogWriter.getLogSegmentId(), txid - 1, txid - 1, 0);
        assertNotNull(zkc.exists(blplm.completedLedgerZNode(txid - 1, txid - 1, perStreamLogWriter.getLogSegmentSequenceNumber()), false));
        Utils.ioResult(blplm.asyncClose());
    }
    BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
    for (long j = 1; j <= DEFAULT_SEGMENT_SIZE / 2; j++) {
        LogRecord op = DLMTestUtil.getLogRecordInstance(txid++);
        out.write(op);
    }
    out.flush();
    out.commit();
    out.close();
    dlm.close();
    dlm = createNewDLM(conf, name);
    AsyncLogReader asyncreader = dlm.getAsyncLogReader(DLSN.InvalidDLSN);
    long numTrans = 0;
    LogRecordWithDLSN record = Utils.ioResult(asyncreader.readNext());
    while (null != record) {
        DLMTestUtil.verifyLogRecord(record);
        numTrans++;
        if (numTrans >= (txid - 1)) {
            break;
        }
        record = Utils.ioResult(asyncreader.readNext());
    }
    assertEquals((txid - 1), numTrans);
    Utils.close(asyncreader);
    LogReader reader = dlm.getInputStream(1);
    numTrans = 0;
    record = reader.readNext(false);
    while (null != record) {
        DLMTestUtil.verifyLogRecord(record);
        numTrans++;
        record = reader.readNext(false);
    }
    assertEquals((txid - 1), numTrans);
    reader.close();
    assertEquals(txid - 1, dlm.getLogRecordCount());
    dlm.close();
}

Example 8 with LogReader

use of org.apache.distributedlog.api.LogReader in project bookkeeper by apache.

the class TestRollLogSegments method testUnableToRollLogSegments.

@Test(timeout = 60000)
public void testUnableToRollLogSegments() throws Exception {
    String name = "distrlog-unable-to-roll-log-segments";
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setOutputBufferSize(0);
    confLocal.setLogSegmentRollingIntervalMinutes(0);
    confLocal.setMaxLogSegmentBytes(1);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    BKAsyncLogWriter writer = (BKAsyncLogWriter) dlm.startAsyncLogSegmentNonPartitioned();
    long txId = 1L;
    // Create Log Segments
    Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(txId)));
    FailpointUtils.setFailpoint(FailpointUtils.FailPointName.FP_StartLogSegmentBeforeLedgerCreate, FailpointUtils.FailPointActions.FailPointAction_Throw);
    try {
        // If we couldn't open new log segment, we should keep using the old one
        final int numRecords = 10;
        final CountDownLatch latch = new CountDownLatch(numRecords);
        for (int i = 0; i < numRecords; i++) {
            writer.write(DLMTestUtil.getLogRecordInstance(++txId)).whenComplete(new FutureEventListener<DLSN>() {

                @Override
                public void onSuccess(DLSN value) {
                    logger.info("Completed entry : {}.", value);
                    latch.countDown();
                }

                @Override
                public void onFailure(Throwable cause) {
                    logger.error("Failed to write entries : ", cause);
                }
            });
        }
        latch.await();
        writer.close();
        List<LogSegmentMetadata> segments = dlm.getLogSegments();
        logger.info("LogSegments: {}", segments);
        assertEquals(1, segments.size());
        long expectedTxID = 1L;
        LogReader reader = dlm.getInputStream(DLSN.InitialDLSN);
        LogRecordWithDLSN record = reader.readNext(false);
        while (null != record) {
            DLMTestUtil.verifyLogRecord(record);
            assertEquals(expectedTxID++, record.getTransactionId());
            assertEquals(record.getTransactionId() - 1, record.getSequenceId());
            record = reader.readNext(false);
        }
        assertEquals(12L, expectedTxID);
        reader.close();
        dlm.close();
    } finally {
        FailpointUtils.removeFailpoint(FailpointUtils.FailPointName.FP_StartLogSegmentBeforeLedgerCreate);
    }
}

Example 9 with LogReader

use of org.apache.distributedlog.api.LogReader in project bookkeeper by apache.

the class TestTruncate method verifyEntries.

private void verifyEntries(String name, long readFromTxId, long startTxId, int numEntries) throws Exception {
    DistributedLogManager dlm = createNewDLM(conf, name);
    LogReader reader = dlm.getInputStream(readFromTxId);
    long txid = startTxId;
    int numRead = 0;
    LogRecord r = reader.readNext(false);
    while (null != r) {
        DLMTestUtil.verifyLogRecord(r);
        assertEquals(txid++, r.getTransactionId());
        ++numRead;
        r = reader.readNext(false);
    }
    assertEquals(numEntries, numRead);
    reader.close();
    dlm.close();
}

Example 10 with LogReader

use of org.apache.distributedlog.api.LogReader in project bookkeeper by apache.

the class DLMTestUtil method getNumberofLogRecords.

static long getNumberofLogRecords(DistributedLogManager bkdlm, long startTxId) throws IOException {
    long numLogRecs = 0;
    LogReader reader = bkdlm.getInputStream(startTxId);
    LogRecord record = reader.readNext(false);
    while (null != record) {
        numLogRecs++;
        verifyLogRecord(record);
        record = reader.readNext(false);
    }
    reader.close();
    return numLogRecs;
}