Examples with LogReader org.apache.distributedlog.api.LogReader used on opensource projects

Example 1 with LogReader

use of org.apache.distributedlog.api.LogReader in project bookkeeper by apache.

the class AppendOnlyStreamReader method skipTo.

/**
 * Position the reader at the given offset. If we fail to skip to the desired position
 * and don't hit end of stream, return false.
 *
 * @throws org.apache.distributedlog.exceptions.EndOfStreamException if we attempt to
 *         skip past the end of the stream.
 */
public boolean skipTo(long position) throws IOException {
    // No need to skip anywhere.
    if (position == position()) {
        return true;
    }
    LogReader skipReader = dlm.getInputStream(position);
    LogRecordWithInputStream logRecord = null;
    try {
        logRecord = nextLogRecord(skipReader);
    } catch (IOException ex) {
        skipReader.close();
        throw ex;
    }
    if (null == logRecord) {
        return false;
    }
    // We may end up with a reader positioned *before* the requested position if
    // we're near the tail and the writer is still active, or if the desired position
    // is not at a log record payload boundary.
    // Transaction ID gives us the starting position of the log record. Read ahead
    // if necessary.
    currentPosition = logRecord.getOffset();
    currentLogRecord = logRecord;
    LogReader oldReader = reader;
    reader = skipReader;
    // Close the oldreader after swapping AppendOnlyStreamReader state. Close may fail
    // and we need to make sure it leaves AppendOnlyStreamReader in a consistent state.
    oldReader.close();
    byte[] skipBuffer = new byte[SKIP_BUFFER_SIZE];
    while (currentPosition < position) {
        long bytesToRead = Math.min(position - currentPosition, SKIP_BUFFER_SIZE);
        long bytesRead = read(skipBuffer, 0, (int) bytesToRead);
        if (bytesRead < bytesToRead) {
            return false;
        }
    }
    return true;
}

Example 2 with LogReader

use of org.apache.distributedlog.api.LogReader in project bookkeeper by apache.

the class TestBKDistributedLogManager method testMarkEndOfStream.

@Test(timeout = 60000)
public void testMarkEndOfStream() throws Exception {
    String name = "distrlog-mark-end-of-stream";
    DistributedLogManager dlm = createNewDLM(conf, name);
    long txid = 1;
    txid = writeAndMarkEndOfStream(dlm, txid);
    LogReader reader = dlm.getInputStream(1);
    long numTrans = 0;
    boolean exceptionEncountered = false;
    LogRecord record = null;
    try {
        record = reader.readNext(false);
        long expectedTxId = 1;
        while (null != record) {
            DLMTestUtil.verifyLogRecord(record);
            assertEquals(expectedTxId, record.getTransactionId());
            expectedTxId++;
            numTrans++;
            record = reader.readNext(false);
        }
    } catch (EndOfStreamException exc) {
        LOG.info("Encountered EndOfStream on reading records after {}", record);
        exceptionEncountered = true;
    }
    assertEquals((txid - 1), numTrans);
    assertTrue(exceptionEncountered);
    exceptionEncountered = false;
    try {
        reader.readNext(false);
    } catch (EndOfStreamException exc) {
        exceptionEncountered = true;
    }
    assertTrue(exceptionEncountered);
    reader.close();
}

Example 3 with LogReader

use of org.apache.distributedlog.api.LogReader in project bookkeeper by apache.

the class TestBKDistributedLogManager method testNonPartitionedWritesInternal.

private void testNonPartitionedWritesInternal(String name, DistributedLogConfiguration conf) throws Exception {
    BKDistributedLogManager dlm = createNewDLM(conf, name);
    long txid = 1;
    for (long i = 0; i < 3; i++) {
        long start = txid;
        BKSyncLogWriter writer = dlm.startLogSegmentNonPartitioned();
        for (long j = 1; j <= DEFAULT_SEGMENT_SIZE; j++) {
            writer.write(DLMTestUtil.getLogRecordInstance(txid++));
        }
        BKLogSegmentWriter perStreamLogWriter = writer.getCachedLogWriter();
        writer.closeAndComplete();
        BKLogWriteHandler blplm = dlm.createWriteHandler(true);
        assertNotNull(zkc.exists(blplm.completedLedgerZNode(start, txid - 1, perStreamLogWriter.getLogSegmentSequenceNumber()), false));
        Utils.ioResult(blplm.asyncClose());
    }
    LogWriter writer = dlm.startLogSegmentNonPartitioned();
    for (long j = 1; j <= DEFAULT_SEGMENT_SIZE / 2; j++) {
        writer.write(DLMTestUtil.getLogRecordInstance(txid++));
    }
    writer.flush();
    writer.commit();
    writer.close();
    LogReader reader = dlm.getInputStream(1);
    long numTrans = 0;
    LogRecord record = reader.readNext(false);
    long lastTxId = -1;
    while (null != record) {
        DLMTestUtil.verifyLogRecord(record);
        assert (lastTxId < record.getTransactionId());
        lastTxId = record.getTransactionId();
        numTrans++;
        record = reader.readNext(false);
    }
    reader.close();
    assertEquals((txid - 1), numTrans);
}

Example 4 with LogReader

use of org.apache.distributedlog.api.LogReader in project bookkeeper by apache.

the class TestBKDistributedLogManager method testContinuousReaderBulk.

@Test(timeout = 60000)
public void testContinuousReaderBulk() throws Exception {
    String name = "distrlog-continuous-bulk";
    DistributedLogManager dlm = createNewDLM(conf, name);
    long txid = 1;
    for (long i = 0; i < 3; i++) {
        BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
        for (long j = 1; j <= DEFAULT_SEGMENT_SIZE; j++) {
            LogRecord op = DLMTestUtil.getLogRecordInstance(txid++);
            out.write(op);
        }
        out.closeAndComplete();
    }
    BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
    for (long j = 1; j <= DEFAULT_SEGMENT_SIZE / 2; j++) {
        LogRecord op = DLMTestUtil.getLogRecordInstance(txid++);
        out.write(op);
    }
    out.flush();
    out.commit();
    out.close();
    dlm.close();
    dlm = createNewDLM(conf, name);
    LogReader reader = dlm.getInputStream(1);
    long numTrans = 0;
    List<LogRecordWithDLSN> recordList = reader.readBulk(false, 13);
    long lastTxId = -1;
    while (!recordList.isEmpty()) {
        for (LogRecord record : recordList) {
            assert (lastTxId < record.getTransactionId());
            lastTxId = record.getTransactionId();
            DLMTestUtil.verifyLogRecord(record);
            numTrans++;
        }
        recordList = reader.readBulk(false, 13);
    }
    reader.close();
    assertEquals((txid - 1), numTrans);
}

Example 5 with LogReader

use of org.apache.distributedlog.api.LogReader in project bookkeeper by apache.

the class TestBKDistributedLogManager method testContinuousReaders.

@Test(timeout = 60000)
public void testContinuousReaders() throws Exception {
    String name = "distrlog-continuous";
    BKDistributedLogManager dlm = createNewDLM(conf, name);
    long txid = 1;
    for (long i = 0; i < 3; i++) {
        long start = txid;
        BKSyncLogWriter out = dlm.startLogSegmentNonPartitioned();
        for (long j = 1; j <= DEFAULT_SEGMENT_SIZE; j++) {
            LogRecord op = DLMTestUtil.getLogRecordInstance(txid++);
            out.write(op);
        }
        BKLogSegmentWriter perStreamLogWriter = out.getCachedLogWriter();
        out.closeAndComplete();
        BKLogWriteHandler blplm = dlm.createWriteHandler(true);
        assertNotNull(zkc.exists(blplm.completedLedgerZNode(start, txid - 1, perStreamLogWriter.getLogSegmentSequenceNumber()), false));
        Utils.ioResult(blplm.asyncClose());
    }
    BKSyncLogWriter out = dlm.startLogSegmentNonPartitioned();
    for (long j = 1; j <= DEFAULT_SEGMENT_SIZE / 2; j++) {
        LogRecord op = DLMTestUtil.getLogRecordInstance(txid++);
        out.write(op);
    }
    out.flush();
    out.commit();
    out.close();
    dlm.close();
    dlm = createNewDLM(conf, name);
    LogReader reader = dlm.getInputStream(1);
    long numTrans = 0;
    LogRecord record = reader.readNext(false);
    while (null != record) {
        DLMTestUtil.verifyLogRecord(record);
        numTrans++;
        record = reader.readNext(false);
    }
    assertEquals((txid - 1), numTrans);
    assertEquals(txid - 1, dlm.getLogRecordCount());
    reader.close();
    dlm.close();
}