Examples with LogReader org.apache.distributedlog.api.LogReader used on opensource projects

Example 11 with LogReader

use of org.apache.distributedlog.api.LogReader in project bookkeeper by apache.

the class TestAsyncReaderWriter method testWriteControlRecord.

/**
 * Test writing control records to writers: writers should be able to write control records, and
 * the readers should skip control records while reading.
 */
@Test(timeout = 60000)
public void testWriteControlRecord() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(testConf);
    confLocal.setOutputBufferSize(1024);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    // Write 3 log segments. For each log segments, write one control record and nine user records.
    int txid = 1;
    for (long i = 0; i < 3; i++) {
        final long currentLogSegmentSeqNo = i + 1;
        BKAsyncLogWriter writer = (BKAsyncLogWriter) (dlm.startAsyncLogSegmentNonPartitioned());
        DLSN dlsn = Utils.ioResult(writer.writeControlRecord(new LogRecord(txid++, "control".getBytes(UTF_8))));
        assertEquals(currentLogSegmentSeqNo, dlsn.getLogSegmentSequenceNo());
        assertEquals(0, dlsn.getEntryId());
        assertEquals(0, dlsn.getSlotId());
        for (long j = 1; j < 10; j++) {
            final LogRecord record = DLMTestUtil.getLargeLogRecordInstance(txid++);
            Utils.ioResult(writer.write(record));
        }
        writer.closeAndComplete();
    }
    dlm.close();
    // Read all the written data: It should skip control records and only return user records.
    DistributedLogManager readDlm = createNewDLM(confLocal, name);
    LogReader reader = readDlm.getInputStream(1);
    long numTrans = 0;
    long expectedTxId = 2;
    LogRecord record = reader.readNext(false);
    while (null != record) {
        DLMTestUtil.verifyLargeLogRecord(record);
        numTrans++;
        assertEquals(expectedTxId, record.getTransactionId());
        if (expectedTxId % 10 == 0) {
            expectedTxId += 2;
        } else {
            ++expectedTxId;
        }
        record = reader.readNext(false);
    }
    reader.close();
    assertEquals(3 * 9, numTrans);
    assertEquals(3 * 9, readDlm.getLogRecordCount());
    readDlm.close();
}

Example 12 with LogReader

use of org.apache.distributedlog.api.LogReader in project bookkeeper by apache.

the class TestBKDistributedLogNamespace method testUseNamespaceAfterCloseShouldFailFast.

@Test(timeout = 60000)
public void testUseNamespaceAfterCloseShouldFailFast() throws Exception {
    URI uri = createDLMURI("/" + runtime.getMethodName());
    Namespace namespace = NamespaceBuilder.newBuilder().conf(conf).uri(uri).build();
    // before closing the namespace, no exception should be thrown
    String logName = "test-stream";
    // create a log
    namespace.createLog(logName);
    // log exists
    Assert.assertTrue(namespace.logExists(logName));
    // create a dlm
    DistributedLogManager dlm = namespace.openLog(logName);
    // do some writes
    BKAsyncLogWriter writer = (BKAsyncLogWriter) (dlm.startAsyncLogSegmentNonPartitioned());
    for (long i = 0; i < 3; i++) {
        LogRecord record = DLMTestUtil.getLargeLogRecordInstance(i);
        writer.write(record);
    }
    writer.closeAndComplete();
    // do some reads
    LogReader reader = dlm.getInputStream(0);
    for (long i = 0; i < 3; i++) {
        Assert.assertEquals(reader.readNext(false).getTransactionId(), i);
    }
    namespace.deleteLog(logName);
    Assert.assertFalse(namespace.logExists(logName));
    // now try to close the namespace
    namespace.close();
    try {
        namespace.createLog(logName);
        fail("Should throw exception after namespace is closed");
    } catch (AlreadyClosedException e) {
    // No-ops
    }
    try {
        namespace.openLog(logName);
        fail("Should throw exception after namespace is closed");
    } catch (AlreadyClosedException e) {
    // No-ops
    }
    try {
        namespace.logExists(logName);
        fail("Should throw exception after namespace is closed");
    } catch (AlreadyClosedException e) {
    // No-ops
    }
    try {
        namespace.getLogs();
        fail("Should throw exception after namespace is closed");
    } catch (AlreadyClosedException e) {
    // No-ops
    }
    try {
        namespace.deleteLog(logName);
        fail("Should throw exception after namespace is closed");
    } catch (AlreadyClosedException e) {
    // No-ops
    }
    try {
        namespace.createAccessControlManager();
        fail("Should throw exception after namespace is closed");
    } catch (AlreadyClosedException e) {
    // No-ops
    }
}

Example 13 with LogReader

use of org.apache.distributedlog.api.LogReader in project bookkeeper by apache.

the class TestBKSyncLogReader method testDeletingLogWhileReading.

@Test(timeout = 60000)
public void testDeletingLogWhileReading() throws Exception {
    String name = testName.getMethodName();
    DistributedLogManager dlm = createNewDLM(conf, name);
    BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
    for (long i = 1; i < 10; i++) {
        LogRecord op = DLMTestUtil.getLogRecordInstance(i);
        out.write(op);
    }
    out.closeAndComplete();
    LogReader reader = dlm.getInputStream(1L);
    for (int i = 1; i < 10; i++) {
        LogRecord record = waitForNextRecord(reader);
        assertEquals((long) i, record.getTransactionId());
    }
    DistributedLogManager deleteDLM = createNewDLM(conf, name);
    deleteDLM.delete();
    LogRecord record;
    try {
        record = reader.readNext(false);
        while (null == record) {
            record = reader.readNext(false);
        }
        fail("Should fail reading next with LogNotFound");
    } catch (LogNotFoundException lnfe) {
    // expected
    }
}

Example 14 with LogReader

use of org.apache.distributedlog.api.LogReader in project bookkeeper by apache.

the class TestInterleavedReaders method testInterleavedReadersWithRollingEdge.

@Test(timeout = 60000)
public void testInterleavedReadersWithRollingEdge() throws Exception {
    String name = "distrlog-interleaved-rolling-edge";
    BKDistributedLogManager dlmwrite0 = createNewDLM(conf, name + "-0");
    BKDistributedLogManager dlmreader0 = createNewDLM(conf, name + "-0");
    BKDistributedLogManager dlmwrite1 = createNewDLM(conf, name + "-1");
    BKDistributedLogManager dlmreader1 = createNewDLM(conf, name + "-1");
    LogReader reader0 = null;
    LogReader reader1 = null;
    long txid = 1;
    int numTrans = 0;
    BKAsyncLogWriter writer0 = dlmwrite0.startAsyncLogSegmentNonPartitioned();
    BKAsyncLogWriter writer1 = dlmwrite1.startAsyncLogSegmentNonPartitioned();
    for (long j = 1; j <= 4; j++) {
        if (j > 1) {
            writer0.setForceRolling(true);
            writer1.setForceRolling(true);
        }
        for (int k = 1; k <= 2; k++) {
            Utils.ioResult(writer1.write(DLMTestUtil.getLogRecordInstance(txid++)));
            Utils.ioResult(writer0.write(DLMTestUtil.getLogRecordInstance(txid++)));
            writer0.setForceRolling(false);
            writer1.setForceRolling(false);
        }
        Utils.ioResult(writer1.writeControlRecord(DLMTestUtil.getLogRecordInstance(txid - 1)));
        Utils.ioResult(writer0.writeControlRecord(DLMTestUtil.getLogRecordInstance(txid - 1)));
        LOG.info("Completed {} write", j);
        if (null == reader0) {
            reader0 = dlmreader0.getInputStream(1);
        }
        if (null == reader1) {
            reader1 = dlmreader1.getInputStream(1);
        }
        numTrans += drainStreams(reader0, 2, reader1, 2);
        assertEquals((txid - 1), numTrans);
    }
    reader0.close();
    reader1.close();
    dlmreader0.close();
    dlmwrite0.close();
    dlmreader1.close();
    dlmwrite1.close();
}

Example 15 with LogReader

use of org.apache.distributedlog.api.LogReader in project bookkeeper by apache.

the class TestInterleavedReaders method testInterleavedReadersWithRollingEdgeUnPartitioned.

@Test(timeout = 60000)
public void testInterleavedReadersWithRollingEdgeUnPartitioned() throws Exception {
    String name = "distrlog-interleaved-rolling-edge-unpartitioned";
    BKDistributedLogManager dlmwrite0 = createNewDLM(conf, name + "-0");
    BKDistributedLogManager dlmreader0 = createNewDLM(conf, name + "-0");
    BKDistributedLogManager dlmwrite1 = createNewDLM(conf, name + "-1");
    BKDistributedLogManager dlmreader1 = createNewDLM(conf, name + "-1");
    LogReader reader0 = null;
    LogReader reader1 = null;
    long txid = 1;
    int numTrans = 0;
    BKAsyncLogWriter writer0 = dlmwrite0.startAsyncLogSegmentNonPartitioned();
    BKAsyncLogWriter writer1 = dlmwrite1.startAsyncLogSegmentNonPartitioned();
    for (long j = 1; j <= 4; j++) {
        if (j > 1) {
            writer0.setForceRolling(true);
            writer1.setForceRolling(true);
        }
        for (int k = 1; k <= 2; k++) {
            Utils.ioResult(writer1.write(DLMTestUtil.getLogRecordInstance(txid++)));
            Utils.ioResult(writer0.write(DLMTestUtil.getLogRecordInstance(txid++)));
            writer0.setForceRolling(false);
            writer1.setForceRolling(false);
        }
        Utils.ioResult(writer1.writeControlRecord(DLMTestUtil.getLogRecordInstance(txid - 1)));
        Utils.ioResult(writer0.writeControlRecord(DLMTestUtil.getLogRecordInstance(txid - 1)));
        if (null == reader0) {
            reader0 = dlmreader0.getInputStream(1);
        }
        if (null == reader1) {
            reader1 = dlmreader1.getInputStream(1);
        }
        numTrans += drainStreams(reader0, 2, reader1, 2);
        assertEquals((txid - 1), numTrans);
    }
    reader0.close();
    reader1.close();
    dlmreader0.close();
    dlmreader1.close();
}