Examples with LogReader org.apache.distributedlog.api.LogReader used on opensource projects

Example 31 with LogReader

use of org.apache.distributedlog.api.LogReader in project bookkeeper by apache.

the class TestInterleavedReaders method testInterleavedReadersWithCleanup.

@Test(timeout = 60000)
public void testInterleavedReadersWithCleanup() throws Exception {
    String name = "distrlog-interleaved-cleanup";
    BKDistributedLogManager dlmwrite0 = createNewDLM(conf, name + "-0");
    BKDistributedLogManager dlmwrite1 = createNewDLM(conf, name + "-1");
    long txid = 1;
    Long retentionPeriodOverride = null;
    BKAsyncLogWriter writer0 = dlmwrite0.startAsyncLogSegmentNonPartitioned();
    BKAsyncLogWriter writer1 = dlmwrite1.startAsyncLogSegmentNonPartitioned();
    for (long j = 1; j <= 4; j++) {
        for (int k = 1; k <= 10; k++) {
            if (k == 5) {
                writer0.setForceRolling(true);
                writer0.overRideMinTimeStampToKeep(retentionPeriodOverride);
                writer1.setForceRolling(true);
                writer1.overRideMinTimeStampToKeep(retentionPeriodOverride);
            }
            DLSN dlsn1 = Utils.ioResult(writer1.write(DLMTestUtil.getLogRecordInstance(txid++)));
            LOG.info("writer1 write record {}", dlsn1);
            DLSN dlsn0 = Utils.ioResult(writer0.write(DLMTestUtil.getLogRecordInstance(txid++)));
            LOG.info("writer0 write record {}", dlsn0);
            if (k == 5) {
                writer0.setForceRolling(false);
                writer1.setForceRolling(false);
                retentionPeriodOverride = System.currentTimeMillis();
            }
            Thread.sleep(5);
        }
        Utils.ioResult(writer1.writeControlRecord(DLMTestUtil.getLogRecordInstance(txid - 1)));
        Utils.ioResult(writer0.writeControlRecord(DLMTestUtil.getLogRecordInstance(txid - 1)));
    }
    writer0.close();
    writer1.close();
    DistributedLogManager dlmreader0 = createNewDLM(conf, name + "-0");
    DistributedLogManager dlmreader1 = createNewDLM(conf, name + "-1");
    LogReader reader0 = dlmreader0.getInputStream(1);
    LogReader reader1 = dlmreader1.getInputStream(1);
    int numTrans = drainStreams(reader0, 15, reader1, 15);
    assertEquals(30, numTrans);
    reader0.close();
    reader1.close();
    dlmreader0.close();
    dlmwrite0.close();
    dlmreader1.close();
    dlmwrite1.close();
}

Example 32 with LogReader

use of org.apache.distributedlog.api.LogReader in project bookkeeper by apache.

the class TestInterleavedReaders method testInterleavedReadersWithRolling.

@Test(timeout = 60000)
public void testInterleavedReadersWithRolling() throws Exception {
    String name = "distrlog-interleaved-rolling";
    BKDistributedLogManager dlmwrite0 = createNewDLM(conf, name + "-0");
    BKDistributedLogManager dlmreader0 = createNewDLM(conf, name + "-0");
    BKDistributedLogManager dlmwrite1 = createNewDLM(conf, name + "-1");
    BKDistributedLogManager dlmreader1 = createNewDLM(conf, name + "-1");
    LogReader reader0 = null;
    LogReader reader1 = null;
    long txid = 1;
    int numTrans = 0;
    BKAsyncLogWriter writer0 = dlmwrite0.startAsyncLogSegmentNonPartitioned();
    BKAsyncLogWriter writer1 = dlmwrite1.startAsyncLogSegmentNonPartitioned();
    for (long j = 1; j <= 2; j++) {
        for (int k = 1; k <= 6; k++) {
            if (k == 3) {
                writer0.setForceRolling(true);
                writer1.setForceRolling(true);
            }
            Utils.ioResult(writer1.write(DLMTestUtil.getLogRecordInstance(txid++)));
            Utils.ioResult(writer0.write(DLMTestUtil.getLogRecordInstance(txid++)));
            writer0.setForceRolling(false);
            writer1.setForceRolling(false);
        }
        Utils.ioResult(writer1.writeControlRecord(DLMTestUtil.getLogRecordInstance(txid - 1)));
        Utils.ioResult(writer0.writeControlRecord(DLMTestUtil.getLogRecordInstance(txid - 1)));
        if (null == reader0) {
            reader0 = dlmreader0.getInputStream(1);
        }
        if (null == reader1) {
            reader1 = dlmreader1.getInputStream(1);
        }
        numTrans += drainStreams(reader0, 6, reader1, 6);
        assertEquals((txid - 1), numTrans);
    }
    reader0.close();
    reader1.close();
    dlmreader0.close();
    dlmwrite0.close();
    dlmreader1.close();
    dlmwrite1.close();
}

Example 33 with LogReader

use of org.apache.distributedlog.api.LogReader in project bookkeeper by apache.

the class TestDistributedLogTool method testToolTruncateStream.

@Test(timeout = 60000)
public void testToolTruncateStream() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setLogSegmentCacheEnabled(false);
    DistributedLogManager dlm = DLMTestUtil.createNewDLM("testToolTruncateStream", confLocal, defaultUri);
    DLMTestUtil.generateCompletedLogSegments(dlm, confLocal, 3, 1000);
    DLSN dlsn = new DLSN(2, 1, 0);
    TruncateStreamCommand cmd = new TruncateStreamCommand();
    cmd.setDlsn(dlsn);
    cmd.setUri(defaultUri);
    cmd.setStreamName("testToolTruncateStream");
    cmd.setForce(true);
    assertEquals(0, cmd.runCmd());
    LogReader reader = dlm.getInputStream(0);
    LogRecordWithDLSN record = reader.readNext(false);
    assertEquals(dlsn, record.getDlsn());
    reader.close();
    dlm.close();
}

Example 34 with LogReader

use of org.apache.distributedlog.api.LogReader in project bookkeeper by apache.

the class TestBKDistributedLogManager method deleteDuringRead.

@Test(timeout = 60000)
public void deleteDuringRead() throws Exception {
    String name = "distrlog-delete-with-reader";
    DistributedLogManager dlm = createNewDLM(conf, name);
    long txid = 1;
    for (long i = 0; i < 3; i++) {
        long start = txid;
        BKSyncLogWriter writer = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
        for (long j = 1; j <= DEFAULT_SEGMENT_SIZE; j++) {
            writer.write(DLMTestUtil.getLogRecordInstance(txid++));
        }
        BKLogSegmentWriter perStreamLogWriter = writer.getCachedLogWriter();
        writer.closeAndComplete();
        BKLogWriteHandler blplm = ((BKDistributedLogManager) (dlm)).createWriteHandler(true);
        assertNotNull(zkc.exists(blplm.completedLedgerZNode(start, txid - 1, perStreamLogWriter.getLogSegmentSequenceNumber()), false));
        Utils.ioResult(blplm.asyncClose());
    }
    LogReader reader = dlm.getInputStream(1);
    LogRecord record = reader.readNext(false);
    assert (null != record);
    DLMTestUtil.verifyLogRecord(record);
    long lastTxId = record.getTransactionId();
    dlm.delete();
    boolean exceptionEncountered;
    try {
        record = reader.readNext(false);
        while (null != record) {
            DLMTestUtil.verifyLogRecord(record);
            assert (lastTxId < record.getTransactionId());
            lastTxId = record.getTransactionId();
            record = reader.readNext(false);
        }
        // make sure the exception is thrown from readahead
        while (true) {
            reader.readNext(false);
        }
    } catch (LogReadException | LogNotFoundException | DLIllegalStateException e) {
        exceptionEncountered = true;
    }
    assertTrue(exceptionEncountered);
    reader.close();
}

Example 35 with LogReader

use of org.apache.distributedlog.api.LogReader in project bookkeeper by apache.

the class TestBKDistributedLogManager method testTruncationValidation.

@Test(timeout = 60000)
public void testTruncationValidation() throws Exception {
    String name = "distrlog-truncation-validation";
    URI uri = createDLMURI("/" + name);
    ZooKeeperClient zookeeperClient = TestZooKeeperClientBuilder.newBuilder().uri(uri).build();
    OrderedScheduler scheduler = OrderedScheduler.newSchedulerBuilder().name("test-truncation-validation").numThreads(1).build();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setDLLedgerMetadataLayoutVersion(LogSegmentMetadata.LEDGER_METADATA_CURRENT_LAYOUT_VERSION);
    confLocal.setOutputBufferSize(0);
    confLocal.setLogSegmentCacheEnabled(false);
    LogSegmentMetadataStore metadataStore = new ZKLogSegmentMetadataStore(confLocal, zookeeperClient, scheduler);
    BKDistributedLogManager dlm = createNewDLM(confLocal, name);
    DLSN truncDLSN = DLSN.InitialDLSN;
    DLSN beyondTruncDLSN = DLSN.InitialDLSN;
    long beyondTruncTxId = 1;
    long txid = 1;
    for (long i = 0; i < 3; i++) {
        long start = txid;
        BKAsyncLogWriter writer = dlm.startAsyncLogSegmentNonPartitioned();
        for (long j = 1; j <= 10; j++) {
            LogRecord record = DLMTestUtil.getLargeLogRecordInstance(txid++);
            CompletableFuture<DLSN> dlsn = writer.write(record);
            if (i == 1 && j == 2) {
                truncDLSN = Utils.ioResult(dlsn);
            } else if (i == 2 && j == 3) {
                beyondTruncDLSN = Utils.ioResult(dlsn);
                beyondTruncTxId = record.getTransactionId();
            } else if (j == 10) {
                Utils.ioResult(dlsn);
            }
        }
        writer.close();
    }
    {
        LogReader reader = dlm.getInputStream(DLSN.InitialDLSN);
        LogRecordWithDLSN record = reader.readNext(false);
        assertTrue((record != null) && (record.getDlsn().compareTo(DLSN.InitialDLSN) == 0));
        reader.close();
    }
    Map<Long, LogSegmentMetadata> segmentList = DLMTestUtil.readLogSegments(zookeeperClient, LogMetadata.getLogSegmentsPath(uri, name, confLocal.getUnpartitionedStreamName()));
    LOG.info("Read segments before truncating first segment : {}", segmentList);
    MetadataUpdater updater = LogSegmentMetadataStoreUpdater.createMetadataUpdater(confLocal, metadataStore);
    Utils.ioResult(updater.setLogSegmentTruncated(segmentList.get(1L)));
    segmentList = DLMTestUtil.readLogSegments(zookeeperClient, LogMetadata.getLogSegmentsPath(uri, name, confLocal.getUnpartitionedStreamName()));
    LOG.info("Read segments after truncated first segment : {}", segmentList);
    {
        LogReader reader = dlm.getInputStream(DLSN.InitialDLSN);
        LogRecordWithDLSN record = reader.readNext(false);
        assertTrue("Unexpected record : " + record, (record != null) && (record.getDlsn().compareTo(new DLSN(2, 0, 0)) == 0));
        reader.close();
    }
    {
        LogReader reader = dlm.getInputStream(1);
        LogRecordWithDLSN record = reader.readNext(false);
        assertTrue((record != null) && (record.getDlsn().compareTo(new DLSN(2, 0, 0)) == 0));
        reader.close();
    }
    updater = LogSegmentMetadataStoreUpdater.createMetadataUpdater(confLocal, metadataStore);
    Utils.ioResult(updater.setLogSegmentActive(segmentList.get(1L)));
    segmentList = DLMTestUtil.readLogSegments(zookeeperClient, LogMetadata.getLogSegmentsPath(uri, name, confLocal.getUnpartitionedStreamName()));
    LOG.info("Read segments after marked first segment as active : {}", segmentList);
    updater = LogSegmentMetadataStoreUpdater.createMetadataUpdater(confLocal, metadataStore);
    Utils.ioResult(updater.setLogSegmentTruncated(segmentList.get(2L)));
    segmentList = DLMTestUtil.readLogSegments(zookeeperClient, LogMetadata.getLogSegmentsPath(uri, name, confLocal.getUnpartitionedStreamName()));
    LOG.info("Read segments after truncated second segment : {}", segmentList);
    {
        AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InitialDLSN);
        long expectedTxId = 1L;
        boolean exceptionEncountered = false;
        try {
            for (int i = 0; i < 3 * 10; i++) {
                LogRecordWithDLSN record = Utils.ioResult(reader.readNext());
                DLMTestUtil.verifyLargeLogRecord(record);
                assertEquals(expectedTxId, record.getTransactionId());
                expectedTxId++;
            }
        } catch (AlreadyTruncatedTransactionException exc) {
            exceptionEncountered = true;
        }
        assertTrue(exceptionEncountered);
        Utils.close(reader);
    }
    updater = LogSegmentMetadataStoreUpdater.createMetadataUpdater(conf, metadataStore);
    Utils.ioResult(updater.setLogSegmentActive(segmentList.get(2L)));
    BKAsyncLogWriter writer = dlm.startAsyncLogSegmentNonPartitioned();
    assertTrue(Utils.ioResult(writer.truncate(truncDLSN)));
    BKLogWriteHandler handler = writer.getCachedWriteHandler();
    List<LogSegmentMetadata> cachedSegments = handler.getCachedLogSegments(LogSegmentMetadata.COMPARATOR);
    for (LogSegmentMetadata segment : cachedSegments) {
        if (segment.getLastDLSN().compareTo(truncDLSN) < 0) {
            assertTrue(segment.isTruncated());
            assertTrue(!segment.isPartiallyTruncated());
        } else if (segment.getFirstDLSN().compareTo(truncDLSN) < 0) {
            assertTrue(!segment.isTruncated());
            assertTrue(segment.isPartiallyTruncated());
        } else {
            assertTrue(!segment.isTruncated());
            assertTrue(!segment.isPartiallyTruncated());
        }
    }
    segmentList = DLMTestUtil.readLogSegments(zookeeperClient, LogMetadata.getLogSegmentsPath(uri, name, conf.getUnpartitionedStreamName()));
    assertTrue(segmentList.get(truncDLSN.getLogSegmentSequenceNo()).getMinActiveDLSN().compareTo(truncDLSN) == 0);
    {
        LogReader reader = dlm.getInputStream(DLSN.InitialDLSN);
        LogRecordWithDLSN record = reader.readNext(false);
        assertTrue(record != null);
        assertEquals(truncDLSN, record.getDlsn());
        reader.close();
    }
    {
        LogReader reader = dlm.getInputStream(1);
        LogRecordWithDLSN record = reader.readNext(false);
        assertTrue(record != null);
        assertEquals(truncDLSN, record.getDlsn());
        reader.close();
    }
    {
        AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InitialDLSN);
        LogRecordWithDLSN record = Utils.ioResult(reader.readNext());
        assertTrue(record != null);
        assertEquals(truncDLSN, record.getDlsn());
        Utils.close(reader);
    }
    {
        LogReader reader = dlm.getInputStream(beyondTruncDLSN);
        LogRecordWithDLSN record = reader.readNext(false);
        assertTrue(record != null);
        assertEquals(beyondTruncDLSN, record.getDlsn());
        reader.close();
    }
    {
        LogReader reader = dlm.getInputStream(beyondTruncTxId);
        LogRecordWithDLSN record = reader.readNext(false);
        assertTrue(record != null);
        assertEquals(beyondTruncDLSN, record.getDlsn());
        assertEquals(beyondTruncTxId, record.getTransactionId());
        reader.close();
    }
    {
        AsyncLogReader reader = dlm.getAsyncLogReader(beyondTruncDLSN);
        LogRecordWithDLSN record = Utils.ioResult(reader.readNext());
        assertTrue(record != null);
        assertEquals(beyondTruncDLSN, record.getDlsn());
        Utils.close(reader);
    }
    zookeeperClient.close();
}