Examples with AsyncLogReader org.apache.distributedlog.api.AsyncLogReader used on opensource projects

Example 16 with AsyncLogReader

use of org.apache.distributedlog.api.AsyncLogReader in project bookkeeper by apache.

the class TestDistributedLogAdmin method testChangeSequenceNumber.

/**
 * {@link https://issues.apache.org/jira/browse/DL-44}.
 */
@DistributedLogAnnotations.FlakyTest
@Ignore
@Test(timeout = 60000)
@SuppressWarnings("deprecation")
public void testChangeSequenceNumber() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setLogSegmentSequenceNumberValidationEnabled(false);
    confLocal.setLogSegmentCacheEnabled(false);
    DistributedLogConfiguration readConf = new DistributedLogConfiguration();
    readConf.addConfiguration(conf);
    readConf.setLogSegmentCacheEnabled(false);
    readConf.setLogSegmentSequenceNumberValidationEnabled(true);
    URI uri = createDLMURI("/change-sequence-number");
    zooKeeperClient.get().create(uri.getPath(), new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    Namespace namespace = NamespaceBuilder.newBuilder().conf(confLocal).uri(uri).build();
    Namespace readNamespace = NamespaceBuilder.newBuilder().conf(readConf).uri(uri).build();
    String streamName = "change-sequence-number";
    // create completed log segments
    DistributedLogManager dlm = namespace.openLog(streamName);
    DLMTestUtil.generateCompletedLogSegments(dlm, confLocal, 4, 10);
    DLMTestUtil.injectLogSegmentWithGivenLogSegmentSeqNo(dlm, confLocal, 5, 41, false, 10, true);
    dlm.close();
    // create a reader
    DistributedLogManager readDLM = readNamespace.openLog(streamName);
    AsyncLogReader reader = readDLM.getAsyncLogReader(DLSN.InitialDLSN);
    // read the records
    long expectedTxId = 1L;
    DLSN lastDLSN = DLSN.InitialDLSN;
    for (int i = 0; i < 4 * 10; i++) {
        LogRecordWithDLSN record = Utils.ioResult(reader.readNext());
        assertNotNull(record);
        DLMTestUtil.verifyLogRecord(record);
        assertEquals(expectedTxId, record.getTransactionId());
        expectedTxId++;
        lastDLSN = record.getDlsn();
    }
    LOG.info("Injecting bad log segment '3'");
    dlm = namespace.openLog(streamName);
    DLMTestUtil.injectLogSegmentWithGivenLogSegmentSeqNo(dlm, confLocal, 3L, 5 * 10 + 1, true, 10, false);
    LOG.info("Injected bad log segment '3'");
    // there isn't records should be read
    CompletableFuture<LogRecordWithDLSN> readFuture = reader.readNext();
    try {
        LogRecordWithDLSN record = Utils.ioResult(readFuture);
        fail("Should fail reading next record " + record + " when there is a corrupted log segment");
    } catch (UnexpectedException ue) {
    // expected
    }
    LOG.info("Dryrun fix inprogress segment that has lower sequence number");
    // Dryrun
    DistributedLogAdmin.fixInprogressSegmentWithLowerSequenceNumber(namespace, new DryrunLogSegmentMetadataStoreUpdater(confLocal, getLogSegmentMetadataStore(namespace)), streamName, false, false);
    try {
        reader = readDLM.getAsyncLogReader(lastDLSN);
        Utils.ioResult(reader.readNext());
        fail("Should fail reading next when there is a corrupted log segment");
    } catch (UnexpectedException ue) {
    // expected
    }
    LOG.info("Actual run fix inprogress segment that has lower sequence number");
    // Actual run
    DistributedLogAdmin.fixInprogressSegmentWithLowerSequenceNumber(namespace, LogSegmentMetadataStoreUpdater.createMetadataUpdater(confLocal, getLogSegmentMetadataStore(namespace)), streamName, false, false);
    // be able to read more after fix
    reader = readDLM.getAsyncLogReader(lastDLSN);
    // skip the first record
    Utils.ioResult(reader.readNext());
    readFuture = reader.readNext();
    expectedTxId = 51L;
    LogRecord record = Utils.ioResult(readFuture);
    assertNotNull(record);
    DLMTestUtil.verifyLogRecord(record);
    assertEquals(expectedTxId, record.getTransactionId());
    expectedTxId++;
    for (int i = 1; i < 10; i++) {
        record = Utils.ioResult(reader.readNext());
        assertNotNull(record);
        DLMTestUtil.verifyLogRecord(record);
        assertEquals(expectedTxId, record.getTransactionId());
        expectedTxId++;
    }
    Utils.close(reader);
    readDLM.close();
    dlm.close();
    namespace.close();
    readNamespace.close();
}

Example 17 with AsyncLogReader

use of org.apache.distributedlog.api.AsyncLogReader in project bookkeeper by apache.

the class TestSequenceID method testSequenceID.

@Test(timeout = 60000)
public void testSequenceID() throws Exception {
    DistributedLogConfiguration confLocalv4 = new DistributedLogConfiguration();
    confLocalv4.addConfiguration(conf);
    confLocalv4.setImmediateFlushEnabled(true);
    confLocalv4.setOutputBufferSize(0);
    confLocalv4.setDLLedgerMetadataLayoutVersion(LogSegmentMetadataVersion.VERSION_V4_ENVELOPED_ENTRIES.value);
    String name = "distrlog-sequence-id";
    BKDistributedLogManager readDLM = (BKDistributedLogManager) createNewDLM(conf, name);
    AsyncLogReader reader = null;
    final LinkedBlockingQueue<LogRecordWithDLSN> readRecords = new LinkedBlockingQueue<LogRecordWithDLSN>();
    BKDistributedLogManager dlm = (BKDistributedLogManager) createNewDLM(confLocalv4, name);
    long txId = 0L;
    for (int i = 0; i < 3; i++) {
        BKAsyncLogWriter writer = dlm.startAsyncLogSegmentNonPartitioned();
        for (int j = 0; j < 2; j++) {
            Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(txId++)));
            if (null == reader) {
                reader = readDLM.getAsyncLogReader(DLSN.InitialDLSN);
                final AsyncLogReader r = reader;
                reader.readNext().whenComplete(new FutureEventListener<LogRecordWithDLSN>() {

                    @Override
                    public void onSuccess(LogRecordWithDLSN record) {
                        readRecords.add(record);
                        r.readNext().whenComplete(this);
                    }

                    @Override
                    public void onFailure(Throwable cause) {
                        logger.error("Encountered exception on reading next : ", cause);
                    }
                });
            }
        }
        writer.closeAndComplete();
    }
    BKAsyncLogWriter writer = dlm.startAsyncLogSegmentNonPartitioned();
    Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(txId++)));
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(4, segments.size());
    for (int i = 0; i < 3; i++) {
        assertFalse(segments.get(i).isInProgress());
        assertTrue(segments.get(i).getStartSequenceId() < 0);
    }
    assertTrue(segments.get(3).isInProgress());
    assertTrue(segments.get(3).getStartSequenceId() < 0);
    dlm.close();
    // simulate upgrading from v4 -> v5
    DistributedLogConfiguration confLocalv5 = new DistributedLogConfiguration();
    confLocalv5.addConfiguration(conf);
    confLocalv5.setImmediateFlushEnabled(true);
    confLocalv5.setOutputBufferSize(0);
    confLocalv5.setDLLedgerMetadataLayoutVersion(LogSegmentMetadataVersion.VERSION_V5_SEQUENCE_ID.value);
    BKDistributedLogManager dlmv5 = (BKDistributedLogManager) createNewDLM(confLocalv5, name);
    for (int i = 0; i < 3; i++) {
        BKAsyncLogWriter writerv5 = dlmv5.startAsyncLogSegmentNonPartitioned();
        for (int j = 0; j < 2; j++) {
            Utils.ioResult(writerv5.write(DLMTestUtil.getLogRecordInstance(txId++)));
        }
        writerv5.closeAndComplete();
    }
    BKAsyncLogWriter writerv5 = dlmv5.startAsyncLogSegmentNonPartitioned();
    Utils.ioResult(writerv5.write(DLMTestUtil.getLogRecordInstance(txId++)));
    List<LogSegmentMetadata> segmentsv5 = dlmv5.getLogSegments();
    assertEquals(8, segmentsv5.size());
    assertFalse(segmentsv5.get(3).isInProgress());
    assertTrue(segmentsv5.get(3).getStartSequenceId() < 0);
    long startSequenceId = 0L;
    for (int i = 4; i < 7; i++) {
        assertFalse(segmentsv5.get(i).isInProgress());
        assertEquals(startSequenceId, segmentsv5.get(i).getStartSequenceId());
        startSequenceId += 2L;
    }
    assertTrue(segmentsv5.get(7).isInProgress());
    assertEquals(startSequenceId, segmentsv5.get(7).getStartSequenceId());
    dlmv5.close();
    // rollback from v5 to v4
    BKDistributedLogManager dlmv4 = (BKDistributedLogManager) createNewDLM(confLocalv4, name);
    for (int i = 0; i < 3; i++) {
        BKAsyncLogWriter writerv4 = dlmv4.startAsyncLogSegmentNonPartitioned();
        for (int j = 0; j < 2; j++) {
            Utils.ioResult(writerv4.write(DLMTestUtil.getLogRecordInstance(txId++)));
        }
        writerv4.closeAndComplete();
    }
    List<LogSegmentMetadata> segmentsv4 = dlmv4.getLogSegments();
    assertEquals(11, segmentsv4.size());
    for (int i = 7; i < 11; i++) {
        assertFalse(segmentsv4.get(i).isInProgress());
        assertTrue(segmentsv4.get(i).getStartSequenceId() < 0);
    }
    dlmv4.close();
    // wait until readers read all records
    while (readRecords.size() < txId) {
        Thread.sleep(100);
    }
    assertEquals(txId, readRecords.size());
    long sequenceId = Long.MIN_VALUE;
    for (LogRecordWithDLSN record : readRecords) {
        if (record.getDlsn().getLogSegmentSequenceNo() <= 4) {
            assertTrue(record.getSequenceId() < 0);
            assertTrue(record.getSequenceId() > sequenceId);
            sequenceId = record.getSequenceId();
        } else if (record.getDlsn().getLogSegmentSequenceNo() <= 7) {
            if (sequenceId < 0L) {
                sequenceId = 0L;
            }
            assertEquals(sequenceId, record.getSequenceId());
            ++sequenceId;
        } else if (record.getDlsn().getLogSegmentSequenceNo() >= 9) {
            if (sequenceId > 0) {
                sequenceId = Long.MIN_VALUE;
            }
            assertTrue(record.getSequenceId() < 0);
            assertTrue(record.getSequenceId() > sequenceId);
            sequenceId = record.getSequenceId();
        }
    }
    readDLM.close();
}

Example 18 with AsyncLogReader

use of org.apache.distributedlog.api.AsyncLogReader in project bookkeeper by apache.

the class TestAsyncReaderWriter method testCancelReadRequestOnReaderClosed.

@Test(timeout = 60000)
public void testCancelReadRequestOnReaderClosed() throws Exception {
    final String name = "distrlog-cancel-read-requests-on-reader-closed";
    DistributedLogManager dlm = createNewDLM(testConf, name);
    BKAsyncLogWriter writer = (BKAsyncLogWriter) (dlm.startAsyncLogSegmentNonPartitioned());
    writer.write(DLMTestUtil.getLogRecordInstance(1L));
    writer.closeAndComplete();
    final AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InitialDLSN);
    LogRecordWithDLSN record = Utils.ioResult(reader.readNext());
    assertEquals(1L, record.getTransactionId());
    DLMTestUtil.verifyLogRecord(record);
    final CountDownLatch readLatch = new CountDownLatch(1);
    final AtomicBoolean receiveExpectedException = new AtomicBoolean(false);
    Thread readThread = new Thread(new Runnable() {

        @Override
        public void run() {
            try {
                Utils.ioResult(reader.readNext());
            } catch (ReadCancelledException rce) {
                receiveExpectedException.set(true);
            } catch (Throwable t) {
                LOG.error("Receive unexpected exception on reading stream {} : ", name, t);
            }
            readLatch.countDown();
        }
    }, "read-thread");
    readThread.start();
    Thread.sleep(1000);
    // close reader should cancel the pending read next
    Utils.close(reader);
    readLatch.await();
    readThread.join();
    assertTrue("Read request should be cancelled.", receiveExpectedException.get());
    // closed reader should reject any readNext
    try {
        Utils.ioResult(reader.readNext());
        fail("Reader should reject readNext if it is closed.");
    } catch (ReadCancelledException rce) {
    // expected
    }
    dlm.close();
}

Example 19 with AsyncLogReader

use of org.apache.distributedlog.api.AsyncLogReader in project bookkeeper by apache.

the class TestAsyncReaderWriter method testReadBrokenEntriesAndLargeBatchSize.

@Test(timeout = 60000)
public void testReadBrokenEntriesAndLargeBatchSize() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setReadAheadWaitTime(10);
    confLocal.setReadAheadBatchSize(5);
    confLocal.setPositionGapDetectionEnabled(false);
    confLocal.setReadAheadSkipBrokenEntries(true);
    confLocal.setEIInjectReadAheadBrokenEntries(true);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    int numLogSegments = 1;
    int numRecordsPerLogSegment = 100;
    long txid = 1L;
    txid = writeRecords(dlm, numLogSegments, numRecordsPerLogSegment, txid, false);
    AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InvalidDLSN);
    // And so on, so 5 records in each 10 will be discarded, for 50 good records.
    for (int i = 0; i < 50; i++) {
        LogRecordWithDLSN record = Utils.ioResult(reader.readNext());
        assertFalse(record.getDlsn().getEntryId() % 10 == 0);
    }
    Utils.close(reader);
    dlm.close();
}

Example 20 with AsyncLogReader

use of org.apache.distributedlog.api.AsyncLogReader in project bookkeeper by apache.

the class TestAsyncReaderWriter method testWriteRecordSet.

@Test(timeout = 60000)
public void testWriteRecordSet() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    URI uri = createDLMURI("/" + name);
    ensureURICreated(uri);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    BKAsyncLogWriter writer = (BKAsyncLogWriter) dlm.startAsyncLogSegmentNonPartitioned();
    List<CompletableFuture<DLSN>> writeFutures = Lists.newArrayList();
    for (int i = 0; i < 5; i++) {
        LogRecord record = DLMTestUtil.getLogRecordInstance(1L + i);
        writeFutures.add(writer.write(record));
    }
    List<CompletableFuture<DLSN>> recordSetFutures = Lists.newArrayList();
    // write another 5 records
    final LogRecordSet.Writer recordSetWriter = LogRecordSet.newWriter(4096, CompressionCodec.Type.LZ4);
    for (int i = 0; i < 5; i++) {
        LogRecord record = DLMTestUtil.getLogRecordInstance(6L + i);
        CompletableFuture<DLSN> writePromise = new CompletableFuture<DLSN>();
        recordSetWriter.writeRecord(ByteBuffer.wrap(record.getPayload()), writePromise);
        recordSetFutures.add(writePromise);
    }
    final ByteBuf recordSetBuffer = recordSetWriter.getBuffer();
    LogRecord setRecord = new LogRecord(6L, recordSetBuffer);
    setRecord.setRecordSet();
    CompletableFuture<DLSN> writeRecordSetFuture = writer.write(setRecord);
    writeRecordSetFuture.whenComplete(new FutureEventListener<DLSN>() {

        @Override
        public void onSuccess(DLSN dlsn) {
            recordSetWriter.completeTransmit(dlsn.getLogSegmentSequenceNo(), dlsn.getEntryId(), dlsn.getSlotId());
        }

        @Override
        public void onFailure(Throwable cause) {
            recordSetWriter.abortTransmit(cause);
        }
    });
    writeFutures.add(writeRecordSetFuture);
    Utils.ioResult(writeRecordSetFuture);
    // write last 5 records
    for (int i = 0; i < 5; i++) {
        LogRecord record = DLMTestUtil.getLogRecordInstance(11L + i);
        CompletableFuture<DLSN> writeFuture = writer.write(record);
        writeFutures.add(writeFuture);
        // make sure get log record count returns the right count
        if (i == 0) {
            Utils.ioResult(writeFuture);
            assertEquals(10, dlm.getLogRecordCount());
        }
    }
    List<DLSN> writeResults = Utils.ioResult(FutureUtils.collect(writeFutures));
    for (int i = 0; i < 5; i++) {
        Assert.assertEquals(new DLSN(1L, i, 0L), writeResults.get(i));
    }
    Assert.assertEquals(new DLSN(1L, 5L, 0L), writeResults.get(5));
    for (int i = 0; i < 5; i++) {
        Assert.assertEquals(new DLSN(1L, 6L + i, 0L), writeResults.get(6 + i));
    }
    List<DLSN> recordSetWriteResults = Utils.ioResult(FutureUtils.collect(recordSetFutures));
    for (int i = 0; i < 5; i++) {
        Assert.assertEquals(new DLSN(1L, 5L, i), recordSetWriteResults.get(i));
    }
    Utils.ioResult(writer.flushAndCommit());
    DistributedLogConfiguration readConf1 = new DistributedLogConfiguration();
    readConf1.addConfiguration(confLocal);
    readConf1.setDeserializeRecordSetOnReads(true);
    DistributedLogManager readDLM1 = createNewDLM(readConf1, name);
    AsyncLogReader reader1 = readDLM1.getAsyncLogReader(DLSN.InitialDLSN);
    for (int i = 0; i < 15; i++) {
        LogRecordWithDLSN record = Utils.ioResult(reader1.readNext());
        if (i < 5) {
            assertEquals(new DLSN(1L, i, 0L), record.getDlsn());
            assertEquals(1L + i, record.getTransactionId());
        } else if (i >= 10) {
            assertEquals(new DLSN(1L, 6L + i - 10, 0L), record.getDlsn());
            assertEquals(11L + i - 10, record.getTransactionId());
        } else {
            assertEquals(new DLSN(1L, 5L, i - 5), record.getDlsn());
            assertEquals(6L, record.getTransactionId());
        }
        assertEquals(i + 1, record.getPositionWithinLogSegment());
        assertArrayEquals(DLMTestUtil.generatePayload(i + 1), record.getPayload());
    }
    Utils.close(reader1);
    readDLM1.close();
    DistributedLogConfiguration readConf2 = new DistributedLogConfiguration();
    readConf2.addConfiguration(confLocal);
    readConf2.setDeserializeRecordSetOnReads(false);
    DistributedLogManager readDLM2 = createNewDLM(readConf2, name);
    AsyncLogReader reader2 = readDLM2.getAsyncLogReader(DLSN.InitialDLSN);
    for (int i = 0; i < 11; i++) {
        LogRecordWithDLSN record = Utils.ioResult(reader2.readNext());
        LOG.info("Read record {}", record);
        if (i < 5) {
            assertEquals(new DLSN(1L, i, 0L), record.getDlsn());
            assertEquals(1L + i, record.getTransactionId());
            assertEquals(i + 1, record.getPositionWithinLogSegment());
            assertArrayEquals(DLMTestUtil.generatePayload(i + 1), record.getPayload());
        } else if (i >= 6L) {
            assertEquals(new DLSN(1L, 6L + i - 6, 0L), record.getDlsn());
            assertEquals(11L + i - 6, record.getTransactionId());
            assertEquals(11 + i - 6, record.getPositionWithinLogSegment());
            assertArrayEquals(DLMTestUtil.generatePayload(11L + i - 6), record.getPayload());
        } else {
            assertEquals(new DLSN(1L, 5L, 0), record.getDlsn());
            assertEquals(6L, record.getTransactionId());
            assertEquals(6, record.getPositionWithinLogSegment());
            assertTrue(record.isRecordSet());
            assertEquals(5, LogRecordSet.numRecords(record));
        }
    }
    Utils.close(reader2);
    readDLM2.close();
}