Examples with AsyncLogReader org.apache.distributedlog.api.AsyncLogReader used on opensource projects

Example 11 with AsyncLogReader

use of org.apache.distributedlog.api.AsyncLogReader in project bookkeeper by apache.

the class TestAsyncReaderWriter method testIdleReaderExceptionWhenKeepAliveIsEnabled.

@Test(timeout = 60000)
public void testIdleReaderExceptionWhenKeepAliveIsEnabled() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setPeriodicKeepAliveMilliSeconds(1000);
    confLocal.setReadLACLongPollTimeout(999);
    confLocal.setReaderIdleWarnThresholdMillis(2000);
    confLocal.setReaderIdleErrorThresholdMillis(4000);
    URI uri = createDLMURI("/" + name);
    ensureURICreated(uri);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    BKAsyncLogWriter writer = (BKAsyncLogWriter) Utils.ioResult(dlm.openAsyncLogWriter());
    writer.write(DLMTestUtil.getLogRecordInstance(1L));
    AsyncLogReader reader = Utils.ioResult(dlm.openAsyncLogReader(DLSN.InitialDLSN));
    LogRecordWithDLSN record = Utils.ioResult(reader.readNext());
    assertEquals(1L, record.getTransactionId());
    DLMTestUtil.verifyLogRecord(record);
    Utils.close(reader);
    writer.close();
    dlm.close();
}

Example 12 with AsyncLogReader

use of org.apache.distributedlog.api.AsyncLogReader in project bookkeeper by apache.

the class TestAsyncReaderWriter method testIdleReaderExceptionWhenKeepAliveIsDisabled.

@Test(timeout = 60000)
public void testIdleReaderExceptionWhenKeepAliveIsDisabled() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setPeriodicKeepAliveMilliSeconds(0);
    confLocal.setReadLACLongPollTimeout(9);
    confLocal.setReaderIdleWarnThresholdMillis(20);
    confLocal.setReaderIdleErrorThresholdMillis(40);
    URI uri = createDLMURI("/" + name);
    ensureURICreated(uri);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    BKAsyncLogWriter writer = (BKAsyncLogWriter) Utils.ioResult(dlm.openAsyncLogWriter());
    writer.write(DLMTestUtil.getLogRecordInstance(1L));
    AsyncLogReader reader = Utils.ioResult(dlm.openAsyncLogReader(DLSN.InitialDLSN));
    try {
        Utils.ioResult(reader.readNext());
        fail("Should fail when stream is idle");
    } catch (IdleReaderException ire) {
    // expected
    }
    Utils.close(reader);
    writer.close();
    dlm.close();
}

Example 13 with AsyncLogReader

use of org.apache.distributedlog.api.AsyncLogReader in project bookkeeper by apache.

the class TestAsyncReaderWriter method testSimpleAsyncReadPosition.

/**
 * Test Async Read by positioning to a given position in the log.
 * @throws Exception
 */
@Test(timeout = 60000)
public void testSimpleAsyncReadPosition() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(testConf);
    confLocal.setOutputBufferSize(1024);
    confLocal.setReadAheadWaitTime(10);
    confLocal.setReadAheadBatchSize(10);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    int numLogSegments = 3;
    int numRecordsPerLogSegment = 10;
    long txid = 1L;
    // write 3 log segments, 10 records per log segment
    txid = writeRecords(dlm, numLogSegments, numRecordsPerLogSegment, txid, false);
    // write another log segment with 5 records
    txid = writeLogSegment(dlm, 5, txid, Integer.MAX_VALUE, false);
    final CountDownLatch syncLatch = new CountDownLatch((int) (txid - 14));
    final CountDownLatch doneLatch = new CountDownLatch(1);
    final AtomicBoolean errorsFound = new AtomicBoolean(false);
    final AsyncLogReader reader = dlm.getAsyncLogReader(new DLSN(2, 2, 4));
    assertEquals(name, reader.getStreamName());
    boolean monotonic = LogSegmentMetadata.supportsSequenceId(confLocal.getDLLedgerMetadataLayoutVersion());
    TestAsyncReaderWriter.readNext(reader, new DLSN(2, 3, 0), monotonic ? 13L : Long.MIN_VALUE, monotonic, syncLatch, doneLatch, errorsFound);
    doneLatch.await();
    assertFalse("Errors found on reading records", errorsFound.get());
    syncLatch.await();
    Utils.close(reader);
    dlm.close();
}

Example 14 with AsyncLogReader

use of org.apache.distributedlog.api.AsyncLogReader in project bookkeeper by apache.

the class TestAsyncReaderWriter method testReadBrokenEntries.

@Test(timeout = 60000)
public void testReadBrokenEntries() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setReadAheadWaitTime(10);
    confLocal.setReadAheadBatchSize(1);
    confLocal.setPositionGapDetectionEnabled(false);
    confLocal.setReadAheadSkipBrokenEntries(true);
    confLocal.setEIInjectReadAheadBrokenEntries(true);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    int numLogSegments = 3;
    int numRecordsPerLogSegment = 10;
    long txid = 1L;
    txid = writeRecords(dlm, numLogSegments, numRecordsPerLogSegment, txid, false);
    AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InvalidDLSN);
    // record in each ledger is discarded, for 30 - 3 = 27 records.
    for (int i = 0; i < 27; i++) {
        LogRecordWithDLSN record = Utils.ioResult(reader.readNext());
        assertFalse(record.getDlsn().getEntryId() % 10 == 0);
    }
    Utils.close(reader);
    dlm.close();
}

Example 15 with AsyncLogReader

use of org.apache.distributedlog.api.AsyncLogReader in project bookkeeper by apache.

the class TestReader method positionReader.

private void positionReader(final DLSN dlsn) {
    positionReaderCount.incrementAndGet();
    Runnable runnable = new Runnable() {

        @Override
        public void run() {
            try {
                AsyncLogReader reader = dlm.getAsyncLogReader(dlsn);
                if (simulateErrors) {
                    ((BKAsyncLogReader) reader).simulateErrors();
                }
                nextDLSN = dlsn;
                LOG.info("Positioned reader {} at {}", readerName, dlsn);
                if (null != TestReader.this.reader) {
                    Utils.close(TestReader.this.reader);
                }
                TestReader.this.reader = reader;
                readNext();
                readyLatch.countDown();
            } catch (IOException exc) {
                int nextMs = nextDelayMs();
                LOG.info("Encountered exception {} on opening reader {} at {}, retrying in {} ms", new Object[] { exc, readerName, dlsn, nextMs });
                positionReader(dlsn);
            }
        }
    };
    executorService.schedule(runnable, delayMs, TimeUnit.MILLISECONDS);
}