Examples with IdleReaderException com.twitter.distributedlog.exceptions.IdleReaderException used on opensource projects

Example 1 with IdleReaderException

use of com.twitter.distributedlog.exceptions.IdleReaderException in project distributedlog by twitter.

the class BKAsyncLogReaderDLSN method scheduleIdleReaderTaskIfNecessary.

private ScheduledFuture<?> scheduleIdleReaderTaskIfNecessary() {
    if (idleErrorThresholdMillis < Integer.MAX_VALUE) {
        // Dont run the task more than once every seconds (for sanity)
        long period = Math.max(idleErrorThresholdMillis / 10, 1000);
        // Except when idle reader threshold is less than a second (tests?)
        period = Math.min(period, idleErrorThresholdMillis / 5);
        return executorService.scheduleAtFixedRate(new Runnable() {

            @Override
            public void run() {
                PendingReadRequest nextRequest = pendingRequests.peek();
                idleReaderCheckCount.inc();
                if (null == nextRequest) {
                    return;
                }
                idleReaderCheckIdleReadRequestCount.inc();
                if (nextRequest.elapsedSinceEnqueue(TimeUnit.MILLISECONDS) < idleErrorThresholdMillis) {
                    return;
                }
                ReadAheadCache cache = bkLedgerManager.getReadAheadCache();
                // read request has been idle
                //   - cache has records but read request are idle,
                //     that means notification was missed between readahead and reader.
                //   - cache is empty and readahead is idle (no records added for a long time)
                idleReaderCheckIdleReadAheadCount.inc();
                if (cache.getNumCachedRecords() <= 0 && !cache.isReadAheadIdle(idleErrorThresholdMillis, TimeUnit.MILLISECONDS)) {
                    return;
                }
                idleReaderError.inc();
                IdleReaderException ire = new IdleReaderException("Reader on stream " + bkLedgerManager.getFullyQualifiedName() + " is idle for " + idleErrorThresholdMillis + " ms");
                setLastException(ire);
                // cancel all pending reads directly rather than notifying on error
                // because idle reader could happen on idle read requests that usually means something wrong
                // in scheduling reads
                cancelAllPendingReads(ire);
            }
        }, period, period, TimeUnit.MILLISECONDS);
    }
    return null;
}

Example 2 with IdleReaderException

use of com.twitter.distributedlog.exceptions.IdleReaderException in project distributedlog by twitter.

the class TestNonBlockingReads method testNonBlockingReadAheadStall.

@Test(timeout = 60000)
public void testNonBlockingReadAheadStall() throws Exception {
    String name = "distrlog-non-blocking-reader-stall";
    final DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setReadAheadBatchSize(1);
    confLocal.setReadAheadMaxRecords(3);
    confLocal.setReaderIdleWarnThresholdMillis(500);
    confLocal.setReaderIdleErrorThresholdMillis(30000);
    final DistributedLogManager dlm = createNewDLM(confLocal, name);
    ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1);
    ScheduledFuture writerClosedFuture = null;
    try {
        final Thread currentThread = Thread.currentThread();
        writerClosedFuture = executor.schedule(new Runnable() {

            @Override
            public void run() {
                try {
                    writeRecordsForNonBlockingReads(confLocal, dlm, false, 3);
                } catch (Exception exc) {
                    currentThread.interrupt();
                }
            }
        }, 10, TimeUnit.MILLISECONDS);
        boolean exceptionEncountered = false;
        try {
            readNonBlocking(dlm, false, 3, false);
        } catch (IdleReaderException exc) {
            LOG.info("Exception encountered", exc);
            exceptionEncountered = true;
        }
        assertFalse(exceptionEncountered);
        assertFalse(currentThread.isInterrupted());
    } finally {
        if (writerClosedFuture != null) {
            // ensure writer.closeAndComplete is done before we close dlm
            writerClosedFuture.get();
        }
        executor.shutdown();
        dlm.close();
    }
}

Example 3 with IdleReaderException

use of com.twitter.distributedlog.exceptions.IdleReaderException in project distributedlog by twitter.

the class TestNonBlockingReads method testNonBlockingReadIdleError.

@Test(timeout = 100000)
public void testNonBlockingReadIdleError() throws Exception {
    String name = "distrlog-non-blocking-reader-error";
    final DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setReadAheadBatchSize(1);
    confLocal.setReadAheadMaxRecords(1);
    confLocal.setReaderIdleWarnThresholdMillis(50);
    confLocal.setReaderIdleErrorThresholdMillis(100);
    final DistributedLogManager dlm = createNewDLM(confLocal, name);
    ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1);
    ScheduledFuture writerClosedFuture = null;
    try {
        final Thread currentThread = Thread.currentThread();
        writerClosedFuture = executor.schedule(new Runnable() {

            @Override
            public void run() {
                try {
                    writeRecordsForNonBlockingReads(confLocal, dlm, false);
                } catch (Exception exc) {
                    currentThread.interrupt();
                }
            }
        }, 100, TimeUnit.MILLISECONDS);
        boolean exceptionEncountered = false;
        try {
            readNonBlocking(dlm, false, DEFAULT_SEGMENT_SIZE, true);
        } catch (IdleReaderException exc) {
            exceptionEncountered = true;
        }
        assertTrue(exceptionEncountered);
        assertFalse(currentThread.isInterrupted());
    } finally {
        if (writerClosedFuture != null) {
            // ensure writer.closeAndComplete is done before we close dlm
            writerClosedFuture.get();
        }
        executor.shutdown();
        dlm.close();
    }
}

Example 4 with IdleReaderException

use of com.twitter.distributedlog.exceptions.IdleReaderException in project distributedlog by twitter.

the class TestAsyncReaderWriter method testAsyncReadMissingZKNotification.

@DistributedLogAnnotations.FlakyTest
@Ignore
@Test(timeout = 60000)
public void testAsyncReadMissingZKNotification() throws Exception {
    String name = "distrlog-async-reader-missing-zk-notification";
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setReadAheadBatchSize(1);
    confLocal.setReadAheadMaxRecords(1);
    confLocal.setReaderIdleWarnThresholdMillis(100);
    confLocal.setReaderIdleErrorThresholdMillis(20000);
    final DistributedLogManager dlm = createNewDLM(confLocal, name);
    final Thread currentThread = Thread.currentThread();
    final int segmentSize = 10;
    final int numSegments = 3;
    final CountDownLatch latch = new CountDownLatch(1);
    final ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1);
    executor.schedule(new Runnable() {

        @Override
        public void run() {
            try {
                int txid = 1;
                for (long i = 0; i < numSegments; i++) {
                    long start = txid;
                    BKSyncLogWriter writer = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
                    for (long j = 1; j <= segmentSize; j++) {
                        writer.write(DLMTestUtil.getLargeLogRecordInstance(txid++));
                        if ((i == 0) && (j == 1)) {
                            latch.countDown();
                        }
                    }
                    writer.closeAndComplete();
                    Thread.sleep(100);
                }
            } catch (Exception exc) {
                if (!executor.isShutdown()) {
                    currentThread.interrupt();
                }
            }
        }
    }, 0, TimeUnit.MILLISECONDS);
    latch.await();
    BKAsyncLogReaderDLSN reader = (BKAsyncLogReaderDLSN) dlm.getAsyncLogReader(DLSN.InitialDLSN);
    reader.disableReadAheadZKNotification();
    boolean exceptionEncountered = false;
    int recordCount = 0;
    try {
        while (true) {
            Future<LogRecordWithDLSN> record = reader.readNext();
            Await.result(record);
            recordCount++;
            if (recordCount >= segmentSize * numSegments) {
                break;
            }
        }
    } catch (IdleReaderException exc) {
        exceptionEncountered = true;
    }
    assert (!exceptionEncountered);
    Assert.assertEquals(recordCount, segmentSize * numSegments);
    assert (!currentThread.isInterrupted());
    executor.shutdown();
}

Example 5 with IdleReaderException

use of com.twitter.distributedlog.exceptions.IdleReaderException in project distributedlog by twitter.

the class TestAsyncReaderWriter method testAsyncReadIdleErrorInternal.

private void testAsyncReadIdleErrorInternal(String name, final int idleReaderErrorThreshold, final boolean heartBeatUsingControlRecs, final boolean simulateReaderStall) throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setReadAheadBatchSize(1);
    confLocal.setReadAheadMaxRecords(1);
    confLocal.setReaderIdleWarnThresholdMillis(50);
    confLocal.setReaderIdleErrorThresholdMillis(idleReaderErrorThreshold);
    final DistributedLogManager dlm = createNewDLM(confLocal, name);
    final Thread currentThread = Thread.currentThread();
    final int segmentSize = 3;
    final int numSegments = 3;
    final CountDownLatch latch = new CountDownLatch(1);
    final ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1);
    executor.schedule(new Runnable() {

        @Override
        public void run() {
            try {
                int txid = 1;
                for (long i = 0; i < numSegments; i++) {
                    long start = txid;
                    BKSyncLogWriter writer = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
                    for (long j = 1; j <= segmentSize; j++) {
                        writer.write(DLMTestUtil.getLargeLogRecordInstance(txid++));
                        if ((i == 0) && (j == 1)) {
                            latch.countDown();
                        }
                    }
                    if (heartBeatUsingControlRecs) {
                        // There should be a control record such that
                        // wait time + commit time (BK) < Idle Reader Threshold
                        int threadSleepTime = idleReaderErrorThreshold - // BK commitTime
                        200 - //safety margin
                        100;
                        for (int iter = 1; iter <= (2 * idleReaderErrorThreshold / threadSleepTime); iter++) {
                            Thread.sleep(threadSleepTime);
                            writer.write(DLMTestUtil.getLargeLogRecordInstance(txid, true));
                            writer.setReadyToFlush();
                        }
                        Thread.sleep(threadSleepTime);
                    }
                    writer.closeAndComplete();
                    if (!heartBeatUsingControlRecs) {
                        Thread.sleep(2 * idleReaderErrorThreshold);
                    }
                }
            } catch (Exception exc) {
                if (!executor.isShutdown()) {
                    currentThread.interrupt();
                }
            }
        }
    }, 0, TimeUnit.MILLISECONDS);
    latch.await();
    BKAsyncLogReaderDLSN reader = (BKAsyncLogReaderDLSN) dlm.getAsyncLogReader(DLSN.InitialDLSN);
    if (simulateReaderStall) {
        reader.disableProcessingReadRequests();
    }
    boolean exceptionEncountered = false;
    int recordCount = 0;
    try {
        while (true) {
            Future<LogRecordWithDLSN> record = reader.readNext();
            Await.result(record);
            recordCount++;
            if (recordCount >= segmentSize * numSegments) {
                break;
            }
        }
    } catch (IdleReaderException exc) {
        exceptionEncountered = true;
    }
    if (simulateReaderStall) {
        assertTrue(exceptionEncountered);
    } else if (heartBeatUsingControlRecs) {
        assertFalse(exceptionEncountered);
        Assert.assertEquals(segmentSize * numSegments, recordCount);
    } else {
        assertTrue(exceptionEncountered);
        Assert.assertEquals(segmentSize, recordCount);
    }
    assertFalse(currentThread.isInterrupted());
    executor.shutdown();
}