use of com.twitter.distributedlog.exceptions.EndOfStreamException in project distributedlog by twitter.
the class BKSyncLogReaderDLSN method readNext.
@Override
public synchronized LogRecordWithDLSN readNext(boolean nonBlocking) throws IOException {
if (null != readerException.get()) {
throw readerException.get();
}
LogRecordWithDLSN record = null;
if (nonBlocking) {
record = readAheadRecords.poll();
} else {
try {
// reader is still catching up, waiting for next record
while (!reader.bkLedgerManager.isReadAheadCaughtUp() && null == readerException.get() && null == record) {
record = readAheadRecords.poll(maxReadAheadWaitTime, TimeUnit.MILLISECONDS);
}
// reader caught up
boolean shallWait = true;
while (shallWait && reader.bkLedgerManager.isReadAheadCaughtUp() && null == record && null == readerException.get()) {
record = readAheadRecords.poll(maxReadAheadWaitTime, TimeUnit.MILLISECONDS);
if (null != record) {
break;
}
DLSN lastDLSNSeenByReadAhead = reader.bkLedgerManager.readAheadCache.getLastReadAheadUserDLSN();
// if last seen DLSN by reader is same as the one seen by ReadAhead
// that means that reader is caught up with ReadAhead and ReadAhead
// is caught up with stream
shallWait = DLSN.InitialDLSN != lastDLSNSeenByReadAhead && lastSeenDLSN.compareTo(lastDLSNSeenByReadAhead) < 0 && startDLSN.compareTo(lastDLSNSeenByReadAhead) <= 0;
}
} catch (InterruptedException e) {
throw new DLInterruptedException("Interrupted on waiting next available log record for stream " + reader.getStreamName(), e);
}
}
if (null != readerException.get()) {
throw readerException.get();
}
if (null != record) {
if (record.isEndOfStream()) {
EndOfStreamException eos = new EndOfStreamException("End of Stream Reached for " + reader.bkLedgerManager.getFullyQualifiedName());
readerException.compareAndSet(null, eos);
throw eos;
}
invokeReadAheadCallback();
}
return record;
}
use of com.twitter.distributedlog.exceptions.EndOfStreamException in project distributedlog by twitter.
the class TestAsyncReaderWriter method testMarkEndOfStreamAtBeginningOfSegment.
@Test(timeout = 60000)
public void testMarkEndOfStreamAtBeginningOfSegment() throws Exception {
String name = runtime.getMethodName();
DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
confLocal.addConfiguration(testConf);
confLocal.setOutputBufferSize(0);
confLocal.setImmediateFlushEnabled(true);
confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
DistributedLogManager dlm = createNewDLM(confLocal, name);
BKAsyncLogWriter writer = (BKAsyncLogWriter) dlm.startAsyncLogSegmentNonPartitioned();
Await.result(writer.markEndOfStream());
try {
Await.result(writer.write(DLMTestUtil.getLogRecordInstance(1)));
fail("Should have thrown");
} catch (EndOfStreamException ex) {
}
BKAsyncLogReaderDLSN reader = (BKAsyncLogReaderDLSN) dlm.getAsyncLogReader(DLSN.InitialDLSN);
try {
LogRecord record = Await.result(reader.readNext());
fail("Should have thrown");
} catch (EndOfStreamException ex) {
}
}
use of com.twitter.distributedlog.exceptions.EndOfStreamException in project distributedlog by twitter.
the class TestBKLogSegmentWriter method testNondurableWriteAfterEndOfStream.
/**
* Non durable write should fail if writer is marked as end of stream.
*
* @throws Exception
*/
@Test(timeout = 60000)
public void testNondurableWriteAfterEndOfStream() throws Exception {
DistributedLogConfiguration confLocal = newLocalConf();
confLocal.setImmediateFlushEnabled(false);
confLocal.setOutputBufferSize(Integer.MAX_VALUE);
confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
confLocal.setDurableWriteEnabled(false);
ZKDistributedLock lock = createLock("/test/lock-" + runtime.getMethodName(), zkc, true);
BKLogSegmentWriter writer = createLogSegmentWriter(confLocal, 0L, -1L, lock);
FutureUtils.result(writer.markEndOfStream());
try {
Await.result(writer.asyncWrite(DLMTestUtil.getLogRecordInstance(1)));
fail("Should fail the write if the writer is marked as end of stream");
} catch (EndOfStreamException we) {
// expected
}
closeWriterAndLock(writer, lock);
}
use of com.twitter.distributedlog.exceptions.EndOfStreamException in project distributedlog by twitter.
the class BKAsyncLogReaderDLSN method run.
@Override
public void run() {
synchronized (scheduleLock) {
if (scheduleDelayStopwatch.isRunning()) {
scheduleLatency.registerSuccessfulEvent(scheduleDelayStopwatch.stop().elapsed(TimeUnit.MICROSECONDS));
}
Stopwatch runTime = Stopwatch.createStarted();
int iterations = 0;
long scheduleCountLocal = scheduleCount.get();
LOG.debug("{}: Scheduled Background Reader", bkLedgerManager.getFullyQualifiedName());
while (true) {
if (LOG.isTraceEnabled()) {
LOG.trace("{}: Executing Iteration: {}", bkLedgerManager.getFullyQualifiedName(), iterations++);
}
PendingReadRequest nextRequest = null;
synchronized (this) {
nextRequest = pendingRequests.peek();
// Queue is empty, nothing to read, return
if (null == nextRequest) {
LOG.trace("{}: Queue Empty waiting for Input", bkLedgerManager.getFullyQualifiedName());
scheduleCount.set(0);
backgroundReaderRunTime.registerSuccessfulEvent(runTime.stop().elapsed(TimeUnit.MICROSECONDS));
return;
}
if (disableProcessingReadRequests) {
LOG.info("Reader of {} is forced to stop processing read requests", bkLedgerManager.getFullyQualifiedName());
return;
}
}
// know the last consumed read
if (null == lastException.get()) {
if (nextRequest.getPromise().isInterrupted().isDefined()) {
setLastException(new DLInterruptedException("Interrupted on reading " + bkLedgerManager.getFullyQualifiedName() + " : ", nextRequest.getPromise().isInterrupted().get()));
}
}
if (checkClosedOrInError("readNext")) {
if (!(lastException.get().getCause() instanceof LogNotFoundException)) {
LOG.warn("{}: Exception", bkLedgerManager.getFullyQualifiedName(), lastException.get());
}
backgroundReaderRunTime.registerFailedEvent(runTime.stop().elapsed(TimeUnit.MICROSECONDS));
return;
}
try {
// Fail 10% of the requests when asked to simulate errors
if (failureInjector.shouldInjectErrors()) {
throw new IOException("Reader Simulated Exception");
}
LogRecordWithDLSN record;
while (!nextRequest.hasReadEnoughRecords()) {
// read single record
do {
record = bkLedgerManager.getNextReadAheadRecord();
} while (null != record && (record.isControl() || (record.getDlsn().compareTo(getStartDLSN()) < 0)));
if (null == record) {
break;
} else {
if (record.isEndOfStream() && !returnEndOfStreamRecord) {
setLastException(new EndOfStreamException("End of Stream Reached for " + bkLedgerManager.getFullyQualifiedName()));
break;
}
// gap detection
if (recordPositionsContainsGap(record, lastPosition)) {
bkDistributedLogManager.raiseAlert("Gap detected between records at dlsn = {}", record.getDlsn());
if (positionGapDetectionEnabled) {
throw new DLIllegalStateException("Gap detected between records at dlsn = " + record.getDlsn());
}
}
lastPosition = record.getLastPositionWithinLogSegment();
nextRequest.addRecord(record);
}
}
;
} catch (IOException exc) {
setLastException(exc);
if (!(exc instanceof LogNotFoundException)) {
LOG.warn("{} : read with skip Exception", bkLedgerManager.getFullyQualifiedName(), lastException.get());
}
continue;
}
if (nextRequest.hasReadRecords()) {
long remainingWaitTime = nextRequest.getRemainingWaitTime();
if (remainingWaitTime > 0 && !nextRequest.hasReadEnoughRecords()) {
backgroundReaderRunTime.registerSuccessfulEvent(runTime.stop().elapsed(TimeUnit.MICROSECONDS));
scheduleDelayStopwatch.reset().start();
scheduleCount.set(0);
// the request could still wait for more records
backgroundScheduleTask = executorService.schedule(BACKGROUND_READ_SCHEDULER, remainingWaitTime, nextRequest.deadlineTimeUnit);
return;
}
PendingReadRequest request = pendingRequests.poll();
if (null != request && nextRequest == request) {
request.complete();
if (null != backgroundScheduleTask) {
backgroundScheduleTask.cancel(true);
backgroundScheduleTask = null;
}
} else {
DLIllegalStateException ise = new DLIllegalStateException("Unexpected condition at dlsn = " + nextRequest.records.get(0).getDlsn());
nextRequest.setException(ise);
if (null != request) {
request.setException(ise);
}
// We should never get here as we should have exited the loop if
// pendingRequests were empty
bkDistributedLogManager.raiseAlert("Unexpected condition at dlsn = {}", nextRequest.records.get(0).getDlsn());
setLastException(ise);
}
} else {
if (0 == scheduleCountLocal) {
LOG.trace("Schedule count dropping to zero", lastException.get());
backgroundReaderRunTime.registerSuccessfulEvent(runTime.stop().elapsed(TimeUnit.MICROSECONDS));
return;
}
scheduleCountLocal = scheduleCount.decrementAndGet();
}
}
}
}
use of com.twitter.distributedlog.exceptions.EndOfStreamException in project distributedlog by twitter.
the class TestAsyncReaderWriter method testMarkEndOfStream.
@Test(timeout = 60000)
public void testMarkEndOfStream() throws Exception {
String name = runtime.getMethodName();
DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
confLocal.addConfiguration(testConf);
confLocal.setOutputBufferSize(0);
confLocal.setImmediateFlushEnabled(true);
confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
DistributedLogManager dlm = createNewDLM(confLocal, name);
BKAsyncLogWriter writer = (BKAsyncLogWriter) dlm.startAsyncLogSegmentNonPartitioned();
final int NUM_RECORDS = 10;
int i = 1;
for (; i <= NUM_RECORDS; i++) {
Await.result(writer.write(DLMTestUtil.getLogRecordInstance(i)));
assertEquals("last tx id should become " + i, i, writer.getLastTxId());
}
Await.result(writer.markEndOfStream());
// Multiple end of streams are ok.
Await.result(writer.markEndOfStream());
try {
Await.result(writer.write(DLMTestUtil.getLogRecordInstance(i)));
fail("Should have thrown");
} catch (EndOfStreamException ex) {
}
BKAsyncLogReaderDLSN reader = (BKAsyncLogReaderDLSN) dlm.getAsyncLogReader(DLSN.InitialDLSN);
LogRecord record = null;
for (int j = 0; j < NUM_RECORDS; j++) {
record = Await.result(reader.readNext());
assertEquals(j + 1, record.getTransactionId());
}
try {
record = Await.result(reader.readNext());
fail("Should have thrown");
} catch (EndOfStreamException ex) {
}
}
Aggregations