use of org.apache.distributedlog.api.AsyncLogReader in project bookkeeper by apache.
the class TestAsyncReaderWriter method testSimpleAsyncReadWritePiggyBack.
@Test(timeout = 60000)
public void testSimpleAsyncReadWritePiggyBack() throws Exception {
String name = runtime.getMethodName();
DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
confLocal.loadConf(testConf);
confLocal.setEnableReadAhead(true);
confLocal.setReadAheadWaitTime(500);
confLocal.setReadAheadBatchSize(10);
confLocal.setReadAheadMaxRecords(100);
confLocal.setOutputBufferSize(1024);
confLocal.setPeriodicFlushFrequencyMilliSeconds(100);
DistributedLogManager dlm = createNewDLM(confLocal, name);
final AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InvalidDLSN);
int numLogSegments = 3;
int numRecordsPerLogSegment = 10;
final CountDownLatch readLatch = new CountDownLatch(30);
final CountDownLatch readDoneLatch = new CountDownLatch(1);
final AtomicBoolean readErrors = new AtomicBoolean(false);
final CountDownLatch writeLatch = new CountDownLatch(30);
final AtomicBoolean writeErrors = new AtomicBoolean(false);
int txid = 1;
for (long i = 0; i < numLogSegments; i++) {
final long currentLogSegmentSeqNo = i + 1;
BKAsyncLogWriter writer = (BKAsyncLogWriter) (dlm.startAsyncLogSegmentNonPartitioned());
for (long j = 0; j < numRecordsPerLogSegment; j++) {
Thread.sleep(50);
final LogRecord record = DLMTestUtil.getLargeLogRecordInstance(txid++);
CompletableFuture<DLSN> dlsnFuture = writer.write(record);
dlsnFuture.whenComplete(new WriteFutureEventListener(record, currentLogSegmentSeqNo, j, writeLatch, writeErrors, false));
if (i == 0 && j == 0) {
boolean monotonic = LogSegmentMetadata.supportsSequenceId(confLocal.getDLLedgerMetadataLayoutVersion());
TestAsyncReaderWriter.readNext(reader, DLSN.InvalidDLSN, monotonic ? 0L : Long.MIN_VALUE, monotonic, readLatch, readDoneLatch, readErrors);
}
}
writer.closeAndComplete();
}
writeLatch.await();
assertFalse("All writes should succeed", writeErrors.get());
readDoneLatch.await();
assertFalse("All reads should succeed", readErrors.get());
readLatch.await();
Utils.close(reader);
dlm.close();
}
use of org.apache.distributedlog.api.AsyncLogReader in project bookkeeper by apache.
the class TestAsyncReaderWriter method testSimpleAsyncReadWriteInternal.
void testSimpleAsyncReadWriteInternal(String name, boolean immediateFlush, int logSegmentVersion) throws Exception {
DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
confLocal.loadConf(testConf);
confLocal.setReadAheadWaitTime(10);
confLocal.setReadAheadBatchSize(10);
confLocal.setOutputBufferSize(1024);
confLocal.setDLLedgerMetadataLayoutVersion(logSegmentVersion);
confLocal.setImmediateFlushEnabled(immediateFlush);
DistributedLogManager dlm = createNewDLM(confLocal, name);
int numLogSegments = 3;
int numRecordsPerLogSegment = 10;
final CountDownLatch readLatch = new CountDownLatch(numLogSegments * numRecordsPerLogSegment);
final CountDownLatch readDoneLatch = new CountDownLatch(1);
final AtomicBoolean readErrors = new AtomicBoolean(false);
final CountDownLatch writeLatch = new CountDownLatch(numLogSegments * numRecordsPerLogSegment);
final AtomicBoolean writeErrors = new AtomicBoolean(false);
final AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InvalidDLSN);
assertEquals(name, reader.getStreamName());
int txid = 1;
for (long i = 0; i < 3; i++) {
final long currentLogSegmentSeqNo = i + 1;
BKAsyncLogWriter writer = (BKAsyncLogWriter) (dlm.startAsyncLogSegmentNonPartitioned());
for (long j = 0; j < 10; j++) {
final long currentEntryId = j;
final LogRecord record = DLMTestUtil.getLargeLogRecordInstance(txid++);
CompletableFuture<DLSN> dlsnFuture = writer.write(record);
dlsnFuture.whenComplete(new WriteFutureEventListener(record, currentLogSegmentSeqNo, currentEntryId, writeLatch, writeErrors, true));
if (i == 0 && j == 0) {
boolean monotonic = LogSegmentMetadata.supportsSequenceId(logSegmentVersion);
TestAsyncReaderWriter.readNext(reader, DLSN.InvalidDLSN, monotonic ? 0L : Long.MIN_VALUE, monotonic, readLatch, readDoneLatch, readErrors);
}
}
writer.closeAndComplete();
}
writeLatch.await();
assertFalse("All writes should succeed", writeErrors.get());
readDoneLatch.await();
assertFalse("All reads should succeed", readErrors.get());
readLatch.await();
Utils.close(reader);
dlm.close();
}
use of org.apache.distributedlog.api.AsyncLogReader in project bookkeeper by apache.
the class TestAsyncReaderWriter method testBulkAsyncRead.
@Test(timeout = 60000)
public void testBulkAsyncRead() throws Exception {
String name = "distrlog-bulkasyncread";
DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
confLocal.loadConf(conf);
confLocal.setOutputBufferSize(0);
confLocal.setImmediateFlushEnabled(true);
confLocal.setReadAheadWaitTime(10);
confLocal.setReadAheadMaxRecords(10000);
confLocal.setReadAheadBatchSize(10);
int numLogSegments = 3;
int numRecordsPerLogSegment = 20;
DistributedLogManager dlm = createNewDLM(confLocal, name);
writeRecords(dlm, numLogSegments, numRecordsPerLogSegment, 1L, false);
final AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InitialDLSN);
int expectedTxID = 1;
int numReads = 0;
while (expectedTxID <= numLogSegments * numRecordsPerLogSegment) {
if (expectedTxID == numLogSegments * numRecordsPerLogSegment) {
break;
}
List<LogRecordWithDLSN> records = Utils.ioResult(reader.readBulk(20));
LOG.info("Bulk read {} entries.", records.size());
assertTrue(records.size() >= 1);
for (LogRecordWithDLSN record : records) {
assertEquals(expectedTxID, record.getTransactionId());
++expectedTxID;
}
++numReads;
}
// we expect bulk read works
assertTrue(numReads < 60);
Utils.close(reader);
dlm.close();
}
use of org.apache.distributedlog.api.AsyncLogReader in project bookkeeper by apache.
the class TestAsyncReaderWriter method simpleAsyncReadTest.
void simpleAsyncReadTest(String name, DistributedLogConfiguration confLocal) throws Exception {
confLocal.setOutputBufferSize(1024);
confLocal.setReadAheadWaitTime(10);
confLocal.setReadAheadBatchSize(10);
DistributedLogManager dlm = createNewDLM(confLocal, name);
int numLogSegments = 3;
int numRecordsPerLogSegment = 10;
// Write 30 records: 3 log segments, 10 records per log segment
long txid = 1L;
txid = writeRecords(dlm, numLogSegments, numRecordsPerLogSegment, txid, false);
// Write another log segment with 5 records and flush every 2 records
txid = writeLogSegment(dlm, 5, txid, 2, false);
final AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InvalidDLSN);
final CountDownLatch syncLatch = new CountDownLatch((int) (txid - 1));
final CountDownLatch completionLatch = new CountDownLatch(1);
final AtomicBoolean errorsFound = new AtomicBoolean(false);
boolean monotonic = LogSegmentMetadata.supportsSequenceId(confLocal.getDLLedgerMetadataLayoutVersion());
TestAsyncReaderWriter.readNext(reader, DLSN.InvalidDLSN, monotonic ? 0L : Long.MIN_VALUE, monotonic, syncLatch, completionLatch, errorsFound);
completionLatch.await();
assertFalse("Errors encountered on reading records", errorsFound.get());
syncLatch.await();
Utils.close(reader);
dlm.close();
}
use of org.apache.distributedlog.api.AsyncLogReader in project bookkeeper by apache.
the class TestAsyncReaderWriter method testAsyncReadEmptyRecords.
@Test(timeout = 60000)
public void testAsyncReadEmptyRecords() throws Exception {
String name = "distrlog-simpleasyncreadempty";
DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
confLocal.loadConf(testConf);
confLocal.setOutputBufferSize(0);
confLocal.setReadAheadWaitTime(10);
confLocal.setReadAheadBatchSize(10);
DistributedLogManager dlm = createNewDLM(confLocal, name);
int numLogSegments = 3;
int numRecordsPerLogSegment = 10;
long txid = 1L;
// write 3 log segments, 10 records per log segment
txid = writeRecords(dlm, numLogSegments, numRecordsPerLogSegment, txid, true);
// write another log segment with 5 records and flush every 2 records
txid = writeLogSegment(dlm, 5, txid, 2, true);
AsyncLogReader asyncReader = dlm.getAsyncLogReader(DLSN.InvalidDLSN);
assertEquals("Expected stream name = " + name + " but " + asyncReader.getStreamName() + " found", name, asyncReader.getStreamName());
long numTrans = 0;
DLSN lastDLSN = DLSN.InvalidDLSN;
LogRecordWithDLSN record = Utils.ioResult(asyncReader.readNext());
while (null != record) {
DLMTestUtil.verifyEmptyLogRecord(record);
assertEquals(0, record.getDlsn().getSlotId());
assertTrue(record.getDlsn().compareTo(lastDLSN) > 0);
lastDLSN = record.getDlsn();
numTrans++;
if (numTrans >= (txid - 1)) {
break;
}
record = Utils.ioResult(asyncReader.readNext());
}
assertEquals((txid - 1), numTrans);
Utils.close(asyncReader);
dlm.close();
}
Aggregations