use of com.twitter.util.Future in project distributedlog by twitter.
the class TestAsyncReaderLock method testReaderLockCloseInAcquireCallback.
@Test(timeout = 60000)
public void testReaderLockCloseInAcquireCallback() throws Exception {
final String name = runtime.getMethodName();
DistributedLogManager dlm = createNewDLM(conf, name);
BKAsyncLogWriter writer = (BKAsyncLogWriter) (dlm.startAsyncLogSegmentNonPartitioned());
writer.write(DLMTestUtil.getLogRecordInstance(1L));
writer.closeAndComplete();
final CountDownLatch latch = new CountDownLatch(1);
Future<AsyncLogReader> futureReader1 = dlm.getAsyncLogReaderWithLock(DLSN.InitialDLSN);
futureReader1.flatMap(new ExceptionalFunction<AsyncLogReader, Future<Void>>() {
@Override
public Future<Void> applyE(AsyncLogReader reader) throws IOException {
return reader.asyncClose().map(new AbstractFunction1<Void, Void>() {
@Override
public Void apply(Void result) {
latch.countDown();
return null;
}
});
}
});
latch.await();
dlm.close();
}
use of com.twitter.util.Future in project distributedlog by twitter.
the class TestAsyncReaderWriter method testWriteRecordSet.
@Test(timeout = 60000)
public void testWriteRecordSet() throws Exception {
String name = runtime.getMethodName();
DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
confLocal.addConfiguration(testConf);
confLocal.setOutputBufferSize(0);
confLocal.setImmediateFlushEnabled(false);
confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
URI uri = createDLMURI("/" + name);
ensureURICreated(uri);
DistributedLogManager dlm = createNewDLM(confLocal, name);
BKAsyncLogWriter writer = (BKAsyncLogWriter) dlm.startAsyncLogSegmentNonPartitioned();
List<Future<DLSN>> writeFutures = Lists.newArrayList();
for (int i = 0; i < 5; i++) {
LogRecord record = DLMTestUtil.getLogRecordInstance(1L + i);
writeFutures.add(writer.write(record));
}
List<Future<DLSN>> recordSetFutures = Lists.newArrayList();
// write another 5 records
final LogRecordSet.Writer recordSetWriter = LogRecordSet.newWriter(4096, CompressionCodec.Type.LZ4);
for (int i = 0; i < 5; i++) {
LogRecord record = DLMTestUtil.getLogRecordInstance(6L + i);
Promise<DLSN> writePromise = new Promise<DLSN>();
recordSetWriter.writeRecord(ByteBuffer.wrap(record.getPayload()), writePromise);
recordSetFutures.add(writePromise);
}
final ByteBuffer recordSetBuffer = recordSetWriter.getBuffer();
byte[] data = new byte[recordSetBuffer.remaining()];
recordSetBuffer.get(data);
LogRecord setRecord = new LogRecord(6L, data);
setRecord.setRecordSet();
Future<DLSN> writeRecordSetFuture = writer.write(setRecord);
writeRecordSetFuture.addEventListener(new FutureEventListener<DLSN>() {
@Override
public void onSuccess(DLSN dlsn) {
recordSetWriter.completeTransmit(dlsn.getLogSegmentSequenceNo(), dlsn.getEntryId(), dlsn.getSlotId());
}
@Override
public void onFailure(Throwable cause) {
recordSetWriter.abortTransmit(cause);
}
});
writeFutures.add(writeRecordSetFuture);
FutureUtils.result(writeRecordSetFuture);
// write last 5 records
for (int i = 0; i < 5; i++) {
LogRecord record = DLMTestUtil.getLogRecordInstance(11L + i);
Future<DLSN> writeFuture = writer.write(record);
writeFutures.add(writeFuture);
// make sure get log record count returns the right count
if (i == 0) {
FutureUtils.result(writeFuture);
assertEquals(10, dlm.getLogRecordCount());
}
}
List<DLSN> writeResults = FutureUtils.result(Future.collect(writeFutures));
for (int i = 0; i < 5; i++) {
Assert.assertEquals(new DLSN(1L, i, 0L), writeResults.get(i));
}
Assert.assertEquals(new DLSN(1L, 5L, 0L), writeResults.get(5));
for (int i = 0; i < 5; i++) {
Assert.assertEquals(new DLSN(1L, 6L + i, 0L), writeResults.get(6 + i));
}
List<DLSN> recordSetWriteResults = Await.result(Future.collect(recordSetFutures));
for (int i = 0; i < 5; i++) {
Assert.assertEquals(new DLSN(1L, 5L, i), recordSetWriteResults.get(i));
}
FutureUtils.result(writer.flushAndCommit());
DistributedLogConfiguration readConf1 = new DistributedLogConfiguration();
readConf1.addConfiguration(confLocal);
readConf1.setDeserializeRecordSetOnReads(true);
DistributedLogManager readDLM1 = createNewDLM(readConf1, name);
AsyncLogReader reader1 = readDLM1.getAsyncLogReader(DLSN.InitialDLSN);
for (int i = 0; i < 15; i++) {
LogRecordWithDLSN record = FutureUtils.result(reader1.readNext());
if (i < 5) {
assertEquals(new DLSN(1L, i, 0L), record.getDlsn());
assertEquals(1L + i, record.getTransactionId());
} else if (i >= 10) {
assertEquals(new DLSN(1L, 6L + i - 10, 0L), record.getDlsn());
assertEquals(11L + i - 10, record.getTransactionId());
} else {
assertEquals(new DLSN(1L, 5L, i - 5), record.getDlsn());
assertEquals(6L, record.getTransactionId());
}
assertEquals(i + 1, record.getPositionWithinLogSegment());
assertArrayEquals(DLMTestUtil.generatePayload(i + 1), record.getPayload());
}
DistributedLogConfiguration readConf2 = new DistributedLogConfiguration();
readConf2.addConfiguration(confLocal);
readConf2.setDeserializeRecordSetOnReads(false);
DistributedLogManager readDLM2 = createNewDLM(readConf2, name);
AsyncLogReader reader2 = readDLM2.getAsyncLogReader(DLSN.InitialDLSN);
for (int i = 0; i < 11; i++) {
LogRecordWithDLSN record = FutureUtils.result(reader2.readNext());
LOG.info("Read record {}", record);
if (i < 5) {
assertEquals(new DLSN(1L, i, 0L), record.getDlsn());
assertEquals(1L + i, record.getTransactionId());
assertEquals(i + 1, record.getPositionWithinLogSegment());
assertArrayEquals(DLMTestUtil.generatePayload(i + 1), record.getPayload());
} else if (i >= 6L) {
assertEquals(new DLSN(1L, 6L + i - 6, 0L), record.getDlsn());
assertEquals(11L + i - 6, record.getTransactionId());
assertEquals(11 + i - 6, record.getPositionWithinLogSegment());
assertArrayEquals(DLMTestUtil.generatePayload(11L + i - 6), record.getPayload());
} else {
assertEquals(new DLSN(1L, 5L, 0), record.getDlsn());
assertEquals(6L, record.getTransactionId());
assertEquals(6, record.getPositionWithinLogSegment());
assertTrue(record.isRecordSet());
assertEquals(5, LogRecordSet.numRecords(record));
}
}
}
use of com.twitter.util.Future in project distributedlog by twitter.
the class TestBKLogSegmentWriter method testAbortShouldNotFlush.
/**
* Abort a segment log writer should just abort pending writes and not flush buffered data.
*
* @throws Exception
*/
@Test(timeout = 60000)
public void testAbortShouldNotFlush() throws Exception {
DistributedLogConfiguration confLocal = newLocalConf();
confLocal.setImmediateFlushEnabled(false);
confLocal.setOutputBufferSize(Integer.MAX_VALUE);
confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
ZKDistributedLock lock = createLock("/test/lock-" + runtime.getMethodName(), zkc, true);
BKLogSegmentWriter writer = createLogSegmentWriter(confLocal, 0L, -1L, lock);
// Use another lock to wait for writer releasing lock
ZKDistributedLock lock0 = createLock("/test/lock-" + runtime.getMethodName(), zkc0, false);
Future<ZKDistributedLock> lockFuture0 = lock0.asyncAcquire();
// add 10 records
int numRecords = 10;
List<Future<DLSN>> futureList = new ArrayList<Future<DLSN>>(numRecords);
for (int i = 0; i < numRecords; i++) {
futureList.add(writer.asyncWrite(DLMTestUtil.getLogRecordInstance(i)));
}
assertEquals("Last tx id should be " + (numRecords - 1), numRecords - 1, writer.getLastTxId());
assertEquals("Last acked tx id should be -1", -1L, writer.getLastTxIdAcknowledged());
assertEquals("Last DLSN should be " + DLSN.InvalidDLSN, DLSN.InvalidDLSN, writer.getLastDLSN());
assertEquals("Position should be " + numRecords, 10, writer.getPositionWithinLogSegment());
// close the writer should flush buffered data and release lock
abortWriterAndLock(writer, lock);
Await.result(lockFuture0);
lock0.checkOwnership();
assertEquals("Last tx id should still be " + (numRecords - 1), numRecords - 1, writer.getLastTxId());
assertEquals("Last acked tx id should still be " + (numRecords - 1), -1L, writer.getLastTxIdAcknowledged());
assertEquals("Last DLSN should still be " + DLSN.InvalidDLSN, DLSN.InvalidDLSN, writer.getLastDLSN());
assertEquals("Position should still be " + numRecords, 10, writer.getPositionWithinLogSegment());
for (int i = 0; i < numRecords; i++) {
try {
Await.result(futureList.get(i));
fail("Should be aborted record " + i + " with transmit exception");
} catch (WriteCancelledException wce) {
assertTrue("Record " + i + " should be aborted because of ledger fenced", wce.getCause() instanceof BKTransmitException);
BKTransmitException bkte = (BKTransmitException) wce.getCause();
assertEquals("Record " + i + " should be aborted", BKException.Code.InterruptedException, bkte.getBKResultCode());
}
}
// check no entries were written
LedgerHandle lh = getLedgerHandle(writer);
LedgerHandle readLh = openLedgerNoRecovery(lh);
assertTrue("Ledger " + lh.getId() + " should not be closed", readLh.isClosed());
assertEquals("There should be no entries in ledger " + lh.getId(), LedgerHandle.INVALID_ENTRY_ID, readLh.getLastAddConfirmed());
}
use of com.twitter.util.Future in project distributedlog by twitter.
the class TestBKLogSegmentWriter method testUpdateLastTxIdForUserRecords.
/**
* Log Segment Writer should only update last tx id only for user records.
*/
@Test(timeout = 60000)
public void testUpdateLastTxIdForUserRecords() throws Exception {
DistributedLogConfiguration confLocal = newLocalConf();
confLocal.setImmediateFlushEnabled(false);
confLocal.setOutputBufferSize(Integer.MAX_VALUE);
confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
ZKDistributedLock lock = createLock("/test/lock-" + runtime.getMethodName(), zkc, true);
BKLogSegmentWriter writer = createLogSegmentWriter(confLocal, 0L, -1L, lock);
// add 10 records
int numRecords = 10;
List<Future<DLSN>> futureList = new ArrayList<Future<DLSN>>(numRecords);
for (int i = 0; i < numRecords; i++) {
futureList.add(writer.asyncWrite(DLMTestUtil.getLogRecordInstance(i)));
}
LogRecord controlRecord = DLMTestUtil.getLogRecordInstance(9999L);
controlRecord.setControl();
futureList.add(writer.asyncWrite(controlRecord));
assertEquals("Last tx id should be " + (numRecords - 1), numRecords - 1, writer.getLastTxId());
assertEquals("Last DLSN should be " + DLSN.InvalidDLSN, DLSN.InvalidDLSN, writer.getLastDLSN());
assertEquals("Position should be " + numRecords, numRecords, writer.getPositionWithinLogSegment());
// close the writer to flush the output buffer
closeWriterAndLock(writer, lock);
List<DLSN> dlsns = Await.result(Future.collect(futureList));
assertEquals("All 11 records should be written", numRecords + 1, dlsns.size());
for (int i = 0; i < numRecords; i++) {
DLSN dlsn = dlsns.get(i);
assertEquals("Incorrent ledger sequence number", 0L, dlsn.getLogSegmentSequenceNo());
assertEquals("Incorrent entry id", 0L, dlsn.getEntryId());
assertEquals("Inconsistent slot id", i, dlsn.getSlotId());
}
DLSN dlsn = dlsns.get(numRecords);
assertEquals("Incorrent ledger sequence number", 0L, dlsn.getLogSegmentSequenceNo());
assertEquals("Incorrent entry id", 1L, dlsn.getEntryId());
assertEquals("Inconsistent slot id", 0L, dlsn.getSlotId());
assertEquals("Last tx id should be " + (numRecords - 1), numRecords - 1, writer.getLastTxId());
assertEquals("Last acked tx id should be " + (numRecords - 1), numRecords - 1, writer.getLastTxIdAcknowledged());
assertEquals("Position should be " + numRecords, numRecords, writer.getPositionWithinLogSegment());
assertEquals("Last DLSN should be " + dlsn, dlsns.get(numRecords - 1), writer.getLastDLSN());
}
use of com.twitter.util.Future in project distributedlog by twitter.
the class TestBKLogSegmentWriter method testAbortShouldFailAllWrites.
/**
* Abort should wait for outstanding transmits to be completed and cancel buffered data.
*
* @throws Exception
*/
@Test(timeout = 60000)
public void testAbortShouldFailAllWrites() throws Exception {
DistributedLogConfiguration confLocal = newLocalConf();
confLocal.setImmediateFlushEnabled(false);
confLocal.setOutputBufferSize(Integer.MAX_VALUE);
confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
ZKDistributedLock lock = createLock("/test/lock-" + runtime.getMethodName(), zkc, true);
BKLogSegmentWriter writer = createLogSegmentWriter(confLocal, 0L, -1L, lock);
// Use another lock to wait for writer releasing lock
ZKDistributedLock lock0 = createLock("/test/lock-" + runtime.getMethodName(), zkc0, false);
Future<ZKDistributedLock> lockFuture0 = lock0.asyncAcquire();
// add 10 records
int numRecords = 10;
List<Future<DLSN>> futureList = new ArrayList<Future<DLSN>>(numRecords);
for (int i = 0; i < numRecords; i++) {
futureList.add(writer.asyncWrite(DLMTestUtil.getLogRecordInstance(i)));
}
assertEquals("Last tx id should be " + (numRecords - 1), numRecords - 1, writer.getLastTxId());
assertEquals("Last acked tx id should be -1", -1L, writer.getLastTxIdAcknowledged());
assertEquals("Last DLSN should be " + DLSN.InvalidDLSN, DLSN.InvalidDLSN, writer.getLastDLSN());
assertEquals("Position should be " + numRecords, numRecords, writer.getPositionWithinLogSegment());
final CountDownLatch deferLatch = new CountDownLatch(1);
writer.getFuturePool().apply(new AbstractFunction0<Object>() {
@Override
public Object apply() {
try {
deferLatch.await();
} catch (InterruptedException e) {
LOG.warn("Interrupted on deferring completion : ", e);
}
return null;
}
});
// transmit the buffered data
FutureUtils.result(writer.flush());
// add another 10 records
List<Future<DLSN>> anotherFutureList = new ArrayList<Future<DLSN>>(numRecords);
for (int i = numRecords; i < 2 * numRecords; i++) {
anotherFutureList.add(writer.asyncWrite(DLMTestUtil.getLogRecordInstance(i)));
}
assertEquals("Last tx id should become " + (2 * numRecords - 1), 2 * numRecords - 1, writer.getLastTxId());
assertEquals("Last acked tx id should become " + (numRecords - 1), (long) (numRecords - 1), writer.getLastTxIdAcknowledged());
assertEquals("Last DLSN should still be " + DLSN.InvalidDLSN, DLSN.InvalidDLSN, writer.getLastDLSN());
assertEquals("Position should become " + (2 * numRecords), 2 * numRecords, writer.getPositionWithinLogSegment());
// abort the writer: it waits for outstanding transmits and abort buffered data
abortWriterAndLock(writer, lock);
Await.result(lockFuture0);
lock0.checkOwnership();
// release defer latch so completion would go through
deferLatch.countDown();
List<DLSN> dlsns = Await.result(Future.collect(futureList));
assertEquals("All first 10 records should be written", numRecords, dlsns.size());
for (int i = 0; i < numRecords; i++) {
DLSN dlsn = dlsns.get(i);
assertEquals("Incorrent ledger sequence number", 0L, dlsn.getLogSegmentSequenceNo());
assertEquals("Incorrent entry id", 0L, dlsn.getEntryId());
assertEquals("Inconsistent slot id", i, dlsn.getSlotId());
}
for (int i = 0; i < numRecords; i++) {
try {
Await.result(anotherFutureList.get(i));
fail("Should be aborted record " + (numRecords + i) + " with transmit exception");
} catch (WriteCancelledException wce) {
// writes should be cancelled.
}
}
assertEquals("Last tx id should still be " + (2 * numRecords - 1), 2 * numRecords - 1, writer.getLastTxId());
assertEquals("Last acked tx id should be still " + (numRecords - 1), (long) (numRecords - 1), writer.getLastTxIdAcknowledged());
assertEquals("Last DLSN should become " + futureList.get(futureList.size() - 1), dlsns.get(futureList.size() - 1), writer.getLastDLSN());
assertEquals("Position should become " + 2 * numRecords, 2 * numRecords, writer.getPositionWithinLogSegment());
// check only 1 entry were written
LedgerHandle lh = getLedgerHandle(writer);
LedgerHandle readLh = openLedgerNoRecovery(lh);
assertTrue("Ledger " + lh.getId() + " should not be closed", readLh.isClosed());
assertEquals("Only one entry is written for ledger " + lh.getId(), 0L, lh.getLastAddPushed());
assertEquals("Only one entry is written for ledger " + lh.getId(), 0L, readLh.getLastAddConfirmed());
}
Aggregations