use of com.twitter.util.Future in project distributedlog by twitter.
the class TestBKLogSegmentWriter method testCloseShouldFailIfLedgerFenced.
/**
* Close the writer when ledger is fenced: it should release the lock, fail on flushing data and throw exception
*
* @throws Exception
*/
@Test(timeout = 60000)
public void testCloseShouldFailIfLedgerFenced() throws Exception {
DistributedLogConfiguration confLocal = newLocalConf();
confLocal.setImmediateFlushEnabled(false);
confLocal.setOutputBufferSize(Integer.MAX_VALUE);
confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
ZKDistributedLock lock = createLock("/test/lock-" + runtime.getMethodName(), zkc, true);
BKLogSegmentWriter writer = createLogSegmentWriter(confLocal, 0L, -1L, lock);
// Use another lock to wait for writer releasing lock
ZKDistributedLock lock0 = createLock("/test/lock-" + runtime.getMethodName(), zkc0, false);
Future<ZKDistributedLock> lockFuture0 = lock0.asyncAcquire();
// add 10 records
int numRecords = 10;
List<Future<DLSN>> futureList = new ArrayList<Future<DLSN>>(numRecords);
for (int i = 0; i < numRecords; i++) {
futureList.add(writer.asyncWrite(DLMTestUtil.getLogRecordInstance(i)));
}
assertEquals("Last tx id should be " + (numRecords - 1), numRecords - 1, writer.getLastTxId());
assertEquals("Last acked tx id should be -1", -1L, writer.getLastTxIdAcknowledged());
assertEquals("Last DLSN should be " + DLSN.InvalidDLSN, DLSN.InvalidDLSN, writer.getLastDLSN());
assertEquals("Position should be " + numRecords, 10, writer.getPositionWithinLogSegment());
// fence the ledger
fenceLedger(getLedgerHandle(writer));
// close the writer: it should release the lock, fail on flushing data and throw exception
try {
closeWriterAndLock(writer, lock);
fail("Close a log segment writer when ledger is fenced should throw exception");
} catch (BKTransmitException bkte) {
assertEquals("Inconsistent rc is thrown", BKException.Code.LedgerFencedException, bkte.getBKResultCode());
}
Await.result(lockFuture0);
lock0.checkOwnership();
assertEquals("Last tx id should still be " + (numRecords - 1), numRecords - 1, writer.getLastTxId());
assertEquals("Last acked tx id should still be " + (numRecords - 1), -1L, writer.getLastTxIdAcknowledged());
assertEquals("Last DLSN should still be " + DLSN.InvalidDLSN, DLSN.InvalidDLSN, writer.getLastDLSN());
assertEquals("Position should still be " + numRecords, 10, writer.getPositionWithinLogSegment());
for (int i = 0; i < numRecords; i++) {
try {
Await.result(futureList.get(i));
fail("Should be aborted record " + i + " with transmit exception");
} catch (BKTransmitException bkte) {
assertEquals("Record " + i + " should be aborted", BKException.Code.LedgerFencedException, bkte.getBKResultCode());
}
}
// check no entries were written
LedgerHandle lh = getLedgerHandle(writer);
LedgerHandle readLh = openLedgerNoRecovery(lh);
assertTrue("Ledger " + lh.getId() + " should be closed", readLh.isClosed());
assertEquals("There should be no entries in ledger " + lh.getId(), LedgerHandle.INVALID_ENTRY_ID, readLh.getLastAddConfirmed());
}
use of com.twitter.util.Future in project distributedlog by twitter.
the class TestBKLogSegmentWriter method testCloseShouldNotFlushIfInErrorState.
/**
* Close a log segment writer that is already in error state, should not flush buffered data.
*
* @throws Exception
*/
void testCloseShouldNotFlushIfInErrorState(int rcToFailComplete) throws Exception {
DistributedLogConfiguration confLocal = newLocalConf();
confLocal.setImmediateFlushEnabled(false);
confLocal.setOutputBufferSize(Integer.MAX_VALUE);
confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
ZKDistributedLock lock = createLock("/test/lock-" + runtime.getMethodName(), zkc, true);
BKLogSegmentWriter writer = createLogSegmentWriter(confLocal, 0L, -1L, lock);
// Use another lock to wait for writer releasing lock
ZKDistributedLock lock0 = createLock("/test/lock-" + runtime.getMethodName(), zkc0, false);
Future<ZKDistributedLock> lockFuture0 = lock0.asyncAcquire();
// add 10 records
int numRecords = 10;
List<Future<DLSN>> futureList = new ArrayList<Future<DLSN>>(numRecords);
for (int i = 0; i < numRecords; i++) {
futureList.add(writer.asyncWrite(DLMTestUtil.getLogRecordInstance(i)));
}
assertEquals("Last tx id should be " + (numRecords - 1), numRecords - 1, writer.getLastTxId());
assertEquals("Last acked tx id should be -1", -1L, writer.getLastTxIdAcknowledged());
assertEquals("Last DLSN should be " + DLSN.InvalidDLSN, DLSN.InvalidDLSN, writer.getLastDLSN());
assertEquals("Position should be " + numRecords, 10, writer.getPositionWithinLogSegment());
writer.setTransmitResult(rcToFailComplete);
// close the writer should release lock but not flush data
try {
closeWriterAndLock(writer, lock);
fail("Close a log segment writer in error state should throw exception");
} catch (BKTransmitException bkte) {
assertEquals("Inconsistent rc is thrown", rcToFailComplete, bkte.getBKResultCode());
}
Await.result(lockFuture0);
lock0.checkOwnership();
assertEquals("Last tx id should still be " + (numRecords - 1), numRecords - 1, writer.getLastTxId());
assertEquals("Last acked tx id should still be " + (numRecords - 1), -1L, writer.getLastTxIdAcknowledged());
assertEquals("Last DLSN should still be " + DLSN.InvalidDLSN, DLSN.InvalidDLSN, writer.getLastDLSN());
assertEquals("Position should still be " + numRecords, 10, writer.getPositionWithinLogSegment());
for (int i = 0; i < numRecords; i++) {
try {
Await.result(futureList.get(i));
fail("Should be aborted record " + i + " with transmit exception");
} catch (WriteCancelledException wce) {
assertTrue("Record " + i + " should be aborted because of ledger fenced", wce.getCause() instanceof BKTransmitException);
BKTransmitException bkte = (BKTransmitException) wce.getCause();
assertEquals("Record " + i + " should be aborted", rcToFailComplete, bkte.getBKResultCode());
}
}
// check no entries were written
LedgerHandle lh = getLedgerHandle(writer);
LedgerHandle readLh = openLedgerNoRecovery(lh);
assertFalse("Ledger " + lh.getId() + " should not be closed", readLh.isClosed());
assertEquals("There should be no entries in ledger " + lh.getId(), LedgerHandle.INVALID_ENTRY_ID, readLh.getLastAddConfirmed());
}
use of com.twitter.util.Future in project distributedlog by twitter.
the class TestEntry method testWriteRecordSet.
@Test(timeout = 20000)
public void testWriteRecordSet() throws Exception {
Writer writer = Entry.newEntry("test-write-recordset", 1024, true, CompressionCodec.Type.NONE, NullStatsLogger.INSTANCE);
assertEquals("zero bytes", 0, writer.getNumBytes());
assertEquals("zero records", 0, writer.getNumRecords());
List<Future<DLSN>> writePromiseList = Lists.newArrayList();
// write first 5 records
for (int i = 0; i < 5; i++) {
LogRecord record = new LogRecord(i, ("record-" + i).getBytes(UTF_8));
record.setPositionWithinLogSegment(i);
Promise<DLSN> writePromise = new Promise<DLSN>();
writer.writeRecord(record, writePromise);
writePromiseList.add(writePromise);
assertEquals((i + 1) + " records", (i + 1), writer.getNumRecords());
}
final LogRecordSet.Writer recordSetWriter = LogRecordSet.newWriter(1024, CompressionCodec.Type.NONE);
List<Future<DLSN>> recordSetPromiseList = Lists.newArrayList();
// write another 5 records as a batch
for (int i = 0; i < 5; i++) {
ByteBuffer record = ByteBuffer.wrap(("record-" + (i + 5)).getBytes(UTF_8));
Promise<DLSN> writePromise = new Promise<DLSN>();
recordSetWriter.writeRecord(record, writePromise);
recordSetPromiseList.add(writePromise);
assertEquals((i + 1) + " records", (i + 1), recordSetWriter.getNumRecords());
}
final ByteBuffer recordSetBuffer = recordSetWriter.getBuffer();
byte[] data = new byte[recordSetBuffer.remaining()];
recordSetBuffer.get(data);
LogRecord setRecord = new LogRecord(5L, data);
setRecord.setPositionWithinLogSegment(5);
setRecord.setRecordSet();
Promise<DLSN> writePromise = new Promise<DLSN>();
writePromise.addEventListener(new FutureEventListener<DLSN>() {
@Override
public void onSuccess(DLSN dlsn) {
recordSetWriter.completeTransmit(dlsn.getLogSegmentSequenceNo(), dlsn.getEntryId(), dlsn.getSlotId());
}
@Override
public void onFailure(Throwable cause) {
recordSetWriter.abortTransmit(cause);
}
});
writer.writeRecord(setRecord, writePromise);
writePromiseList.add(writePromise);
// write last 5 records
for (int i = 0; i < 5; i++) {
LogRecord record = new LogRecord(i + 10, ("record-" + (i + 10)).getBytes(UTF_8));
record.setPositionWithinLogSegment(i + 10);
writePromise = new Promise<DLSN>();
writer.writeRecord(record, writePromise);
writePromiseList.add(writePromise);
assertEquals((i + 11) + " records", (i + 11), writer.getNumRecords());
}
Buffer buffer = writer.getBuffer();
// Test transmit complete
writer.completeTransmit(1L, 1L);
List<DLSN> writeResults = Await.result(Future.collect(writePromiseList));
for (int i = 0; i < 5; i++) {
Assert.assertEquals(new DLSN(1L, 1L, i), writeResults.get(i));
}
Assert.assertEquals(new DLSN(1L, 1L, 5), writeResults.get(5));
for (int i = 0; i < 5; i++) {
Assert.assertEquals(new DLSN(1L, 1L, (10 + i)), writeResults.get(6 + i));
}
List<DLSN> recordSetWriteResults = Await.result(Future.collect(recordSetPromiseList));
for (int i = 0; i < 5; i++) {
Assert.assertEquals(new DLSN(1L, 1L, (5 + i)), recordSetWriteResults.get(i));
}
// Test reading from buffer
verifyReadResult(buffer, 1L, 1L, 1L, true, new DLSN(1L, 1L, 2L), 3, 5, 5, new DLSN(1L, 1L, 2L), 2L);
verifyReadResult(buffer, 1L, 1L, 1L, true, new DLSN(1L, 1L, 7L), 0, 3, 5, new DLSN(1L, 1L, 7L), 7L);
verifyReadResult(buffer, 1L, 1L, 1L, true, new DLSN(1L, 1L, 12L), 0, 0, 3, new DLSN(1L, 1L, 12L), 12L);
verifyReadResult(buffer, 1L, 1L, 1L, false, new DLSN(1L, 1L, 2L), 3, 5, 5, new DLSN(1L, 1L, 2L), 2L);
verifyReadResult(buffer, 1L, 1L, 1L, false, new DLSN(1L, 1L, 7L), 0, 3, 5, new DLSN(1L, 1L, 7L), 7L);
verifyReadResult(buffer, 1L, 1L, 1L, false, new DLSN(1L, 1L, 12L), 0, 0, 3, new DLSN(1L, 1L, 12L), 12L);
}
use of com.twitter.util.Future in project distributedlog by twitter.
the class BulkWriteOp method executeOp.
@Override
protected Future<BulkWriteResponse> executeOp(AsyncLogWriter writer, Sequencer sequencer, Object txnLock) {
// Need to convert input buffers to LogRecords.
List<LogRecord> records;
Future<List<Future<DLSN>>> futureList;
synchronized (txnLock) {
records = asRecordList(buffers, sequencer);
futureList = writer.writeBulk(records);
}
// Collect into a list of tries to make it easier to extract exception or DLSN.
Future<List<Try<DLSN>>> writes = asTryList(futureList);
Future<BulkWriteResponse> response = writes.flatMap(new AbstractFunction1<List<Try<DLSN>>, Future<BulkWriteResponse>>() {
@Override
public Future<BulkWriteResponse> apply(List<Try<DLSN>> results) {
// Considered a success at batch level even if no individual writes succeeed.
// The reason is that its impossible to make an appropriate decision re retries without
// individual buffer failure reasons.
List<WriteResponse> writeResponses = new ArrayList<WriteResponse>(results.size());
BulkWriteResponse bulkWriteResponse = ResponseUtils.bulkWriteSuccess().setWriteResponses(writeResponses);
// failed.
if (results.size() > 0) {
Try<DLSN> firstResult = results.get(0);
if (isDefiniteFailure(firstResult)) {
return new ConstFuture(firstResult);
}
}
// Translate all futures to write responses.
Iterator<Try<DLSN>> iterator = results.iterator();
while (iterator.hasNext()) {
Try<DLSN> completedFuture = iterator.next();
try {
DLSN dlsn = completedFuture.get();
WriteResponse writeResponse = ResponseUtils.writeSuccess().setDlsn(dlsn.serialize());
writeResponses.add(writeResponse);
successRecordCounter.inc();
} catch (Exception ioe) {
WriteResponse writeResponse = ResponseUtils.write(ResponseUtils.exceptionToHeader(ioe));
writeResponses.add(writeResponse);
if (StatusCode.FOUND == writeResponse.getHeader().getCode()) {
redirectRecordCounter.inc();
} else {
failureRecordCounter.inc();
}
}
}
return Future.value(bulkWriteResponse);
}
});
return response;
}
use of com.twitter.util.Future in project distributedlog by twitter.
the class TestSafeQueueingFuturePool method testRejectedBackupFailure.
@Test
public void testRejectedBackupFailure() throws Exception {
TestFuturePool<Void> pool = new TestFuturePool<Void>();
final AtomicBoolean result = new AtomicBoolean(false);
pool.executor.shutdownNow();
final CountDownLatch latch1 = new CountDownLatch(1);
final CountDownLatch latch2 = new CountDownLatch(1);
Future<Void> future1 = pool.wrapper.apply(new Function0<Void>() {
public Void apply() {
try {
latch1.await();
} catch (Exception ex) {
}
return null;
}
});
// Enqueue a set of futures behind.
final int blockedCount = 100;
final ArrayList<Future<Void>> blockedFutures = new ArrayList<Future<Void>>(blockedCount);
final int[] doneArray = new int[blockedCount];
final AtomicInteger doneCount = new AtomicInteger(0);
for (int i = 0; i < blockedCount; i++) {
final int index = i;
blockedFutures.add(pool.wrapper.apply(new Function0<Void>() {
public Void apply() {
doneArray[index] = doneCount.getAndIncrement();
return null;
}
}));
}
// All the futures fail when the executor is force closed.
latch1.countDown();
pool.executor.shutdownNow();
for (int i = 0; i < blockedCount; i++) {
try {
Await.result(blockedFutures.get(i));
fail("should have thrown");
} catch (RejectedExecutionException ex) {
}
}
// None of them have completed.
for (int i = 0; i < blockedCount; i++) {
assertEquals(0, doneArray[i]);
}
// Close cleans up all pending ops in order.
pool.wrapper.close();
for (int i = 0; i < blockedCount; i++) {
assertEquals(i, doneArray[i]);
}
pool.shutdown();
}
Aggregations