Examples with Future com.twitter.util.Future used on opensource projects

Example 11 with Future

use of com.twitter.util.Future in project distributedlog by twitter.

the class TestEntry method testWriteRecords.

@Test(timeout = 20000)
public void testWriteRecords() throws Exception {
    Writer writer = Entry.newEntry("test-write-records", 1024, true, CompressionCodec.Type.NONE, NullStatsLogger.INSTANCE);
    assertEquals("zero bytes", 0, writer.getNumBytes());
    assertEquals("zero records", 0, writer.getNumRecords());
    List<Future<DLSN>> writePromiseList = Lists.newArrayList();
    // write first 5 records
    for (int i = 0; i < 5; i++) {
        LogRecord record = new LogRecord(i, ("record-" + i).getBytes(UTF_8));
        record.setPositionWithinLogSegment(i);
        Promise<DLSN> writePromise = new Promise<DLSN>();
        writer.writeRecord(record, writePromise);
        writePromiseList.add(writePromise);
        assertEquals((i + 1) + " records", (i + 1), writer.getNumRecords());
    }
    // write large record
    LogRecord largeRecord = new LogRecord(1L, new byte[MAX_LOGRECORD_SIZE + 1]);
    try {
        writer.writeRecord(largeRecord, new Promise<DLSN>());
        Assert.fail("Should fail on writing large record");
    } catch (LogRecordTooLongException lrtle) {
    // expected
    }
    assertEquals("5 records", 5, writer.getNumRecords());
    // write another 5 records
    for (int i = 0; i < 5; i++) {
        LogRecord record = new LogRecord(i + 5, ("record-" + (i + 5)).getBytes(UTF_8));
        record.setPositionWithinLogSegment(i + 5);
        Promise<DLSN> writePromise = new Promise<DLSN>();
        writer.writeRecord(record, writePromise);
        writePromiseList.add(writePromise);
        assertEquals((i + 6) + " records", (i + 6), writer.getNumRecords());
    }
    Buffer buffer = writer.getBuffer();
    // Test transmit complete
    writer.completeTransmit(1L, 1L);
    List<DLSN> writeResults = Await.result(Future.collect(writePromiseList));
    for (int i = 0; i < 10; i++) {
        Assert.assertEquals(new DLSN(1L, 1L, i), writeResults.get(i));
    }
    // Test reading from buffer
    Entry recordSet = Entry.newBuilder().setData(buffer.getData(), 0, buffer.size()).setLogSegmentInfo(1L, 1L).setEntryId(0L).build();
    Reader reader = recordSet.reader();
    LogRecordWithDLSN record = reader.nextRecord();
    int numReads = 0;
    long expectedTxid = 0L;
    while (null != record) {
        Assert.assertEquals(expectedTxid, record.getTransactionId());
        Assert.assertEquals(expectedTxid, record.getSequenceId());
        Assert.assertEquals(new DLSN(1L, 0L, expectedTxid), record.getDlsn());
        ++numReads;
        ++expectedTxid;
        record = reader.nextRecord();
    }
    Assert.assertEquals(10, numReads);
}

Example 12 with Future

use of com.twitter.util.Future in project distributedlog by twitter.

the class TestAsyncReaderWriter method writeRecordsWithOutstandingWriteLimit.

public void writeRecordsWithOutstandingWriteLimit(int stream, int global, boolean shouldFail) throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setPerWriterOutstandingWriteLimit(stream);
    confLocal.setOutstandingWriteLimitDarkmode(false);
    DistributedLogManager dlm;
    if (global > -1) {
        dlm = createNewDLM(confLocal, runtime.getMethodName(), new SimplePermitLimiter(false, global, new NullStatsLogger(), true, new FixedValueFeature("", 0)));
    } else {
        dlm = createNewDLM(confLocal, runtime.getMethodName());
    }
    BKAsyncLogWriter writer = (BKAsyncLogWriter) (dlm.startAsyncLogSegmentNonPartitioned());
    ArrayList<Future<DLSN>> results = new ArrayList<Future<DLSN>>(1000);
    for (int i = 0; i < 1000; i++) {
        results.add(writer.write(DLMTestUtil.getLogRecordInstance(1L)));
    }
    for (Future<DLSN> result : results) {
        try {
            Await.result(result);
            if (shouldFail) {
                fail("should fail due to no outstanding writes permitted");
            }
        } catch (OverCapacityException ex) {
            assertTrue(shouldFail);
        }
    }
    writer.closeAndComplete();
    dlm.close();
}

Example 13 with Future

use of com.twitter.util.Future in project distributedlog by twitter.

the class BKLogReadHandler method lockStream.

/**
     * Elective stream lock--readers are not required to acquire the lock before using the stream.
     */
synchronized Future<Void> lockStream() {
    if (null == lockAcquireFuture) {
        final Function0<DistributedLock> lockFunction = new ExceptionalFunction0<DistributedLock>() {

            @Override
            public DistributedLock applyE() throws IOException {
                // Unfortunately this has a blocking call which we should not execute on the
                // ZK completion thread
                BKLogReadHandler.this.readLock = new ZKDistributedLock(lockStateExecutor, lockFactory, readLockPath, conf.getLockTimeoutMilliSeconds(), statsLogger.scope("read_lock"));
                LOG.info("acquiring readlock {} at {}", getLockClientId(), readLockPath);
                return BKLogReadHandler.this.readLock;
            }
        };
        lockAcquireFuture = ensureReadLockPathExist().flatMap(new ExceptionalFunction<Void, Future<Void>>() {

            @Override
            public Future<Void> applyE(Void in) throws Throwable {
                return scheduler.apply(lockFunction).flatMap(new ExceptionalFunction<DistributedLock, Future<Void>>() {

                    @Override
                    public Future<Void> applyE(DistributedLock lock) throws IOException {
                        return acquireLockOnExecutorThread(lock);
                    }
                });
            }
        });
    }
    return lockAcquireFuture;
}

Example 14 with Future

use of com.twitter.util.Future in project distributedlog by twitter.

the class TestDistributedLogServer method testBulkWriteTotalFailureFirstWriteFailed.

@Test(timeout = 60000)
public void testBulkWriteTotalFailureFirstWriteFailed() throws Exception {
    String name = String.format("dlserver-bulk-write-%s", "first-write-failed");
    dlClient.routingService.addHost(name, dlServer.getAddress());
    final int writeCount = 100;
    List<ByteBuffer> writes = new ArrayList<ByteBuffer>(writeCount + 1);
    ByteBuffer buf = ByteBuffer.allocate(MAX_LOGRECORD_SIZE + 1);
    writes.add(buf);
    for (long i = 1; i <= writeCount; i++) {
        writes.add(ByteBuffer.wrap(("" + i).getBytes()));
    }
    List<Future<DLSN>> futures = dlClient.dlClient.writeBulk(name, writes);
    validateFailedAsLogRecordTooLong(futures.get(0));
    FutureUtils.result(Futures.collect(futures.subList(1, writeCount + 1)));
}

Example 15 with Future

use of com.twitter.util.Future in project distributedlog by twitter.

the class TestDistributedLogServer method testBulkWritePartialFailure.

@Test(timeout = 60000)
public void testBulkWritePartialFailure() throws Exception {
    String name = String.format("dlserver-bulk-write-%s", "partial-failure");
    dlClient.routingService.addHost(name, dlServer.getAddress());
    final int writeCount = 100;
    List<ByteBuffer> writes = new ArrayList<ByteBuffer>(writeCount * 2 + 1);
    for (long i = 1; i <= writeCount; i++) {
        writes.add(ByteBuffer.wrap(("" + i).getBytes()));
    }
    // Too big, will cause partial failure.
    ByteBuffer buf = ByteBuffer.allocate(MAX_LOGRECORD_SIZE + 1);
    writes.add(buf);
    for (long i = 1; i <= writeCount; i++) {
        writes.add(ByteBuffer.wrap(("" + i).getBytes()));
    }
    // Count succeeded.
    List<Future<DLSN>> futures = dlClient.dlClient.writeBulk(name, writes);
    int succeeded = 0;
    for (int i = 0; i < writeCount; i++) {
        Future<DLSN> future = futures.get(i);
        try {
            DLSN dlsn = Await.result(future, Duration.fromSeconds(10));
            ++succeeded;
        } catch (Exception ex) {
            failDueToWrongException(ex);
        }
    }
    validateFailedAsLogRecordTooLong(futures.get(writeCount));
    FutureUtils.result(Futures.collect(futures.subList(writeCount + 1, 2 * writeCount + 1)));
    assertEquals(writeCount, succeeded);
}