Examples with Future com.twitter.util.Future used on opensource projects

Example 31 with Future

use of com.twitter.util.Future in project distributedlog by twitter.

the class TestLogRecordSet method testWriteRecords.

void testWriteRecords(Type codec) throws Exception {
    Writer writer = LogRecordSet.newWriter(1024, codec);
    assertEquals("zero user bytes", HEADER_LEN, writer.getNumBytes());
    assertEquals("zero records", 0, writer.getNumRecords());
    List<Future<DLSN>> writePromiseList = Lists.newArrayList();
    /// write first 5 records
    for (int i = 0; i < 5; i++) {
        ByteBuffer record = ByteBuffer.wrap(("record-" + i).getBytes(UTF_8));
        Promise<DLSN> writePromise = new Promise<DLSN>();
        writer.writeRecord(record, writePromise);
        writePromiseList.add(writePromise);
        assertEquals((i + 1) + " records", (i + 1), writer.getNumRecords());
    }
    ByteBuffer dataBuf = ByteBuffer.allocate(MAX_LOGRECORD_SIZE + 1);
    try {
        writer.writeRecord(dataBuf, new Promise<DLSN>());
        fail("Should fail on writing large record");
    } catch (LogRecordTooLongException lrtle) {
    // expected
    }
    assertEquals("5 records", 5, writer.getNumRecords());
    /// write another 5 records
    for (int i = 0; i < 5; i++) {
        ByteBuffer record = ByteBuffer.wrap(("record-" + (i + 5)).getBytes(UTF_8));
        Promise<DLSN> writePromise = new Promise<DLSN>();
        writer.writeRecord(record, writePromise);
        writePromiseList.add(writePromise);
        assertEquals((i + 6) + " records", (i + 6), writer.getNumRecords());
    }
    ByteBuffer buffer = writer.getBuffer();
    assertEquals("10 records", 10, writer.getNumRecords());
    // Test transmit complete
    writer.completeTransmit(1L, 1L, 10L);
    List<DLSN> writeResults = Await.result(Future.collect(writePromiseList));
    for (int i = 0; i < 10; i++) {
        assertEquals(new DLSN(1L, 1L, 10L + i), writeResults.get(i));
    }
    // Test reading from buffer
    byte[] data = new byte[buffer.remaining()];
    buffer.get(data);
    LogRecordWithDLSN record = new LogRecordWithDLSN(new DLSN(1L, 1L, 10L), 99L, data, 999L);
    record.setPositionWithinLogSegment(888);
    record.setRecordSet();
    Reader reader = LogRecordSet.of(record);
    LogRecordWithDLSN readRecord = reader.nextRecord();
    int numReads = 0;
    while (null != readRecord) {
        assertEquals(new DLSN(1L, 1L, 10L + numReads), readRecord.getDlsn());
        assertEquals(99L, readRecord.getTransactionId());
        assertEquals(888 + numReads, readRecord.getPositionWithinLogSegment());
        assertEquals(999L, readRecord.getStartSequenceIdOfCurrentSegment());
        assertEquals(999L + 888 + numReads - 1, readRecord.getSequenceId());
        // read next
        ++numReads;
        readRecord = reader.nextRecord();
    }
    assertEquals(10, numReads);
}

Example 32 with Future

use of com.twitter.util.Future in project distributedlog by twitter.

the class ReadUtils method getLogRecordNotLessThanTxIdFromEntries.

/**
     * Find the log record whose transaction id is not less than provided <code>transactionId</code> from
     * entries between <code>startEntryId</code> and <code>endEntryId</code>.
     *
     * @param logName
     *          name of the log
     * @param segment
     *          log segment
     * @param transactionId
     *          provided transaction id to search
     * @param executorService
     *          executor service
     * @param handleCache
     *          handle cache
     * @param entriesToSearch
     *          list of entries to search
     * @param nWays
     *          how many entries to search in parallel
     * @param prevFoundRecord
     *          the log record found in previous search
     * @param promise
     *          promise to satisfy the result
     */
private static void getLogRecordNotLessThanTxIdFromEntries(final String logName, final LedgerDescriptor ld, final LogSegmentMetadata segment, final long transactionId, final ExecutorService executorService, final LedgerHandleCache handleCache, final List<Long> entriesToSearch, final int nWays, final Optional<LogRecordWithDLSN> prevFoundRecord, final Promise<Optional<LogRecordWithDLSN>> promise) {
    final List<Future<LogRecordWithDLSN>> searchResults = Lists.newArrayListWithExpectedSize(entriesToSearch.size());
    for (Long entryId : entriesToSearch) {
        LogRecordSelector selector = new FirstTxIdNotLessThanSelector(transactionId);
        Future<LogRecordWithDLSN> searchResult = asyncReadRecordFromEntries(logName, ld, handleCache, segment, executorService, new SingleEntryScanContext(entryId), selector);
        searchResults.add(searchResult);
    }
    FutureEventListener<List<LogRecordWithDLSN>> processSearchResultsListener = new FutureEventListener<List<LogRecordWithDLSN>>() {

        @Override
        public void onSuccess(List<LogRecordWithDLSN> resultList) {
            processSearchResults(logName, ld, segment, transactionId, executorService, handleCache, resultList, nWays, prevFoundRecord, promise);
        }

        @Override
        public void onFailure(Throwable cause) {
            promise.setException(cause);
        }
    };
    Future.collect(searchResults).addEventListener(FutureEventListenerRunnable.of(processSearchResultsListener, executorService));
}

Example 33 with Future

use of com.twitter.util.Future in project distributedlog by twitter.

the class FederatedZKLogMetadataStore method fetchLogLocation.

private Future<Optional<URI>> fetchLogLocation(final String logName) {
    final Promise<Optional<URI>> fetchPromise = new Promise<Optional<URI>>();
    Set<URI> uris = subNamespaces.keySet();
    List<Future<Optional<URI>>> fetchFutures = Lists.newArrayListWithExpectedSize(uris.size());
    for (URI uri : uris) {
        fetchFutures.add(fetchLogLocation(uri, logName));
    }
    Future.collect(fetchFutures).addEventListener(new FutureEventListener<List<Optional<URI>>>() {

        @Override
        public void onSuccess(List<Optional<URI>> fetchResults) {
            Optional<URI> result = Optional.absent();
            for (Optional<URI> fetchResult : fetchResults) {
                if (result.isPresent()) {
                    if (fetchResult.isPresent()) {
                        logger.error("Log {} is found in multiple sub namespaces : {} & {}.", new Object[] { logName, result.get(), fetchResult.get() });
                        duplicatedLogName.compareAndSet(null, logName);
                        duplicatedLogFound.set(true);
                        fetchPromise.setException(new UnexpectedException("Log " + logName + " is found in multiple sub namespaces : " + result.get() + " & " + fetchResult.get()));
                        return;
                    }
                } else {
                    result = fetchResult;
                }
            }
            fetchPromise.setValue(result);
        }

        @Override
        public void onFailure(Throwable cause) {
            fetchPromise.setException(cause);
        }
    });
    return fetchPromise;
}

Example 34 with Future

use of com.twitter.util.Future in project distributedlog by twitter.

the class FederatedZKLogMetadataStore method findSubNamespaceToCreateLog.

private void findSubNamespaceToCreateLog(final String logName, final Set<URI> uris, final Promise<URI> createPromise) {
    final List<URI> uriList = Lists.newArrayListWithExpectedSize(uris.size());
    List<Future<Set<String>>> futureList = Lists.newArrayListWithExpectedSize(uris.size());
    for (URI uri : uris) {
        SubNamespace subNs = subNamespaces.get(uri);
        if (null == subNs) {
            createPromise.setException(new UnexpectedException("No sub namespace " + uri + " found"));
            return;
        }
        futureList.add(subNs.getLogs());
        uriList.add(uri);
    }
    Future.collect(futureList).addEventListener(new FutureEventListener<List<Set<String>>>() {

        @Override
        public void onSuccess(List<Set<String>> resultList) {
            for (int i = resultList.size() - 1; i >= 0; i--) {
                Set<String> logs = resultList.get(i);
                if (logs.size() < maxLogsPerSubnamespace) {
                    URI uri = uriList.get(i);
                    createLogInNamespace(uri, logName, createPromise);
                    return;
                }
            }
            // All sub namespaces are full
            createSubNamespace().addEventListener(new FutureEventListener<URI>() {

                @Override
                public void onSuccess(URI uri) {
                    // the new namespace will be propagated to the namespace cache by the namespace listener
                    // so we don't need to cache it here. we could go ahead to create the stream under this
                    // namespace, as we are using sequential znode. we are mostly the first guy who create
                    // the log under this namespace.
                    createLogInNamespace(uri, logName, createPromise);
                }

                @Override
                public void onFailure(Throwable cause) {
                    createPromise.setException(cause);
                }
            });
        }

        @Override
        public void onFailure(Throwable cause) {
            createPromise.setException(cause);
        }
    });
}

Example 35 with Future

use of com.twitter.util.Future in project distributedlog by twitter.

the class AtomicWriter method main.

public static void main(String[] args) throws Exception {
    if (args.length < 3) {
        System.out.println(HELP);
        return;
    }
    String finagleNameStr = args[0];
    String streamName = args[1];
    String[] messages = new String[args.length - 2];
    System.arraycopy(args, 2, messages, 0, messages.length);
    DistributedLogClient client = DistributedLogClientBuilder.newBuilder().clientId(ClientId$.MODULE$.apply("atomic-writer")).name("atomic-writer").thriftmux(true).finagleNameStr(finagleNameStr).build();
    final LogRecordSet.Writer recordSetWriter = LogRecordSet.newWriter(16 * 1024, Type.NONE);
    List<Future<DLSN>> writeFutures = Lists.newArrayListWithExpectedSize(messages.length);
    for (String msg : messages) {
        final String message = msg;
        ByteBuffer msgBuf = ByteBuffer.wrap(msg.getBytes(UTF_8));
        Promise<DLSN> writeFuture = new Promise<DLSN>();
        writeFuture.addEventListener(new FutureEventListener<DLSN>() {

            @Override
            public void onFailure(Throwable cause) {
                System.out.println("Encountered error on writing data");
                cause.printStackTrace(System.err);
                Runtime.getRuntime().exit(0);
            }

            @Override
            public void onSuccess(DLSN dlsn) {
                System.out.println("Write '" + message + "' as record " + dlsn);
            }
        });
        recordSetWriter.writeRecord(msgBuf, writeFuture);
        writeFutures.add(writeFuture);
    }
    FutureUtils.result(client.writeRecordSet(streamName, recordSetWriter).addEventListener(new FutureEventListener<DLSN>() {

        @Override
        public void onFailure(Throwable cause) {
            recordSetWriter.abortTransmit(cause);
            System.out.println("Encountered error on writing data");
            cause.printStackTrace(System.err);
            Runtime.getRuntime().exit(0);
        }

        @Override
        public void onSuccess(DLSN dlsn) {
            recordSetWriter.completeTransmit(dlsn.getLogSegmentSequenceNo(), dlsn.getEntryId(), dlsn.getSlotId());
        }
    }));
    FutureUtils.result(Future.collect(writeFutures));
    client.close();
}