Examples with ListenableFutureTask com.google.common.util.concurrent.ListenableFutureTask used on opensource projects

Example 1 with ListenableFutureTask

use of com.google.common.util.concurrent.ListenableFutureTask in project cassandra by apache.

the class LongBTreeTest method testInsertions.

private static void testInsertions(int tests, int perTestCount, float testKeyRatio, int modificationBatchSize, boolean quickEquality) throws ExecutionException, InterruptedException {
    int batchesPerTest = perTestCount / modificationBatchSize;
    int testKeyRange = (int) (perTestCount * testKeyRatio);
    long totalCount = (long) perTestCount * tests;
    log("Performing %d tests of %d operations, with %.2f max size/key-range ratio in batches of ~%d ops", tests, perTestCount, 1 / testKeyRatio, modificationBatchSize);
    // if we're not doing quick-equality, we can spam with garbage for all the checks we perform, so we'll split the work into smaller chunks
    int chunkSize = quickEquality ? tests : (int) (100000 / Math.pow(perTestCount, 2));
    for (int chunk = 0; chunk < tests; chunk += chunkSize) {
        final List<ListenableFutureTask<List<ListenableFuture<?>>>> outer = new ArrayList<>();
        for (int i = 0; i < chunkSize; i++) {
            int maxRunLength = modificationBatchSize == 1 ? 1 : ThreadLocalRandom.current().nextInt(1, modificationBatchSize);
            outer.add(doOneTestInsertions(testKeyRange, maxRunLength, modificationBatchSize, batchesPerTest, quickEquality));
        }
        final List<ListenableFuture<?>> inner = new ArrayList<>();
        long complete = 0;
        int reportInterval = Math.max(1000, (int) (totalCount / 10000));
        long lastReportAt = 0;
        for (ListenableFutureTask<List<ListenableFuture<?>>> f : outer) {
            inner.addAll(f.get());
            complete += perTestCount;
            if (complete - lastReportAt >= reportInterval) {
                long done = (chunk * perTestCount) + complete;
                float ratio = done / (float) totalCount;
                log("Completed %.1f%% (%d of %d operations)", ratio * 100, done, totalCount);
                lastReportAt = complete;
            }
        }
        Futures.allAsList(inner).get();
    }
    Snapshot snap = BTREE_TIMER.getSnapshot();
    log("btree: %.2fns, %.2fns, %.2fns", snap.getMedian(), snap.get95thPercentile(), snap.get999thPercentile());
    snap = TREE_TIMER.getSnapshot();
    log("java: %.2fns, %.2fns, %.2fns", snap.getMedian(), snap.get95thPercentile(), snap.get999thPercentile());
    log("Done");
}

Example 2 with ListenableFutureTask

use of com.google.common.util.concurrent.ListenableFutureTask in project jackrabbit-oak by apache.

the class AtomicCounterIT method concurrentSegmentIncrements.

@Test
public void concurrentSegmentIncrements() throws RepositoryException, InterruptedException, ExecutionException {
    // ensuring the run only on allowed fix
    assumeTrue(NodeStoreFixtures.SEGMENT_TAR.equals(fixture));
    // setting-up
    Session session = getAdminSession();
    try {
        Node counter = session.getRootNode().addNode("counter");
        counter.addMixin(MIX_ATOMIC_COUNTER);
        session.save();
        final AtomicLong expected = new AtomicLong(0);
        final String counterPath = counter.getPath();
        final Random rnd = new Random(11);
        // ensuring initial state
        assertEquals(expected.get(), counter.getProperty(PROP_COUNTER).getLong());
        List<ListenableFutureTask<Void>> tasks = Lists.newArrayList();
        for (int t = 0; t < 100; t++) {
            tasks.add(updateCounter(counterPath, rnd.nextInt(10) + 1, expected));
        }
        Futures.allAsList(tasks).get();
        session.refresh(false);
        assertEquals(expected.get(), session.getNode(counterPath).getProperty(PROP_COUNTER).getLong());
    } finally {
        session.logout();
    }
}

Example 3 with ListenableFutureTask

use of com.google.common.util.concurrent.ListenableFutureTask in project cassandra by apache.

the class PendingAntiCompaction method run.

public ListenableFuture run() {
    ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(prsId);
    List<ListenableFutureTask<AcquireResult>> tasks = new ArrayList<>();
    for (ColumnFamilyStore cfs : prs.getColumnFamilyStores()) {
        cfs.forceBlockingFlush();
        ListenableFutureTask<AcquireResult> task = ListenableFutureTask.create(new AcquisitionCallable(cfs, ranges, prsId));
        executor.submit(task);
        tasks.add(task);
    }
    ListenableFuture<List<AcquireResult>> acquisitionResults = Futures.successfulAsList(tasks);
    ListenableFuture compactionResult = Futures.transform(acquisitionResults, new AcquisitionCallback(prsId, ranges));
    return compactionResult;
}

Example 4 with ListenableFutureTask

use of com.google.common.util.concurrent.ListenableFutureTask in project pinot by linkedin.

the class BaseClusterIntegrationTest method buildSegmentsFromAvro.

public static Future<Map<File, File>> buildSegmentsFromAvro(final List<File> avroFiles, Executor executor, int baseSegmentIndex, final File baseDirectory, final File segmentTarDir, final String tableName, final boolean createStarTreeIndex, final com.linkedin.pinot.common.data.Schema inputPinotSchema) {
    int segmentCount = avroFiles.size();
    LOGGER.info("Building " + segmentCount + " segments in parallel");
    List<ListenableFutureTask<Pair<File, File>>> futureTasks = new ArrayList<ListenableFutureTask<Pair<File, File>>>();
    for (int i = 1; i <= segmentCount; ++i) {
        final int segmentIndex = i - 1;
        final int segmentNumber = i + baseSegmentIndex;
        final ListenableFutureTask<Pair<File, File>> buildSegmentFutureTask = ListenableFutureTask.<Pair<File, File>>create(new Callable<Pair<File, File>>() {

            @Override
            public Pair<File, File> call() throws Exception {
                try {
                    // Build segment
                    LOGGER.info("Starting to build segment " + segmentNumber);
                    File outputDir = new File(baseDirectory, "segment-" + segmentNumber);
                    final File inputAvroFile = avroFiles.get(segmentIndex);
                    final SegmentGeneratorConfig genConfig = SegmentTestUtils.getSegmentGenSpecWithSchemAndProjectedColumns(inputAvroFile, outputDir, TimeUnit.DAYS, tableName, inputPinotSchema);
                    if (inputPinotSchema != null) {
                        genConfig.setSchema(inputPinotSchema);
                    }
                    // jfim: We add a space and a special character to do a regression test for PINOT-3296 Segments with spaces
                    // in their filename don't work properly
                    genConfig.setSegmentNamePostfix(Integer.toString(segmentNumber) + " %");
                    genConfig.setEnableStarTreeIndex(createStarTreeIndex);
                    // Enable off heap star tree format in the integration test.
                    StarTreeIndexSpec starTreeIndexSpec = null;
                    if (createStarTreeIndex) {
                        starTreeIndexSpec = new StarTreeIndexSpec();
                        starTreeIndexSpec.setEnableOffHeapFormat(true);
                    }
                    genConfig.setStarTreeIndexSpec(starTreeIndexSpec);
                    final SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null);
                    driver.init(genConfig);
                    driver.build();
                    // Tar segment
                    String segmentName = outputDir.list()[0];
                    final String tarGzPath = TarGzCompressionUtils.createTarGzOfDirectory(outputDir.getAbsolutePath() + "/" + segmentName, new File(segmentTarDir, segmentName).getAbsolutePath());
                    LOGGER.info("Completed segment " + segmentNumber + " : " + segmentName + " from file " + inputAvroFile.getName());
                    return new ImmutablePair<File, File>(inputAvroFile, new File(tarGzPath));
                } catch (Exception e) {
                    LOGGER.error("Exception while building segment input: {} output {} ", avroFiles.get(segmentIndex), "segment-" + segmentNumber);
                    throw new RuntimeException(e);
                }
            }
        });
        futureTasks.add(buildSegmentFutureTask);
        executor.execute(buildSegmentFutureTask);
    }
    ListenableFuture<List<Pair<File, File>>> pairListFuture = Futures.allAsList(futureTasks);
    return Futures.transform(pairListFuture, new AsyncFunction<List<Pair<File, File>>, Map<File, File>>() {

        @Override
        public ListenableFuture<Map<File, File>> apply(List<Pair<File, File>> input) throws Exception {
            Map<File, File> avroToSegmentMap = new HashMap<File, File>();
            for (Pair<File, File> avroToSegmentPair : input) {
                avroToSegmentMap.put(avroToSegmentPair.getLeft(), avroToSegmentPair.getRight());
            }
            return Futures.immediateFuture(avroToSegmentMap);
        }
    });
}

Example 5 with ListenableFutureTask

use of com.google.common.util.concurrent.ListenableFutureTask in project jackrabbit-oak by apache.

the class AtomicCounterClusterIT method increments.

@Test
public void increments() throws Exception {
    setUpCluster(this.getClass(), mks, repos, NOT_PROVIDED);
    assertEquals("repositories and executors should match", repos.size(), executors.size());
    final String counterPath;
    final Random rnd = new Random(14);
    final AtomicLong expected = new AtomicLong(0);
    final Map<String, Exception> exceptions = Collections.synchronizedMap(new HashMap<String, Exception>());
    // setting-up the repo state
    Repository repo = repos.get(0);
    Session session = repo.login(ADMIN);
    Node counter;
    try {
        counter = session.getRootNode().addNode("counter");
        counter.addMixin(MIX_ATOMIC_COUNTER);
        session.save();
        counterPath = counter.getPath();
    } finally {
        session.logout();
    }
    // allow the cluster to align
    alignCluster(mks);
    // asserting the initial state
    assertFalse("Path to the counter node should be set", Strings.isNullOrEmpty(counterPath));
    for (Repository r : repos) {
        try {
            session = r.login(ADMIN);
            counter = session.getNode(counterPath);
            assertEquals("Nothing should have touched the `expected`", 0, expected.get());
            assertEquals("Wrong initial counter", expected.get(), counter.getProperty(PROP_COUNTER).getLong());
        } finally {
            session.logout();
        }
    }
    // number of threads per cluster node
    final int numIncrements = Integer.getInteger("oak.test.it.atomiccounter.threads", 100);
    LOG.debug("pushing {} increments per each of the {} cluster nodes for a total of {} concurrent updates", numIncrements, repos.size(), numIncrements * repos.size());
    // for each cluster node, `numIncrements` sessions pushing random increments
    long start = LOG_PERF.start("Firing the threads");
    List<ListenableFutureTask<Void>> tasks = Lists.newArrayList();
    for (Repository rep : repos) {
        final Repository r = rep;
        for (int i = 0; i < numIncrements; i++) {
            ListenableFutureTask<Void> task = ListenableFutureTask.create(new Callable<Void>() {

                @Override
                public Void call() throws Exception {
                    Session s = r.login(ADMIN);
                    try {
                        try {
                            Node n = s.getNode(counterPath);
                            int increment = rnd.nextInt(10) + 1;
                            n.setProperty(PROP_INCREMENT, increment);
                            expected.addAndGet(increment);
                            s.save();
                        } finally {
                            s.logout();
                        }
                    } catch (Exception e) {
                        exceptions.put(Thread.currentThread().getName(), e);
                    }
                    return null;
                }
            });
            new Thread(task).start();
            tasks.add(task);
        }
    }
    LOG_PERF.end(start, -1, "Firing threads completed", "");
    Futures.allAsList(tasks).get();
    LOG_PERF.end(start, -1, "Futures completed", "");
    waitForTaskCompletion();
    LOG_PERF.end(start, -1, "All tasks completed", "");
    // let the time for the async process to kick in and run.
    Thread.sleep(5000);
    raiseExceptions(exceptions, LOG);
    // assert the final situation
    for (int i = 0; i < repos.size(); i++) {
        Repository r = repos.get(i);
        try {
            session = r.login(ADMIN);
            counter = session.getNode(counterPath);
            LOG.debug("Cluster node: {}, actual counter: {}, expected counter: {}", i + 1, expected.get(), counter.getProperty(PROP_COUNTER).getLong());
            assertEquals("Wrong counter on node " + (i + 1), expected.get(), counter.getProperty(PROP_COUNTER).getLong());
        } finally {
            session.logout();
        }
    }
}