Examples with SettableFuture com.google.common.util.concurrent.SettableFuture used on opensource projects

Example 11 with SettableFuture

use of com.google.common.util.concurrent.SettableFuture in project presto by prestodb.

the class TestThriftIndexPageSource method testGetNextPageTwoConcurrentRequests.

@Test
public void testGetNextPageTwoConcurrentRequests() throws Exception {
    final int splits = 3;
    final int lookupRequestsConcurrency = 2;
    final int rowsPerSplit = 1;
    List<SettableFuture<PrestoThriftPageResult>> futures = IntStream.range(0, splits).mapToObj(i -> SettableFuture.<PrestoThriftPageResult>create()).collect(toImmutableList());
    List<CountDownLatch> signals = IntStream.range(0, splits).mapToObj(i -> new CountDownLatch(1)).collect(toImmutableList());
    TestingThriftService client = new TestingThriftService(rowsPerSplit, false, false) {

        @Override
        public ListenableFuture<PrestoThriftPageResult> getRows(PrestoThriftId splitId, List<String> columns, long maxBytes, PrestoThriftNullableToken nextToken) {
            int key = Ints.fromByteArray(splitId.getId());
            signals.get(key).countDown();
            return futures.get(key);
        }
    };
    ThriftConnectorStats stats = new ThriftConnectorStats();
    long pageSizeReceived = 0;
    ThriftIndexPageSource pageSource = new ThriftIndexPageSource((context, headers) -> client, ImmutableMap.of(), stats, new ThriftIndexHandle(new SchemaTableName("default", "table1"), TupleDomain.all()), ImmutableList.of(column("a", INTEGER)), ImmutableList.of(column("b", INTEGER)), new InMemoryRecordSet(ImmutableList.of(INTEGER), generateKeys(0, splits)), MAX_BYTES_PER_RESPONSE, lookupRequestsConcurrency);
    assertNull(pageSource.getNextPage());
    assertEquals((long) stats.getIndexPageSize().getAllTime().getTotal(), 0);
    signals.get(0).await(1, SECONDS);
    signals.get(1).await(1, SECONDS);
    signals.get(2).await(1, SECONDS);
    assertEquals(signals.get(0).getCount(), 0, "first request wasn't sent");
    assertEquals(signals.get(1).getCount(), 0, "second request wasn't sent");
    assertEquals(signals.get(2).getCount(), 1, "third request shouldn't be sent");
    // at this point first two requests were sent
    assertFalse(pageSource.isFinished());
    assertNull(pageSource.getNextPage());
    assertEquals((long) stats.getIndexPageSize().getAllTime().getTotal(), 0);
    // completing the second request
    futures.get(1).set(pageResult(20, null));
    Page page = pageSource.getNextPage();
    pageSizeReceived += page.getSizeInBytes();
    assertEquals((long) stats.getIndexPageSize().getAllTime().getTotal(), pageSizeReceived);
    assertNotNull(page);
    assertEquals(page.getPositionCount(), 1);
    assertEquals(page.getBlock(0).getInt(0), 20);
    // not complete yet
    assertFalse(pageSource.isFinished());
    // once one of the requests completes the next one should be sent
    signals.get(2).await(1, SECONDS);
    assertEquals(signals.get(2).getCount(), 0, "third request wasn't sent");
    // completing the first request
    futures.get(0).set(pageResult(10, null));
    page = pageSource.getNextPage();
    assertNotNull(page);
    pageSizeReceived += page.getSizeInBytes();
    assertEquals((long) stats.getIndexPageSize().getAllTime().getTotal(), pageSizeReceived);
    assertEquals(page.getPositionCount(), 1);
    assertEquals(page.getBlock(0).getInt(0), 10);
    // still not complete
    assertFalse(pageSource.isFinished());
    // completing the third request
    futures.get(2).set(pageResult(30, null));
    page = pageSource.getNextPage();
    assertNotNull(page);
    pageSizeReceived += page.getSizeInBytes();
    assertEquals((long) stats.getIndexPageSize().getAllTime().getTotal(), pageSizeReceived);
    assertEquals(page.getPositionCount(), 1);
    assertEquals(page.getBlock(0).getInt(0), 30);
    // finished now
    assertTrue(pageSource.isFinished());
    // after completion
    assertNull(pageSource.getNextPage());
    pageSource.close();
}

Example 12 with SettableFuture

use of com.google.common.util.concurrent.SettableFuture in project presto by prestodb.

the class ExchangeClient method addPages.

private boolean addPages(List<SerializedPage> pages) {
    // Compute stats before acquiring the lock
    long pagesRetainedSizeInBytes = 0;
    long responseSize = 0;
    for (SerializedPage page : pages) {
        pagesRetainedSizeInBytes += page.getRetainedSizeInBytes();
        responseSize += page.getSizeInBytes();
    }
    List<SettableFuture<?>> notify = ImmutableList.of();
    synchronized (this) {
        if (isClosed() || isFailed()) {
            return false;
        }
        if (!pages.isEmpty()) {
            pageBuffer.addAll(pages);
            bufferRetainedSizeInBytes += pagesRetainedSizeInBytes;
            maxBufferRetainedSizeInBytes = max(maxBufferRetainedSizeInBytes, bufferRetainedSizeInBytes);
            systemMemoryContext.setBytes(bufferRetainedSizeInBytes);
            // Notify pending listeners that a page has been added
            notify = ImmutableList.copyOf(blockedCallers);
            blockedCallers.clear();
        }
        successfulRequests++;
        responseSizeExponentialMovingAverage.update(responseSize);
    }
    // Trigger notifications after releasing the lock
    notifyListeners(notify);
    return true;
}

Example 13 with SettableFuture

use of com.google.common.util.concurrent.SettableFuture in project presto by prestodb.

the class HivePageSink method doFinish.

private ListenableFuture<Collection<Slice>> doFinish() {
    ImmutableList.Builder<PartitionUpdate> partitionUpdatesBuilder = ImmutableList.builder();
    List<Callable<Object>> verificationTasks = new ArrayList<>();
    for (HiveWriter writer : writers) {
        writer.commit();
        partitionUpdatesBuilder.add(writer.getPartitionUpdate());
        writer.getVerificationTask().map(Executors::callable).ifPresent(verificationTasks::add);
    }
    List<PartitionUpdate> partitionUpdates = partitionUpdatesBuilder.build();
    boolean optimizedPartitionUpdateSerializationEnabled = isOptimizedPartitionUpdateSerializationEnabled(session);
    if (optimizedPartitionUpdateSerializationEnabled) {
        // Merge multiple partition updates for a single partition into one.
        // Multiple partition updates for a single partition are produced when writing into a bucketed table.
        // Merged partition updates will contain multiple items in the fileWriteInfos list (one per bucket).
        // This optimization should be enabled only together with the optimized serialization (compression + binary encoding).
        // Since serialized fragments will be transmitted as Presto pages serializing a merged partition update to JSON without
        // compression is unsafe, as it may cross the maximum page size limit.
        partitionUpdates = mergePartitionUpdates(partitionUpdates);
    }
    ImmutableList.Builder<Slice> serializedPartitionUpdatesBuilder = ImmutableList.builder();
    for (PartitionUpdate partitionUpdate : partitionUpdates) {
        byte[] serializedBytes;
        if (optimizedPartitionUpdateSerializationEnabled) {
            serializedBytes = serializeZstdCompressed(partitionUpdateSmileCodec, partitionUpdate);
        } else {
            serializedBytes = partitionUpdateCodec.toBytes(partitionUpdate);
        }
        serializedPartitionUpdatesBuilder.add(wrappedBuffer(serializedBytes));
    }
    List<Slice> serializedPartitionUpdates = serializedPartitionUpdatesBuilder.build();
    writtenBytes = writers.stream().mapToLong(HiveWriter::getWrittenBytes).sum();
    validationCpuNanos = writers.stream().mapToLong(HiveWriter::getValidationCpuNanos).sum();
    if (waitForFileRenaming && verificationTasks.isEmpty()) {
        // Use CopyOnWriteArrayList to prevent race condition when callbacks try to add partitionUpdates to this list
        List<Slice> partitionUpdatesWithRenamedFileNames = new CopyOnWriteArrayList<>();
        List<ListenableFuture<?>> futures = new ArrayList<>();
        for (int i = 0; i < writers.size(); i++) {
            int writerIndex = i;
            ListenableFuture<?> fileNameFuture = toListenableFuture(hiveMetadataUpdater.getMetadataResult(writerIndex));
            SettableFuture renamingFuture = SettableFuture.create();
            futures.add(renamingFuture);
            addSuccessCallback(fileNameFuture, obj -> renameFiles((String) obj, writerIndex, renamingFuture, partitionUpdatesWithRenamedFileNames));
        }
        return Futures.transform(Futures.allAsList(futures), input -> partitionUpdatesWithRenamedFileNames, directExecutor());
    }
    if (verificationTasks.isEmpty()) {
        return Futures.immediateFuture(serializedPartitionUpdates);
    }
    try {
        List<ListenableFuture<?>> futures = writeVerificationExecutor.invokeAll(verificationTasks).stream().map(future -> (ListenableFuture<?>) future).collect(toList());
        return Futures.transform(Futures.allAsList(futures), input -> serializedPartitionUpdates, directExecutor());
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        throw new RuntimeException(e);
    }
}

Example 14 with SettableFuture

use of com.google.common.util.concurrent.SettableFuture in project azure-tools-for-java by Microsoft.

the class AddDependencyAction method onActionPerformed.

@Override
public boolean onActionPerformed(@NotNull AnActionEvent event, @Nullable Operation operation) {
    final Module module = event.getData(LangDataKeys.MODULE);
    final Project project = module.getProject();
    final MavenProjectsManager projectsManager = MavenProjectsManager.getInstance(project);
    final MavenProject mavenProject = projectsManager.findProject(module);
    if (mavenProject == null) {
        PluginUtil.showErrorNotificationProject(project, "Error", String.format("Project '%s' is not a maven project.", project.getName()));
        return true;
    }
    final AzureString title = AzureOperationBundle.title("springcloud.update_dependency", project.getName());
    AzureTaskManager.getInstance().runInBackground(new AzureTask(project, title, true, () -> {
        ProgressIndicator progressIndicator = ProgressManager.getInstance().getProgressIndicator();
        progressIndicator.setText("Syncing maven project " + project.getName());
        final SettableFuture<Boolean> isDirty = SettableFuture.create();
        AzureTaskManager.getInstance().runAndWait(() -> {
            ProjectNotificationAware notificationAware = ProjectNotificationAware.getInstance(project);
            isDirty.set(notificationAware.isNotificationVisible());
            if (notificationAware.isNotificationVisible()) {
                ExternalSystemProjectTracker projectTracker = ExternalSystemProjectTracker.getInstance(project);
                projectTracker.scheduleProjectRefresh();
            }
        });
        try {
            if (isDirty.get().booleanValue()) {
                projectsManager.forceUpdateProjects(Collections.singletonList(mavenProject)).get();
            }
        } catch (InterruptedException | ExecutionException e) {
            PluginUtil.showErrorNotification("Error", "Failed to update project due to error: " + e.getMessage());
            return;
        }
        try {
            progressIndicator.setText("Check existing dependencies");
            final String evaluateEffectivePom = MavenUtils.evaluateEffectivePom(project, mavenProject);
            ProgressManager.checkCanceled();
            if (StringUtils.isEmpty(evaluateEffectivePom)) {
                PluginUtil.showErrorNotificationProject(project, "Error", "Failed to evaluate effective pom.");
                return;
            }
            final String springBootVer = getMavenLibraryVersion(mavenProject, SPRING_BOOT_GROUP_ID, "spring-boot-autoconfigure");
            if (StringUtils.isEmpty(springBootVer)) {
                throw new AzureExecutionException(String.format("Module %s is not a spring-boot application.", module.getName()));
            }
            progressIndicator.setText("Get latest versions ...");
            SpringCloudDependencyManager dependencyManager = new SpringCloudDependencyManager(evaluateEffectivePom);
            Map<String, DependencyArtifact> versionMaps = dependencyManager.getDependencyVersions();
            Map<String, DependencyArtifact> managerDependencyVersionsMaps = dependencyManager.getDependencyManagementVersions();
            // given the spring-cloud-commons is greater or equal to 2.2.5.RELEASE, we should not add spring-cloud-starter-azure-spring-cloud-client
            // because the code is already merged into spring repo: https://github.com/spring-cloud/spring-cloud-commons/pull/803
            boolean noAzureSpringCloudClientDependency = shouldNotAddAzureSpringCloudClientDependency(versionMaps) || shouldNotAddAzureSpringCloudClientDependency(managerDependencyVersionsMaps);
            ProgressManager.checkCanceled();
            final List<DependencyArtifact> versionChanges = calculateVersionChanges(springBootVer, noAzureSpringCloudClientDependency, versionMaps);
            if (versionChanges.isEmpty()) {
                PluginUtil.showInfoNotificationProject(project, "Your project is update-to-date.", "No updates are needed.");
                return;
            }
            progressIndicator.setText("Applying versions ...");
            final File pomFile = new File(mavenProject.getFile().getCanonicalPath());
            ProgressManager.checkCanceled();
            if (applyVersionChanges(dependencyManager, pomFile, springBootVer, managerDependencyVersionsMaps, versionChanges)) {
                noticeUserVersionChanges(project, pomFile, versionChanges);
            } else {
                PluginUtil.showInfoNotificationProject(project, "Your project is update-to-date.", "No updates are needed.");
            }
        } catch (DocumentException | IOException | AzureExecutionException | MavenProcessCanceledException e) {
            PluginUtil.showErrorNotification("Error", "Failed to update Azure Spring Cloud dependencies due to error: " + e.getMessage());
        }
    }));
    return false;
}

Example 15 with SettableFuture

use of com.google.common.util.concurrent.SettableFuture in project storm by apache.

the class AsyncExecutor method execAsync.

/**
 * Asynchronously executes all statements associated to the specified input.
 * The input will be passed to handler#onSuccess once all queries succeed or to handler#onFailure if any one of them fails.
 */
public List<SettableFuture<T>> execAsync(List<Statement> statements, final T input) {
    List<SettableFuture<T>> settableFutures = new ArrayList<>(statements.size());
    for (Statement s : statements) {
        settableFutures.add(execAsync(s, input, AsyncResultHandler.NO_OP_HANDLER));
    }
    ListenableFuture<List<T>> allAsList = Futures.allAsList(settableFutures);
    Futures.addCallback(allAsList, new FutureCallback<List<T>>() {

        @Override
        public void onSuccess(List<T> inputs) {
            handler.success(input);
        }

        @Override
        public void onFailure(Throwable t) {
            handler.failure(t, input);
        }
    }, executorService);
    return settableFutures;
}