Examples with ExecutorCompletionService java.util.concurrent.ExecutorCompletionService used on opensource projects

Example 41 with ExecutorCompletionService

use of java.util.concurrent.ExecutorCompletionService in project pinot by linkedin.

the class MultiGetRequest method execute.

/**
   * GET urls in parallel using the executor service.
   * @param urls absolute URLs to GET
   * @param timeoutMs timeout in milliseconds for each GET request
   * @return instance of CompletionService. Completion service will provide
   *   results as they arrive. The order is NOT same as the order of URLs
   */
public CompletionService<GetMethod> execute(@Nonnull List<String> urls, final int timeoutMs) {
    Preconditions.checkNotNull(urls);
    Preconditions.checkArgument(timeoutMs > 0, "Timeout value for multi-get must be greater than 0");
    CompletionService<GetMethod> completionService = new ExecutorCompletionService<>(executor);
    for (final String url : urls) {
        completionService.submit(new Callable<GetMethod>() {

            @Override
            public GetMethod call() throws Exception {
                HttpClient client = new HttpClient(connectionManager);
                GetMethod getMethod = new GetMethod(url);
                getMethod.getParams().setSoTimeout(timeoutMs);
                // if all connections in the connection manager are busy this will wait to retrieve a connection
                // set time to wait to retrieve a connection from connection manager
                client.getParams().setConnectionManagerTimeout(timeoutMs);
                client.executeMethod(getMethod);
                return getMethod;
            }
        });
    }
    return completionService;
}

Example 42 with ExecutorCompletionService

use of java.util.concurrent.ExecutorCompletionService in project guava by hceylan.

the class AbstractListeningExecutorService method doInvokeAny.

/**
   * the main mechanics of invokeAny.
   */
private <T> T doInvokeAny(Collection<? extends Callable<T>> tasks, boolean timed, long nanos) throws InterruptedException, ExecutionException, TimeoutException {
    int ntasks = tasks.size();
    checkArgument(ntasks > 0);
    List<Future<T>> futures = new ArrayList<Future<T>>(ntasks);
    ExecutorCompletionService<T> ecs = new ExecutorCompletionService<T>(this);
    try {
        // Record exceptions so that if we fail to obtain any
        // result, we can throw the last exception we got.
        ExecutionException ee = null;
        long lastTime = timed ? System.nanoTime() : 0;
        Iterator<? extends Callable<T>> it = tasks.iterator();
        // Start one task for sure; the rest incrementally
        futures.add(ecs.submit(it.next()));
        --ntasks;
        int active = 1;
        for (; ; ) {
            Future<T> f = ecs.poll();
            if (f == null) {
                if (ntasks > 0) {
                    --ntasks;
                    futures.add(ecs.submit(it.next()));
                    ++active;
                } else if (active == 0) {
                    break;
                } else if (timed) {
                    f = ecs.poll(nanos, TimeUnit.NANOSECONDS);
                    if (f == null) {
                        throw new TimeoutException();
                    }
                    long now = System.nanoTime();
                    nanos -= now - lastTime;
                    lastTime = now;
                } else {
                    f = ecs.take();
                }
            }
            if (f != null) {
                --active;
                try {
                    return f.get();
                } catch (ExecutionException eex) {
                    ee = eex;
                } catch (RuntimeException rex) {
                    ee = new ExecutionException(rex);
                }
            }
        }
        if (ee == null) {
            ee = new ExecutionException(null);
        }
        throw ee;
    } finally {
        for (Future<T> f : futures) {
            f.cancel(true);
        }
    }
}

Example 43 with ExecutorCompletionService

use of java.util.concurrent.ExecutorCompletionService in project otter by alibaba.

the class GlobalMonitor method concurrentProcess.

private void concurrentProcess(Map<Long, List<AlarmRule>> rules) {
    ExecutorCompletionService completionExecutor = new ExecutorCompletionService(executor);
    List<Future> futures = new ArrayList<Future>();
    for (Entry<Long, List<AlarmRule>> entry : rules.entrySet()) {
        final List<AlarmRule> alarmRules = entry.getValue();
        futures.add(completionExecutor.submit(new Callable<Object>() {

            @Override
            public Object call() throws Exception {
                pipelineMonitor.explore(alarmRules);
                return null;
            }
        }));
    }
    List<Throwable> exceptions = new ArrayList<Throwable>();
    int index = 0;
    int size = futures.size();
    while (index < size) {
        try {
            Future<?> future = completionExecutor.take();
            future.get();
        } catch (InterruptedException e) {
            exceptions.add(e);
        } catch (ExecutionException e) {
            exceptions.add(e);
        }
        index++;
    }
    if (!exceptions.isEmpty()) {
        StringBuilder sb = new StringBuilder(exceptions.size() + " exception happens in global monitor\n");
        sb.append("exception stack start :\n");
        for (Throwable t : exceptions) {
            sb.append(ExceptionUtils.getStackTrace(t));
        }
        sb.append("exception stack end \n");
        throw new IllegalStateException(sb.toString());
    }
}

Example 44 with ExecutorCompletionService

use of java.util.concurrent.ExecutorCompletionService in project otter by alibaba.

the class GlobalMonitor method concurrentProcess.

private void concurrentProcess(List<Long> channelIds) {
    ExecutorCompletionService completionExecutor = new ExecutorCompletionService(executor);
    List<Future> futures = new ArrayList<Future>();
    for (final Long channelId : channelIds) {
        futures.add(completionExecutor.submit(new Callable<Object>() {

            @Override
            public Object call() throws Exception {
                ChannelStatus status = arbitrateManageService.channelEvent().status(channelId);
                if (status.isPause()) {
                    restartAlarmRecovery.recovery(channelId);
                }
                return null;
            }
        }));
    }
    List<Throwable> exceptions = new ArrayList<Throwable>();
    int index = 0;
    int size = futures.size();
    while (index < size) {
        try {
            Future<?> future = completionExecutor.take();
            future.get();
        } catch (InterruptedException e) {
            exceptions.add(e);
        } catch (ExecutionException e) {
            exceptions.add(e);
        }
        index++;
    }
    if (!exceptions.isEmpty()) {
        StringBuilder sb = new StringBuilder(exceptions.size() + " exception happens in global monitor\n");
        sb.append("exception stack start :\n");
        for (Throwable t : exceptions) {
            sb.append(ExceptionUtils.getStackTrace(t));
        }
        sb.append("exception stack end \n");
        throw new IllegalStateException(sb.toString());
    }
}

Example 45 with ExecutorCompletionService

use of java.util.concurrent.ExecutorCompletionService in project otter by alibaba.

the class DataBatchLoader method load.

public List<LoadContext> load(DbBatch data) {
    final RowBatch rowBatch = data.getRowBatch();
    final FileBatch fileBatch = data.getFileBatch();
    boolean existFileBatch = (rowBatch != null && !CollectionUtils.isEmpty(fileBatch.getFiles()) && data.getRoot() != null);
    boolean existRowBatch = (rowBatch != null && !CollectionUtils.isEmpty(rowBatch.getDatas()));
    int count = 0;
    List<RowBatch> rowBatchs = null;
    if (existRowBatch) {
        // 根据介质内容进行分类合并，每个介质一个载入通道
        rowBatchs = split(rowBatch);
        count += rowBatchs.size();
    }
    if (existFileBatch) {
        count += 1;
    }
    WeightController controller = new WeightController(count);
    List<Future> futures = new ArrayList<Future>();
    ExecutorCompletionService completionService = new ExecutorCompletionService(executorService);
    if (existFileBatch) {
        submitFileBatch(futures, completionService, fileBatch, data.getRoot(), controller);
    }
    if (existRowBatch) {
        submitRowBatch(futures, completionService, rowBatchs, controller);
    }
    // 先获取一下异步处理的结果，记录一下出错的index
    List<LoadContext> processedContexts = new ArrayList<LoadContext>();
    int index = 0;
    LoadException exception = null;
    while (index < futures.size()) {
        try {
            // 它也可能被打断
            Future future = completionService.take();
            future.get();
        } catch (InterruptedException e) {
            exception = new LoadException(e);
            break;
        } catch (ExecutionException e) {
            exception = new LoadException(e);
            break;
        }
        index++;
    }
    // 任何一个线程返回，出现了异常，就退出整个调度
    if (index < futures.size()) {
        // 小于代表有错误，需要对未完成的记录进行cancel操作，对已完成的结果进行收集，做重复录入过滤记录
        for (int errorIndex = 0; errorIndex < futures.size(); errorIndex++) {
            Future future = futures.get(errorIndex);
            if (future.isDone()) {
                try {
                    LoadContext loadContext = (LoadContext) future.get();
                    if (loadContext instanceof DbLoadContext) {
                        // 做一下出错处理，记录到store中
                        dbInterceptor.error((DbLoadContext) loadContext);
                    }
                } catch (InterruptedException e) {
                // ignore
                } catch (ExecutionException e) {
                // ignore
                } catch (Exception e) {
                    logger.error("interceptor process error failed", e);
                }
            } else {
                // 对未完成的进行取消
                future.cancel(true);
            }
        }
    } else {
        for (int i = 0; i < futures.size(); i++) {
            // 收集一下正确处理完成的结果
            Future future = futures.get(i);
            try {
                LoadContext loadContext = (LoadContext) future.get();
                if (loadContext instanceof DbLoadContext) {
                    processedContexts.add((DbLoadContext) loadContext);
                }
            } catch (InterruptedException e) {
            // ignore
            } catch (ExecutionException e) {
            // ignore
            }
        }
    }
    if (exception != null) {
        throw exception;
    } else {
        return processedContexts;
    }
}