Examples with SeekableStreamIndexTaskTuningConfig org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskTuningConfig used on opensource projects

Example 1 with SeekableStreamIndexTaskTuningConfig

use of org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskTuningConfig in project druid by druid-io.

the class SeekableStreamSupervisor method getCurrentParseErrors.

/**
 * Collect parse errors from all tasks managed by this supervisor.
 *
 * @return A list of parse error strings
 *
 * @throws InterruptedException
 * @throws ExecutionException
 * @throws TimeoutException
 */
private List<ParseExceptionReport> getCurrentParseErrors() throws InterruptedException, ExecutionException, TimeoutException {
    final List<ListenableFuture<ErrorsFromTaskResult>> futures = new ArrayList<>();
    final List<Pair<Integer, String>> groupAndTaskIds = new ArrayList<>();
    for (int groupId : activelyReadingTaskGroups.keySet()) {
        TaskGroup group = activelyReadingTaskGroups.get(groupId);
        for (String taskId : group.taskIds()) {
            futures.add(Futures.transform(taskClient.getParseErrorsAsync(taskId), (Function<List<ParseExceptionReport>, ErrorsFromTaskResult>) (taskErrors) -> new ErrorsFromTaskResult(groupId, taskId, taskErrors)));
            groupAndTaskIds.add(new Pair<>(groupId, taskId));
        }
    }
    for (int groupId : pendingCompletionTaskGroups.keySet()) {
        List<TaskGroup> pendingGroups = pendingCompletionTaskGroups.get(groupId);
        for (TaskGroup pendingGroup : pendingGroups) {
            for (String taskId : pendingGroup.taskIds()) {
                futures.add(Futures.transform(taskClient.getParseErrorsAsync(taskId), (Function<List<ParseExceptionReport>, ErrorsFromTaskResult>) (taskErrors) -> new ErrorsFromTaskResult(groupId, taskId, taskErrors)));
                groupAndTaskIds.add(new Pair<>(groupId, taskId));
            }
        }
    }
    // We use a tree set to sort the parse errors by time, and eliminate duplicates across calls to this method
    TreeSet<ParseExceptionReport> parseErrorsTreeSet = new TreeSet<>(PARSE_EXCEPTION_REPORT_COMPARATOR);
    parseErrorsTreeSet.addAll(lastKnownParseErrors);
    List<ErrorsFromTaskResult> results = Futures.successfulAsList(futures).get(futureTimeoutInSeconds, TimeUnit.SECONDS);
    for (int i = 0; i < results.size(); i++) {
        ErrorsFromTaskResult result = results.get(i);
        if (result != null) {
            parseErrorsTreeSet.addAll(result.getErrors());
        } else {
            Pair<Integer, String> groupAndTaskId = groupAndTaskIds.get(i);
            log.error("Failed to get errors for group[%d]-task[%s]", groupAndTaskId.lhs, groupAndTaskId.rhs);
        }
    }
    SeekableStreamIndexTaskTuningConfig ss = spec.getSpec().getTuningConfig().convertToTaskTuningConfig();
    SeekableStreamSupervisorIOConfig oo = spec.getSpec().getIOConfig();
    // store a limited number of parse exceptions, keeping the most recent ones
    int parseErrorLimit = spec.getSpec().getTuningConfig().convertToTaskTuningConfig().getMaxSavedParseExceptions() * spec.getSpec().getIOConfig().getTaskCount();
    parseErrorLimit = Math.min(parseErrorLimit, parseErrorsTreeSet.size());
    final List<ParseExceptionReport> limitedParseErrors = new ArrayList<>();
    Iterator<ParseExceptionReport> descendingIterator = parseErrorsTreeSet.descendingIterator();
    for (int i = 0; i < parseErrorLimit; i++) {
        limitedParseErrors.add(descendingIterator.next());
    }
    return limitedParseErrors;
}