Examples with TaskInfo com.facebook.presto.execution.TaskInfo used on opensource projects

Example 1 with TaskInfo

use of com.facebook.presto.execution.TaskInfo in project presto by prestodb.

the class PlanPrinter method textDistributedPlan.

public static String textDistributedPlan(List<StageInfo> stages, Metadata metadata, Session session) {
    StringBuilder builder = new StringBuilder();
    List<StageInfo> allStages = stages.stream().flatMap(stage -> getAllStages(Optional.of(stage)).stream()).collect(toImmutableList());
    for (StageInfo stageInfo : allStages) {
        Map<PlanNodeId, PlanNodeStats> aggregatedStats = new HashMap<>();
        List<PlanNodeStats> planNodeStats = stageInfo.getTasks().stream().map(TaskInfo::getStats).flatMap(taskStats -> getPlanNodeStats(taskStats).stream()).collect(toList());
        for (PlanNodeStats stats : planNodeStats) {
            aggregatedStats.merge(stats.getPlanNodeId(), stats, PlanNodeStats::merge);
        }
        builder.append(formatFragment(metadata, session, stageInfo.getPlan(), Optional.of(stageInfo.getStageStats()), Optional.of(aggregatedStats)));
    }
    return builder.toString();
}

Example 2 with TaskInfo

use of com.facebook.presto.execution.TaskInfo in project presto by prestodb.

the class QueryMonitor method queryCompletedEvent.

public void queryCompletedEvent(QueryInfo queryInfo) {
    try {
        Optional<QueryFailureInfo> queryFailureInfo = Optional.empty();
        if (queryInfo.getFailureInfo() != null) {
            FailureInfo failureInfo = queryInfo.getFailureInfo();
            Optional<TaskInfo> failedTask = queryInfo.getOutputStage().flatMap(QueryMonitor::findFailedTask);
            queryFailureInfo = Optional.of(new QueryFailureInfo(queryInfo.getErrorCode(), Optional.ofNullable(failureInfo.getType()), Optional.ofNullable(failureInfo.getMessage()), failedTask.map(task -> task.getTaskStatus().getTaskId().toString()), failedTask.map(task -> task.getTaskStatus().getSelf().getHost()), objectMapper.writeValueAsString(queryInfo.getFailureInfo())));
        }
        ImmutableList.Builder<QueryInputMetadata> inputs = ImmutableList.builder();
        for (Input input : queryInfo.getInputs()) {
            inputs.add(new QueryInputMetadata(input.getConnectorId().getCatalogName(), input.getSchema(), input.getTable(), input.getColumns().stream().map(Column::toString).collect(Collectors.toList()), input.getConnectorInfo()));
        }
        QueryStats queryStats = queryInfo.getQueryStats();
        Optional<QueryOutputMetadata> output = Optional.empty();
        if (queryInfo.getOutput().isPresent()) {
            Optional<TableFinishInfo> tableFinishInfo = queryStats.getOperatorSummaries().stream().map(OperatorStats::getInfo).filter(TableFinishInfo.class::isInstance).map(TableFinishInfo.class::cast).findFirst();
            output = Optional.of(new QueryOutputMetadata(queryInfo.getOutput().get().getConnectorId().getCatalogName(), queryInfo.getOutput().get().getSchema(), queryInfo.getOutput().get().getTable(), tableFinishInfo.map(TableFinishInfo::getConnectorOutputMetadata), tableFinishInfo.map(TableFinishInfo::isJsonLengthLimitExceeded)));
        }
        eventListenerManager.queryCompleted(new QueryCompletedEvent(new QueryMetadata(queryInfo.getQueryId().toString(), queryInfo.getSession().getTransactionId().map(TransactionId::toString), queryInfo.getQuery(), queryInfo.getState().toString(), queryInfo.getSelf(), queryInfo.getOutputStage().flatMap(stage -> stageInfoCodec.toJsonWithLengthLimit(stage, toIntExact(config.getMaxOutputStageJsonSize().toBytes())))), new QueryStatistics(ofMillis(queryStats.getTotalCpuTime().toMillis()), ofMillis(queryStats.getTotalScheduledTime().toMillis()), ofMillis(queryStats.getQueuedTime().toMillis()), Optional.ofNullable(queryStats.getAnalysisTime()).map(duration -> ofMillis(duration.toMillis())), Optional.ofNullable(queryStats.getDistributedPlanningTime()).map(duration -> ofMillis(duration.toMillis())), queryStats.getPeakMemoryReservation().toBytes(), queryStats.getRawInputDataSize().toBytes(), queryStats.getRawInputPositions(), queryStats.getCompletedDrivers(), queryInfo.isCompleteInfo(), objectMapper.writeValueAsString(queryInfo.getQueryStats().getOperatorSummaries())), new QueryContext(queryInfo.getSession().getUser(), queryInfo.getSession().getPrincipal(), queryInfo.getSession().getRemoteUserAddress(), queryInfo.getSession().getUserAgent(), queryInfo.getSession().getClientInfo(), queryInfo.getSession().getSource(), queryInfo.getSession().getCatalog(), queryInfo.getSession().getSchema(), mergeSessionAndCatalogProperties(queryInfo), serverAddress, serverVersion, environment), new QueryIOMetadata(inputs.build(), output), queryFailureInfo, ofEpochMilli(queryStats.getCreateTime().getMillis()), ofEpochMilli(queryStats.getExecutionStartTime().getMillis()), ofEpochMilli(queryStats.getEndTime().getMillis())));
        logQueryTimeline(queryInfo);
    } catch (JsonProcessingException e) {
        throw Throwables.propagate(e);
    }
}

Example 3 with TaskInfo

use of com.facebook.presto.execution.TaskInfo in project presto by prestodb.

the class QueryMonitor method logQueryTimeline.

private static void logQueryTimeline(QueryInfo queryInfo) {
    try {
        QueryStats queryStats = queryInfo.getQueryStats();
        DateTime queryStartTime = queryStats.getCreateTime();
        DateTime queryEndTime = queryStats.getEndTime();
        // query didn't finish cleanly
        if (queryStartTime == null || queryEndTime == null) {
            return;
        }
        // planning duration -- start to end of planning
        long planning = queryStats.getTotalPlanningTime() == null ? 0 : queryStats.getTotalPlanningTime().toMillis();
        List<StageInfo> stages = StageInfo.getAllStages(queryInfo.getOutputStage());
        // long lastSchedulingCompletion = 0;
        long firstTaskStartTime = queryEndTime.getMillis();
        long lastTaskStartTime = queryStartTime.getMillis() + planning;
        long lastTaskEndTime = queryStartTime.getMillis() + planning;
        for (StageInfo stage : stages) {
            // only consider leaf stages
            if (!stage.getSubStages().isEmpty()) {
                continue;
            }
            for (TaskInfo taskInfo : stage.getTasks()) {
                TaskStats taskStats = taskInfo.getStats();
                DateTime firstStartTime = taskStats.getFirstStartTime();
                if (firstStartTime != null) {
                    firstTaskStartTime = Math.min(firstStartTime.getMillis(), firstTaskStartTime);
                }
                DateTime lastStartTime = taskStats.getLastStartTime();
                if (lastStartTime != null) {
                    lastTaskStartTime = max(lastStartTime.getMillis(), lastTaskStartTime);
                }
                DateTime endTime = taskStats.getEndTime();
                if (endTime != null) {
                    lastTaskEndTime = max(endTime.getMillis(), lastTaskEndTime);
                }
            }
        }
        long elapsed = queryEndTime.getMillis() - queryStartTime.getMillis();
        long scheduling = firstTaskStartTime - queryStartTime.getMillis() - planning;
        long running = lastTaskEndTime - firstTaskStartTime;
        long finishing = queryEndTime.getMillis() - lastTaskEndTime;
        log.info("TIMELINE: Query %s :: Transaction:[%s] :: elapsed %sms :: planning %sms :: scheduling %sms :: running %sms :: finishing %sms :: begin %s :: end %s", queryInfo.getQueryId(), queryInfo.getSession().getTransactionId().map(TransactionId::toString).orElse(""), max(elapsed, 0), max(planning, 0), max(scheduling, 0), max(running, 0), max(finishing, 0), queryStartTime, queryEndTime);
    } catch (Exception e) {
        log.error(e, "Error logging query timeline");
    }
}

Example 4 with TaskInfo

use of com.facebook.presto.execution.TaskInfo in project presto by prestodb.

the class GracefulShutdownHandler method requestShutdown.

public synchronized void requestShutdown() {
    log.info("Shutdown requested");
    if (isCoordinator) {
        throw new UnsupportedOperationException("Cannot shutdown coordinator");
    }
    if (isShutdownRequested()) {
        return;
    }
    setShutdownRequested(true);
    //wait for a grace period to start the shutdown sequence
    shutdownHandler.schedule(() -> {
        List<TaskInfo> activeTasks = getActiveTasks();
        while (activeTasks.size() > 0) {
            CountDownLatch countDownLatch = new CountDownLatch(activeTasks.size());
            for (TaskInfo taskInfo : activeTasks) {
                sqlTaskManager.addStateChangeListener(taskInfo.getTaskStatus().getTaskId(), newState -> {
                    if (newState.isDone()) {
                        countDownLatch.countDown();
                    }
                });
            }
            log.info("Waiting for all tasks to finish");
            try {
                countDownLatch.await();
            } catch (InterruptedException e) {
                log.warn("Interrupted while waiting for all tasks to finish");
                currentThread().interrupt();
            }
            activeTasks = getActiveTasks();
        }
        // wait for another grace period for all task states to be observed by the coordinator
        sleepUninterruptibly(gracePeriod.toMillis(), MILLISECONDS);
        Future<?> shutdownFuture = lifeCycleStopper.submit(() -> {
            lifeCycleManager.stop();
            return null;
        });
        // terminate the jvm if life cycle cannot be stopped in a timely manner
        try {
            shutdownFuture.get(LIFECYCLE_STOP_TIMEOUT.toMillis(), MILLISECONDS);
        } catch (TimeoutException e) {
            log.warn(e, "Timed out waiting for the life cycle to stop");
        } catch (InterruptedException e) {
            log.warn(e, "Interrupted while waiting for the life cycle to stop");
            currentThread().interrupt();
        } catch (ExecutionException e) {
            log.warn(e, "Problem stopping the life cycle");
        }
        shutdownAction.onShutdown();
    }, gracePeriod.toMillis(), MILLISECONDS);
}

Example 5 with TaskInfo

use of com.facebook.presto.execution.TaskInfo in project presto by prestodb.

the class TestTaskInfoResource method testGetTaskInfo.

@Test
public void testGetTaskInfo() {
    String sql = "SELECT * FROM tpch.sf1.customer WHERE tpch.sf1.customer.nationkey = 1";
    ResultWithQueryId<MaterializedResult> result = queryRunner.executeWithQueryId(queryRunner.getDefaultSession(), sql);
    QueryId queryId = result.getQueryId();
    Optional<StageInfo> stageInfo = server.getQueryManager().getFullQueryInfo(queryId).getOutputStage();
    if (stageInfo.isPresent()) {
        Stream<TaskInfo> latestTaskInfo = stageInfo.get().getAllStages().stream().flatMap(stage -> stage.getLatestAttemptExecutionInfo().getTasks().stream());
        Iterable<TaskInfo> iterableTaskInfo = latestTaskInfo::iterator;
        for (TaskInfo taskInfo : iterableTaskInfo) {
            URI taskURI = taskUri("v1/taskInfo/", taskInfo.getTaskId().toString());
            Request taskInfoRequest = prepareGet().setUri(taskURI).build();
            TaskInfo responseTaskInfo = client.execute(taskInfoRequest, createJsonResponseHandler(jsonCodec(TaskInfo.class)));
            compareTasks(taskInfo, responseTaskInfo);
        }
    } else {
        fail("StageInfo not present");
    }
}