Examples with TaskInfo com.facebook.presto.execution.TaskInfo used on opensource projects

Example 16 with TaskInfo

use of com.facebook.presto.execution.TaskInfo in project presto by prestodb.

the class QueryExecutionResource method getTaskInfo.

@GET
@Path("/v1/query-execution/{queryId}")
@Produces(MediaType.APPLICATION_JSON)
public Response getTaskInfo(@PathParam("queryId") String queryId) {
    QueryInfo query;
    try {
        query = manager.getQueryInfo(QueryId.valueOf(queryId));
    } catch (NoSuchElementException e) {
        return Response.status(Response.Status.NOT_FOUND).build();
    }
    List<StageInfo> stages = collectStages(query.getOutputStage());
    List<Task> tasks = new ArrayList<>();
    List<Flow> flows = new ArrayList<>();
    for (StageInfo stage : stages) {
        for (TaskInfo task : stage.getTasks()) {
            int bufferedPages = 0;
            TaskStatus taskStatus = task.getTaskStatus();
            for (BufferInfo bufferInfo : task.getOutputBuffers().getBuffers()) {
                bufferedPages += bufferInfo.getBufferedPages();
                if (!bufferInfo.getBufferId().equals(OUTPUT_TASK_ID)) {
                    flows.add(new Flow(taskStatus.getTaskId().toString(), bufferInfo.getBufferId().toString(), bufferInfo.getPageBufferInfo().getPagesAdded(), bufferInfo.getBufferedPages(), bufferInfo.isFinished()));
                }
            }
            long last = TimeUnit.NANOSECONDS.toMillis(System.nanoTime());
            if (task.getStats().getEndTime() != null) {
                last = task.getStats().getEndTime().getMillis();
            }
            tasks.add(new Task(taskStatus.getTaskId().toString(), taskStatus.getState().toString(), taskStatus.getSelf().getHost(), last - task.getStats().getCreateTime().getMillis(), task.getStats().getTotalCpuTime().roundTo(TimeUnit.MILLISECONDS), task.getStats().getTotalBlockedTime().roundTo(TimeUnit.MILLISECONDS), task.getStats().getRawInputDataSize().roundTo(DataSize.Unit.BYTE), task.getStats().getRawInputPositions(), task.getStats().getOutputDataSize().roundTo(DataSize.Unit.BYTE), task.getStats().getOutputPositions(), task.getStats().getMemoryReservation().roundTo(DataSize.Unit.BYTE), task.getStats().getQueuedDrivers(), task.getStats().getRunningDrivers(), task.getStats().getCompletedDrivers(), bufferedPages));
        }
    }
    Map<String, Object> result = ImmutableMap.<String, Object>builder().put("tasks", tasks).put("flows", flows).build();
    return Response.ok(result).build();
}

Example 17 with TaskInfo

use of com.facebook.presto.execution.TaskInfo in project presto by prestodb.

the class TaskResource method abortResults.

@DELETE
@Path("{taskId}/results/{bufferId}")
@Produces(MediaType.APPLICATION_JSON)
public Response abortResults(@PathParam("taskId") TaskId taskId, @PathParam("bufferId") OutputBufferId bufferId, @Context UriInfo uriInfo) {
    requireNonNull(taskId, "taskId is null");
    requireNonNull(bufferId, "bufferId is null");
    TaskInfo taskInfo = taskManager.abortTaskResults(taskId, bufferId);
    if (shouldSummarize(uriInfo)) {
        taskInfo = taskInfo.summarize();
    }
    return Response.ok(taskInfo).build();
}

Example 18 with TaskInfo

use of com.facebook.presto.execution.TaskInfo in project presto by prestodb.

the class TestHttpRemoteTask method testRemoteTaskMismatch.

// This timeout should never be reached because a daemon thread in test should fail the test and do proper cleanup.
@Test(timeOut = 30000)
public void testRemoteTaskMismatch() throws InterruptedException, ExecutionException {
    Duration idleTimeout = new Duration(3, SECONDS);
    Duration failTimeout = new Duration(20, SECONDS);
    JsonCodec<TaskStatus> taskStatusCodec = JsonCodec.jsonCodec(TaskStatus.class);
    JsonCodec<TaskInfo> taskInfoCodec = JsonCodec.jsonCodec(TaskInfo.class);
    TaskManagerConfig taskManagerConfig = new TaskManagerConfig();
    // Shorten status refresh wait and info update interval so that we can have a shorter test timeout
    taskManagerConfig.setStatusRefreshMaxWait(new Duration(idleTimeout.roundTo(MILLISECONDS) / 100, MILLISECONDS));
    taskManagerConfig.setInfoUpdateInterval(new Duration(idleTimeout.roundTo(MILLISECONDS) / 10, MILLISECONDS));
    AtomicLong lastActivityNanos = new AtomicLong(System.nanoTime());
    HttpProcessor httpProcessor = new HttpProcessor(taskStatusCodec, taskInfoCodec, lastActivityNanos);
    TestingHttpClient testingHttpClient = new TestingHttpClient(httpProcessor);
    HttpRemoteTaskFactory httpRemoteTaskFactory = new HttpRemoteTaskFactory(new QueryManagerConfig(), taskManagerConfig, testingHttpClient, new TestSqlTaskManager.MockLocationFactory(), taskStatusCodec, taskInfoCodec, JsonCodec.jsonCodec(TaskUpdateRequest.class), new RemoteTaskStats());
    RemoteTask remoteTask = httpRemoteTaskFactory.createRemoteTask(TEST_SESSION, new TaskId("test", 1, 2), new PrestoNode("node-id", URI.create("http://192.0.1.2"), new NodeVersion("version"), false), TaskTestUtils.PLAN_FRAGMENT, ImmutableMultimap.of(), createInitialEmptyOutputBuffers(OutputBuffers.BufferType.BROADCAST), new NodeTaskMap.PartitionedSplitCountTracker(i -> {
    }), true);
    httpProcessor.setInitialTaskInfo(remoteTask.getTaskInfo());
    remoteTask.start();
    CompletableFuture<Void> testComplete = new CompletableFuture<>();
    asyncRun(idleTimeout.roundTo(MILLISECONDS), failTimeout.roundTo(MILLISECONDS), lastActivityNanos, () -> testComplete.complete(null), (message, cause) -> testComplete.completeExceptionally(new AssertionError(message, cause)));
    testComplete.get();
    httpRemoteTaskFactory.stop();
    assertTrue(remoteTask.getTaskStatus().getState().isDone(), format("TaskStatus is not in a done state: %s", remoteTask.getTaskStatus()));
    assertEquals(getOnlyElement(remoteTask.getTaskStatus().getFailures()).getErrorCode(), REMOTE_TASK_MISMATCH.toErrorCode());
    assertTrue(remoteTask.getTaskInfo().getTaskStatus().getState().isDone(), format("TaskInfo is not in a done state: %s", remoteTask.getTaskInfo()));
}

Example 19 with TaskInfo

use of com.facebook.presto.execution.TaskInfo in project presto by prestodb.

the class QueryMonitor method logQueryTimeline.

private static void logQueryTimeline(QueryInfo queryInfo) {
    try {
        QueryStats queryStats = queryInfo.getQueryStats();
        DateTime queryStartTime = queryStats.getCreateTime();
        DateTime queryEndTime = queryStats.getEndTime();
        // query didn't finish cleanly
        if (queryStartTime == null || queryEndTime == null) {
            return;
        }
        // planning duration -- start to end of planning
        long planning = queryStats.getTotalPlanningTime().toMillis();
        List<StageInfo> stages = getAllStages(queryInfo.getOutputStage());
        // long lastSchedulingCompletion = 0;
        long firstTaskStartTime = queryEndTime.getMillis();
        long lastTaskStartTime = queryStartTime.getMillis() + planning;
        long lastTaskEndTime = queryStartTime.getMillis() + planning;
        for (StageInfo stage : stages) {
            // only consider leaf stages
            if (!stage.getSubStages().isEmpty()) {
                continue;
            }
            for (TaskInfo taskInfo : stage.getLatestAttemptExecutionInfo().getTasks()) {
                TaskStats taskStats = taskInfo.getStats();
                DateTime firstStartTime = taskStats.getFirstStartTime();
                if (firstStartTime != null) {
                    firstTaskStartTime = Math.min(firstStartTime.getMillis(), firstTaskStartTime);
                }
                DateTime lastStartTime = taskStats.getLastStartTime();
                if (lastStartTime != null) {
                    lastTaskStartTime = max(lastStartTime.getMillis(), lastTaskStartTime);
                }
                DateTime endTime = taskStats.getEndTime();
                if (endTime != null) {
                    lastTaskEndTime = max(endTime.getMillis(), lastTaskEndTime);
                }
            }
        }
        long elapsed = max(queryEndTime.getMillis() - queryStartTime.getMillis(), 0);
        long scheduling = max(firstTaskStartTime - queryStartTime.getMillis() - planning, 0);
        long running = max(lastTaskEndTime - firstTaskStartTime, 0);
        long finishing = max(queryEndTime.getMillis() - lastTaskEndTime, 0);
        logQueryTimeline(queryInfo.getQueryId(), queryInfo.getSession().getTransactionId().map(TransactionId::toString).orElse(""), elapsed, planning, scheduling, running, finishing, queryStartTime, queryEndTime);
    } catch (Exception e) {
        log.error(e, "Error logging query timeline");
    }
}

Example 20 with TaskInfo

use of com.facebook.presto.execution.TaskInfo in project presto by prestodb.

the class TaskSystemTable method cursor.

@Override
public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, ConnectorSession session, TupleDomain<Integer> constraint) {
    Builder table = InMemoryRecordSet.builder(TASK_TABLE);
    for (TaskInfo taskInfo : taskManager.getAllTaskInfo()) {
        TaskStats stats = taskInfo.getStats();
        TaskStatus taskStatus = taskInfo.getTaskStatus();
        table.addRow(nodeId, taskInfo.getTaskId().toString(), taskInfo.getTaskId().getStageExecutionId().toString(), taskInfo.getTaskId().getStageExecutionId().getStageId().toString(), taskInfo.getTaskId().getQueryId().toString(), taskStatus.getState().toString(), (long) stats.getTotalDrivers(), (long) stats.getQueuedDrivers(), (long) stats.getRunningDrivers(), (long) stats.getCompletedDrivers(), NANOSECONDS.toMillis(stats.getTotalScheduledTimeInNanos()), NANOSECONDS.toMillis(stats.getTotalCpuTimeInNanos()), NANOSECONDS.toMillis(stats.getTotalBlockedTimeInNanos()), stats.getRawInputDataSizeInBytes(), stats.getRawInputPositions(), stats.getProcessedInputDataSizeInBytes(), stats.getProcessedInputPositions(), stats.getOutputDataSizeInBytes(), stats.getOutputPositions(), stats.getPhysicalWrittenDataSizeInBytes(), toTimeStamp(stats.getCreateTime()), toTimeStamp(stats.getFirstStartTime()), toTimeStamp(taskInfo.getLastHeartbeat()), toTimeStamp(stats.getEndTime()));
    }
    return table.build().cursor();
}