Examples with TaskInfo com.facebook.presto.execution.TaskInfo used on opensource projects

Example 21 with TaskInfo

use of com.facebook.presto.execution.TaskInfo in project presto by prestodb.

the class HttpRemoteTask method doScheduleAsyncCleanupRequest.

private void doScheduleAsyncCleanupRequest(Backoff cleanupBackoff, Request request, String action) {
    ResponseHandler responseHandler;
    if (binaryTransportEnabled) {
        responseHandler = createFullSmileResponseHandler((SmileCodec<TaskInfo>) taskInfoCodec);
    } else {
        responseHandler = createAdaptingJsonResponseHandler((JsonCodec<TaskInfo>) taskInfoCodec);
    }
    Futures.addCallback(httpClient.executeAsync(request, responseHandler), new FutureCallback<BaseResponse<TaskInfo>>() {

        @Override
        public void onSuccess(BaseResponse<TaskInfo> result) {
            try {
                updateTaskInfo(result.getValue());
            } finally {
                if (!getTaskInfo().getTaskStatus().getState().isDone()) {
                    cleanUpLocally();
                }
            }
        }

        @Override
        public void onFailure(Throwable t) {
            if (t instanceof RejectedExecutionException && httpClient.isClosed()) {
                logError(t, "Unable to %s task at %s. HTTP client is closed.", action, request.getUri());
                cleanUpLocally();
                return;
            }
            // record failure
            if (cleanupBackoff.failure()) {
                logError(t, "Unable to %s task at %s. Back off depleted.", action, request.getUri());
                cleanUpLocally();
                return;
            }
            // reschedule
            long delayNanos = cleanupBackoff.getBackoffDelayNanos();
            if (delayNanos == 0) {
                doScheduleAsyncCleanupRequest(cleanupBackoff, request, action);
            } else {
                errorScheduledExecutor.schedule(() -> doScheduleAsyncCleanupRequest(cleanupBackoff, request, action), delayNanos, NANOSECONDS);
            }
        }

        private void cleanUpLocally() {
            // Update the taskInfo with the new taskStatus.
            // Generally, we send a cleanup request to the worker, and update the TaskInfo on
            // the coordinator based on what we fetched from the worker. If we somehow cannot
            // get the cleanup request to the worker, the TaskInfo that we fetch for the worker
            // likely will not say the task is done however many times we try. In this case,
            // we have to set the local query info directly so that we stop trying to fetch
            // updated TaskInfo from the worker. This way, the task on the worker eventually
            // expires due to lack of activity.
            // This is required because the query state machine depends on TaskInfo (instead of task status)
            // to transition its own state.
            // TODO: Update the query state machine and stage state machine to depend on TaskStatus instead
            // Since this TaskInfo is updated in the client the "complete" flag will not be set,
            // indicating that the stats may not reflect the final stats on the worker.
            updateTaskInfo(getTaskInfo().withTaskStatus(getTaskStatus()));
        }
    }, executor);
}

Example 22 with TaskInfo

use of com.facebook.presto.execution.TaskInfo in project presto by prestodb.

the class TaskResource method createOrUpdateTask.

@POST
@Path("{taskId}")
@Consumes({ APPLICATION_JSON, APPLICATION_JACKSON_SMILE })
@Produces({ APPLICATION_JSON, APPLICATION_JACKSON_SMILE })
public Response createOrUpdateTask(@PathParam("taskId") TaskId taskId, TaskUpdateRequest taskUpdateRequest, @Context UriInfo uriInfo) {
    requireNonNull(taskUpdateRequest, "taskUpdateRequest is null");
    Session session = taskUpdateRequest.getSession().toSession(sessionPropertyManager, taskUpdateRequest.getExtraCredentials());
    TaskInfo taskInfo = taskManager.updateTask(session, taskId, taskUpdateRequest.getFragment().map(planFragmentCodec::fromBytes), taskUpdateRequest.getSources(), taskUpdateRequest.getOutputIds(), taskUpdateRequest.getTableWriteInfo());
    if (shouldSummarize(uriInfo)) {
        taskInfo = taskInfo.summarize();
    }
    return Response.ok().entity(taskInfo).build();
}

Example 23 with TaskInfo

use of com.facebook.presto.execution.TaskInfo in project presto by prestodb.

the class QueryResourceUtil method toStageStats.

private static StageStats toStageStats(StageInfo stageInfo) {
    if (stageInfo == null) {
        return null;
    }
    StageExecutionInfo currentStageExecutionInfo = stageInfo.getLatestAttemptExecutionInfo();
    StageExecutionStats stageExecutionStats = currentStageExecutionInfo.getStats();
    ImmutableList.Builder<StageStats> subStages = ImmutableList.builder();
    for (StageInfo subStage : stageInfo.getSubStages()) {
        subStages.add(toStageStats(subStage));
    }
    Set<String> uniqueNodes = new HashSet<>();
    for (TaskInfo task : currentStageExecutionInfo.getTasks()) {
        // todo add nodeId to TaskInfo
        URI uri = task.getTaskStatus().getSelf();
        uniqueNodes.add(uri.getHost() + ":" + uri.getPort());
    }
    return StageStats.builder().setStageId(String.valueOf(stageInfo.getStageId().getId())).setState(currentStageExecutionInfo.getState().toString()).setDone(currentStageExecutionInfo.getState().isDone()).setNodes(uniqueNodes.size()).setTotalSplits(stageExecutionStats.getTotalDrivers()).setQueuedSplits(stageExecutionStats.getQueuedDrivers()).setRunningSplits(stageExecutionStats.getRunningDrivers() + stageExecutionStats.getBlockedDrivers()).setCompletedSplits(stageExecutionStats.getCompletedDrivers()).setCpuTimeMillis(stageExecutionStats.getTotalCpuTime().toMillis()).setWallTimeMillis(stageExecutionStats.getTotalScheduledTime().toMillis()).setProcessedRows(stageExecutionStats.getRawInputPositions()).setProcessedBytes(stageExecutionStats.getRawInputDataSize().toBytes()).setSubStages(subStages.build()).build();
}

Example 24 with TaskInfo

use of com.facebook.presto.execution.TaskInfo in project presto by prestodb.

the class TaskInfoFetcher method sendNextRequest.

private synchronized void sendNextRequest() {
    TaskInfo taskInfo = getTaskInfo();
    TaskStatus taskStatus = taskInfo.getTaskStatus();
    if (!running) {
        return;
    }
    // we already have the final task info
    if (isDone(getTaskInfo())) {
        stop();
        return;
    }
    // if we have an outstanding request
    if (future != null && !future.isDone()) {
        return;
    }
    // if throttled due to error, asynchronously wait for timeout and try again
    ListenableFuture<?> errorRateLimit = errorTracker.acquireRequestPermit();
    if (!errorRateLimit.isDone()) {
        errorRateLimit.addListener(this::sendNextRequest, executor);
        return;
    }
    MetadataUpdates metadataUpdateRequests = taskInfo.getMetadataUpdates();
    if (!metadataUpdateRequests.getMetadataUpdates().isEmpty()) {
        scheduleMetadataUpdates(metadataUpdateRequests);
    }
    HttpUriBuilder httpUriBuilder = uriBuilderFrom(taskStatus.getSelf());
    URI uri = summarizeTaskInfo ? httpUriBuilder.addParameter("summarize").build() : httpUriBuilder.build();
    Request.Builder uriBuilder = setContentTypeHeaders(isBinaryTransportEnabled, prepareGet());
    if (taskInfoRefreshMaxWait.toMillis() != 0L) {
        uriBuilder.setHeader(PRESTO_CURRENT_STATE, taskStatus.getState().toString()).setHeader(PRESTO_MAX_WAIT, taskInfoRefreshMaxWait.toString());
    }
    Request request = uriBuilder.setUri(uri).build();
    ResponseHandler responseHandler;
    if (isBinaryTransportEnabled) {
        responseHandler = createFullSmileResponseHandler((SmileCodec<TaskInfo>) taskInfoCodec);
    } else {
        responseHandler = createAdaptingJsonResponseHandler((JsonCodec<TaskInfo>) taskInfoCodec);
    }
    errorTracker.startRequest();
    future = httpClient.executeAsync(request, responseHandler);
    currentRequestStartNanos.set(System.nanoTime());
    Futures.addCallback(future, new SimpleHttpResponseHandler<>(this, request.getUri(), stats.getHttpResponseStats(), REMOTE_TASK_ERROR), executor);
}

Example 25 with TaskInfo

use of com.facebook.presto.execution.TaskInfo in project presto by prestodb.

the class GcStatusMonitor method logTaskStats.

private static void logTaskStats(List<QueryId> queryIds, ListMultimap<QueryId, SqlTask> tasksByQueryId) {
    List<List<String>> rows = queryIds.stream().flatMap(queryId -> {
        List<SqlTask> sqlTasks = tasksByQueryId.get(queryId);
        Comparator<SqlTask> comparator = (first, second) -> {
            TaskStats aTaskStats = first.getTaskInfo().getStats();
            TaskStats bTaskStats = second.getTaskInfo().getStats();
            return Long.compare(aTaskStats.getUserMemoryReservationInBytes() + aTaskStats.getSystemMemoryReservationInBytes(), bTaskStats.getUserMemoryReservationInBytes() + bTaskStats.getSystemMemoryReservationInBytes());
        };
        return sqlTasks.stream().sorted(comparator.reversed()).map(task -> {
            TaskInfo taskInfo = task.getTaskInfo();
            SqlTaskIoStats taskIOStats = task.getIoStats();
            TaskStatus taskStatus = taskInfo.getTaskStatus();
            TaskStats taskStats = taskInfo.getStats();
            return ImmutableList.of(task.getQueryContext().getQueryId().toString(), task.getTaskId().toString(), taskStatus.getState().toString(), taskStats.getCreateTime().toString(), Long.toString(taskStats.getUserMemoryReservationInBytes()), Long.toString(taskStats.getSystemMemoryReservationInBytes()), Long.toString(taskIOStats.getInputDataSize().getTotalCount()), Long.toString(taskIOStats.getOutputDataSize().getTotalCount()), Long.toString(taskIOStats.getInputPositions().getTotalCount()), Long.toString(taskIOStats.getOutputPositions().getTotalCount()));
        });
    }).collect(toImmutableList());
    if (!rows.isEmpty()) {
        logInfoTable(ImmutableList.<List<String>>builder().add(ImmutableList.of("Query ID", "Task ID", "State", "Created Ts", "User Memory", "System Memory", "Input Bytes", "Output Bytes", "Input Row Count", "Output Row Count")).addAll(rows).build());
    }
}