Examples with TaskInfo io.prestosql.execution.TaskInfo used on opensource projects

Example 1 with TaskInfo

use of io.prestosql.execution.TaskInfo in project hetu-core by openlookeng.

the class TestNodeStateChange method testWorkerIsolation.

@Test(timeOut = 30_000)
public void testWorkerIsolation() throws Exception {
    try (DistributedQueryRunner queryRunner = createQueryRunner(TINY_SESSION, defaultProperties)) {
        List<ListenableFuture<?>> queryFutures = new ArrayList<>();
        for (int i = 0; i < 5; i++) {
            queryFutures.add(executor.submit(() -> queryRunner.execute("SELECT COUNT(*), clerk FROM orders GROUP BY clerk")));
        }
        TestingPrestoServer worker = queryRunner.getServers().stream().filter(server -> !server.isCoordinator()).findFirst().get();
        assertEquals(worker.getNodeStateChangeHandler().getState(), NodeState.ACTIVE);
        TaskManager taskManager = worker.getTaskManager();
        // wait until tasks show up on the worker
        while (taskManager.getAllTaskInfo().isEmpty()) {
            MILLISECONDS.sleep(500);
        }
        worker.getNodeStateChangeHandler().doStateTransition(NodeState.ISOLATING);
        Futures.allAsList(queryFutures).get();
        List<TaskInfo> taskInfos = worker.getTaskManager().getAllTaskInfo();
        for (TaskInfo info : taskInfos) {
            assertEquals(info.getTaskStatus().getState(), TaskState.FINISHED);
        }
        while (!worker.getNodeStateChangeHandler().getState().equals(NodeState.ISOLATED)) {
            Thread.sleep(1000);
        }
    }
}

Example 2 with TaskInfo

use of io.prestosql.execution.TaskInfo in project hetu-core by openlookeng.

the class TaskSystemTable method cursor.

@Override
public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, ConnectorSession session, TupleDomain<Integer> constraint) {
    Builder table = InMemoryRecordSet.builder(TASK_TABLE);
    for (TaskInfo taskInfo : taskManager.getAllTaskInfo()) {
        TaskStats stats = taskInfo.getStats();
        TaskStatus taskStatus = taskInfo.getTaskStatus();
        table.addRow(nodeId, taskStatus.getTaskId().toString(), taskStatus.getTaskId().getStageId().toString(), taskStatus.getTaskId().getQueryId().toString(), taskStatus.getState().toString(), (long) stats.getTotalDrivers(), (long) stats.getQueuedDrivers(), (long) stats.getRunningDrivers(), (long) stats.getCompletedDrivers(), toMillis(stats.getTotalScheduledTime()), toMillis(stats.getTotalCpuTime()), toMillis(stats.getTotalBlockedTime()), toBytes(stats.getRawInputDataSize()), stats.getRawInputPositions(), toBytes(stats.getProcessedInputDataSize()), stats.getProcessedInputPositions(), toBytes(stats.getOutputDataSize()), stats.getOutputPositions(), toBytes(stats.getPhysicalWrittenDataSize()), toTimeStamp(stats.getCreateTime()), toTimeStamp(stats.getFirstStartTime()), toTimeStamp(taskInfo.getLastHeartbeat()), toTimeStamp(stats.getEndTime()));
    }
    return table.build().cursor();
}

Example 3 with TaskInfo

use of io.prestosql.execution.TaskInfo in project hetu-core by openlookeng.

the class HttpRemoteTask method doScheduleAsyncCleanupRequest.

private void doScheduleAsyncCleanupRequest(Backoff cleanupBackoff, Request request, String action) {
    ResponseHandler responseHandler;
    if (isBinaryEncoding) {
        responseHandler = createFullSmileResponseHandler((SmileCodec<TaskInfo>) taskInfoCodec);
    } else {
        responseHandler = createAdaptingJsonResponseHandler(unwrapJsonCodec(taskInfoCodec));
    }
    Futures.addCallback(httpClient.executeAsync(request, responseHandler), new FutureCallback<BaseResponse<TaskInfo>>() {

        @Override
        public void onSuccess(BaseResponse<TaskInfo> result) {
            try {
                updateTaskInfo(result.getValue());
            } finally {
                if (!getTaskInfo().getTaskStatus().getState().isDone()) {
                    cleanUpLocally();
                }
            }
        }

        @Override
        public void onFailure(Throwable t) {
            if (cancelledToResume.get()) {
                // Remote worker is probably unreachable. Don't make additional attempts.
                cleanUpLocally();
                return;
            }
            if (t instanceof RejectedExecutionException && httpClient.isClosed()) {
                logError(t, "Unable to %s task at %s. HTTP client is closed.", action, request.getUri());
                cleanUpLocally();
                return;
            }
            // record failure
            if (cleanupBackoff.failure()) {
                logError(t, "Unable to %s task at %s. Back off depleted.", action, request.getUri());
                cleanUpLocally();
                return;
            }
            // reschedule
            long delayNanos = cleanupBackoff.getBackoffDelayNanos();
            if (delayNanos == 0) {
                doScheduleAsyncCleanupRequest(cleanupBackoff, request, action);
            } else {
                errorScheduledExecutor.schedule(() -> doScheduleAsyncCleanupRequest(cleanupBackoff, request, action), delayNanos, NANOSECONDS);
            }
        }

        private void cleanUpLocally() {
            // Update the taskInfo with the new taskStatus.
            // Generally, we send a cleanup request to the worker, and update the TaskInfo on
            // the coordinator based on what we fetched from the worker. If we somehow cannot
            // get the cleanup request to the worker, the TaskInfo that we fetch for the worker
            // likely will not say the task is done however many times we try. In this case,
            // we have to set the local query info directly so that we stop trying to fetch
            // updated TaskInfo from the worker. This way, the task on the worker eventually
            // expires due to lack of activity.
            // This is required because the query state machine depends on TaskInfo (instead of task status)
            // to transition its own state.
            // TODO: Update the query state machine and stage state machine to depend on TaskStatus instead
            // Since this TaskInfo is updated in the client the "complete" flag will not be set,
            // indicating that the stats may not reflect the final stats on the worker.
            TaskStatus taskStatus = getTaskStatus();
            if (cancelledToResume.get()) {
                // When the task is cancelled to resume, then make sure it gets the new state, so query can start resuming.
                // Check for task state is in QueryInfo#areAllStagesDone.
                taskStatus = TaskStatus.failWith(taskStatus, CANCELED_TO_RESUME, ImmutableList.of());
            }
            updateTaskInfo(getTaskInfo().withTaskStatus(taskStatus));
        }
    }, executor);
}

Example 4 with TaskInfo

use of io.prestosql.execution.TaskInfo in project hetu-core by openlookeng.

the class Query method globalUniqueNodes.

public static Set<String> globalUniqueNodes(StageInfo stageInfo, boolean getSpilledNodes) {
    boolean isSpilledNode = false;
    if (stageInfo == null) {
        return ImmutableSet.of();
    }
    ImmutableSet.Builder<String> nodes = ImmutableSet.builder();
    for (TaskInfo task : stageInfo.getTasks()) {
        if (getSpilledNodes) {
            for (PipelineStats pipeline : task.getStats().getPipelines()) {
                isSpilledNode = pipeline.getOperatorSummaries().stream().filter(operatorStats -> operatorStats.getSpilledDataSize().toBytes() > 0).collect(Collectors.toList()).size() > 0;
                if (isSpilledNode) {
                    break;
                }
            }
        }
        // todo add nodeId to TaskInfo
        if (isSpilledNode || !getSpilledNodes) {
            URI uri = task.getTaskStatus().getSelf();
            nodes.add(uri.getHost() + ":" + uri.getPort());
        }
    }
    for (StageInfo subStage : stageInfo.getSubStages()) {
        nodes.addAll(globalUniqueNodes(subStage, getSpilledNodes));
    }
    return nodes.build();
}

Example 5 with TaskInfo

use of io.prestosql.execution.TaskInfo in project hetu-core by openlookeng.

the class TaskInfoFetcher method addFinalTaskInfoListener.

/**
 * Add a listener for the final task info.  This notification is guaranteed to be fired only once.
 * Listener is always notified asynchronously using a dedicated notification thread pool so, care should
 * be taken to avoid leaking {@code this} when adding a listener in a constructor. Additionally, it is
 * possible notifications are observed out of order due to the asynchronous execution.
 */
public void addFinalTaskInfoListener(StateChangeListener<TaskInfo> stateChangeListener) {
    AtomicBoolean done = new AtomicBoolean();
    StateChangeListener<Optional<TaskInfo>> fireOnceStateChangeListener = finalTaskInfo -> {
        if (finalTaskInfo.isPresent() && done.compareAndSet(false, true)) {
            stateChangeListener.stateChanged(finalTaskInfo.get());
        }
    };
    finalTaskInfo.addStateChangeListener(fireOnceStateChangeListener);
    fireOnceStateChangeListener.stateChanged(finalTaskInfo.get());
}