Examples with Lifespan io.trino.execution.Lifespan used on opensource projects

Example 1 with Lifespan

use of io.trino.execution.Lifespan in project trino by trinodb.

the class HttpRemoteTask method getSplitAssignment.

private synchronized SplitAssignment getSplitAssignment(PlanNodeId planNodeId) {
    Set<ScheduledSplit> splits = pendingSplits.get(planNodeId);
    boolean pendingNoMoreSplits = Boolean.TRUE.equals(this.noMoreSplits.get(planNodeId));
    boolean noMoreSplits = this.noMoreSplits.containsKey(planNodeId);
    Set<Lifespan> noMoreSplitsForLifespan = pendingNoMoreSplitsForLifespan.get(planNodeId);
    SplitAssignment assignment = null;
    if (!splits.isEmpty() || !noMoreSplitsForLifespan.isEmpty() || pendingNoMoreSplits) {
        assignment = new SplitAssignment(planNodeId, splits, noMoreSplitsForLifespan, noMoreSplits);
    }
    return assignment;
}

Example 2 with Lifespan

use of io.trino.execution.Lifespan in project trino by trinodb.

the class FixedSourcePartitionedScheduler method schedule.

@Override
public ScheduleResult schedule() {
    // schedule a task on every node in the distribution
    List<RemoteTask> newTasks = ImmutableList.of();
    if (scheduledTasks.isEmpty()) {
        ImmutableList.Builder<RemoteTask> newTasksBuilder = ImmutableList.builder();
        for (InternalNode node : nodes) {
            Optional<RemoteTask> task = stageExecution.scheduleTask(node, partitionIdAllocator.getNextId(), ImmutableMultimap.of(), ImmutableMultimap.of());
            if (task.isPresent()) {
                scheduledTasks.put(node, task.get());
                newTasksBuilder.add(task.get());
            }
        }
        newTasks = newTasksBuilder.build();
    }
    boolean allBlocked = true;
    List<ListenableFuture<Void>> blocked = new ArrayList<>();
    BlockedReason blockedReason = BlockedReason.NO_ACTIVE_DRIVER_GROUP;
    if (groupedLifespanScheduler.isPresent()) {
        // Start new driver groups on the first scheduler if necessary,
        // i.e. when previous ones have finished execution (not finished scheduling).
        // 
        // Invoke schedule method to get a new SettableFuture every time.
        // Reusing previously returned SettableFuture could lead to the ListenableFuture retaining too many listeners.
        blocked.add(groupedLifespanScheduler.get().schedule(sourceSchedulers.get(0)));
    }
    int splitsScheduled = 0;
    Iterator<SourceScheduler> schedulerIterator = sourceSchedulers.iterator();
    List<Lifespan> driverGroupsToStart = ImmutableList.of();
    boolean shouldInvokeNoMoreDriverGroups = false;
    while (schedulerIterator.hasNext()) {
        SourceScheduler sourceScheduler = schedulerIterator.next();
        for (Lifespan lifespan : driverGroupsToStart) {
            sourceScheduler.startLifespan(lifespan, partitionHandleFor(lifespan));
        }
        if (shouldInvokeNoMoreDriverGroups) {
            sourceScheduler.noMoreLifespans();
        }
        ScheduleResult schedule = sourceScheduler.schedule();
        splitsScheduled += schedule.getSplitsScheduled();
        if (schedule.getBlockedReason().isPresent()) {
            blocked.add(schedule.getBlocked());
            blockedReason = blockedReason.combineWith(schedule.getBlockedReason().get());
        } else {
            verify(schedule.getBlocked().isDone(), "blockedReason not provided when scheduler is blocked");
            allBlocked = false;
        }
        driverGroupsToStart = sourceScheduler.drainCompletedLifespans();
        if (schedule.isFinished()) {
            stageExecution.schedulingComplete(sourceScheduler.getPlanNodeId());
            schedulerIterator.remove();
            sourceScheduler.close();
            shouldInvokeNoMoreDriverGroups = true;
        } else {
            shouldInvokeNoMoreDriverGroups = false;
        }
    }
    if (allBlocked) {
        return new ScheduleResult(sourceSchedulers.isEmpty(), newTasks, whenAnyComplete(blocked), blockedReason, splitsScheduled);
    } else {
        return new ScheduleResult(sourceSchedulers.isEmpty(), newTasks, splitsScheduled);
    }
}

Example 3 with Lifespan

use of io.trino.execution.Lifespan in project trino by trinodb.

the class DynamicLifespanScheduler method onLifespanFinished.

@Override
public void onLifespanFinished(Iterable<Lifespan> newlyCompletedDriverGroups) {
    checkState(initialScheduled);
    SettableFuture<Void> newDriverGroupReady;
    synchronized (this) {
        for (Lifespan newlyCompletedDriverGroup : newlyCompletedDriverGroups) {
            checkArgument(!newlyCompletedDriverGroup.isTaskWide());
            recentlyCompletedDriverGroups.add(newlyCompletedDriverGroup);
        }
        newDriverGroupReady = this.newDriverGroupReady;
    }
    newDriverGroupReady.set(null);
}

Example 4 with Lifespan

use of io.trino.execution.Lifespan in project trino by trinodb.

the class DynamicLifespanScheduler method schedule.

@Override
public SettableFuture<Void> schedule(SourceScheduler scheduler) {
    // Return a new future even if newDriverGroupReady has not finished.
    // Returning the same SettableFuture instance could lead to ListenableFuture retaining too many listener objects.
    checkState(initialScheduled);
    List<Lifespan> recentlyCompletedDriverGroups;
    synchronized (this) {
        recentlyCompletedDriverGroups = ImmutableList.copyOf(this.recentlyCompletedDriverGroups);
        this.recentlyCompletedDriverGroups.clear();
        newDriverGroupReady = SettableFuture.create();
    }
    for (Lifespan driverGroup : recentlyCompletedDriverGroups) {
        if (!driverGroups.hasNext()) {
            break;
        }
        int driverGroupId = driverGroups.nextInt();
        InternalNode nodeForCompletedDriverGroup = bucketNodeMap.getAssignedNode(driverGroup.getId()).orElseThrow(IllegalStateException::new);
        bucketNodeMap.assignBucketToNode(driverGroupId, nodeForCompletedDriverGroup);
        scheduler.startLifespan(Lifespan.driverGroup(driverGroupId), partitionHandles.get(driverGroupId));
    }
    if (!driverGroups.hasNext()) {
        scheduler.noMoreLifespans();
    }
    return newDriverGroupReady;
}

Example 5 with Lifespan

use of io.trino.execution.Lifespan in project trino by trinodb.

the class FixedLifespanScheduler method schedule.

@Override
public SettableFuture<Void> schedule(SourceScheduler scheduler) {
    // Return a new future even if newDriverGroupReady has not finished.
    // Returning the same SettableFuture instance could lead to ListenableFuture retaining too many listener objects.
    checkState(initialScheduled);
    List<Lifespan> recentlyCompletedDriverGroups;
    synchronized (this) {
        recentlyCompletedDriverGroups = ImmutableList.copyOf(this.recentlyCompletedDriverGroups);
        this.recentlyCompletedDriverGroups.clear();
        newDriverGroupReady = SettableFuture.create();
    }
    for (Lifespan driverGroup : recentlyCompletedDriverGroups) {
        IntListIterator driverGroupsIterator = nodeToDriverGroupsMap.get(driverGroupToNodeMap.get(driverGroup.getId()));
        if (!driverGroupsIterator.hasNext()) {
            continue;
        }
        int driverGroupId = driverGroupsIterator.nextInt();
        scheduler.startLifespan(Lifespan.driverGroup(driverGroupId), partitionHandles.get(driverGroupId));
        totalDriverGroupsScheduled++;
    }
    verify(totalDriverGroupsScheduled <= driverGroupToNodeMap.size());
    if (totalDriverGroupsScheduled == driverGroupToNodeMap.size()) {
        scheduler.noMoreLifespans();
    }
    return newDriverGroupReady;
}