Examples with Lifespan com.facebook.presto.execution.Lifespan used on opensource projects

Example 6 with Lifespan

use of com.facebook.presto.execution.Lifespan in project presto by prestodb.

the class LifespanSerializedPageTracker method incrementLifespanPageCount.

public void incrementLifespanPageCount(Lifespan lifespan, int pagesAdded) {
    // JDK-8 acquires the write lock unconditionally in computeIfAbsent
    // TODO: Remove this extra get call once Presto no longer supports JDK-8
    AtomicLong counter = outstandingPageCountPerLifespan.get(lifespan);
    if (counter == null) {
        counter = outstandingPageCountPerLifespan.computeIfAbsent(lifespan, ignore -> new AtomicLong());
    }
    counter.addAndGet(pagesAdded);
}

Example 7 with Lifespan

use of com.facebook.presto.execution.Lifespan in project presto by prestodb.

the class FixedSourcePartitionedScheduler method schedule.

@Override
public ScheduleResult schedule() {
    // schedule a task on every node in the distribution
    List<RemoteTask> newTasks = ImmutableList.of();
    if (!scheduledTasks) {
        newTasks = Streams.mapWithIndex(nodes.stream(), (node, id) -> stage.scheduleTask(node, toIntExact(id))).filter(Optional::isPresent).map(Optional::get).collect(toImmutableList());
        scheduledTasks = true;
        // notify listeners that we have scheduled all tasks so they can set no more buffers or exchange splits
        stage.transitionToFinishedTaskScheduling();
    }
    boolean allBlocked = true;
    List<ListenableFuture<?>> blocked = new ArrayList<>();
    BlockedReason blockedReason = BlockedReason.NO_ACTIVE_DRIVER_GROUP;
    if (groupedLifespanScheduler.isPresent()) {
        while (!tasksToRecover.isEmpty()) {
            if (anySourceSchedulingFinished) {
                throw new IllegalStateException("Recover after any source scheduling finished is not supported");
            }
            groupedLifespanScheduler.get().onTaskFailed(tasksToRecover.poll(), sourceSchedulers);
        }
        if (groupedLifespanScheduler.get().allLifespanExecutionFinished()) {
            for (SourceScheduler sourceScheduler : sourceSchedulers) {
                sourceScheduler.notifyAllLifespansFinishedExecution();
            }
        } else {
            // Start new driver groups on the first scheduler if necessary,
            // i.e. when previous ones have finished execution (not finished scheduling).
            // 
            // Invoke schedule method to get a new SettableFuture every time.
            // Reusing previously returned SettableFuture could lead to the ListenableFuture retaining too many listeners.
            blocked.add(groupedLifespanScheduler.get().schedule(sourceSchedulers.get(0)));
        }
    }
    int splitsScheduled = 0;
    Iterator<SourceScheduler> schedulerIterator = sourceSchedulers.iterator();
    List<Lifespan> driverGroupsToStart = ImmutableList.of();
    while (schedulerIterator.hasNext()) {
        synchronized (this) {
            // prevent that by checking if scheduling has been cancelled first.
            if (closed) {
                break;
            }
            SourceScheduler sourceScheduler = schedulerIterator.next();
            for (Lifespan lifespan : driverGroupsToStart) {
                sourceScheduler.startLifespan(lifespan, partitionHandleFor(lifespan));
            }
            ScheduleResult schedule = sourceScheduler.schedule();
            if (schedule.getSplitsScheduled() > 0) {
                stage.transitionToSchedulingSplits();
            }
            splitsScheduled += schedule.getSplitsScheduled();
            if (schedule.getBlockedReason().isPresent()) {
                blocked.add(schedule.getBlocked());
                blockedReason = blockedReason.combineWith(schedule.getBlockedReason().get());
            } else {
                verify(schedule.getBlocked().isDone(), "blockedReason not provided when scheduler is blocked");
                allBlocked = false;
            }
            driverGroupsToStart = sourceScheduler.drainCompletelyScheduledLifespans();
            if (schedule.isFinished()) {
                stage.schedulingComplete(sourceScheduler.getPlanNodeId());
                sourceSchedulers.remove(sourceScheduler);
                sourceScheduler.close();
                anySourceSchedulingFinished = true;
            }
        }
    }
    if (allBlocked) {
        return ScheduleResult.blocked(sourceSchedulers.isEmpty(), newTasks, whenAnyComplete(blocked), blockedReason, splitsScheduled);
    } else {
        return ScheduleResult.nonBlocked(sourceSchedulers.isEmpty(), newTasks, splitsScheduled);
    }
}

Example 8 with Lifespan

use of com.facebook.presto.execution.Lifespan in project presto by prestodb.

the class SourcePartitionedScheduler method assignSplits.

private Set<RemoteTask> assignSplits(Multimap<InternalNode, Split> splitAssignment, Multimap<InternalNode, Lifespan> noMoreSplitsNotification) {
    ImmutableSet.Builder<RemoteTask> newTasks = ImmutableSet.builder();
    ImmutableSet<InternalNode> nodes = ImmutableSet.<InternalNode>builder().addAll(splitAssignment.keySet()).addAll(noMoreSplitsNotification.keySet()).build();
    for (InternalNode node : nodes) {
        ImmutableMultimap<PlanNodeId, Split> splits = ImmutableMultimap.<PlanNodeId, Split>builder().putAll(partitionedNode, splitAssignment.get(node)).build();
        ImmutableMultimap.Builder<PlanNodeId, Lifespan> noMoreSplits = ImmutableMultimap.builder();
        if (noMoreSplitsNotification.containsKey(node)) {
            noMoreSplits.putAll(partitionedNode, noMoreSplitsNotification.get(node));
        }
        newTasks.addAll(stage.scheduleSplits(node, splits, noMoreSplits.build()));
    }
    return newTasks.build();
}

Example 9 with Lifespan

use of com.facebook.presto.execution.Lifespan in project presto by prestodb.

the class FixedLifespanScheduler method onLifespanExecutionFinished.

public void onLifespanExecutionFinished(Iterable<Lifespan> newlyCompletelyExecutedDriverGroups) {
    checkState(initialScheduled);
    SettableFuture<?> newDriverGroupReady;
    synchronized (this) {
        for (Lifespan newlyCompletelyExecutedDriverGroup : newlyCompletelyExecutedDriverGroups) {
            checkArgument(!newlyCompletelyExecutedDriverGroup.isTaskWide());
            recentlyCompletelyExecutedDriverGroups.add(newlyCompletelyExecutedDriverGroup);
            totalLifespanExecutionFinished++;
        }
        newDriverGroupReady = this.newDriverGroupReady;
    }
    newDriverGroupReady.set(null);
    verify(totalLifespanExecutionFinished <= partitionHandles.size());
}

Example 10 with Lifespan

use of com.facebook.presto.execution.Lifespan in project presto by prestodb.

the class TestClientBuffer method testReferenceCount.

@Test
public void testReferenceCount() {
    AtomicInteger releasedPages = new AtomicInteger(0);
    PagesReleasedListener onPagesReleased = (lifespan, releasedPagesCount, releasedSizeInBytes) -> {
        releasedPages.addAndGet(releasedPagesCount);
    };
    ClientBuffer buffer = new ClientBuffer(TASK_INSTANCE_ID, BUFFER_ID, onPagesReleased);
    // add 2 pages and verify they are referenced
    addPage(buffer, createPage(0));
    addPage(buffer, createPage(1));
    assertEquals(releasedPages.get(), 0);
    assertBufferInfo(buffer, 2, 0);
    // read one page
    assertBufferResultEquals(TYPES, getBufferResult(buffer, 0, sizeOfPages(0), NO_WAIT), bufferResult(0, createPage(0)));
    assertEquals(releasedPages.get(), 0);
    assertBufferInfo(buffer, 2, 0);
    // acknowledge first page
    assertBufferResultEquals(TYPES, getBufferResult(buffer, 1, sizeOfPages(1), NO_WAIT), bufferResult(1, createPage(1)));
    assertEquals(releasedPages.get(), 1);
    assertBufferInfo(buffer, 1, 1);
    // destroy the buffer
    buffer.destroy();
    assertEquals(releasedPages.get(), 2);
    assertBufferDestroyed(buffer, 1);
}