Examples with ScheduledSplit io.prestosql.execution.ScheduledSplit used on opensource projects

Example 1 with ScheduledSplit

use of io.prestosql.execution.ScheduledSplit in project hetu-core by openlookeng.

the class TestSystemMemoryBlocking method testTableScanSystemMemoryBlocking.

@Test
public void testTableScanSystemMemoryBlocking() {
    PlanNodeId sourceId = new PlanNodeId("source");
    final List<Type> types = ImmutableList.of(VARCHAR);
    TableScanOperator source = new TableScanOperator(driverContext.addOperatorContext(1, new PlanNodeId("test"), "values"), sourceId, (session, split, table, columns, dynamicFilter) -> new FixedPageSource(rowPagesBuilder(types).addSequencePage(10, 1).addSequencePage(10, 1).addSequencePage(10, 1).addSequencePage(10, 1).addSequencePage(10, 1).build()), TEST_TABLE_HANDLE, ImmutableList.of(), ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_DEFAULT, new UUID(0, 0), types, false, Optional.empty(), 0, 0);
    PageConsumerOperator sink = createSinkOperator(types);
    Driver driver = Driver.createDriver(driverContext, source, sink);
    assertSame(driver.getDriverContext(), driverContext);
    assertFalse(driver.isFinished());
    Split testSplit = new Split(new CatalogName("test"), new TestSplit(), Lifespan.taskWide());
    driver.updateSource(new TaskSource(sourceId, ImmutableSet.of(new ScheduledSplit(0, sourceId, testSplit)), true));
    ListenableFuture<?> blocked = driver.processFor(new Duration(1, NANOSECONDS));
    // the driver shouldn't block in the first call as it will be able to move a page between source and the sink operator
    // but the operator should be blocked
    assertTrue(blocked.isDone());
    assertFalse(source.getOperatorContext().isWaitingForMemory().isDone());
    // and they should stay blocked until more memory becomes available
    for (int i = 0; i < 10; i++) {
        blocked = driver.processFor(new Duration(1, NANOSECONDS));
        assertFalse(blocked.isDone());
        assertFalse(source.getOperatorContext().isWaitingForMemory().isDone());
    }
    // free up some memory
    memoryPool.free(QUERY_ID, "test", memoryPool.getReservedBytes());
    // the operator should be unblocked
    assertTrue(source.getOperatorContext().isWaitingForMemory().isDone());
    // the driver shouldn't be blocked
    blocked = driver.processFor(new Duration(1, NANOSECONDS));
    assertTrue(blocked.isDone());
}

Example 2 with ScheduledSplit

use of io.prestosql.execution.ScheduledSplit in project hetu-core by openlookeng.

the class HttpRemoteTask method processTaskUpdate.

private synchronized void processTaskUpdate(TaskInfo newValue, List<TaskSource> sources) {
    updateTaskInfo(newValue);
    // remove acknowledged splits, which frees memory
    for (TaskSource source : sources) {
        PlanNodeId planNodeId = source.getPlanNodeId();
        int removed = 0;
        for (ScheduledSplit split : source.getSplits()) {
            if (pendingSplits.remove(planNodeId, split)) {
                removed++;
            }
        }
        if (source.isNoMoreSplits()) {
            noMoreSplits.put(planNodeId, false);
        }
        for (Lifespan lifespan : source.getNoMoreSplitsForLifespan()) {
            pendingNoMoreSplitsForLifespan.remove(planNodeId, lifespan);
        }
        if (planFragment.isPartitionedSources(planNodeId)) {
            pendingSourceSplitCount -= removed;
        }
    }
    updateSplitQueueSpace();
    partitionedSplitCountTracker.setPartitionedSplitCount(getPartitionedSplitCount());
}

Example 3 with ScheduledSplit

use of io.prestosql.execution.ScheduledSplit in project hetu-core by openlookeng.

the class HttpRemoteTask method getSource.

private synchronized TaskSource getSource(PlanNodeId planNodeId) {
    Set<ScheduledSplit> splits = ImmutableSet.copyOf(pendingSplits.get(planNodeId));
    boolean pendingNoMoreSplits = Boolean.TRUE.equals(this.noMoreSplits.get(planNodeId));
    boolean tmpNoMoreSplits = this.noMoreSplits.containsKey(planNodeId);
    Set<Lifespan> noMoreSplitsForLifespan = pendingNoMoreSplitsForLifespan.get(planNodeId);
    TaskSource element = null;
    if (!splits.isEmpty() || !noMoreSplitsForLifespan.isEmpty() || pendingNoMoreSplits) {
        element = new TaskSource(planNodeId, splits, noMoreSplitsForLifespan, tmpNoMoreSplits);
    }
    return element;
}

Example 4 with ScheduledSplit

use of io.prestosql.execution.ScheduledSplit in project hetu-core by openlookeng.

the class HttpRemoteTask method addSplits.

@Override
public synchronized void addSplits(Multimap<PlanNodeId, Split> splitsBySource) {
    requireNonNull(splitsBySource, "splitsBySource is null");
    // only add pending split if not done
    if (getTaskStatus().getState().isDone()) {
        return;
    }
    boolean tmpNeedsUpdate = false;
    for (Entry<PlanNodeId, Collection<Split>> entry : splitsBySource.asMap().entrySet()) {
        PlanNodeId sourceId = entry.getKey();
        Collection<Split> splits = entry.getValue();
        checkState(!noMoreSplits.containsKey(sourceId), "noMoreSplits has already been set for %s", sourceId);
        int added = 0;
        for (Split split : splits) {
            if (pendingSplits.put(sourceId, new ScheduledSplit(nextSplitId.getAndIncrement(), sourceId, split))) {
                added++;
            }
        }
        if (planFragment.isPartitionedSources(sourceId)) {
            pendingSourceSplitCount += added;
            partitionedSplitCountTracker.setPartitionedSplitCount(getPartitionedSplitCount());
        }
        tmpNeedsUpdate = true;
    }
    updateSplitQueueSpace();
    if (tmpNeedsUpdate) {
        this.needsUpdate.set(true);
        scheduleUpdate();
    }
}

Example 5 with ScheduledSplit

use of io.prestosql.execution.ScheduledSplit in project hetu-core by openlookeng.

the class TestDriver method testBrokenOperatorAddSource.

@Test
public void testBrokenOperatorAddSource() throws Exception {
    PlanNodeId sourceId = new PlanNodeId("source");
    final List<Type> types = ImmutableList.of(VARCHAR, BIGINT, BIGINT);
    // create a table scan operator that does not block, which will cause the driver loop to busy wait
    TableScanOperator source = new NotBlockedTableScanOperator(driverContext.addOperatorContext(99, new PlanNodeId("test"), "values"), sourceId, (session, split, table, columns, dynamicFilter) -> new FixedPageSource(rowPagesBuilder(types).addSequencePage(10, 20, 30, 40).build()), TEST_TABLE_HANDLE, ImmutableList.of());
    BrokenOperator brokenOperator = new BrokenOperator(driverContext.addOperatorContext(0, new PlanNodeId("test"), "source"));
    final Driver driver = Driver.createDriver(driverContext, source, brokenOperator);
    // block thread in operator processing
    Future<Boolean> driverProcessFor = executor.submit(new Callable<Boolean>() {

        @Override
        public Boolean call() {
            return driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone();
        }
    });
    brokenOperator.waitForLocked();
    assertSame(driver.getDriverContext(), driverContext);
    assertFalse(driver.isFinished());
    // processFor always returns NOT_BLOCKED, because DriveLockResult was not acquired
    assertTrue(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone());
    assertFalse(driver.isFinished());
    driver.updateSource(new TaskSource(sourceId, ImmutableSet.of(new ScheduledSplit(0, sourceId, newMockSplit())), true));
    assertFalse(driver.isFinished());
    // processFor always returns NOT_BLOCKED, because DriveLockResult was not acquired
    assertTrue(driver.processFor(new Duration(1, TimeUnit.SECONDS)).isDone());
    assertFalse(driver.isFinished());
    driver.close();
    assertTrue(driver.isFinished());
    try {
        driverProcessFor.get(1, TimeUnit.SECONDS);
        fail("Expected InterruptedException");
    } catch (ExecutionException e) {
        assertDriverInterrupted(e.getCause());
    }
}