Examples with TaskId io.prestosql.execution.TaskId used on opensource projects

Example 21 with TaskId

use of io.prestosql.execution.TaskId in project hetu-core by openlookeng.

the class TestDynamicFilterSourceOperator method createOperatorFactory.

private DynamicFilterSourceOperatorFactory createOperatorFactory(DynamicFilter.Type dfType, FeaturesConfig.DynamicFilterDataType dataType, int partitionCount, Channel... buildChannels) {
    NodeInfo nodeInfo = new NodeInfo("test");
    Multimap<String, Symbol> probeSymbols = MultimapBuilder.treeKeys().arrayListValues().build();
    Map<String, Integer> buildChannelMap = new HashMap<>();
    Arrays.stream(buildChannels).map(channel -> buildChannelMap.put(channel.getFilterId(), channel.getIndex()));
    Arrays.stream(buildChannels).map(channel -> probeSymbols.put(channel.getFilterId(), new Symbol(String.valueOf(channel.getIndex()))));
    TaskId taskId = new TaskId("test0.0");
    LocalDynamicFilter localDynamicFilter = new LocalDynamicFilter(probeSymbols, buildChannelMap, partitionCount, dfType, dataType, 0.1D, taskId, stateStoreProvider);
    return new DynamicFilterSourceOperatorFactory(0, new PlanNodeId("PLAN_NODE_ID"), localDynamicFilter.getValueConsumer(), Arrays.stream(buildChannels).collect(toList()), getDynamicFilteringMaxPerDriverValueCount(TEST_SESSION), getDynamicFilteringMaxPerDriverSize(TEST_SESSION));
}

Example 22 with TaskId

use of io.prestosql.execution.TaskId in project hetu-core by openlookeng.

the class LocalDynamicFilter method create.

public static Optional<LocalDynamicFilter> create(JoinNode planNode, int partitionCount, Session session, TaskId taskId, StateStoreProvider stateStoreProvider) {
    Set<String> joinDynamicFilters = planNode.getDynamicFilters().keySet();
    // Mapping from probe-side dynamic filters' IDs to their matching probe symbols.
    Multimap<String, Symbol> localProbeSymbols = MultimapBuilder.treeKeys().arrayListValues().build();
    PlanNode buildNode;
    DynamicFilter.Type localType = DynamicFilter.Type.LOCAL;
    List<FilterNode> filterNodes = findFilterNodeInStage(planNode);
    if (filterNodes.isEmpty()) {
        buildNode = planNode.getRight();
        mapProbeSymbolsFromCriteria(planNode.getDynamicFilters(), localProbeSymbols, planNode.getCriteria());
        localType = DynamicFilter.Type.GLOBAL;
    } else {
        buildNode = planNode.getRight();
        for (FilterNode filterNode : filterNodes) {
            mapProbeSymbols(filterNode.getPredicate(), joinDynamicFilters, localProbeSymbols);
        }
    }
    final List<Symbol> buildSideSymbols = buildNode.getOutputSymbols();
    Map<String, Integer> localBuildChannels = planNode.getDynamicFilters().entrySet().stream().filter(entry -> localProbeSymbols.containsKey(entry.getKey())).collect(toMap(// Dynamic filter ID
    entry -> entry.getKey(), // Build-side channel index
    entry -> {
        Symbol buildSymbol = entry.getValue();
        int buildChannelIndex = buildSideSymbols.indexOf(buildSymbol);
        verify(buildChannelIndex >= 0);
        return buildChannelIndex;
    }));
    if (localBuildChannels.isEmpty()) {
        return Optional.empty();
    }
    return Optional.of(new LocalDynamicFilter(localProbeSymbols, localBuildChannels, partitionCount, localType, session, taskId, stateStoreProvider));
}

Example 23 with TaskId

use of io.prestosql.execution.TaskId in project hetu-core by openlookeng.

the class MemoryLocalQueryRunner method execute.

public List<Page> execute(@Language("SQL") String query) {
    MemoryPool memoryPool = new MemoryPool(new MemoryPoolId("test"), new DataSize(2, GIGABYTE));
    SpillSpaceTracker spillSpaceTracker = new SpillSpaceTracker(new DataSize(1, GIGABYTE));
    QueryContext queryContext = new QueryContext(new QueryId("test"), new DataSize(1, GIGABYTE), new DataSize(2, GIGABYTE), memoryPool, new TestingGcMonitor(), localQueryRunner.getExecutor(), localQueryRunner.getScheduler(), new DataSize(4, GIGABYTE), spillSpaceTracker, NOOP_SNAPSHOT_UTILS);
    TaskContext taskContext = queryContext.addTaskContext(new TaskStateMachine(new TaskId("query", 0, 0), localQueryRunner.getExecutor()), localQueryRunner.getDefaultSession(), false, false, OptionalInt.empty(), Optional.empty(), TESTING_SERDE_FACTORY);
    // Use NullOutputFactory to avoid coping out results to avoid affecting benchmark results
    ImmutableList.Builder<Page> output = ImmutableList.builder();
    List<Driver> drivers = localQueryRunner.createDrivers(query, new PageConsumerOperator.PageConsumerOutputFactory(types -> output::add), taskContext);
    boolean done = false;
    while (!done) {
        boolean processed = false;
        for (Driver driver : drivers) {
            if (!driver.isFinished()) {
                driver.process();
                processed = true;
            }
        }
        done = !processed;
    }
    return output.build();
}

Example 24 with TaskId

use of io.prestosql.execution.TaskId in project hetu-core by openlookeng.

the class TestTaskExecutor method testTaskHandle.

@Test
public void testTaskHandle() {
    TestingTicker ticker = new TestingTicker();
    TaskExecutor taskExecutor = new TaskExecutor(4, 8, 3, 4, ticker);
    taskExecutor.start();
    try {
        TaskId taskId = new TaskId("test", 0, 0);
        TaskHandle taskHandle = taskExecutor.addTask(taskId, () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty());
        Phaser beginPhase = new Phaser();
        beginPhase.register();
        Phaser verificationComplete = new Phaser();
        verificationComplete.register();
        TestingJob driver1 = new TestingJob(ticker, new Phaser(), beginPhase, verificationComplete, 10, 0);
        TestingJob driver2 = new TestingJob(ticker, new Phaser(), beginPhase, verificationComplete, 10, 0);
        // force enqueue a split
        taskExecutor.enqueueSplits(taskHandle, true, ImmutableList.of(driver1));
        assertEquals(taskHandle.getRunningLeafSplits(), 0);
        // normal enqueue a split
        taskExecutor.enqueueSplits(taskHandle, false, ImmutableList.of(driver2));
        assertEquals(taskHandle.getRunningLeafSplits(), 1);
        // let the split continue to run
        beginPhase.arriveAndDeregister();
        verificationComplete.arriveAndDeregister();
    } finally {
        taskExecutor.stop();
    }
}

Example 25 with TaskId

use of io.prestosql.execution.TaskId in project hetu-core by openlookeng.

the class TestTaskExecutor method testTasksComplete.

@Test(invocationCount = 100)
public void testTasksComplete() throws Exception {
    TestingTicker ticker = new TestingTicker();
    TaskExecutor taskExecutor = new TaskExecutor(4, 8, 3, 4, ticker);
    taskExecutor.start();
    ticker.increment(20, MILLISECONDS);
    try {
        TaskId taskId = new TaskId("test", 0, 0);
        TaskHandle taskHandle = taskExecutor.addTask(taskId, () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty());
        Phaser beginPhase = new Phaser();
        beginPhase.register();
        Phaser verificationComplete = new Phaser();
        verificationComplete.register();
        // add two jobs
        TestingJob driver1 = new TestingJob(ticker, new Phaser(1), beginPhase, verificationComplete, 10, 0);
        ListenableFuture<?> future1 = getOnlyElement(taskExecutor.enqueueSplits(taskHandle, true, ImmutableList.of(driver1)));
        TestingJob driver2 = new TestingJob(ticker, new Phaser(1), beginPhase, verificationComplete, 10, 0);
        ListenableFuture<?> future2 = getOnlyElement(taskExecutor.enqueueSplits(taskHandle, true, ImmutableList.of(driver2)));
        assertEquals(driver1.getCompletedPhases(), 0);
        assertEquals(driver2.getCompletedPhases(), 0);
        // verify worker have arrived but haven't processed yet
        beginPhase.arriveAndAwaitAdvance();
        assertEquals(driver1.getCompletedPhases(), 0);
        assertEquals(driver2.getCompletedPhases(), 0);
        ticker.increment(60, SECONDS);
        assertEquals(taskExecutor.getRunAwaySplitCount(), 0);
        ticker.increment(600, SECONDS);
        assertEquals(taskExecutor.getRunAwaySplitCount(), 2);
        verificationComplete.arriveAndAwaitAdvance();
        // advance one phase and verify
        beginPhase.arriveAndAwaitAdvance();
        assertEquals(driver1.getCompletedPhases(), 1);
        assertEquals(driver2.getCompletedPhases(), 1);
        verificationComplete.arriveAndAwaitAdvance();
        // add one more job
        TestingJob driver3 = new TestingJob(ticker, new Phaser(1), beginPhase, verificationComplete, 10, 0);
        ListenableFuture<?> future3 = getOnlyElement(taskExecutor.enqueueSplits(taskHandle, false, ImmutableList.of(driver3)));
        // advance one phase and verify
        beginPhase.arriveAndAwaitAdvance();
        assertEquals(driver1.getCompletedPhases(), 2);
        assertEquals(driver2.getCompletedPhases(), 2);
        assertEquals(driver3.getCompletedPhases(), 0);
        verificationComplete.arriveAndAwaitAdvance();
        // advance to the end of the first two task and verify
        beginPhase.arriveAndAwaitAdvance();
        for (int i = 0; i < 7; i++) {
            verificationComplete.arriveAndAwaitAdvance();
            beginPhase.arriveAndAwaitAdvance();
            assertEquals(beginPhase.getPhase(), verificationComplete.getPhase() + 1);
        }
        assertEquals(driver1.getCompletedPhases(), 10);
        assertEquals(driver2.getCompletedPhases(), 10);
        assertEquals(driver3.getCompletedPhases(), 8);
        future1.get(1, SECONDS);
        future2.get(1, SECONDS);
        verificationComplete.arriveAndAwaitAdvance();
        // advance two more times and verify
        beginPhase.arriveAndAwaitAdvance();
        verificationComplete.arriveAndAwaitAdvance();
        beginPhase.arriveAndAwaitAdvance();
        assertEquals(driver1.getCompletedPhases(), 10);
        assertEquals(driver2.getCompletedPhases(), 10);
        assertEquals(driver3.getCompletedPhases(), 10);
        future3.get(1, SECONDS);
        verificationComplete.arriveAndAwaitAdvance();
        assertEquals(driver1.getFirstPhase(), 0);
        assertEquals(driver2.getFirstPhase(), 0);
        assertEquals(driver3.getFirstPhase(), 2);
        assertEquals(driver1.getLastPhase(), 10);
        assertEquals(driver2.getLastPhase(), 10);
        assertEquals(driver3.getLastPhase(), 12);
        // no splits remaining
        ticker.increment(610, SECONDS);
        assertEquals(taskExecutor.getRunAwaySplitCount(), 0);
    } finally {
        taskExecutor.stop();
    }
}