Examples with Operator io.trino.operator.Operator used on opensource projects

Example 11 with Operator

use of io.trino.operator.Operator in project trino by trinodb.

the class TestHashJoinOperator method testYield.

@Test
public void testYield() {
    // create a filter function that yields for every probe match
    // verify we will yield #match times totally
    TaskContext taskContext = createTaskContext();
    DriverContext driverContext = taskContext.addPipelineContext(0, true, true, false).addDriverContext();
    // force a yield for every match
    AtomicInteger filterFunctionCalls = new AtomicInteger();
    InternalJoinFilterFunction filterFunction = new TestInternalJoinFilterFunction(((leftPosition, leftPage, rightPosition, rightPage) -> {
        filterFunctionCalls.incrementAndGet();
        driverContext.getYieldSignal().forceYieldForTesting();
        return true;
    }));
    // build with 40 entries
    int entries = 40;
    RowPagesBuilder buildPages = rowPagesBuilder(true, Ints.asList(0), ImmutableList.of(BIGINT)).addSequencePage(entries, 42);
    BuildSideSetup buildSideSetup = setupBuildSide(nodePartitioningManager, true, taskContext, buildPages, Optional.of(filterFunction), false, SINGLE_STREAM_SPILLER_FACTORY);
    JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactory = buildSideSetup.getLookupSourceFactoryManager();
    // probe matching the above 40 entries
    RowPagesBuilder probePages = rowPagesBuilder(false, Ints.asList(0), ImmutableList.of(BIGINT));
    List<Page> probeInput = probePages.addSequencePage(100, 0).build();
    OperatorFactory joinOperatorFactory = operatorFactories.innerJoin(0, new PlanNodeId("test"), lookupSourceFactory, false, false, true, probePages.getTypes(), Ints.asList(0), getHashChannelAsInt(probePages), Optional.empty(), OptionalInt.of(1), PARTITIONING_SPILLER_FACTORY, TYPE_OPERATOR_FACTORY);
    instantiateBuildDrivers(buildSideSetup, taskContext);
    buildLookupSource(executor, buildSideSetup);
    Operator operator = joinOperatorFactory.createOperator(driverContext);
    assertTrue(operator.needsInput());
    operator.addInput(probeInput.get(0));
    operator.finish();
    // we will yield 40 times due to filterFunction
    for (int i = 0; i < entries; i++) {
        driverContext.getYieldSignal().setWithDelay(5 * SECONDS.toNanos(1), driverContext.getYieldExecutor());
        filterFunctionCalls.set(0);
        assertNull(operator.getOutput());
        assertEquals(filterFunctionCalls.get(), 1, "Expected join to stop processing (yield) after calling filter function once");
        driverContext.getYieldSignal().reset();
    }
    // delayed yield is not going to prevent operator from producing a page now (yield won't be forced because filter function won't be called anymore)
    driverContext.getYieldSignal().setWithDelay(5 * SECONDS.toNanos(1), driverContext.getYieldExecutor());
    // expect output page to be produced within few calls to getOutput(), e.g. to facilitate spill
    Page output = null;
    for (int i = 0; output == null && i < 5; i++) {
        output = operator.getOutput();
    }
    assertNotNull(output);
    driverContext.getYieldSignal().reset();
    // make sure we have all 4 entries
    assertEquals(output.getPositionCount(), entries);
}

Example 12 with Operator

use of io.trino.operator.Operator in project trino by trinodb.

the class TestHashJoinOperator method testInnerJoinWithBlockingLookupSourceAndEmptyProbe.

@Test(dataProvider = "hashJoinTestValues")
public void testInnerJoinWithBlockingLookupSourceAndEmptyProbe(boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled) throws Exception {
    // join that waits for build side to be collected
    TaskContext taskContext = createTaskContext();
    OperatorFactory joinOperatorFactory = createJoinOperatorFactoryWithBlockingLookupSource(taskContext, parallelBuild, probeHashEnabled, buildHashEnabled, true);
    DriverContext driverContext = taskContext.addPipelineContext(0, true, true, false).addDriverContext();
    try (Operator joinOperator = joinOperatorFactory.createOperator(driverContext)) {
        joinOperatorFactory.noMoreOperators();
        assertFalse(joinOperator.needsInput());
        joinOperator.finish();
        assertNull(joinOperator.getOutput());
        // lookup join operator got blocked waiting for build side
        assertFalse(joinOperator.isBlocked().isDone());
        assertFalse(joinOperator.isFinished());
    }
    // join that doesn't wait for build side to be collected
    taskContext = createTaskContext();
    joinOperatorFactory = createJoinOperatorFactoryWithBlockingLookupSource(taskContext, parallelBuild, probeHashEnabled, buildHashEnabled, false);
    driverContext = taskContext.addPipelineContext(0, true, true, false).addDriverContext();
    try (Operator joinOperator = joinOperatorFactory.createOperator(driverContext)) {
        joinOperatorFactory.noMoreOperators();
        assertTrue(joinOperator.needsInput());
        joinOperator.finish();
        assertNull(joinOperator.getOutput());
        // lookup join operator will yield once before finishing
        assertNull(joinOperator.getOutput());
        assertTrue(joinOperator.isBlocked().isDone());
        assertTrue(joinOperator.isFinished());
    }
}

Example 13 with Operator

use of io.trino.operator.Operator in project trino by trinodb.

the class TestNestedLoopJoinOperator method newJoinOperatorFactoryWithCompletedBuild.

private static NestedLoopJoinOperatorFactory newJoinOperatorFactoryWithCompletedBuild(TaskContext taskContext, RowPagesBuilder buildPages, List<Integer> probeChannels, List<Integer> buildChannels) {
    DriverContext driverContext = taskContext.addPipelineContext(0, true, true, false).addDriverContext();
    ValuesOperatorFactory valuesOperatorFactory = new ValuesOperatorFactory(0, new PlanNodeId("test"), buildPages.build());
    JoinBridgeManager<NestedLoopJoinBridge> nestedLoopJoinBridgeManager = new JoinBridgeManager<>(false, PipelineExecutionStrategy.UNGROUPED_EXECUTION, PipelineExecutionStrategy.UNGROUPED_EXECUTION, lifespan -> new NestedLoopJoinPagesSupplier(), buildPages.getTypes());
    NestedLoopBuildOperatorFactory nestedLoopBuildOperatorFactory = new NestedLoopBuildOperatorFactory(1, new PlanNodeId("test"), nestedLoopJoinBridgeManager);
    NestedLoopJoinOperatorFactory joinOperatorFactory = new NestedLoopJoinOperatorFactory(3, new PlanNodeId("test"), nestedLoopJoinBridgeManager, probeChannels, buildChannels);
    Operator valuesOperator = valuesOperatorFactory.createOperator(driverContext);
    Operator nestedLoopBuildOperator = nestedLoopBuildOperatorFactory.createOperator(driverContext);
    Driver driver = Driver.createDriver(driverContext, valuesOperator, nestedLoopBuildOperator);
    valuesOperatorFactory.noMoreOperators();
    nestedLoopBuildOperatorFactory.noMoreOperators();
    while (nestedLoopBuildOperator.isBlocked().isDone()) {
        driver.process();
    }
    return joinOperatorFactory;
}