Search in sources :

Example 1 with Operator

use of io.prestosql.operator.Operator in project hetu-core by openlookeng.

the class TestLocalExchangeSourceOperator method createOperator.

private LocalExchangeSourceOperator createOperator(LocalExchange.LocalExchangeFactory localExchangeFactory, int totalInputChannels, Session session, int driverId) {
    if (pipelineContext == null) {
        ScheduledExecutorService scheduler = newScheduledThreadPool(4, daemonThreadsNamed("test-%s"));
        ScheduledExecutorService scheduledExecutor = newScheduledThreadPool(2, daemonThreadsNamed("test-scheduledExecutor-%s"));
        OperatorFactory operatorFactory = new LocalExchangeSourceOperator.LocalExchangeSourceOperatorFactory(0, new PlanNodeId("test"), localExchangeFactory, totalInputChannels);
        pipelineContext = createTaskContext(scheduler, scheduledExecutor, session).addPipelineContext(0, true, true, false);
    }
    DriverContext driverContext = pipelineContext.addDriverContext(Lifespan.taskWide(), driverId);
    OperatorFactory operatorFactory = new LocalExchangeSourceOperator.LocalExchangeSourceOperatorFactory(0, new PlanNodeId("test"), localExchangeFactory, totalInputChannels);
    Operator operator = operatorFactory.createOperator(driverContext);
    assertEquals(operator.getOperatorContext().getOperatorStats().getSystemMemoryReservation().toBytes(), 0);
    return (LocalExchangeSourceOperator) operator;
}
Also used : PlanNodeId(io.prestosql.spi.plan.PlanNodeId) Operator(io.prestosql.operator.Operator) DriverContext(io.prestosql.operator.DriverContext) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) OperatorFactory(io.prestosql.operator.OperatorFactory)

Example 2 with Operator

use of io.prestosql.operator.Operator in project hetu-core by openlookeng.

the class TestSpatialJoinOperator method buildIndexSnapshot.

private PagesSpatialIndexFactory buildIndexSnapshot(DriverContext driverContext, SpatialPredicate spatialRelationshipTest, Optional<Integer> radiusChannel, Optional<Integer> partitionChannel, Optional<String> kdbTreeJson, Optional<InternalJoinFilterFunction> filterFunction, RowPagesBuilder buildPages) {
    Optional<JoinFilterFunctionCompiler.JoinFilterFunctionFactory> filterFunctionFactory = filterFunction.map(function -> (session, addresses, pages) -> new StandardJoinFilterFunction(function, addresses, pages));
    SpatialIndexBuilderOperatorFactory buildOperatorFactory = new SpatialIndexBuilderOperatorFactory(1, new PlanNodeId("test"), buildPages.getTypes(), Ints.asList(1), 0, radiusChannel, partitionChannel, spatialRelationshipTest, kdbTreeJson, filterFunctionFactory, 10_000, new TestingFactory(false));
    Operator operator = buildOperatorFactory.createOperator(driverContext);
    PagesSpatialIndexFactory pagesSpatialIndexFactory = buildOperatorFactory.getPagesSpatialIndexFactory();
    ListenableFuture<PagesSpatialIndex> pagesSpatialIndex = pagesSpatialIndexFactory.createPagesSpatialIndex();
    List<Page> buildSideInputs = buildPages.build();
    int inputIndex = 0;
    boolean restored = false;
    Object snapshot = null;
    // When build side is not done, keep looping
    while (!pagesSpatialIndex.isDone()) {
        if (operator.needsInput() && inputIndex < buildSideInputs.size()) {
            operator.addInput(buildSideInputs.get(inputIndex));
            inputIndex++;
        }
        // Take snapshot in the middle
        if (inputIndex == buildSideInputs.size() / 2 && !restored) {
            snapshot = operator.capture(operator.getOperatorContext().getDriverContext().getSerde());
        }
        // When input pages are used up, restore operator to snapshot and move inputIndex back to when snapshot was taken
        if (inputIndex == buildSideInputs.size() && !restored) {
            assertTrue(snapshot != null);
            operator.restore(snapshot, operator.getOperatorContext().getDriverContext().getSerde());
            restored = true;
            inputIndex = buildSideInputs.size() / 2;
        }
        // Used up all provided build side inputs and have done rollback process, finish build operator, this will cause future to be done
        if (inputIndex >= buildSideInputs.size() && restored) {
            operator.finish();
        }
    }
    return pagesSpatialIndexFactory;
}
Also used : ValuesOperator(io.prestosql.operator.ValuesOperator) Operator(io.prestosql.operator.Operator) SpatialIndexBuilderOperatorFactory(io.prestosql.operator.SpatialIndexBuilderOperator.SpatialIndexBuilderOperatorFactory) StandardJoinFilterFunction(io.prestosql.operator.StandardJoinFilterFunction) PagesSpatialIndex(io.prestosql.operator.PagesSpatialIndex) Page(io.prestosql.spi.Page) GeoFunctions.stPoint(io.prestosql.plugin.geospatial.GeoFunctions.stPoint) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) TestingFactory(io.prestosql.operator.PagesIndex.TestingFactory) PagesSpatialIndexFactory(io.prestosql.operator.PagesSpatialIndexFactory)

Example 3 with Operator

use of io.prestosql.operator.Operator in project hetu-core by openlookeng.

the class TestSpatialJoinOperator method testYield.

@Test
public void testYield() {
    // create a filter function that yields for every probe match
    // verify we will yield #match times totally
    TaskContext taskContext = createTaskContext();
    DriverContext driverContext = taskContext.addPipelineContext(0, true, true, false).addDriverContext();
    // force a yield for every match
    AtomicInteger filterFunctionCalls = new AtomicInteger();
    InternalJoinFilterFunction filterFunction = new TestInternalJoinFilterFunction(((leftPosition, leftPage, rightPosition, rightPage) -> {
        filterFunctionCalls.incrementAndGet();
        driverContext.getYieldSignal().forceYieldForTesting();
        return true;
    }));
    RowPagesBuilder buildPages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR)).row(POLYGON_A, "A").pageBreak().row(POLYGON_B, "B");
    PagesSpatialIndexFactory pagesSpatialIndexFactory = buildIndex(driverContext, (build, probe, r) -> build.contains(probe), Optional.empty(), Optional.of(filterFunction), buildPages);
    // 10 points in polygon A (x0...x9)
    // 10 points in polygons A and B (y0...y9)
    // 10 points in polygon B (z0...z9)
    // 40 total matches
    RowPagesBuilder probePages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR));
    for (int i = 0; i < 10; i++) {
        probePages.row(stPoint(1 + 0.1 * i, 1 + 0.1 * i), "x" + i);
    }
    for (int i = 0; i < 10; i++) {
        probePages.row(stPoint(4.5 + 0.01 * i, 4.5 + 0.01 * i), "y" + i);
    }
    for (int i = 0; i < 10; i++) {
        probePages.row(stPoint(6 + 0.1 * i, 6 + 0.1 * i), "z" + i);
    }
    List<Page> probeInput = probePages.build();
    OperatorFactory joinOperatorFactory = new SpatialJoinOperatorFactory(2, new PlanNodeId("test"), INNER, probePages.getTypes(), Ints.asList(1), 0, Optional.empty(), pagesSpatialIndexFactory);
    Operator operator = joinOperatorFactory.createOperator(driverContext);
    assertTrue(operator.needsInput());
    operator.addInput(probeInput.get(0));
    operator.finish();
    // we will yield 40 times due to filterFunction
    for (int i = 0; i < 40; i++) {
        driverContext.getYieldSignal().setWithDelay(5 * SECONDS.toNanos(1), driverContext.getYieldExecutor());
        assertNull(operator.getOutput());
        assertEquals(filterFunctionCalls.get(), i + 1, "Expected join to stop processing (yield) after calling filter function once");
        driverContext.getYieldSignal().reset();
    }
    // delayed yield is not going to prevent operator from producing a page now (yield won't be forced because filter function won't be called anymore)
    driverContext.getYieldSignal().setWithDelay(5 * SECONDS.toNanos(1), driverContext.getYieldExecutor());
    Page output = operator.getOutput();
    assertNotNull(output);
    // make sure we have 40 matches
    assertEquals(output.getPositionCount(), 40);
}
Also used : Test(org.testng.annotations.Test) AfterMethod(org.testng.annotations.AfterMethod) PagesSpatialIndexFactory(io.prestosql.operator.PagesSpatialIndexFactory) MaterializedResult(io.prestosql.testing.MaterializedResult) GeoFunctions.stPoint(io.prestosql.plugin.geospatial.GeoFunctions.stPoint) KdbTree(io.prestosql.geospatial.KdbTree) KdbTreeUtils(io.prestosql.geospatial.KdbTreeUtils) Type(io.prestosql.sql.planner.plan.SpatialJoinNode.Type) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Executors.newScheduledThreadPool(java.util.concurrent.Executors.newScheduledThreadPool) OperatorAssertion.assertOperatorEquals(io.prestosql.operator.OperatorAssertion.assertOperatorEquals) Slices(io.airlift.slice.Slices) PipelineContext(io.prestosql.operator.PipelineContext) SpatialJoinOperatorFactory(io.prestosql.operator.SpatialJoinOperator.SpatialJoinOperatorFactory) Node.newInternal(io.prestosql.geospatial.KdbTree.Node.newInternal) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) PrestoException(io.prestosql.spi.PrestoException) SynchronousQueue(java.util.concurrent.SynchronousQueue) TestingFactory(io.prestosql.operator.PagesIndex.TestingFactory) RowPagesBuilder.rowPagesBuilder(io.prestosql.RowPagesBuilder.rowPagesBuilder) BeforeMethod(org.testng.annotations.BeforeMethod) ValuesOperator(io.prestosql.operator.ValuesOperator) Collections.emptyIterator(java.util.Collections.emptyIterator) Assert.assertNotNull(org.testng.Assert.assertNotNull) List(java.util.List) Driver(io.prestosql.operator.Driver) Optional(java.util.Optional) MaterializedResult.resultBuilder(io.prestosql.testing.MaterializedResult.resultBuilder) TEST_SESSION(io.prestosql.SessionTestUtils.TEST_SESSION) PagesSpatialIndex(io.prestosql.operator.PagesSpatialIndex) Operator(io.prestosql.operator.Operator) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) Slice(io.airlift.slice.Slice) DataProvider(org.testng.annotations.DataProvider) Assert.assertNull(org.testng.Assert.assertNull) Rectangle(io.prestosql.geospatial.Rectangle) ThreadPoolExecutor(java.util.concurrent.ThreadPoolExecutor) TaskContext(io.prestosql.operator.TaskContext) LEFT(io.prestosql.sql.planner.plan.SpatialJoinNode.Type.LEFT) Assert.assertEquals(org.testng.Assert.assertEquals) OperatorAssertion.toMaterializedResult(io.prestosql.operator.OperatorAssertion.toMaterializedResult) INTEGER(io.prestosql.spi.type.IntegerType.INTEGER) OperatorFactory(io.prestosql.operator.OperatorFactory) InternalJoinFilterFunction(io.prestosql.operator.InternalJoinFilterFunction) GeoFunctions.stGeometryFromText(io.prestosql.plugin.geospatial.GeoFunctions.stGeometryFromText) OperatorAssertion.toPages(io.prestosql.operator.OperatorAssertion.toPages) INNER(io.prestosql.sql.planner.plan.SpatialJoinNode.Type.INNER) VARCHAR(io.prestosql.spi.type.VarcharType.VARCHAR) ImmutableList(com.google.common.collect.ImmutableList) Threads.daemonThreadsNamed(io.airlift.concurrent.Threads.daemonThreadsNamed) StandardJoinFilterFunction(io.prestosql.operator.StandardJoinFilterFunction) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) DOUBLE(io.prestosql.spi.type.DoubleType.DOUBLE) SpatialPredicate(io.prestosql.operator.SpatialIndexBuilderOperator.SpatialPredicate) ExecutorService(java.util.concurrent.ExecutorService) DriverContext(io.prestosql.operator.DriverContext) Page(io.prestosql.spi.Page) TestingTaskContext(io.prestosql.testing.TestingTaskContext) Ints(com.google.common.primitives.Ints) SpatialIndexBuilderOperatorFactory(io.prestosql.operator.SpatialIndexBuilderOperator.SpatialIndexBuilderOperatorFactory) JoinFilterFunctionCompiler(io.prestosql.sql.gen.JoinFilterFunctionCompiler) Node.newLeaf(io.prestosql.geospatial.KdbTree.Node.newLeaf) Assert(io.prestosql.testing.assertions.Assert) GEOMETRY(io.prestosql.plugin.geospatial.GeometryType.GEOMETRY) RowPagesBuilder(io.prestosql.RowPagesBuilder) Assert.assertTrue(org.testng.Assert.assertTrue) SECONDS(java.util.concurrent.TimeUnit.SECONDS) ValuesOperator(io.prestosql.operator.ValuesOperator) Operator(io.prestosql.operator.Operator) DriverContext(io.prestosql.operator.DriverContext) TaskContext(io.prestosql.operator.TaskContext) TestingTaskContext(io.prestosql.testing.TestingTaskContext) RowPagesBuilder(io.prestosql.RowPagesBuilder) Page(io.prestosql.spi.Page) InternalJoinFilterFunction(io.prestosql.operator.InternalJoinFilterFunction) GeoFunctions.stPoint(io.prestosql.plugin.geospatial.GeoFunctions.stPoint) SpatialJoinOperatorFactory(io.prestosql.operator.SpatialJoinOperator.SpatialJoinOperatorFactory) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) SpatialJoinOperatorFactory(io.prestosql.operator.SpatialJoinOperator.SpatialJoinOperatorFactory) OperatorFactory(io.prestosql.operator.OperatorFactory) SpatialIndexBuilderOperatorFactory(io.prestosql.operator.SpatialIndexBuilderOperator.SpatialIndexBuilderOperatorFactory) PagesSpatialIndexFactory(io.prestosql.operator.PagesSpatialIndexFactory) Test(org.testng.annotations.Test)

Example 4 with Operator

use of io.prestosql.operator.Operator in project hetu-core by openlookeng.

the class TestSpatialJoinOperator method testDuplicateProbeFactory.

@Test(dataProvider = "testDuplicateProbeFactoryDataProvider")
public void testDuplicateProbeFactory(boolean createSecondaryOperators) throws Exception {
    TaskContext taskContext = createTaskContext();
    PipelineContext pipelineContext = taskContext.addPipelineContext(0, true, true, false);
    DriverContext probeDriver = pipelineContext.addDriverContext();
    DriverContext buildDriver = pipelineContext.addDriverContext();
    RowPagesBuilder buildPages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR, DOUBLE)).row(stPoint(0, 0), "0_0", 1.5);
    PagesSpatialIndexFactory pagesSpatialIndexFactory = buildIndex(buildDriver, (build, probe, r) -> build.distance(probe) <= r.getAsDouble(), Optional.of(2), Optional.empty(), buildPages);
    RowPagesBuilder probePages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR)).row(stPoint(0, 1), "0_1");
    OperatorFactory firstFactory = new SpatialJoinOperatorFactory(2, new PlanNodeId("test"), INNER, probePages.getTypes(), Ints.asList(1), 0, Optional.empty(), pagesSpatialIndexFactory);
    for (int i = 0; i < 3; i++) {
        DriverContext secondDriver = pipelineContext.addDriverContext();
        OperatorFactory secondFactory = firstFactory.duplicate();
        if (createSecondaryOperators) {
            try (Operator secondOperator = secondFactory.createOperator(secondDriver)) {
                assertEquals(toPages(secondOperator, emptyIterator()), ImmutableList.of());
            }
        }
        secondFactory.noMoreOperators();
    }
    MaterializedResult expected = resultBuilder(taskContext.getSession(), ImmutableList.of(VARCHAR, VARCHAR)).row("0_1", "0_0").build();
    assertOperatorEquals(firstFactory, probeDriver, probePages.build(), expected);
}
Also used : SpatialJoinOperatorFactory(io.prestosql.operator.SpatialJoinOperator.SpatialJoinOperatorFactory) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) ValuesOperator(io.prestosql.operator.ValuesOperator) Operator(io.prestosql.operator.Operator) DriverContext(io.prestosql.operator.DriverContext) TaskContext(io.prestosql.operator.TaskContext) TestingTaskContext(io.prestosql.testing.TestingTaskContext) RowPagesBuilder(io.prestosql.RowPagesBuilder) PipelineContext(io.prestosql.operator.PipelineContext) SpatialJoinOperatorFactory(io.prestosql.operator.SpatialJoinOperator.SpatialJoinOperatorFactory) OperatorFactory(io.prestosql.operator.OperatorFactory) SpatialIndexBuilderOperatorFactory(io.prestosql.operator.SpatialIndexBuilderOperator.SpatialIndexBuilderOperatorFactory) PagesSpatialIndexFactory(io.prestosql.operator.PagesSpatialIndexFactory) MaterializedResult(io.prestosql.testing.MaterializedResult) OperatorAssertion.toMaterializedResult(io.prestosql.operator.OperatorAssertion.toMaterializedResult) GeoFunctions.stPoint(io.prestosql.plugin.geospatial.GeoFunctions.stPoint) Test(org.testng.annotations.Test)

Example 5 with Operator

use of io.prestosql.operator.Operator in project hetu-core by openlookeng.

the class TestSpatialJoinOperator method assetResultEqualsSnapshot.

private void assetResultEqualsSnapshot(OperatorFactory operatorFactory, DriverContext driverContext, List<Page> input, MaterializedResult expected) {
    Operator operator = operatorFactory.createOperator(driverContext);
    ImmutableList.Builder<Page> outputPages = ImmutableList.builder();
    int inputIdx = 0;
    Object snapshot = null;
    boolean restored = false;
    for (int loopsSinceLastPage = 0; loopsSinceLastPage < 1_000; loopsSinceLastPage++) {
        if (inputIdx < input.size() && operator.needsInput()) {
            if (inputIdx == input.size() / 2) {
                snapshot = operator.capture(operator.getOperatorContext().getDriverContext().getSerde());
            } else if (inputIdx == input.size() - 1 && snapshot != null && !restored) {
                operator.restore(snapshot, operator.getOperatorContext().getDriverContext().getSerde());
                inputIdx = input.size() / 2;
                restored = true;
            }
            operator.addInput(input.get(inputIdx));
            inputIdx++;
            loopsSinceLastPage = 0;
            // Operator doesn't produce output in the process, but needs getOutput call to clear for next input.
            operator.getOutput();
        }
    }
    for (int loopsSinceLastPage = 0; !operator.isFinished() && loopsSinceLastPage < 1_000; loopsSinceLastPage++) {
        operator.finish();
        Page outputPage = operator.getOutput();
        if (outputPage != null && outputPage.getPositionCount() != 0) {
            outputPages.add(outputPage);
            loopsSinceLastPage = 0;
        }
    }
    assertEquals(operator.isFinished(), true, "Operator did not finish");
    assertEquals(operator.needsInput(), false, "Operator still wants input");
    assertEquals(operator.isBlocked().isDone(), true, "Operator is blocked");
    List<Page> output = outputPages.build();
    MaterializedResult actual = toMaterializedResult(driverContext.getSession(), expected.getTypes(), output);
    Assert.assertEquals(actual, expected);
}
Also used : ValuesOperator(io.prestosql.operator.ValuesOperator) Operator(io.prestosql.operator.Operator) ImmutableList(com.google.common.collect.ImmutableList) Page(io.prestosql.spi.Page) MaterializedResult(io.prestosql.testing.MaterializedResult) OperatorAssertion.toMaterializedResult(io.prestosql.operator.OperatorAssertion.toMaterializedResult) GeoFunctions.stPoint(io.prestosql.plugin.geospatial.GeoFunctions.stPoint)

Aggregations

Operator (io.prestosql.operator.Operator)8 Page (io.prestosql.spi.Page)5 PlanNodeId (io.prestosql.spi.plan.PlanNodeId)5 DriverContext (io.prestosql.operator.DriverContext)4 ValuesOperator (io.prestosql.operator.ValuesOperator)4 GeoFunctions.stPoint (io.prestosql.plugin.geospatial.GeoFunctions.stPoint)4 Test (org.testng.annotations.Test)4 ImmutableList (com.google.common.collect.ImmutableList)3 OperatorAssertion.toMaterializedResult (io.prestosql.operator.OperatorAssertion.toMaterializedResult)3 OperatorFactory (io.prestosql.operator.OperatorFactory)3 PagesSpatialIndexFactory (io.prestosql.operator.PagesSpatialIndexFactory)3 SpatialIndexBuilderOperatorFactory (io.prestosql.operator.SpatialIndexBuilderOperator.SpatialIndexBuilderOperatorFactory)3 TaskContext (io.prestosql.operator.TaskContext)3 MaterializedResult (io.prestosql.testing.MaterializedResult)3 ListenableFuture (com.google.common.util.concurrent.ListenableFuture)2 RowPagesBuilder (io.prestosql.RowPagesBuilder)2 Driver (io.prestosql.operator.Driver)2 PipelineContext (io.prestosql.operator.PipelineContext)2 SpatialJoinOperatorFactory (io.prestosql.operator.SpatialJoinOperator.SpatialJoinOperatorFactory)2 TestingTaskContext (io.prestosql.testing.TestingTaskContext)2