Search in sources :

Example 11 with OrderingCompiler

use of io.prestosql.sql.gen.OrderingCompiler in project hetu-core by openlookeng.

the class TestOrderByOperator method testSingleFieldKey.

@Test(dataProvider = "spillEnabled")
public void testSingleFieldKey(boolean spillEnabled, boolean revokeMemoryWhenAddingPages, long memoryLimit) {
    List<Page> input = rowPagesBuilder(BIGINT, DOUBLE).row(1L, 0.1).row(2L, 0.2).pageBreak().row(-1L, -0.1).row(4L, 0.4).build();
    OrderByOperatorFactory operatorFactory = new OrderByOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(BIGINT, DOUBLE), ImmutableList.of(1), 10, ImmutableList.of(0), ImmutableList.of(ASC_NULLS_LAST), new PagesIndex.TestingFactory(false), spillEnabled, Optional.of(spillerFactory), new OrderingCompiler(), false);
    DriverContext driverContext = createDriverContext(memoryLimit, TEST_SESSION);
    MaterializedResult expected = resultBuilder(driverContext.getSession(), DOUBLE).row(-0.1).row(0.1).row(0.2).row(0.4).build();
    assertOperatorEquals(operatorFactory, driverContext, input, expected, revokeMemoryWhenAddingPages);
}
Also used : PlanNodeId(io.prestosql.spi.plan.PlanNodeId) OrderByOperatorFactory(io.prestosql.operator.OrderByOperator.OrderByOperatorFactory) OrderingCompiler(io.prestosql.sql.gen.OrderingCompiler) MarkerPage(io.prestosql.spi.snapshot.MarkerPage) Page(io.prestosql.spi.Page) MaterializedResult(io.prestosql.testing.MaterializedResult) OperatorAssertion.toMaterializedResult(io.prestosql.operator.OperatorAssertion.toMaterializedResult) Test(org.testng.annotations.Test)

Example 12 with OrderingCompiler

use of io.prestosql.sql.gen.OrderingCompiler in project hetu-core by openlookeng.

the class TestOrderByOperator method testReverseOrder.

@Test(dataProvider = "spillEnabled")
public void testReverseOrder(boolean spillEnabled, boolean revokeMemoryWhenAddingPages, long memoryLimit) {
    List<Page> input = rowPagesBuilder(BIGINT, DOUBLE).row(1L, 0.1).row(2L, 0.2).pageBreak().row(-1L, -0.1).row(4L, 0.4).build();
    OrderByOperatorFactory operatorFactory = new OrderByOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(BIGINT, DOUBLE), ImmutableList.of(0), 10, ImmutableList.of(0), ImmutableList.of(DESC_NULLS_LAST), new PagesIndex.TestingFactory(false), spillEnabled, Optional.of(spillerFactory), new OrderingCompiler(), false);
    DriverContext driverContext = createDriverContext(memoryLimit, TEST_SESSION);
    MaterializedResult expected = resultBuilder(driverContext.getSession(), BIGINT).row(4L).row(2L).row(1L).row(-1L).build();
    assertOperatorEquals(operatorFactory, driverContext, input, expected, revokeMemoryWhenAddingPages);
}
Also used : PlanNodeId(io.prestosql.spi.plan.PlanNodeId) OrderByOperatorFactory(io.prestosql.operator.OrderByOperator.OrderByOperatorFactory) OrderingCompiler(io.prestosql.sql.gen.OrderingCompiler) MarkerPage(io.prestosql.spi.snapshot.MarkerPage) Page(io.prestosql.spi.Page) MaterializedResult(io.prestosql.testing.MaterializedResult) OperatorAssertion.toMaterializedResult(io.prestosql.operator.OperatorAssertion.toMaterializedResult) Test(org.testng.annotations.Test)

Example 13 with OrderingCompiler

use of io.prestosql.sql.gen.OrderingCompiler in project hetu-core by openlookeng.

the class TestOrderByOperator method testReverseOrderWithSnapshot.

@Test
public void testReverseOrderWithSnapshot() {
    List<Page> input = rowPagesBuilder(BIGINT, DOUBLE).row(1L, 0.1).row(2L, 0.2).pageBreak().row(-1L, -0.1).row(4L, 0.4).build();
    OrderByOperatorFactory operatorFactory = new OrderByOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(BIGINT, DOUBLE), ImmutableList.of(0), 10, ImmutableList.of(0), ImmutableList.of(DESC_NULLS_LAST), new PagesIndex.TestingFactory(false), true, Optional.of(spillerFactory), new OrderingCompiler(), true);
    DriverContext driverContext = createDriverContext(8, TEST_SESSION);
    MaterializedResult expected = resultBuilder(driverContext.getSession(), BIGINT).row(4L).row(2L).row(1L).row(-1L).build();
    assertOperatorEqualsWithSimpleSelfStateComparison(operatorFactory, driverContext, input, expected, true, createExpectedMappingRevoke());
}
Also used : PlanNodeId(io.prestosql.spi.plan.PlanNodeId) OrderByOperatorFactory(io.prestosql.operator.OrderByOperator.OrderByOperatorFactory) OrderingCompiler(io.prestosql.sql.gen.OrderingCompiler) MarkerPage(io.prestosql.spi.snapshot.MarkerPage) Page(io.prestosql.spi.Page) MaterializedResult(io.prestosql.testing.MaterializedResult) OperatorAssertion.toMaterializedResult(io.prestosql.operator.OperatorAssertion.toMaterializedResult) Test(org.testng.annotations.Test)

Example 14 with OrderingCompiler

use of io.prestosql.sql.gen.OrderingCompiler in project hetu-core by openlookeng.

the class TestOrderByOperator method testMultiFieldKey.

@Test(dataProvider = "spillEnabled")
public void testMultiFieldKey(boolean spillEnabled, boolean revokeMemoryWhenAddingPages, long memoryLimit) {
    List<Page> input = rowPagesBuilder(VARCHAR, BIGINT).row("a", 1L).row("b", 2L).pageBreak().row("b", 3L).row("a", 4L).build();
    OrderByOperatorFactory operatorFactory = new OrderByOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(VARCHAR, BIGINT), ImmutableList.of(0, 1), 10, ImmutableList.of(0, 1), ImmutableList.of(ASC_NULLS_LAST, DESC_NULLS_LAST), new PagesIndex.TestingFactory(false), spillEnabled, Optional.of(spillerFactory), new OrderingCompiler(), false);
    DriverContext driverContext = createDriverContext(memoryLimit, TEST_SESSION);
    MaterializedResult expected = resultBuilder(driverContext.getSession(), VARCHAR, BIGINT).row("a", 4L).row("a", 1L).row("b", 3L).row("b", 2L).build();
    assertOperatorEquals(operatorFactory, driverContext, input, expected, revokeMemoryWhenAddingPages);
}
Also used : PlanNodeId(io.prestosql.spi.plan.PlanNodeId) OrderByOperatorFactory(io.prestosql.operator.OrderByOperator.OrderByOperatorFactory) OrderingCompiler(io.prestosql.sql.gen.OrderingCompiler) MarkerPage(io.prestosql.spi.snapshot.MarkerPage) Page(io.prestosql.spi.Page) MaterializedResult(io.prestosql.testing.MaterializedResult) OperatorAssertion.toMaterializedResult(io.prestosql.operator.OperatorAssertion.toMaterializedResult) Test(org.testng.annotations.Test)

Example 15 with OrderingCompiler

use of io.prestosql.sql.gen.OrderingCompiler in project hetu-core by openlookeng.

the class TestWindowOperator method testCaptureRestoreWithSpill.

@Test
public void testCaptureRestoreWithSpill() throws Exception {
    // Initialization
    Path spillPath = Paths.get("/tmp/hetu/snapshot/");
    GenericSpillerFactory genericSpillerFactory = createGenericSpillerFactory(spillPath, fileSystemClientManager, false, null);
    SnapshotConfig snapshotConfig = new SnapshotConfig();
    snapshotUtils = new SnapshotUtils(fileSystemClientManager, snapshotConfig, new InMemoryNodeManager());
    snapshotUtils.initialize();
    ImmutableList.Builder<Page> outputPages = ImmutableList.builder();
    List<Page> input1 = rowPagesBuilder(VARCHAR, BIGINT, DOUBLE, BOOLEAN).row("b", -1L, -0.1, true).row("a", 2L, 0.3, false).row("a", 4L, 0.2, true).pageBreak().row("b", 5L, 0.4, false).row("a", 6L, 0.1, true).build();
    List<Page> input2 = rowPagesBuilder(VARCHAR, BIGINT, DOUBLE, BOOLEAN).row("c", -1L, -0.1, true).row("d", 2L, 0.3, false).row("c", 4L, 0.2, true).pageBreak().row("d", 5L, 0.4, false).build();
    WindowOperatorFactory operatorFactory = new WindowOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(VARCHAR, BIGINT, DOUBLE, BOOLEAN), Ints.asList(0, 1, 2, 3), ROW_NUMBER, Ints.asList(0), ImmutableList.of(), Ints.asList(1), ImmutableList.copyOf(new SortOrder[] { SortOrder.ASC_NULLS_LAST }), 0, 10, new PagesIndex.TestingFactory(false), true, genericSpillerFactory, new OrderingCompiler());
    DriverContext driverContext = createDriverContext(defaultMemoryLimit, TEST_SNAPSHOT_SESSION);
    WindowOperator windowOperator = (WindowOperator) operatorFactory.createOperator(driverContext);
    // Step1: add the first 2 pages
    for (Page page : input1) {
        windowOperator.addInput(page);
        windowOperator.getOutput();
    }
    // Step2: spilling happened here
    getFutureValue(windowOperator.startMemoryRevoke());
    windowOperator.finishMemoryRevoke();
    // Step3: add a marker page to make 'capture1' happened
    MarkerPage marker = MarkerPage.snapshotPage(1);
    windowOperator.addInput(marker);
    windowOperator.getOutput();
    // Step4: add another 2 pages
    for (Page page : input2) {
        windowOperator.addInput(page);
        windowOperator.getOutput();
    }
    // Step5: assume the task is rescheduled due to failure and everything is re-constructed
    driverContext = createDriverContext(8, TEST_SNAPSHOT_SESSION);
    operatorFactory = new WindowOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(VARCHAR, BIGINT, DOUBLE, BOOLEAN), Ints.asList(0, 1, 2, 3), ROW_NUMBER, Ints.asList(0), ImmutableList.of(), Ints.asList(1), ImmutableList.copyOf(new SortOrder[] { SortOrder.ASC_NULLS_LAST }), 0, 10, new PagesIndex.TestingFactory(false), true, genericSpillerFactory, new OrderingCompiler());
    windowOperator = (WindowOperator) operatorFactory.createOperator(driverContext);
    // Step6: restore to 'capture1', the spiller should contains the reference of the first 2 pages for now.
    MarkerPage resumeMarker = MarkerPage.resumePage(1);
    windowOperator.addInput(resumeMarker);
    windowOperator.getOutput();
    // Step7: continue to add another 2 pages
    for (Page page : input2) {
        windowOperator.addInput(page);
        windowOperator.getOutput();
    }
    windowOperator.finish();
    // Compare the results
    MaterializedResult expected = resultBuilder(driverContext.getSession(), VARCHAR, BIGINT, DOUBLE, BOOLEAN, BIGINT).row("a", 2L, 0.3, false, 1L).row("a", 4L, 0.2, true, 2L).row("a", 6L, 0.1, true, 3L).row("b", -1L, -0.1, true, 1L).row("b", 5L, 0.4, false, 2L).row("c", -1L, -0.1, true, 1L).row("c", 4L, 0.2, true, 2L).row("d", 2L, 0.3, false, 1L).row("d", 5L, 0.4, false, 2L).build();
    Page p = windowOperator.getOutput();
    while (p == null) {
        p = windowOperator.getOutput();
    }
    outputPages.add(p);
    MaterializedResult actual = toMaterializedResult(driverContext.getSession(), expected.getTypes(), outputPages.build());
    Assert.assertEquals(actual, expected);
}
Also used : Path(java.nio.file.Path) WindowOperatorFactory(io.prestosql.operator.WindowOperator.WindowOperatorFactory) MarkerPage(io.prestosql.spi.snapshot.MarkerPage) ImmutableList(com.google.common.collect.ImmutableList) SortOrder(io.prestosql.spi.block.SortOrder) MarkerPage(io.prestosql.spi.snapshot.MarkerPage) Page(io.prestosql.spi.Page) InMemoryNodeManager(io.prestosql.metadata.InMemoryNodeManager) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) SnapshotConfig(io.prestosql.snapshot.SnapshotConfig) SnapshotUtils(io.prestosql.snapshot.SnapshotUtils) OrderingCompiler(io.prestosql.sql.gen.OrderingCompiler) MaterializedResult(io.prestosql.testing.MaterializedResult) OperatorAssertion.toMaterializedResult(io.prestosql.operator.OperatorAssertion.toMaterializedResult) GenericSpillerFactory(io.prestosql.spiller.GenericSpillerFactory) Test(org.testng.annotations.Test)

Aggregations

OrderingCompiler (io.prestosql.sql.gen.OrderingCompiler)18 PlanNodeId (io.prestosql.spi.plan.PlanNodeId)16 Page (io.prestosql.spi.Page)12 Test (org.testng.annotations.Test)12 MarkerPage (io.prestosql.spi.snapshot.MarkerPage)11 OperatorAssertion.toMaterializedResult (io.prestosql.operator.OperatorAssertion.toMaterializedResult)10 OrderByOperatorFactory (io.prestosql.operator.OrderByOperator.OrderByOperatorFactory)10 MaterializedResult (io.prestosql.testing.MaterializedResult)10 ImmutableList (com.google.common.collect.ImmutableList)5 PagesIndex (io.prestosql.operator.PagesIndex)5 NodeInfo (io.airlift.node.NodeInfo)4 DynamicFilterCacheManager (io.prestosql.dynamicfilter.DynamicFilterCacheManager)4 FileSystemClientManager (io.prestosql.filesystem.FileSystemClientManager)4 InMemoryNodeManager (io.prestosql.metadata.InMemoryNodeManager)4 LookupJoinOperators (io.prestosql.operator.LookupJoinOperators)4 IndexJoinLookupStats (io.prestosql.operator.index.IndexJoinLookupStats)4 SeedStoreManager (io.prestosql.seedstore.SeedStoreManager)4 GenericSpillerFactory (io.prestosql.spiller.GenericSpillerFactory)4 LocalExecutionPlanner (io.prestosql.sql.planner.LocalExecutionPlanner)4 TypeAnalyzer (io.prestosql.sql.planner.TypeAnalyzer)4