Search in sources :

Example 11 with WindowOperatorFactory

use of io.prestosql.operator.WindowOperator.WindowOperatorFactory in project hetu-core by openlookeng.

the class TestWindowOperator method testFullyPreGroupedPartitionSnapshot.

@Test(dataProvider = "spillEnabledSnapshot")
public void testFullyPreGroupedPartitionSnapshot(boolean spillEnabled, boolean revokeMemoryWhenAddingPages, long memoryLimit) {
    List<Page> input = rowPagesBuilder(BIGINT, VARCHAR, BIGINT, VARCHAR).pageBreak().row(1L, "a", 100L, "A").pageBreak().row(2L, "a", 101L, "B").pageBreak().row(2L, "b", 102L, "D").row(2L, "b", 103L, "C").row(1L, "b", 104L, "E").pageBreak().row(1L, "b", 105L, "F").row(3L, "c", 106L, "G").build();
    WindowOperatorFactory operatorFactory = createFactoryUnbounded(ImmutableList.of(BIGINT, VARCHAR, BIGINT, VARCHAR), Ints.asList(0, 1, 2, 3), ROW_NUMBER, Ints.asList(1, 0), Ints.asList(0, 1), Ints.asList(3), ImmutableList.of(SortOrder.ASC_NULLS_LAST), 0, spillEnabled);
    DriverContext driverContext = createDriverContext(memoryLimit);
    MaterializedResult expected = resultBuilder(driverContext.getSession(), BIGINT, VARCHAR, BIGINT, VARCHAR, BIGINT).row(1L, "a", 100L, "A", 1L).row(2L, "a", 101L, "B", 1L).row(2L, "b", 103L, "C", 1L).row(2L, "b", 102L, "D", 2L).row(1L, "b", 104L, "E", 1L).row(1L, "b", 105L, "F", 2L).row(3L, "c", 106L, "G", 1L).build();
    assertOperatorEqualsIgnoreOrderWithRestoreToNewOperator(operatorFactory, driverContext, () -> createDriverContext(memoryLimit), input, expected, revokeMemoryWhenAddingPages);
}
Also used : WindowOperatorFactory(io.prestosql.operator.WindowOperator.WindowOperatorFactory) MarkerPage(io.prestosql.spi.snapshot.MarkerPage) Page(io.prestosql.spi.Page) MaterializedResult(io.prestosql.testing.MaterializedResult) OperatorAssertion.toMaterializedResult(io.prestosql.operator.OperatorAssertion.toMaterializedResult) Test(org.testng.annotations.Test)

Example 12 with WindowOperatorFactory

use of io.prestosql.operator.WindowOperator.WindowOperatorFactory in project hetu-core by openlookeng.

the class TestWindowOperator method testPartiallyPreGroupedPartition.

@Test(dataProvider = "spillEnabled")
public void testPartiallyPreGroupedPartition(boolean spillEnabled, boolean revokeMemoryWhenAddingPages, long memoryLimit) {
    List<Page> input = rowPagesBuilder(BIGINT, VARCHAR, BIGINT, VARCHAR).pageBreak().row(1L, "a", 100L, "A").row(2L, "a", 101L, "B").pageBreak().row(3L, "b", 102L, "E").row(1L, "b", 103L, "D").pageBreak().row(3L, "b", 104L, "C").row(1L, "c", 105L, "F").pageBreak().build();
    WindowOperatorFactory operatorFactory = createFactoryUnbounded(ImmutableList.of(BIGINT, VARCHAR, BIGINT, VARCHAR), Ints.asList(0, 1, 2, 3), ROW_NUMBER, Ints.asList(0, 1), Ints.asList(1), Ints.asList(3), ImmutableList.of(SortOrder.ASC_NULLS_LAST), 0, spillEnabled);
    DriverContext driverContext = createDriverContext(memoryLimit, TEST_SESSION);
    MaterializedResult expected = resultBuilder(driverContext.getSession(), BIGINT, VARCHAR, BIGINT, VARCHAR, BIGINT).row(1L, "a", 100L, "A", 1L).row(2L, "a", 101L, "B", 1L).row(3L, "b", 104L, "C", 1L).row(3L, "b", 102L, "E", 2L).row(1L, "b", 103L, "D", 1L).row(1L, "c", 105L, "F", 1L).build();
    assertOperatorEqualsIgnoreOrder(operatorFactory, driverContext, input, expected, revokeMemoryWhenAddingPages);
}
Also used : WindowOperatorFactory(io.prestosql.operator.WindowOperator.WindowOperatorFactory) MarkerPage(io.prestosql.spi.snapshot.MarkerPage) Page(io.prestosql.spi.Page) MaterializedResult(io.prestosql.testing.MaterializedResult) OperatorAssertion.toMaterializedResult(io.prestosql.operator.OperatorAssertion.toMaterializedResult) Test(org.testng.annotations.Test)

Example 13 with WindowOperatorFactory

use of io.prestosql.operator.WindowOperator.WindowOperatorFactory in project hetu-core by openlookeng.

the class TestWindowOperator method testFullyPreGroupedAndFullySortedPartition.

@Test(dataProvider = "spillEnabled")
public void testFullyPreGroupedAndFullySortedPartition(boolean spillEnabled, boolean revokeMemoryWhenAddingPages, long memoryLimit) {
    List<Page> input = rowPagesBuilder(BIGINT, VARCHAR, BIGINT, VARCHAR).pageBreak().row(1L, "a", 100L, "A").pageBreak().row(2L, "a", 101L, "A").pageBreak().row(2L, "b", 102L, "A").row(2L, "b", 103L, "A").row(2L, "b", 104L, "B").row(1L, "b", 105L, "A").pageBreak().row(1L, "b", 106L, "A").row(3L, "c", 107L, "A").build();
    WindowOperatorFactory operatorFactory = createFactoryUnbounded(ImmutableList.of(BIGINT, VARCHAR, BIGINT, VARCHAR), Ints.asList(0, 1, 2, 3), ROW_NUMBER, Ints.asList(1, 0), Ints.asList(0, 1), Ints.asList(3), ImmutableList.of(SortOrder.ASC_NULLS_LAST), 1, spillEnabled);
    DriverContext driverContext = createDriverContext(memoryLimit, TEST_SESSION);
    MaterializedResult expected = resultBuilder(driverContext.getSession(), BIGINT, VARCHAR, BIGINT, VARCHAR, BIGINT).row(1L, "a", 100L, "A", 1L).row(2L, "a", 101L, "A", 1L).row(2L, "b", 102L, "A", 1L).row(2L, "b", 103L, "A", 2L).row(2L, "b", 104L, "B", 3L).row(1L, "b", 105L, "A", 1L).row(1L, "b", 106L, "A", 2L).row(3L, "c", 107L, "A", 1L).build();
    // Since fully grouped and sorted already, should respect original input order
    assertOperatorEquals(operatorFactory, driverContext, input, expected, revokeMemoryWhenAddingPages);
}
Also used : WindowOperatorFactory(io.prestosql.operator.WindowOperator.WindowOperatorFactory) MarkerPage(io.prestosql.spi.snapshot.MarkerPage) Page(io.prestosql.spi.Page) MaterializedResult(io.prestosql.testing.MaterializedResult) OperatorAssertion.toMaterializedResult(io.prestosql.operator.OperatorAssertion.toMaterializedResult) Test(org.testng.annotations.Test)

Example 14 with WindowOperatorFactory

use of io.prestosql.operator.WindowOperator.WindowOperatorFactory in project hetu-core by openlookeng.

the class TestWindowOperator method testRowNumberPartitionSnapshot.

@Test(dataProvider = "spillEnabledSnapshot")
public void testRowNumberPartitionSnapshot(boolean spillEnabled, boolean revokeMemoryWhenAddingPages, long memoryLimit) {
    List<Page> input = rowPagesBuilder(VARCHAR, BIGINT, DOUBLE, BOOLEAN).row("b", -1L, -0.1, true).row("a", 2L, 0.3, false).row("a", 4L, 0.2, true).pageBreak().row("b", 5L, 0.4, false).row("a", 6L, 0.1, true).build();
    WindowOperatorFactory operatorFactory = createFactoryUnbounded(ImmutableList.of(VARCHAR, BIGINT, DOUBLE, BOOLEAN), Ints.asList(0, 1, 2, 3), ROW_NUMBER, Ints.asList(0), Ints.asList(1), ImmutableList.copyOf(new SortOrder[] { SortOrder.ASC_NULLS_LAST }), spillEnabled);
    DriverContext driverContext = createDriverContext(memoryLimit);
    MaterializedResult expected = resultBuilder(driverContext.getSession(), VARCHAR, BIGINT, DOUBLE, BOOLEAN, BIGINT).row("a", 2L, 0.3, false, 1L).row("a", 4L, 0.2, true, 2L).row("a", 6L, 0.1, true, 3L).row("b", -1L, -0.1, true, 1L).row("b", 5L, 0.4, false, 2L).build();
    assertOperatorEqualsWithRestoreToNewOperator(operatorFactory, driverContext, () -> createDriverContext(memoryLimit), input, expected, revokeMemoryWhenAddingPages);
}
Also used : WindowOperatorFactory(io.prestosql.operator.WindowOperator.WindowOperatorFactory) SortOrder(io.prestosql.spi.block.SortOrder) MarkerPage(io.prestosql.spi.snapshot.MarkerPage) Page(io.prestosql.spi.Page) MaterializedResult(io.prestosql.testing.MaterializedResult) OperatorAssertion.toMaterializedResult(io.prestosql.operator.OperatorAssertion.toMaterializedResult) Test(org.testng.annotations.Test)

Example 15 with WindowOperatorFactory

use of io.prestosql.operator.WindowOperator.WindowOperatorFactory in project hetu-core by openlookeng.

the class TestWindowOperator method testMultipleOutputPages.

@Test(dataProvider = "spillEnabled")
public void testMultipleOutputPages(boolean spillEnabled, boolean revokeMemoryWhenAddingPages, long memoryLimit) {
    // make operator produce multiple pages during finish phase
    int numberOfRows = 80_000;
    List<Page> input = rowPagesBuilder(BIGINT, DOUBLE).addSequencePage(numberOfRows, 0, 0).build();
    WindowOperatorFactory operatorFactory = createFactoryUnbounded(ImmutableList.of(BIGINT, DOUBLE), Ints.asList(1, 0), ROW_NUMBER, Ints.asList(), Ints.asList(0), ImmutableList.copyOf(new SortOrder[] { SortOrder.DESC_NULLS_FIRST }), spillEnabled);
    DriverContext driverContext = createDriverContext(memoryLimit, TEST_SESSION);
    MaterializedResult.Builder expectedBuilder = resultBuilder(driverContext.getSession(), DOUBLE, BIGINT, BIGINT);
    for (int i = 0; i < numberOfRows; ++i) {
        expectedBuilder.row((double) numberOfRows - i - 1, (long) numberOfRows - i - 1, (long) i + 1);
    }
    MaterializedResult expected = expectedBuilder.build();
    List<Page> pages = toPages(operatorFactory, driverContext, input, revokeMemoryWhenAddingPages);
    assertGreaterThan(pages.size(), 1, "Expected more than one output page");
    MaterializedResult actual = toMaterializedResult(driverContext.getSession(), expected.getTypes(), pages);
    assertEquals(actual.getMaterializedRows(), expected.getMaterializedRows());
    assertTrue(spillEnabled == (spillerFactory.getSpillsCount() > 0), format("Spill state mismatch. Expected spill: %s, spill count: %s", spillEnabled, spillerFactory.getSpillsCount()));
}
Also used : WindowOperatorFactory(io.prestosql.operator.WindowOperator.WindowOperatorFactory) SortOrder(io.prestosql.spi.block.SortOrder) MarkerPage(io.prestosql.spi.snapshot.MarkerPage) Page(io.prestosql.spi.Page) MaterializedResult(io.prestosql.testing.MaterializedResult) OperatorAssertion.toMaterializedResult(io.prestosql.operator.OperatorAssertion.toMaterializedResult) Test(org.testng.annotations.Test)

Aggregations

WindowOperatorFactory (io.prestosql.operator.WindowOperator.WindowOperatorFactory)27 Page (io.prestosql.spi.Page)27 MarkerPage (io.prestosql.spi.snapshot.MarkerPage)27 Test (org.testng.annotations.Test)27 OperatorAssertion.toMaterializedResult (io.prestosql.operator.OperatorAssertion.toMaterializedResult)26 MaterializedResult (io.prestosql.testing.MaterializedResult)26 SortOrder (io.prestosql.spi.block.SortOrder)19 ImmutableList (com.google.common.collect.ImmutableList)3 InMemoryNodeManager (io.prestosql.metadata.InMemoryNodeManager)3 SnapshotConfig (io.prestosql.snapshot.SnapshotConfig)3 SnapshotUtils (io.prestosql.snapshot.SnapshotUtils)3 PlanNodeId (io.prestosql.spi.plan.PlanNodeId)2 GenericSpillerFactory (io.prestosql.spiller.GenericSpillerFactory)2 OrderingCompiler (io.prestosql.sql.gen.OrderingCompiler)2 Path (java.nio.file.Path)2 DataSize (io.airlift.units.DataSize)1 HetuHdfsFileSystemClient (io.hetu.core.filesystem.HetuHdfsFileSystemClient)1