use of io.prestosql.operator.WindowOperator.WindowOperatorFactory in project hetu-core by openlookeng.
the class TestWindowOperator method testRowNumberArbitraryWithSpill.
@Test
public void testRowNumberArbitraryWithSpill() {
List<Page> input = rowPagesBuilder(BIGINT).row(1L).row(3L).row(5L).row(7L).pageBreak().row(2L).row(4L).row(6L).row(8L).build();
WindowOperatorFactory operatorFactory = createFactoryUnbounded(ImmutableList.of(BIGINT), Ints.asList(0), ROW_NUMBER, Ints.asList(), Ints.asList(), ImmutableList.copyOf(new SortOrder[] {}), true);
DriverContext driverContext = createDriverContext();
MaterializedResult expected = resultBuilder(driverContext.getSession(), BIGINT, BIGINT).row(1L, 1L).row(2L, 2L).row(3L, 3L).row(4L, 4L).row(5L, 5L).row(6L, 6L).row(7L, 7L).row(8L, 8L).build();
assertOperatorEquals(operatorFactory, driverContext, input, expected);
}
use of io.prestosql.operator.WindowOperator.WindowOperatorFactory in project hetu-core by openlookeng.
the class TestWindowOperator method testNthValuePartition.
@Test(dataProvider = "spillEnabled")
public void testNthValuePartition(boolean spillEnabled, boolean revokeMemoryWhenAddingPages, long memoryLimit) {
List<Page> input = rowPagesBuilder(VARCHAR, VARCHAR, BIGINT, BIGINT, BOOLEAN, VARCHAR).row("b", "A1", 1L, 2L, true, "").row("a", "A2", 1L, 3L, false, "").row("a", "B1", 2L, 2L, true, "").pageBreak().row("b", "C1", 2L, 3L, false, "").row("a", "C2", 3L, 1L, true, "").row("c", "A3", 1L, null, true, "").build();
WindowOperatorFactory operatorFactory = createFactoryUnbounded(ImmutableList.of(VARCHAR, VARCHAR, BIGINT, BIGINT, BOOLEAN, VARCHAR), Ints.asList(0, 1, 2, 4), NTH_VALUE, Ints.asList(0), Ints.asList(2), ImmutableList.copyOf(new SortOrder[] { SortOrder.ASC_NULLS_LAST }), spillEnabled);
DriverContext driverContext = createDriverContext(memoryLimit, TEST_SESSION);
MaterializedResult expected = resultBuilder(driverContext.getSession(), VARCHAR, VARCHAR, BIGINT, BOOLEAN, VARCHAR).row("a", "A2", 1L, false, "C2").row("a", "B1", 2L, true, "B1").row("a", "C2", 3L, true, "A2").row("b", "A1", 1L, true, "C1").row("b", "C1", 2L, false, null).row("c", "A3", 1L, true, null).build();
assertOperatorEquals(operatorFactory, driverContext, input, expected, revokeMemoryWhenAddingPages);
}
use of io.prestosql.operator.WindowOperator.WindowOperatorFactory in project hetu-core by openlookeng.
the class TestWindowOperator method testLastValuePartition.
@Test(dataProvider = "spillEnabled")
public void testLastValuePartition(boolean spillEnabled, boolean revokeMemoryWhenAddingPages, long memoryLimit) {
List<Page> input = rowPagesBuilder(VARCHAR, VARCHAR, BIGINT, BOOLEAN, VARCHAR).row("b", "A1", 1L, true, "").row("a", "A2", 1L, false, "").row("a", "B1", 2L, true, "").pageBreak().row("b", "C1", 2L, false, "").row("a", "C2", 3L, true, "").row("c", "A3", 1L, true, "").build();
DriverContext driverContext = createDriverContext(memoryLimit, TEST_SESSION);
WindowOperatorFactory operatorFactory = createFactoryUnbounded(ImmutableList.of(VARCHAR, VARCHAR, BIGINT, BOOLEAN, VARCHAR), Ints.asList(0, 1, 2, 3), LAST_VALUE, Ints.asList(0), Ints.asList(2), ImmutableList.copyOf(new SortOrder[] { SortOrder.ASC_NULLS_LAST }), spillEnabled);
MaterializedResult expected = resultBuilder(driverContext.getSession(), VARCHAR, VARCHAR, BIGINT, BOOLEAN, VARCHAR).row("a", "A2", 1L, false, "C2").row("a", "B1", 2L, true, "C2").row("a", "C2", 3L, true, "C2").row("b", "A1", 1L, true, "C1").row("b", "C1", 2L, false, "C1").row("c", "A3", 1L, true, "A3").build();
assertOperatorEquals(operatorFactory, driverContext, input, expected, revokeMemoryWhenAddingPages);
}
use of io.prestosql.operator.WindowOperator.WindowOperatorFactory in project hetu-core by openlookeng.
the class TestWindowOperator method testFirstValuePartition.
@Test(dataProvider = "spillEnabled")
public void testFirstValuePartition(boolean spillEnabled, boolean revokeMemoryWhenAddingPages, long memoryLimit) {
List<Page> input = rowPagesBuilder(VARCHAR, VARCHAR, BIGINT, BOOLEAN, VARCHAR).row("b", "A1", 1L, true, "").row("a", "A2", 1L, false, "").row("a", "B1", 2L, true, "").pageBreak().row("b", "C1", 2L, false, "").row("a", "C2", 3L, true, "").row("c", "A3", 1L, true, "").build();
WindowOperatorFactory operatorFactory = createFactoryUnbounded(ImmutableList.of(VARCHAR, VARCHAR, BIGINT, BOOLEAN, VARCHAR), Ints.asList(0, 1, 2, 3), FIRST_VALUE, Ints.asList(0), Ints.asList(2), ImmutableList.copyOf(new SortOrder[] { SortOrder.ASC_NULLS_LAST }), spillEnabled);
DriverContext driverContext = createDriverContext(memoryLimit, TEST_SESSION);
MaterializedResult expected = resultBuilder(driverContext.getSession(), VARCHAR, VARCHAR, BIGINT, BOOLEAN, VARCHAR).row("a", "A2", 1L, false, "A2").row("a", "B1", 2L, true, "A2").row("a", "C2", 3L, true, "A2").row("b", "A1", 1L, true, "A1").row("b", "C1", 2L, false, "A1").row("c", "A3", 1L, true, "A3").build();
assertOperatorEquals(operatorFactory, driverContext, input, expected, revokeMemoryWhenAddingPages);
}
use of io.prestosql.operator.WindowOperator.WindowOperatorFactory in project hetu-core by openlookeng.
the class TestWindowOperator method testCaptureRestoreWithoutSpill.
@Test
public void testCaptureRestoreWithoutSpill() {
SnapshotConfig snapshotConfig = new SnapshotConfig();
snapshotUtils = new SnapshotUtils(fileSystemClientManager, snapshotConfig, new InMemoryNodeManager());
snapshotUtils.initialize();
ImmutableList.Builder<Page> outputPages = ImmutableList.builder();
List<Page> input1 = rowPagesBuilder(VARCHAR, BIGINT, DOUBLE, BOOLEAN).row("b", -1L, -0.1, true).row("a", 2L, 0.3, false).row("a", 4L, 0.2, true).pageBreak().row("b", 5L, 0.4, false).row("a", 6L, 0.1, true).build();
List<Page> input2 = rowPagesBuilder(VARCHAR, BIGINT, DOUBLE, BOOLEAN).row("c", -1L, -0.1, true).row("d", 2L, 0.3, false).row("c", 4L, 0.2, true).pageBreak().row("d", 5L, 0.4, false).build();
WindowOperatorFactory operatorFactory = createFactoryUnbounded(ImmutableList.of(VARCHAR, BIGINT, DOUBLE, BOOLEAN), Ints.asList(0, 1, 2, 3), ROW_NUMBER, Ints.asList(0), Ints.asList(1), ImmutableList.copyOf(new SortOrder[] { SortOrder.ASC_NULLS_LAST }), false);
DriverContext driverContext = createDriverContext(defaultMemoryLimit, TEST_SNAPSHOT_SESSION);
WindowOperator windowOperator = (WindowOperator) operatorFactory.createOperator(driverContext);
// Step1: add the first 2 pages
for (Page page : input1) {
windowOperator.addInput(page);
windowOperator.getOutput();
}
// Step2: add a marker page to make 'capture1' happened
MarkerPage marker = MarkerPage.snapshotPage(1);
windowOperator.addInput(marker);
windowOperator.getOutput();
// Step3: add another 2 pages
for (Page page : input2) {
windowOperator.addInput(page);
windowOperator.getOutput();
}
// Step4: assume the task is rescheduled due to failure and everything is re-constructed
driverContext = createDriverContext(8, TEST_SNAPSHOT_SESSION);
operatorFactory = createFactoryUnbounded(ImmutableList.of(VARCHAR, BIGINT, DOUBLE, BOOLEAN), Ints.asList(0, 1, 2, 3), ROW_NUMBER, Ints.asList(0), Ints.asList(1), ImmutableList.copyOf(new SortOrder[] { SortOrder.ASC_NULLS_LAST }), false);
windowOperator = (WindowOperator) operatorFactory.createOperator(driverContext);
// Step5: restore to 'capture1'
MarkerPage resumeMarker = MarkerPage.resumePage(1);
windowOperator.addInput(resumeMarker);
windowOperator.getOutput();
// Step6: continue to add another 2 pages
for (Page page : input2) {
windowOperator.addInput(page);
windowOperator.getOutput();
}
windowOperator.finish();
// Compare the results
MaterializedResult expected = resultBuilder(driverContext.getSession(), VARCHAR, BIGINT, DOUBLE, BOOLEAN, BIGINT).row("a", 2L, 0.3, false, 1L).row("a", 4L, 0.2, true, 2L).row("a", 6L, 0.1, true, 3L).row("b", -1L, -0.1, true, 1L).row("b", 5L, 0.4, false, 2L).row("c", -1L, -0.1, true, 1L).row("c", 4L, 0.2, true, 2L).row("d", 2L, 0.3, false, 1L).row("d", 5L, 0.4, false, 2L).build();
Page p = windowOperator.getOutput();
while (p == null) {
p = windowOperator.getOutput();
}
outputPages.add(p);
MaterializedResult actual = toMaterializedResult(driverContext.getSession(), expected.getTypes(), outputPages.build());
Assert.assertEquals(actual, expected);
}
Aggregations