Examples with RowPagesBuilder io.prestosql.RowPagesBuilder used on opensource projects

Example 1 with RowPagesBuilder

use of io.prestosql.RowPagesBuilder in project hetu-core by openlookeng.

the class TestSortBuffer method testPageSorterSmallPages.

@Test
public void testPageSorterSmallPages() {
    List<Type> types = ImmutableList.of(BIGINT, DOUBLE, VARCHAR);
    List<Integer> sortChannels = Ints.asList(0);
    List<SortOrder> sortOrders = ImmutableList.of(ASC_NULLS_FIRST);
    SortBuffer sortBuffer = new SortBuffer(new DataSize(1, DataSize.Unit.GIGABYTE), types, sortChannels, sortOrders, sorter);
    RowPagesBuilder inputPagesBuilder = RowPagesBuilder.rowPagesBuilder(types);
    for (long i = 1; i < 100000; i++) {
        inputPagesBuilder.row(i, 1.1, "a").pageBreak();
    }
    List<Page> inputPages = inputPagesBuilder.build();
    RowPagesBuilder expectedPagesBuilder = RowPagesBuilder.rowPagesBuilder(types);
    for (long i = 1; i < 100000; i++) {
        expectedPagesBuilder.row(i, 1.1, "a");
        // based on default max page size, each page will be full when it has 43691 rows
        if (i % 43691 == 0) {
            expectedPagesBuilder.pageBreak();
        }
    }
    List<Page> expectedPages = expectedPagesBuilder.build();
    inputPages.stream().forEach(sortBuffer::add);
    List<Page> actualPages = new LinkedList<>();
    sortBuffer.flushTo(actualPages::add);
    assertSorted(expectedPages, actualPages, types);
}

Example 2 with RowPagesBuilder

use of io.prestosql.RowPagesBuilder in project hetu-core by openlookeng.

the class TestHashJoinOperator method testInnerJoin.

@Test(dataProvider = "hashJoinTestValues")
public void testInnerJoin(boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled) {
    TaskContext taskContext = createTaskContext();
    // build factory
    RowPagesBuilder buildPages = rowPagesBuilder(buildHashEnabled, Ints.asList(0), ImmutableList.of(VARCHAR, BIGINT, BIGINT)).addSequencePage(10, 20, 30, 40);
    BuildSideSetup buildSideSetup = setupBuildSide(parallelBuild, taskContext, Ints.asList(0), buildPages, Optional.empty(), false, SINGLE_STREAM_SPILLER_FACTORY);
    JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactory = buildSideSetup.getLookupSourceFactoryManager();
    // probe factory
    RowPagesBuilder probePages = rowPagesBuilder(probeHashEnabled, Ints.asList(0), ImmutableList.of(VARCHAR, BIGINT, BIGINT));
    List<Page> probeInput = probePages.addSequencePage(1000, 0, 1000, 2000).build();
    OperatorFactory joinOperatorFactory = innerJoinOperatorFactory(lookupSourceFactory, probePages, PARTITIONING_SPILLER_FACTORY);
    // build drivers and operators
    instantiateBuildDrivers(buildSideSetup, taskContext);
    buildLookupSource(buildSideSetup);
    // expected
    MaterializedResult expected = MaterializedResult.resultBuilder(taskContext.getSession(), concat(probePages.getTypesWithoutHash(), buildPages.getTypesWithoutHash())).row("20", 1020L, 2020L, "20", 30L, 40L).row("21", 1021L, 2021L, "21", 31L, 41L).row("22", 1022L, 2022L, "22", 32L, 42L).row("23", 1023L, 2023L, "23", 33L, 43L).row("24", 1024L, 2024L, "24", 34L, 44L).row("25", 1025L, 2025L, "25", 35L, 45L).row("26", 1026L, 2026L, "26", 36L, 46L).row("27", 1027L, 2027L, "27", 37L, 47L).row("28", 1028L, 2028L, "28", 38L, 48L).row("29", 1029L, 2029L, "29", 39L, 49L).build();
    assertOperatorEquals(joinOperatorFactory, taskContext.addPipelineContext(0, true, true, false).addDriverContext(), probeInput, expected, true, getHashChannels(probePages, buildPages));
}

Example 3 with RowPagesBuilder

use of io.prestosql.RowPagesBuilder in project hetu-core by openlookeng.

the class TestHashJoinOperator method testOuterJoinWithNullOnBothSides.

@Test(dataProvider = "hashJoinTestValues")
public void testOuterJoinWithNullOnBothSides(boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled) {
    TaskContext taskContext = createTaskContext();
    // build factory
    RowPagesBuilder buildPages = rowPagesBuilder(buildHashEnabled, Ints.asList(0), ImmutableList.of(VARCHAR)).row("a").row((String) null).row((String) null).row("a").row("b");
    BuildSideSetup buildSideSetup = setupBuildSide(parallelBuild, taskContext, Ints.asList(0), buildPages, Optional.empty(), false, SINGLE_STREAM_SPILLER_FACTORY);
    JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactory = buildSideSetup.getLookupSourceFactoryManager();
    // probe factory
    List<Type> probeTypes = ImmutableList.of(VARCHAR);
    RowPagesBuilder probePages = rowPagesBuilder(probeHashEnabled, Ints.asList(0), probeTypes);
    List<Page> probeInput = probePages.row("a").row("b").row((String) null).row("c").build();
    OperatorFactory joinOperatorFactory = probeOuterJoinOperatorFactory(lookupSourceFactory, probePages);
    // build drivers and operators
    instantiateBuildDrivers(buildSideSetup, taskContext);
    buildLookupSource(buildSideSetup);
    // expected
    MaterializedResult expected = MaterializedResult.resultBuilder(taskContext.getSession(), concat(probeTypes, buildPages.getTypesWithoutHash())).row("a", "a").row("a", "a").row("b", "b").row(null, null).row("c", null).build();
    assertOperatorEquals(joinOperatorFactory, taskContext.addPipelineContext(0, true, true, false).addDriverContext(), probeInput, expected, true, getHashChannels(probePages, buildPages));
}

Example 4 with RowPagesBuilder

use of io.prestosql.RowPagesBuilder in project hetu-core by openlookeng.

the class TestHashJoinOperator method testInnerJoinSnapshot.

@Test
public void testInnerJoinSnapshot() {
    TaskContext taskContext = createTaskContext();
    // build factory
    RowPagesBuilder buildPages = rowPagesBuilder(true, Ints.asList(0), ImmutableList.of(VARCHAR, BIGINT, BIGINT)).addSequencePage(10, 20, 30, 40);
    BuildSideSetup buildSideSetup = setupBuildSide(true, taskContext, Ints.asList(0), buildPages, Optional.empty(), false, SINGLE_STREAM_SPILLER_FACTORY);
    JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactory = buildSideSetup.getLookupSourceFactoryManager();
    // probe factory
    RowPagesBuilder probePages = rowPagesBuilder(true, Ints.asList(0), ImmutableList.of(VARCHAR, BIGINT, BIGINT));
    List<Page> probeInput = probePages.addSequencePage(1000, 0, 1000, 2000).build();
    OperatorFactory joinOperatorFactory = innerJoinOperatorFactory(lookupSourceFactory, probePages, PARTITIONING_SPILLER_FACTORY);
    // build drivers and operators
    instantiateBuildDrivers(buildSideSetup, taskContext);
    buildLookupSourceSnapshot(buildSideSetup);
    // expected
    MaterializedResult expected = MaterializedResult.resultBuilder(taskContext.getSession(), concat(probePages.getTypesWithoutHash(), buildPages.getTypesWithoutHash())).row("20", 1020L, 2020L, "20", 30L, 40L).row("21", 1021L, 2021L, "21", 31L, 41L).row("22", 1022L, 2022L, "22", 32L, 42L).row("23", 1023L, 2023L, "23", 33L, 43L).row("24", 1024L, 2024L, "24", 34L, 44L).row("25", 1025L, 2025L, "25", 35L, 45L).row("26", 1026L, 2026L, "26", 36L, 46L).row("27", 1027L, 2027L, "27", 37L, 47L).row("28", 1028L, 2028L, "28", 38L, 48L).row("29", 1029L, 2029L, "29", 39L, 49L).build();
    assertOperatorEqualsWithSimpleStateComparison(joinOperatorFactory, taskContext.addPipelineContext(0, true, true, false).addDriverContext(), probeInput, expected, true, getHashChannels(probePages, buildPages), createProbeSideExpectedMapping());
}

Example 5 with RowPagesBuilder

use of io.prestosql.RowPagesBuilder in project hetu-core by openlookeng.

the class TestHashJoinOperator method testYield.

@Test
public void testYield() {
    // create a filter function that yields for every probe match
    // verify we will yield #match times totally
    TaskContext taskContext = createTaskContext();
    DriverContext driverContext = taskContext.addPipelineContext(0, true, true, false).addDriverContext();
    // force a yield for every match
    AtomicInteger filterFunctionCalls = new AtomicInteger();
    InternalJoinFilterFunction filterFunction = new TestInternalJoinFilterFunction(((leftPosition, leftPage, rightPosition, rightPage) -> {
        filterFunctionCalls.incrementAndGet();
        driverContext.getYieldSignal().forceYieldForTesting();
        return true;
    }));
    // build with 40 entries
    int entries = 40;
    RowPagesBuilder buildPages = rowPagesBuilder(true, Ints.asList(0), ImmutableList.of(BIGINT)).addSequencePage(entries, 42);
    BuildSideSetup buildSideSetup = setupBuildSide(true, taskContext, Ints.asList(0), buildPages, Optional.of(filterFunction), false, SINGLE_STREAM_SPILLER_FACTORY);
    JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactory = buildSideSetup.getLookupSourceFactoryManager();
    // probe matching the above 40 entries
    RowPagesBuilder probePages = rowPagesBuilder(false, Ints.asList(0), ImmutableList.of(BIGINT));
    List<Page> probeInput = probePages.addSequencePage(100, 0).build();
    OperatorFactory joinOperatorFactory = LOOKUP_JOIN_OPERATORS.innerJoin(0, new PlanNodeId("test"), lookupSourceFactory, probePages.getTypes(), Ints.asList(0), getHashChannelAsInt(probePages), Optional.empty(), OptionalInt.of(1), PARTITIONING_SPILLER_FACTORY);
    instantiateBuildDrivers(buildSideSetup, taskContext);
    buildLookupSource(buildSideSetup);
    Operator operator = joinOperatorFactory.createOperator(driverContext);
    assertTrue(operator.needsInput());
    operator.addInput(probeInput.get(0));
    operator.finish();
    // we will yield 40 times due to filterFunction
    for (int i = 0; i < entries; i++) {
        driverContext.getYieldSignal().setWithDelay(5 * SECONDS.toNanos(1), driverContext.getYieldExecutor());
        filterFunctionCalls.set(0);
        assertNull(operator.getOutput());
        assertEquals(filterFunctionCalls.get(), 1, "Expected join to stop processing (yield) after calling filter function once");
        driverContext.getYieldSignal().reset();
    }
    // delayed yield is not going to prevent operator from producing a page now (yield won't be forced because filter function won't be called anymore)
    driverContext.getYieldSignal().setWithDelay(5 * SECONDS.toNanos(1), driverContext.getYieldExecutor());
    // expect output page to be produced within few calls to getOutput(), e.g. to facilitate spill
    Page output = null;
    for (int i = 0; output == null && i < 5; i++) {
        output = operator.getOutput();
    }
    assertNotNull(output);
    driverContext.getYieldSignal().reset();
    // make sure we have all 4 entries
    assertEquals(output.getPositionCount(), entries);
}