Search in sources :

Example 21 with BlockTypeOperators

use of io.trino.type.BlockTypeOperators in project trino by trinodb.

the class TestGroupedTopNRankBuilder method testSingleGroupTopN.

@Test(dataProvider = "produceRanking")
public void testSingleGroupTopN(boolean produceRanking) {
    TypeOperators typeOperators = new TypeOperators();
    BlockTypeOperators blockTypeOperators = new BlockTypeOperators(typeOperators);
    List<Type> types = ImmutableList.of(DOUBLE);
    GroupedTopNBuilder groupedTopNBuilder = new GroupedTopNRankBuilder(types, new SimplePageWithPositionComparator(types, ImmutableList.of(0), ImmutableList.of(ASC_NULLS_LAST), typeOperators), new SimplePageWithPositionEqualsAndHash(types, ImmutableList.of(0), blockTypeOperators), 3, produceRanking, new NoChannelGroupByHash());
    // Expected effect: [0.2 x 1 => rank=1, 0.3 x 2 => rank=2]
    assertTrue(groupedTopNBuilder.processPage(rowPageBuilder(types).row(0.3).row(0.3).row(0.2).build()).process());
    // Page should be dropped, because single value 0.4 is too large to be considered
    assertTrue(groupedTopNBuilder.processPage(rowPageBuilder(types).row(0.4).build()).process());
    // Next page should cause 0.3 values to be evicted (first page will be compacted)
    // Expected effect: [0.1 x 2 => rank 1, 0.2 x 3 => rank 3]
    assertTrue(groupedTopNBuilder.processPage(rowPageBuilder(types).row(0.1).row(0.2).row(0.3).row(0.2).row(0.1).build()).process());
    List<Page> output = ImmutableList.copyOf(groupedTopNBuilder.buildResult());
    assertEquals(output.size(), 1);
    List<Type> outputTypes = ImmutableList.of(DOUBLE, BIGINT);
    Page expected = rowPageBuilder(outputTypes).row(0.1, 1).row(0.1, 1).row(0.2, 3).row(0.2, 3).row(0.2, 3).build();
    if (!produceRanking) {
        outputTypes = outputTypes.subList(0, outputTypes.size() - 1);
        expected = dropLastColumn(expected);
    }
    assertPageEquals(outputTypes, getOnlyElement(output), expected);
}
Also used : Type(io.trino.spi.type.Type) BlockTypeOperators(io.trino.type.BlockTypeOperators) Page(io.trino.spi.Page) TypeOperators(io.trino.spi.type.TypeOperators) BlockTypeOperators(io.trino.type.BlockTypeOperators) Test(org.testng.annotations.Test)

Example 22 with BlockTypeOperators

use of io.trino.type.BlockTypeOperators in project trino by trinodb.

the class HashBuildBenchmark method createDrivers.

@Override
protected List<Driver> createDrivers(TaskContext taskContext) {
    // hash build
    List<Type> ordersTypes = getColumnTypes("orders", "orderkey", "totalprice");
    OperatorFactory ordersTableScan = createTableScanOperator(0, new PlanNodeId("test"), "orders", "orderkey", "totalprice");
    BlockTypeOperators blockTypeOperators = new BlockTypeOperators(new TypeOperators());
    JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactoryManager = JoinBridgeManager.lookupAllAtOnce(new PartitionedLookupSourceFactory(ordersTypes, ImmutableList.of(0, 1).stream().map(ordersTypes::get).collect(toImmutableList()), Ints.asList(0).stream().map(ordersTypes::get).collect(toImmutableList()), 1, false, blockTypeOperators));
    HashBuilderOperatorFactory hashBuilder = new HashBuilderOperatorFactory(1, new PlanNodeId("test"), lookupSourceFactoryManager, ImmutableList.of(0, 1), Ints.asList(0), OptionalInt.empty(), Optional.empty(), Optional.empty(), ImmutableList.of(), 1_500_000, new PagesIndex.TestingFactory(false), false, SingleStreamSpillerFactory.unsupportedSingleStreamSpillerFactory(), incrementalLoadFactorHashArraySizeSupplier(session));
    DriverFactory hashBuildDriverFactory = new DriverFactory(0, true, true, ImmutableList.of(ordersTableScan, hashBuilder), OptionalInt.empty(), UNGROUPED_EXECUTION);
    // empty join so build finishes
    ImmutableList.Builder<OperatorFactory> joinDriversBuilder = ImmutableList.builder();
    joinDriversBuilder.add(new ValuesOperatorFactory(0, new PlanNodeId("values"), ImmutableList.of()));
    OperatorFactory joinOperator = operatorFactories.innerJoin(2, new PlanNodeId("test"), lookupSourceFactoryManager, false, false, false, ImmutableList.of(BIGINT), Ints.asList(0), OptionalInt.empty(), Optional.empty(), OptionalInt.empty(), unsupportedPartitioningSpillerFactory(), blockTypeOperators);
    joinDriversBuilder.add(joinOperator);
    joinDriversBuilder.add(new NullOutputOperatorFactory(3, new PlanNodeId("test")));
    DriverFactory joinDriverFactory = new DriverFactory(1, true, true, joinDriversBuilder.build(), OptionalInt.empty(), UNGROUPED_EXECUTION);
    Driver hashBuildDriver = hashBuildDriverFactory.createDriver(taskContext.addPipelineContext(0, true, true, false).addDriverContext());
    hashBuildDriverFactory.noMoreDrivers();
    Driver joinDriver = joinDriverFactory.createDriver(taskContext.addPipelineContext(1, true, true, false).addDriverContext());
    joinDriverFactory.noMoreDrivers();
    return ImmutableList.of(hashBuildDriver, joinDriver);
}
Also used : ImmutableList(com.google.common.collect.ImmutableList) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) HashBuilderOperatorFactory(io.trino.operator.join.HashBuilderOperator.HashBuilderOperatorFactory) Driver(io.trino.operator.Driver) PagesIndex(io.trino.operator.PagesIndex) PlanNodeId(io.trino.sql.planner.plan.PlanNodeId) Type(io.trino.spi.type.Type) BlockTypeOperators(io.trino.type.BlockTypeOperators) ValuesOperatorFactory(io.trino.operator.ValuesOperator.ValuesOperatorFactory) HashBuilderOperatorFactory(io.trino.operator.join.HashBuilderOperator.HashBuilderOperatorFactory) NullOutputOperatorFactory(io.trino.testing.NullOutputOperator.NullOutputOperatorFactory) OperatorFactory(io.trino.operator.OperatorFactory) DriverFactory(io.trino.operator.DriverFactory) ValuesOperatorFactory(io.trino.operator.ValuesOperator.ValuesOperatorFactory) PartitionedLookupSourceFactory(io.trino.operator.join.PartitionedLookupSourceFactory) NullOutputOperatorFactory(io.trino.testing.NullOutputOperator.NullOutputOperatorFactory) TypeOperators(io.trino.spi.type.TypeOperators) BlockTypeOperators(io.trino.type.BlockTypeOperators)

Example 23 with BlockTypeOperators

use of io.trino.type.BlockTypeOperators in project trino by trinodb.

the class HashJoinBenchmark method createDrivers.

/*
    select orderkey, quantity, totalprice
    from lineitem join orders using (orderkey)
     */
@Override
protected List<Driver> createDrivers(TaskContext taskContext) {
    if (probeDriverFactory == null) {
        List<Type> ordersTypes = getColumnTypes("orders", "orderkey", "totalprice");
        OperatorFactory ordersTableScan = createTableScanOperator(0, new PlanNodeId("test"), "orders", "orderkey", "totalprice");
        BlockTypeOperators blockTypeOperators = new BlockTypeOperators(new TypeOperators());
        JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactoryManager = JoinBridgeManager.lookupAllAtOnce(new PartitionedLookupSourceFactory(ordersTypes, ImmutableList.of(0, 1).stream().map(ordersTypes::get).collect(toImmutableList()), Ints.asList(0).stream().map(ordersTypes::get).collect(toImmutableList()), 1, false, blockTypeOperators));
        HashBuilderOperatorFactory hashBuilder = new HashBuilderOperatorFactory(1, new PlanNodeId("test"), lookupSourceFactoryManager, ImmutableList.of(0, 1), Ints.asList(0), OptionalInt.empty(), Optional.empty(), Optional.empty(), ImmutableList.of(), 1_500_000, new PagesIndex.TestingFactory(false), false, SingleStreamSpillerFactory.unsupportedSingleStreamSpillerFactory(), incrementalLoadFactorHashArraySizeSupplier(session));
        DriverContext driverContext = taskContext.addPipelineContext(0, false, false, false).addDriverContext();
        DriverFactory buildDriverFactory = new DriverFactory(0, false, false, ImmutableList.of(ordersTableScan, hashBuilder), OptionalInt.empty(), UNGROUPED_EXECUTION);
        List<Type> lineItemTypes = getColumnTypes("lineitem", "orderkey", "quantity");
        OperatorFactory lineItemTableScan = createTableScanOperator(0, new PlanNodeId("test"), "lineitem", "orderkey", "quantity");
        OperatorFactory joinOperator = operatorFactories.innerJoin(1, new PlanNodeId("test"), lookupSourceFactoryManager, false, false, false, lineItemTypes, Ints.asList(0), OptionalInt.empty(), Optional.empty(), OptionalInt.empty(), unsupportedPartitioningSpillerFactory(), blockTypeOperators);
        NullOutputOperatorFactory output = new NullOutputOperatorFactory(2, new PlanNodeId("test"));
        this.probeDriverFactory = new DriverFactory(1, true, true, ImmutableList.of(lineItemTableScan, joinOperator, output), OptionalInt.empty(), UNGROUPED_EXECUTION);
        Driver driver = buildDriverFactory.createDriver(driverContext);
        Future<LookupSourceProvider> lookupSourceProvider = lookupSourceFactoryManager.getJoinBridge(Lifespan.taskWide()).createLookupSourceProvider();
        while (!lookupSourceProvider.isDone()) {
            driver.process();
        }
        getFutureValue(lookupSourceProvider).close();
    }
    DriverContext driverContext = taskContext.addPipelineContext(1, true, true, false).addDriverContext();
    Driver driver = probeDriverFactory.createDriver(driverContext);
    return ImmutableList.of(driver);
}
Also used : DriverContext(io.trino.operator.DriverContext) HashBuilderOperatorFactory(io.trino.operator.join.HashBuilderOperator.HashBuilderOperatorFactory) Driver(io.trino.operator.Driver) PagesIndex(io.trino.operator.PagesIndex) PlanNodeId(io.trino.sql.planner.plan.PlanNodeId) Type(io.trino.spi.type.Type) BlockTypeOperators(io.trino.type.BlockTypeOperators) HashBuilderOperatorFactory(io.trino.operator.join.HashBuilderOperator.HashBuilderOperatorFactory) NullOutputOperatorFactory(io.trino.testing.NullOutputOperator.NullOutputOperatorFactory) OperatorFactory(io.trino.operator.OperatorFactory) DriverFactory(io.trino.operator.DriverFactory) PartitionedLookupSourceFactory(io.trino.operator.join.PartitionedLookupSourceFactory) NullOutputOperatorFactory(io.trino.testing.NullOutputOperator.NullOutputOperatorFactory) LookupSourceProvider(io.trino.operator.join.LookupSourceProvider) TypeOperators(io.trino.spi.type.TypeOperators) BlockTypeOperators(io.trino.type.BlockTypeOperators)

Aggregations

BlockTypeOperators (io.trino.type.BlockTypeOperators)23 TypeOperators (io.trino.spi.type.TypeOperators)20 Type (io.trino.spi.type.Type)9 Test (org.testng.annotations.Test)9 Page (io.trino.spi.Page)6 BeforeMethod (org.testng.annotations.BeforeMethod)6 ImmutableList (com.google.common.collect.ImmutableList)5 JoinCompiler (io.trino.sql.gen.JoinCompiler)5 PlanNodeId (io.trino.sql.planner.plan.PlanNodeId)5 PagesIndex (io.trino.operator.PagesIndex)4 FeaturesConfig (io.trino.FeaturesConfig)3 NodeScheduler (io.trino.execution.scheduler.NodeScheduler)3 NodeSchedulerConfig (io.trino.execution.scheduler.NodeSchedulerConfig)3 UniformNodeSelectorFactory (io.trino.execution.scheduler.UniformNodeSelectorFactory)3 InMemoryNodeManager (io.trino.metadata.InMemoryNodeManager)3 Driver (io.trino.operator.Driver)3 GroupByHashPageIndexerFactory (io.trino.operator.GroupByHashPageIndexerFactory)3 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)2 NodeTaskMap (io.trino.execution.NodeTaskMap)2 DriverFactory (io.trino.operator.DriverFactory)2