Examples with BlockTypeOperators io.trino.type.BlockTypeOperators used on opensource projects

Example 11 with BlockTypeOperators

use of io.trino.type.BlockTypeOperators in project trino by trinodb.

the class HashBuildAndJoinBenchmark method createDrivers.

/*
    select orderkey, quantity, totalprice
    from lineitem join orders using (orderkey)
     */
@Override
protected List<Driver> createDrivers(TaskContext taskContext) {
    ImmutableList.Builder<OperatorFactory> driversBuilder = ImmutableList.builder();
    driversBuilder.add(ordersTableScan);
    List<Type> sourceTypes = ordersTableTypes;
    OptionalInt hashChannel = OptionalInt.empty();
    if (hashEnabled) {
        driversBuilder.add(createHashProjectOperator(1, new PlanNodeId("test"), sourceTypes));
        sourceTypes = ImmutableList.<Type>builder().addAll(sourceTypes).add(BIGINT).build();
        hashChannel = OptionalInt.of(sourceTypes.size() - 1);
    }
    // hash build
    BlockTypeOperators blockTypeOperators = new BlockTypeOperators(new TypeOperators());
    JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactoryManager = JoinBridgeManager.lookupAllAtOnce(new PartitionedLookupSourceFactory(sourceTypes, ImmutableList.of(0, 1).stream().map(sourceTypes::get).collect(toImmutableList()), Ints.asList(0).stream().map(sourceTypes::get).collect(toImmutableList()), 1, false, blockTypeOperators));
    HashBuilderOperatorFactory hashBuilder = new HashBuilderOperatorFactory(2, new PlanNodeId("test"), lookupSourceFactoryManager, ImmutableList.of(0, 1), Ints.asList(0), hashChannel, Optional.empty(), Optional.empty(), ImmutableList.of(), 1_500_000, new PagesIndex.TestingFactory(false), false, SingleStreamSpillerFactory.unsupportedSingleStreamSpillerFactory(), incrementalLoadFactorHashArraySizeSupplier(session));
    driversBuilder.add(hashBuilder);
    DriverFactory hashBuildDriverFactory = new DriverFactory(0, true, false, driversBuilder.build(), OptionalInt.empty(), UNGROUPED_EXECUTION);
    // join
    ImmutableList.Builder<OperatorFactory> joinDriversBuilder = ImmutableList.builder();
    joinDriversBuilder.add(lineItemTableScan);
    sourceTypes = lineItemTableTypes;
    hashChannel = OptionalInt.empty();
    if (hashEnabled) {
        joinDriversBuilder.add(createHashProjectOperator(1, new PlanNodeId("test"), sourceTypes));
        sourceTypes = ImmutableList.<Type>builder().addAll(sourceTypes).add(BIGINT).build();
        hashChannel = OptionalInt.of(sourceTypes.size() - 1);
    }
    OperatorFactory joinOperator = operatorFactories.innerJoin(2, new PlanNodeId("test"), lookupSourceFactoryManager, false, false, false, sourceTypes, Ints.asList(0), hashChannel, Optional.empty(), OptionalInt.empty(), unsupportedPartitioningSpillerFactory(), blockTypeOperators);
    joinDriversBuilder.add(joinOperator);
    joinDriversBuilder.add(new NullOutputOperatorFactory(3, new PlanNodeId("test")));
    DriverFactory joinDriverFactory = new DriverFactory(1, true, true, joinDriversBuilder.build(), OptionalInt.empty(), UNGROUPED_EXECUTION);
    Driver hashBuildDriver = hashBuildDriverFactory.createDriver(taskContext.addPipelineContext(0, true, false, false).addDriverContext());
    hashBuildDriverFactory.noMoreDrivers();
    Driver joinDriver = joinDriverFactory.createDriver(taskContext.addPipelineContext(1, true, true, false).addDriverContext());
    joinDriverFactory.noMoreDrivers();
    return ImmutableList.of(hashBuildDriver, joinDriver);
}

Example 12 with BlockTypeOperators

use of io.trino.type.BlockTypeOperators in project trino by trinodb.

the class TestGlobalFunctionCatalog method testConflictingScalarAggregation.

@Test
public void testConflictingScalarAggregation() {
    FunctionBundle functions = extractFunctions(ScalarSum.class);
    TypeOperators typeOperators = new TypeOperators();
    GlobalFunctionCatalog globalFunctionCatalog = new GlobalFunctionCatalog();
    globalFunctionCatalog.addFunctions(SystemFunctionBundle.create(new FeaturesConfig(), typeOperators, new BlockTypeOperators(typeOperators), NodeVersion.UNKNOWN));
    assertThatThrownBy(() -> globalFunctionCatalog.addFunctions(functions)).isInstanceOf(IllegalStateException.class).hasMessage("'sum' is both an aggregation and a scalar function");
}

Example 13 with BlockTypeOperators

use of io.trino.type.BlockTypeOperators in project trino by trinodb.

the class TestHivePageSink method createPageSink.

private static ConnectorPageSink createPageSink(HiveTransactionHandle transaction, HiveConfig config, HiveMetastore metastore, Path outputPath, HiveWriterStats stats) {
    LocationHandle locationHandle = new LocationHandle(outputPath, outputPath, false, DIRECT_TO_TARGET_NEW_DIRECTORY);
    HiveOutputTableHandle handle = new HiveOutputTableHandle(SCHEMA_NAME, TABLE_NAME, getColumnHandles(), new HivePageSinkMetadata(new SchemaTableName(SCHEMA_NAME, TABLE_NAME), metastore.getTable(SCHEMA_NAME, TABLE_NAME), ImmutableMap.of()), locationHandle, config.getHiveStorageFormat(), config.getHiveStorageFormat(), ImmutableList.of(), Optional.empty(), "test", ImmutableMap.of(), NO_ACID_TRANSACTION, false, false);
    JsonCodec<PartitionUpdate> partitionUpdateCodec = JsonCodec.jsonCodec(PartitionUpdate.class);
    TypeOperators typeOperators = new TypeOperators();
    BlockTypeOperators blockTypeOperators = new BlockTypeOperators(typeOperators);
    HivePageSinkProvider provider = new HivePageSinkProvider(getDefaultHiveFileWriterFactories(config, HDFS_ENVIRONMENT), HDFS_ENVIRONMENT, PAGE_SORTER, HiveMetastoreFactory.ofInstance(metastore), new GroupByHashPageIndexerFactory(new JoinCompiler(typeOperators), blockTypeOperators), TESTING_TYPE_MANAGER, config, new HiveLocationService(HDFS_ENVIRONMENT), partitionUpdateCodec, new TestingNodeManager("fake-environment"), new HiveEventClient(), getHiveSessionProperties(config), stats);
    return provider.createPageSink(transaction, getHiveSession(config), handle);
}

Example 14 with BlockTypeOperators

use of io.trino.type.BlockTypeOperators in project trino by trinodb.

the class FunctionManager method createTestingFunctionManager.

public static FunctionManager createTestingFunctionManager() {
    TypeOperators typeOperators = new TypeOperators();
    GlobalFunctionCatalog functionCatalog = new GlobalFunctionCatalog();
    functionCatalog.addFunctions(SystemFunctionBundle.create(new FeaturesConfig(), typeOperators, new BlockTypeOperators(typeOperators), UNKNOWN));
    functionCatalog.addFunctions(new InternalFunctionBundle(new LiteralFunction(new InternalBlockEncodingSerde(new BlockEncodingManager(), TESTING_TYPE_MANAGER))));
    return new FunctionManager(functionCatalog);
}

Example 15 with BlockTypeOperators

use of io.trino.type.BlockTypeOperators in project trino by trinodb.

the class TestJoinCompiler method testSingleChannel.

@Test(dataProvider = "hashEnabledValues")
public void testSingleChannel(boolean hashEnabled) {
    List<Type> joinTypes = ImmutableList.of(VARCHAR);
    List<Integer> joinChannels = Ints.asList(0);
    // compile a single channel hash strategy
    PagesHashStrategyFactory pagesHashStrategyFactory = joinCompiler.compilePagesHashStrategyFactory(joinTypes, joinChannels);
    // create hash strategy with a single channel blocks -- make sure there is some overlap in values
    List<Block> channel = ImmutableList.of(BlockAssertions.createStringSequenceBlock(10, 20), BlockAssertions.createStringSequenceBlock(20, 30), BlockAssertions.createStringSequenceBlock(15, 25));
    OptionalInt hashChannel = OptionalInt.empty();
    List<List<Block>> channels = ImmutableList.of(channel);
    if (hashEnabled) {
        ImmutableList.Builder<Block> hashChannelBuilder = ImmutableList.builder();
        for (Block block : channel) {
            hashChannelBuilder.add(TypeTestUtils.getHashBlock(joinTypes, block));
        }
        hashChannel = OptionalInt.of(1);
        channels = ImmutableList.of(channel, hashChannelBuilder.build());
    }
    PagesHashStrategy hashStrategy = pagesHashStrategyFactory.createPagesHashStrategy(channels, hashChannel);
    // verify channel count
    assertEquals(hashStrategy.getChannelCount(), 1);
    BlockTypeOperators blockTypeOperators = new BlockTypeOperators();
    BlockPositionEqual equalOperator = blockTypeOperators.getEqualOperator(VARCHAR);
    BlockPositionIsDistinctFrom distinctFromOperator = blockTypeOperators.getDistinctFromOperator(VARCHAR);
    BlockPositionHashCode hashCodeOperator = blockTypeOperators.getHashCodeOperator(VARCHAR);
    // verify hashStrategy is consistent with equals and hash code from block
    for (int leftBlockIndex = 0; leftBlockIndex < channel.size(); leftBlockIndex++) {
        Block leftBlock = channel.get(leftBlockIndex);
        PageBuilder pageBuilder = new PageBuilder(ImmutableList.of(VARCHAR));
        for (int leftBlockPosition = 0; leftBlockPosition < leftBlock.getPositionCount(); leftBlockPosition++) {
            // hash code of position must match block hash
            assertEquals(hashStrategy.hashPosition(leftBlockIndex, leftBlockPosition), hashCodeOperator.hashCodeNullSafe(leftBlock, leftBlockPosition));
            // position must be equal to itself
            assertTrue(hashStrategy.positionEqualsPositionIgnoreNulls(leftBlockIndex, leftBlockPosition, leftBlockIndex, leftBlockPosition));
            // check equality of every position against every other position in the block
            for (int rightBlockIndex = 0; rightBlockIndex < channel.size(); rightBlockIndex++) {
                Block rightBlock = channel.get(rightBlockIndex);
                for (int rightBlockPosition = 0; rightBlockPosition < rightBlock.getPositionCount(); rightBlockPosition++) {
                    boolean expected = equalOperator.equalNullSafe(leftBlock, leftBlockPosition, rightBlock, rightBlockPosition);
                    boolean expectedNotDistinct = !distinctFromOperator.isDistinctFrom(leftBlock, leftBlockPosition, rightBlock, rightBlockPosition);
                    assertEquals(hashStrategy.positionEqualsRow(leftBlockIndex, leftBlockPosition, rightBlockPosition, new Page(rightBlock)), expected);
                    assertEquals(hashStrategy.positionNotDistinctFromRow(leftBlockIndex, leftBlockPosition, rightBlockPosition, new Page(rightBlock)), expectedNotDistinct);
                    assertEquals(hashStrategy.rowEqualsRow(leftBlockPosition, new Page(leftBlock), rightBlockPosition, new Page(rightBlock)), expected);
                    assertEquals(hashStrategy.rowNotDistinctFromRow(leftBlockPosition, new Page(leftBlock), rightBlockPosition, new Page(rightBlock)), expectedNotDistinct);
                    assertEquals(hashStrategy.positionEqualsRowIgnoreNulls(leftBlockIndex, leftBlockPosition, rightBlockPosition, new Page(rightBlock)), expected);
                    assertEquals(hashStrategy.positionEqualsPositionIgnoreNulls(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expected);
                    assertEquals(hashStrategy.positionEqualsPosition(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expected);
                    assertEquals(hashStrategy.positionNotDistinctFromPosition(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expectedNotDistinct);
                }
            }
            // check equality of every position against every other position in the block cursor
            for (int rightBlockIndex = 0; rightBlockIndex < channel.size(); rightBlockIndex++) {
                Block rightBlock = channel.get(rightBlockIndex);
                for (int rightBlockPosition = 0; rightBlockPosition < rightBlock.getPositionCount(); rightBlockPosition++) {
                    boolean expected = equalOperator.equalNullSafe(leftBlock, leftBlockPosition, rightBlock, rightBlockPosition);
                    boolean expectedNotDistinct = !distinctFromOperator.isDistinctFrom(leftBlock, leftBlockPosition, rightBlock, rightBlockPosition);
                    assertEquals(hashStrategy.positionEqualsRow(leftBlockIndex, leftBlockPosition, rightBlockPosition, new Page(rightBlock)), expected);
                    assertEquals(hashStrategy.positionNotDistinctFromRow(leftBlockIndex, leftBlockPosition, rightBlockPosition, new Page(rightBlock)), expectedNotDistinct);
                    assertEquals(hashStrategy.rowEqualsRow(leftBlockPosition, new Page(leftBlock), rightBlockPosition, new Page(rightBlock)), expected);
                    assertEquals(hashStrategy.rowNotDistinctFromRow(leftBlockPosition, new Page(leftBlock), rightBlockPosition, new Page(rightBlock)), expectedNotDistinct);
                    assertEquals(hashStrategy.positionEqualsRowIgnoreNulls(leftBlockIndex, leftBlockPosition, rightBlockPosition, new Page(rightBlock)), expected);
                    assertEquals(hashStrategy.positionEqualsPositionIgnoreNulls(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expected);
                    assertEquals(hashStrategy.positionEqualsPosition(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expected);
                    assertEquals(hashStrategy.positionNotDistinctFromPosition(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expectedNotDistinct);
                }
            }
            // write position to output block
            pageBuilder.declarePosition();
            hashStrategy.appendTo(leftBlockIndex, leftBlockPosition, pageBuilder, 0);
        }
        // verify output block matches
        assertBlockEquals(VARCHAR, pageBuilder.build().getBlock(0), leftBlock);
    }
}