Examples with PageBuilder io.trino.spi.PageBuilder used on opensource projects

Example 51 with PageBuilder

use of io.trino.spi.PageBuilder in project trino by trinodb.

the class FunctionAssertions method getRetainedSizeOf.

private long getRetainedSizeOf(Object object) {
    if (object instanceof PageBuilder) {
        return ((PageBuilder) object).getRetainedSizeInBytes();
    }
    if (object instanceof Block) {
        return ((Block) object).getRetainedSizeInBytes();
    }
    Class<?> type = object.getClass();
    if (type.isArray()) {
        if (type == int[].class) {
            return sizeOf((int[]) object);
        }
        if (type == boolean[].class) {
            return sizeOf((boolean[]) object);
        }
        if (type == byte[].class) {
            return sizeOf((byte[]) object);
        }
        if (type == long[].class) {
            return sizeOf((long[]) object);
        }
        if (type == short[].class) {
            return sizeOf((short[]) object);
        }
        if (type == Block[].class) {
            Object[] objects = (Object[]) object;
            return Arrays.stream(objects).mapToLong(this::getRetainedSizeOf).sum();
        }
        throw new IllegalArgumentException(format("Unknown type encountered: %s", type));
    }
    long retainedSize = ClassLayout.parseClass(type).instanceSize();
    Field[] fields = type.getDeclaredFields();
    for (Field field : fields) {
        try {
            if (field.getType().isPrimitive() || Modifier.isStatic(field.getModifiers())) {
                continue;
            }
            field.setAccessible(true);
            retainedSize += getRetainedSizeOf(field.get(object));
        } catch (IllegalAccessException t) {
            throw new RuntimeException(t);
        }
    }
    return retainedSize;
}

Example 52 with PageBuilder

use of io.trino.spi.PageBuilder in project trino by trinodb.

the class TestJoinCompiler method testMultiChannel.

@Test(dataProvider = "hashEnabledValues")
public void testMultiChannel(boolean hashEnabled) {
    // compile a single channel hash strategy
    List<Type> types = ImmutableList.of(VARCHAR, VARCHAR, BIGINT, DOUBLE, BOOLEAN, VARCHAR);
    List<Type> joinTypes = ImmutableList.of(VARCHAR, BIGINT, DOUBLE, BOOLEAN);
    List<Type> outputTypes = ImmutableList.of(VARCHAR, BIGINT, DOUBLE, BOOLEAN, VARCHAR);
    List<Integer> joinChannels = Ints.asList(1, 2, 3, 4);
    List<Integer> outputChannels = Ints.asList(1, 2, 3, 4, 0);
    // crate hash strategy with a single channel blocks -- make sure there is some overlap in values
    List<Block> extraChannel = ImmutableList.of(BlockAssertions.createStringSequenceBlock(10, 20), BlockAssertions.createStringSequenceBlock(20, 30), BlockAssertions.createStringSequenceBlock(15, 25));
    List<Block> varcharChannel = ImmutableList.of(BlockAssertions.createStringSequenceBlock(10, 20), BlockAssertions.createStringSequenceBlock(20, 30), BlockAssertions.createStringSequenceBlock(15, 25));
    List<Block> longChannel = ImmutableList.of(BlockAssertions.createLongSequenceBlock(10, 20), BlockAssertions.createLongSequenceBlock(20, 30), BlockAssertions.createLongSequenceBlock(15, 25));
    List<Block> doubleChannel = ImmutableList.of(BlockAssertions.createDoubleSequenceBlock(10, 20), BlockAssertions.createDoubleSequenceBlock(20, 30), BlockAssertions.createDoubleSequenceBlock(15, 25));
    List<Block> booleanChannel = ImmutableList.of(BlockAssertions.createBooleanSequenceBlock(10, 20), BlockAssertions.createBooleanSequenceBlock(20, 30), BlockAssertions.createBooleanSequenceBlock(15, 25));
    List<Block> extraUnusedChannel = ImmutableList.of(BlockAssertions.createBooleanSequenceBlock(10, 20), BlockAssertions.createBooleanSequenceBlock(20, 30), BlockAssertions.createBooleanSequenceBlock(15, 25));
    OptionalInt hashChannel = OptionalInt.empty();
    ImmutableList<List<Block>> channels = ImmutableList.of(extraChannel, varcharChannel, longChannel, doubleChannel, booleanChannel, extraUnusedChannel);
    List<Block> precomputedHash = ImmutableList.of();
    if (hashEnabled) {
        ImmutableList.Builder<Block> hashChannelBuilder = ImmutableList.builder();
        for (int i = 0; i < 3; i++) {
            hashChannelBuilder.add(TypeTestUtils.getHashBlock(joinTypes, varcharChannel.get(i), longChannel.get(i), doubleChannel.get(i), booleanChannel.get(i)));
        }
        hashChannel = OptionalInt.of(6);
        precomputedHash = hashChannelBuilder.build();
        channels = ImmutableList.of(extraChannel, varcharChannel, longChannel, doubleChannel, booleanChannel, extraUnusedChannel, precomputedHash);
        types = ImmutableList.of(VARCHAR, VARCHAR, BIGINT, DOUBLE, BOOLEAN, VARCHAR, BIGINT);
        outputTypes = ImmutableList.of(VARCHAR, BIGINT, DOUBLE, BOOLEAN, VARCHAR, BIGINT);
        outputChannels = Ints.asList(1, 2, 3, 4, 0, 6);
    }
    PagesHashStrategyFactory pagesHashStrategyFactory = joinCompiler.compilePagesHashStrategyFactory(types, joinChannels, Optional.of(outputChannels));
    PagesHashStrategy hashStrategy = pagesHashStrategyFactory.createPagesHashStrategy(channels, hashChannel);
    // todo add tests for filter function
    PagesHashStrategy expectedHashStrategy = new SimplePagesHashStrategy(types, outputChannels, channels, joinChannels, hashChannel, Optional.empty(), blockTypeOperators);
    // verify channel count
    assertEquals(hashStrategy.getChannelCount(), outputChannels.size());
    // verify size
    int instanceSize = ClassLayout.parseClass(hashStrategy.getClass()).instanceSize();
    long sizeInBytes = instanceSize + channels.stream().flatMap(List::stream).mapToLong(Block::getRetainedSizeInBytes).sum();
    assertEquals(hashStrategy.getSizeInBytes(), sizeInBytes);
    // verify hashStrategy is consistent with equals and hash code from block
    for (int leftBlockIndex = 0; leftBlockIndex < varcharChannel.size(); leftBlockIndex++) {
        PageBuilder pageBuilder = new PageBuilder(outputTypes);
        Block[] leftBlocks = new Block[4];
        leftBlocks[0] = varcharChannel.get(leftBlockIndex);
        leftBlocks[1] = longChannel.get(leftBlockIndex);
        leftBlocks[2] = doubleChannel.get(leftBlockIndex);
        leftBlocks[3] = booleanChannel.get(leftBlockIndex);
        int leftPositionCount = varcharChannel.get(leftBlockIndex).getPositionCount();
        for (int leftBlockPosition = 0; leftBlockPosition < leftPositionCount; leftBlockPosition++) {
            // hash code of position must match block hash
            assertEquals(hashStrategy.hashPosition(leftBlockIndex, leftBlockPosition), expectedHashStrategy.hashPosition(leftBlockIndex, leftBlockPosition));
            // position must be equal to itself
            assertTrue(hashStrategy.positionEqualsPositionIgnoreNulls(leftBlockIndex, leftBlockPosition, leftBlockIndex, leftBlockPosition));
            assertTrue(hashStrategy.positionEqualsPosition(leftBlockIndex, leftBlockPosition, leftBlockIndex, leftBlockPosition));
            assertTrue(hashStrategy.positionNotDistinctFromPosition(leftBlockIndex, leftBlockPosition, leftBlockIndex, leftBlockPosition));
            // check equality of every position against every other position in the block
            for (int rightBlockIndex = 0; rightBlockIndex < varcharChannel.size(); rightBlockIndex++) {
                Block rightBlock = varcharChannel.get(rightBlockIndex);
                for (int rightBlockPosition = 0; rightBlockPosition < rightBlock.getPositionCount(); rightBlockPosition++) {
                    assertEquals(hashStrategy.positionEqualsPositionIgnoreNulls(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expectedHashStrategy.positionEqualsPositionIgnoreNulls(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition));
                    assertEquals(hashStrategy.positionEqualsPosition(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expectedHashStrategy.positionEqualsPosition(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition));
                    assertEquals(hashStrategy.positionNotDistinctFromPosition(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expectedHashStrategy.positionNotDistinctFromPosition(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition));
                }
            }
            // check equality of every position against every other position in the block cursor
            for (int rightBlockIndex = 0; rightBlockIndex < varcharChannel.size(); rightBlockIndex++) {
                Block[] rightBlocks = new Block[4];
                rightBlocks[0] = varcharChannel.get(rightBlockIndex);
                rightBlocks[1] = longChannel.get(rightBlockIndex);
                rightBlocks[2] = doubleChannel.get(rightBlockIndex);
                rightBlocks[3] = booleanChannel.get(rightBlockIndex);
                int rightPositionCount = varcharChannel.get(rightBlockIndex).getPositionCount();
                for (int rightPosition = 0; rightPosition < rightPositionCount; rightPosition++) {
                    boolean expected = expectedHashStrategy.positionEqualsRow(leftBlockIndex, leftBlockPosition, rightPosition, new Page(rightBlocks));
                    boolean expectedNotDistinct = expectedHashStrategy.positionNotDistinctFromRow(leftBlockIndex, leftBlockPosition, rightPosition, new Page(rightBlocks));
                    assertEquals(hashStrategy.positionEqualsRow(leftBlockIndex, leftBlockPosition, rightPosition, new Page(rightBlocks)), expected);
                    assertEquals(hashStrategy.positionNotDistinctFromRow(leftBlockIndex, leftBlockPosition, rightPosition, new Page(rightBlocks)), expectedNotDistinct);
                    assertEquals(hashStrategy.rowEqualsRow(leftBlockPosition, new Page(leftBlocks), rightPosition, new Page(rightBlocks)), expected);
                    assertEquals(hashStrategy.rowNotDistinctFromRow(leftBlockPosition, new Page(leftBlocks), rightPosition, new Page(rightBlocks)), expectedNotDistinct);
                    assertEquals(hashStrategy.positionEqualsRowIgnoreNulls(leftBlockIndex, leftBlockPosition, rightPosition, new Page(rightBlocks)), expected);
                }
            }
            // write position to output block
            pageBuilder.declarePosition();
            hashStrategy.appendTo(leftBlockIndex, leftBlockPosition, pageBuilder, 0);
        }
        // verify output block matches
        Page page = pageBuilder.build();
        if (hashEnabled) {
            assertPageEquals(outputTypes, page, new Page(varcharChannel.get(leftBlockIndex), longChannel.get(leftBlockIndex), doubleChannel.get(leftBlockIndex), booleanChannel.get(leftBlockIndex), extraChannel.get(leftBlockIndex), precomputedHash.get(leftBlockIndex)));
        } else {
            assertPageEquals(outputTypes, page, new Page(varcharChannel.get(leftBlockIndex), longChannel.get(leftBlockIndex), doubleChannel.get(leftBlockIndex), booleanChannel.get(leftBlockIndex), extraChannel.get(leftBlockIndex)));
        }
    }
}

Example 53 with PageBuilder

use of io.trino.spi.PageBuilder in project trino by trinodb.

the class TestJoinCompiler method testSingleChannel.

@Test(dataProvider = "hashEnabledValues")
public void testSingleChannel(boolean hashEnabled) {
    List<Type> joinTypes = ImmutableList.of(VARCHAR);
    List<Integer> joinChannels = Ints.asList(0);
    // compile a single channel hash strategy
    PagesHashStrategyFactory pagesHashStrategyFactory = joinCompiler.compilePagesHashStrategyFactory(joinTypes, joinChannels);
    // create hash strategy with a single channel blocks -- make sure there is some overlap in values
    List<Block> channel = ImmutableList.of(BlockAssertions.createStringSequenceBlock(10, 20), BlockAssertions.createStringSequenceBlock(20, 30), BlockAssertions.createStringSequenceBlock(15, 25));
    OptionalInt hashChannel = OptionalInt.empty();
    List<List<Block>> channels = ImmutableList.of(channel);
    if (hashEnabled) {
        ImmutableList.Builder<Block> hashChannelBuilder = ImmutableList.builder();
        for (Block block : channel) {
            hashChannelBuilder.add(TypeTestUtils.getHashBlock(joinTypes, block));
        }
        hashChannel = OptionalInt.of(1);
        channels = ImmutableList.of(channel, hashChannelBuilder.build());
    }
    PagesHashStrategy hashStrategy = pagesHashStrategyFactory.createPagesHashStrategy(channels, hashChannel);
    // verify channel count
    assertEquals(hashStrategy.getChannelCount(), 1);
    BlockTypeOperators blockTypeOperators = new BlockTypeOperators();
    BlockPositionEqual equalOperator = blockTypeOperators.getEqualOperator(VARCHAR);
    BlockPositionIsDistinctFrom distinctFromOperator = blockTypeOperators.getDistinctFromOperator(VARCHAR);
    BlockPositionHashCode hashCodeOperator = blockTypeOperators.getHashCodeOperator(VARCHAR);
    // verify hashStrategy is consistent with equals and hash code from block
    for (int leftBlockIndex = 0; leftBlockIndex < channel.size(); leftBlockIndex++) {
        Block leftBlock = channel.get(leftBlockIndex);
        PageBuilder pageBuilder = new PageBuilder(ImmutableList.of(VARCHAR));
        for (int leftBlockPosition = 0; leftBlockPosition < leftBlock.getPositionCount(); leftBlockPosition++) {
            // hash code of position must match block hash
            assertEquals(hashStrategy.hashPosition(leftBlockIndex, leftBlockPosition), hashCodeOperator.hashCodeNullSafe(leftBlock, leftBlockPosition));
            // position must be equal to itself
            assertTrue(hashStrategy.positionEqualsPositionIgnoreNulls(leftBlockIndex, leftBlockPosition, leftBlockIndex, leftBlockPosition));
            // check equality of every position against every other position in the block
            for (int rightBlockIndex = 0; rightBlockIndex < channel.size(); rightBlockIndex++) {
                Block rightBlock = channel.get(rightBlockIndex);
                for (int rightBlockPosition = 0; rightBlockPosition < rightBlock.getPositionCount(); rightBlockPosition++) {
                    boolean expected = equalOperator.equalNullSafe(leftBlock, leftBlockPosition, rightBlock, rightBlockPosition);
                    boolean expectedNotDistinct = !distinctFromOperator.isDistinctFrom(leftBlock, leftBlockPosition, rightBlock, rightBlockPosition);
                    assertEquals(hashStrategy.positionEqualsRow(leftBlockIndex, leftBlockPosition, rightBlockPosition, new Page(rightBlock)), expected);
                    assertEquals(hashStrategy.positionNotDistinctFromRow(leftBlockIndex, leftBlockPosition, rightBlockPosition, new Page(rightBlock)), expectedNotDistinct);
                    assertEquals(hashStrategy.rowEqualsRow(leftBlockPosition, new Page(leftBlock), rightBlockPosition, new Page(rightBlock)), expected);
                    assertEquals(hashStrategy.rowNotDistinctFromRow(leftBlockPosition, new Page(leftBlock), rightBlockPosition, new Page(rightBlock)), expectedNotDistinct);
                    assertEquals(hashStrategy.positionEqualsRowIgnoreNulls(leftBlockIndex, leftBlockPosition, rightBlockPosition, new Page(rightBlock)), expected);
                    assertEquals(hashStrategy.positionEqualsPositionIgnoreNulls(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expected);
                    assertEquals(hashStrategy.positionEqualsPosition(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expected);
                    assertEquals(hashStrategy.positionNotDistinctFromPosition(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expectedNotDistinct);
                }
            }
            // check equality of every position against every other position in the block cursor
            for (int rightBlockIndex = 0; rightBlockIndex < channel.size(); rightBlockIndex++) {
                Block rightBlock = channel.get(rightBlockIndex);
                for (int rightBlockPosition = 0; rightBlockPosition < rightBlock.getPositionCount(); rightBlockPosition++) {
                    boolean expected = equalOperator.equalNullSafe(leftBlock, leftBlockPosition, rightBlock, rightBlockPosition);
                    boolean expectedNotDistinct = !distinctFromOperator.isDistinctFrom(leftBlock, leftBlockPosition, rightBlock, rightBlockPosition);
                    assertEquals(hashStrategy.positionEqualsRow(leftBlockIndex, leftBlockPosition, rightBlockPosition, new Page(rightBlock)), expected);
                    assertEquals(hashStrategy.positionNotDistinctFromRow(leftBlockIndex, leftBlockPosition, rightBlockPosition, new Page(rightBlock)), expectedNotDistinct);
                    assertEquals(hashStrategy.rowEqualsRow(leftBlockPosition, new Page(leftBlock), rightBlockPosition, new Page(rightBlock)), expected);
                    assertEquals(hashStrategy.rowNotDistinctFromRow(leftBlockPosition, new Page(leftBlock), rightBlockPosition, new Page(rightBlock)), expectedNotDistinct);
                    assertEquals(hashStrategy.positionEqualsRowIgnoreNulls(leftBlockIndex, leftBlockPosition, rightBlockPosition, new Page(rightBlock)), expected);
                    assertEquals(hashStrategy.positionEqualsPositionIgnoreNulls(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expected);
                    assertEquals(hashStrategy.positionEqualsPosition(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expected);
                    assertEquals(hashStrategy.positionNotDistinctFromPosition(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expectedNotDistinct);
                }
            }
            // write position to output block
            pageBuilder.declarePosition();
            hashStrategy.appendTo(leftBlockIndex, leftBlockPosition, pageBuilder, 0);
        }
        // verify output block matches
        assertBlockEquals(VARCHAR, pageBuilder.build().getBlock(0), leftBlock);
    }
}

Example 54 with PageBuilder

use of io.trino.spi.PageBuilder in project trino by trinodb.

the class TestDeltaLakePageSink method testPageSinkStats.

@Test
public void testPageSinkStats() throws Exception {
    File tempDir = Files.createTempDir();
    try {
        DeltaLakeWriterStats stats = new DeltaLakeWriterStats();
        String tablePath = tempDir.getAbsolutePath() + "/test_table";
        ConnectorPageSink pageSink = createPageSink(new Path(tablePath), stats);
        List<LineItemColumn> columns = ImmutableList.copyOf(LineItemColumn.values());
        List<Type> columnTypes = columns.stream().map(LineItemColumn::getType).map(TestDeltaLakePageSink::getTrinoType).collect(toList());
        PageBuilder pageBuilder = new PageBuilder(columnTypes);
        long rows = 0;
        for (LineItem lineItem : new LineItemGenerator(0.01, 1, 1)) {
            if (rows >= NUM_ROWS) {
                break;
            }
            rows++;
            pageBuilder.declarePosition();
            for (int i = 0; i < columns.size(); i++) {
                LineItemColumn column = columns.get(i);
                BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(i);
                writeToBlock(blockBuilder, column, lineItem);
            }
        }
        Page page = pageBuilder.build();
        pageSink.appendPage(page);
        JsonCodec<DataFileInfo> dataFileInfoCodec = new JsonCodecFactory().jsonCodec(DataFileInfo.class);
        Collection<Slice> fragments = getFutureValue(pageSink.finish());
        List<DataFileInfo> dataFileInfos = fragments.stream().map(Slice::getBytes).map(dataFileInfoCodec::fromJson).collect(toImmutableList());
        assertEquals(dataFileInfos.size(), 1);
        DataFileInfo dataFileInfo = dataFileInfos.get(0);
        List<File> files = ImmutableList.copyOf(new File(tablePath).listFiles((dir, name) -> !name.endsWith(".crc")));
        assertEquals(files.size(), 1);
        File outputFile = files.get(0);
        assertEquals(round(stats.getInputPageSizeInBytes().getAllTime().getMax()), page.getRetainedSizeInBytes());
        assertEquals(dataFileInfo.getStatistics().getNumRecords(), Optional.of(rows));
        assertEquals(dataFileInfo.getPartitionValues(), ImmutableList.of());
        assertEquals(dataFileInfo.getSize(), outputFile.length());
        assertEquals(dataFileInfo.getPath(), outputFile.getName());
        Instant now = Instant.now();
        assertTrue(dataFileInfo.getCreationTime() < now.toEpochMilli());
        assertTrue(dataFileInfo.getCreationTime() > now.minus(1, MINUTES).toEpochMilli());
    } finally {
        deleteRecursively(tempDir.toPath(), ALLOW_INSECURE);
    }
}

Example 55 with PageBuilder

use of io.trino.spi.PageBuilder in project trino by trinodb.

the class JoinCompiler method generateAppendToMethod.

private static void generateAppendToMethod(ClassDefinition classDefinition, CallSiteBinder callSiteBinder, List<Type> types, List<Integer> outputChannels, List<FieldDefinition> channelFields) {
    Parameter blockIndex = arg("blockIndex", int.class);
    Parameter blockPosition = arg("blockPosition", int.class);
    Parameter pageBuilder = arg("pageBuilder", PageBuilder.class);
    Parameter outputChannelOffset = arg("outputChannelOffset", int.class);
    MethodDefinition appendToMethod = classDefinition.declareMethod(a(PUBLIC), "appendTo", type(void.class), blockIndex, blockPosition, pageBuilder, outputChannelOffset);
    Variable thisVariable = appendToMethod.getThis();
    BytecodeBlock appendToBody = appendToMethod.getBody();
    int pageBuilderOutputChannel = 0;
    for (int outputChannel : outputChannels) {
        Type type = types.get(outputChannel);
        BytecodeExpression typeExpression = constantType(callSiteBinder, type);
        BytecodeExpression block = thisVariable.getField(channelFields.get(outputChannel)).invoke("get", Object.class, blockIndex).cast(Block.class);
        appendToBody.comment("%s.appendTo(channel_%s.get(outputChannel), blockPosition, pageBuilder.getBlockBuilder(outputChannelOffset + %s));", type.getClass(), outputChannel, pageBuilderOutputChannel).append(typeExpression).append(block).append(blockPosition).append(pageBuilder).append(outputChannelOffset).push(pageBuilderOutputChannel++).append(OpCode.IADD).invokeVirtual(PageBuilder.class, "getBlockBuilder", BlockBuilder.class, int.class).invokeInterface(Type.class, "appendTo", void.class, Block.class, int.class, BlockBuilder.class);
    }
    appendToBody.ret();
}