Examples with PageBuilder com.facebook.presto.spi.PageBuilder used on opensource projects

Example 16 with PageBuilder

use of com.facebook.presto.spi.PageBuilder in project presto by prestodb.

the class TestJoinCompiler method testSingleChannel.

@Test(dataProvider = "hashEnabledValues")
public void testSingleChannel(boolean hashEnabled) throws Exception {
    List<Type> joinTypes = ImmutableList.of(VARCHAR);
    List<Integer> joinChannels = Ints.asList(0);
    // compile a single channel hash strategy
    PagesHashStrategyFactory pagesHashStrategyFactory = joinCompiler.compilePagesHashStrategyFactory(joinTypes, joinChannels);
    // create hash strategy with a single channel blocks -- make sure there is some overlap in values
    List<Block> channel = ImmutableList.of(BlockAssertions.createStringSequenceBlock(10, 20), BlockAssertions.createStringSequenceBlock(20, 30), BlockAssertions.createStringSequenceBlock(15, 25));
    Optional<Integer> hashChannel = Optional.empty();
    List<List<Block>> channels = ImmutableList.of(channel);
    if (hashEnabled) {
        ImmutableList.Builder<Block> hashChannelBuilder = ImmutableList.builder();
        for (Block block : channel) {
            hashChannelBuilder.add(TypeUtils.getHashBlock(joinTypes, block));
        }
        hashChannel = Optional.of(1);
        channels = ImmutableList.of(channel, hashChannelBuilder.build());
    }
    PagesHashStrategy hashStrategy = pagesHashStrategyFactory.createPagesHashStrategy(channels, hashChannel);
    // verify channel count
    assertEquals(hashStrategy.getChannelCount(), 1);
    // verify hashStrategy is consistent with equals and hash code from block
    for (int leftBlockIndex = 0; leftBlockIndex < channel.size(); leftBlockIndex++) {
        Block leftBlock = channel.get(leftBlockIndex);
        PageBuilder pageBuilder = new PageBuilder(ImmutableList.of(VARCHAR));
        for (int leftBlockPosition = 0; leftBlockPosition < leftBlock.getPositionCount(); leftBlockPosition++) {
            // hash code of position must match block hash
            assertEquals(hashStrategy.hashPosition(leftBlockIndex, leftBlockPosition), hashPosition(VARCHAR, leftBlock, leftBlockPosition));
            // position must be equal to itself
            assertTrue(hashStrategy.positionEqualsPositionIgnoreNulls(leftBlockIndex, leftBlockPosition, leftBlockIndex, leftBlockPosition));
            // check equality of every position against every other position in the block
            for (int rightBlockIndex = 0; rightBlockIndex < channel.size(); rightBlockIndex++) {
                Block rightBlock = channel.get(rightBlockIndex);
                for (int rightBlockPosition = 0; rightBlockPosition < rightBlock.getPositionCount(); rightBlockPosition++) {
                    boolean expected = positionEqualsPosition(VARCHAR, leftBlock, leftBlockPosition, rightBlock, rightBlockPosition);
                    assertEquals(hashStrategy.positionEqualsRow(leftBlockIndex, leftBlockPosition, rightBlockPosition, new Page(rightBlock)), expected);
                    assertEquals(hashStrategy.rowEqualsRow(leftBlockPosition, new Page(leftBlock), rightBlockPosition, new Page(rightBlock)), expected);
                    assertEquals(hashStrategy.positionEqualsRowIgnoreNulls(leftBlockIndex, leftBlockPosition, rightBlockPosition, new Page(rightBlock)), expected);
                    assertEquals(hashStrategy.positionEqualsPositionIgnoreNulls(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expected);
                    assertEquals(hashStrategy.positionEqualsPosition(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expected);
                }
            }
            // check equality of every position against every other position in the block cursor
            for (int rightBlockIndex = 0; rightBlockIndex < channel.size(); rightBlockIndex++) {
                Block rightBlock = channel.get(rightBlockIndex);
                for (int rightBlockPosition = 0; rightBlockPosition < rightBlock.getPositionCount(); rightBlockPosition++) {
                    boolean expected = positionEqualsPosition(VARCHAR, leftBlock, leftBlockPosition, rightBlock, rightBlockPosition);
                    assertEquals(hashStrategy.positionEqualsRow(leftBlockIndex, leftBlockPosition, rightBlockPosition, new Page(rightBlock)), expected);
                    assertEquals(hashStrategy.rowEqualsRow(leftBlockPosition, new Page(leftBlock), rightBlockPosition, new Page(rightBlock)), expected);
                    assertEquals(hashStrategy.positionEqualsRowIgnoreNulls(leftBlockIndex, leftBlockPosition, rightBlockPosition, new Page(rightBlock)), expected);
                    assertEquals(hashStrategy.positionEqualsPositionIgnoreNulls(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expected);
                    assertEquals(hashStrategy.positionEqualsPosition(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expected);
                }
            }
            // write position to output block
            pageBuilder.declarePosition();
            hashStrategy.appendTo(leftBlockIndex, leftBlockPosition, pageBuilder, 0);
        }
        // verify output block matches
        assertBlockEquals(VARCHAR, pageBuilder.build().getBlock(0), leftBlock);
    }
}

Example 17 with PageBuilder

use of com.facebook.presto.spi.PageBuilder in project presto by prestodb.

the class TestJoinCompiler method testMultiChannel.

@Test(dataProvider = "hashEnabledValues")
public void testMultiChannel(boolean hashEnabled) throws Exception {
    // compile a single channel hash strategy
    JoinCompiler joinCompiler = new JoinCompiler();
    List<Type> types = ImmutableList.of(VARCHAR, VARCHAR, BIGINT, DOUBLE, BOOLEAN, VARCHAR);
    List<Type> joinTypes = ImmutableList.of(VARCHAR, BIGINT, DOUBLE, BOOLEAN);
    List<Type> outputTypes = ImmutableList.of(VARCHAR, BIGINT, DOUBLE, BOOLEAN, VARCHAR);
    List<Integer> joinChannels = Ints.asList(1, 2, 3, 4);
    List<Integer> outputChannels = Ints.asList(1, 2, 3, 4, 0);
    // crate hash strategy with a single channel blocks -- make sure there is some overlap in values
    List<Block> extraChannel = ImmutableList.of(BlockAssertions.createStringSequenceBlock(10, 20), BlockAssertions.createStringSequenceBlock(20, 30), BlockAssertions.createStringSequenceBlock(15, 25));
    List<Block> varcharChannel = ImmutableList.of(BlockAssertions.createStringSequenceBlock(10, 20), BlockAssertions.createStringSequenceBlock(20, 30), BlockAssertions.createStringSequenceBlock(15, 25));
    List<Block> longChannel = ImmutableList.of(BlockAssertions.createLongSequenceBlock(10, 20), BlockAssertions.createLongSequenceBlock(20, 30), BlockAssertions.createLongSequenceBlock(15, 25));
    List<Block> doubleChannel = ImmutableList.of(BlockAssertions.createDoubleSequenceBlock(10, 20), BlockAssertions.createDoubleSequenceBlock(20, 30), BlockAssertions.createDoubleSequenceBlock(15, 25));
    List<Block> booleanChannel = ImmutableList.of(BlockAssertions.createBooleanSequenceBlock(10, 20), BlockAssertions.createBooleanSequenceBlock(20, 30), BlockAssertions.createBooleanSequenceBlock(15, 25));
    List<Block> extraUnusedChannel = ImmutableList.of(BlockAssertions.createBooleanSequenceBlock(10, 20), BlockAssertions.createBooleanSequenceBlock(20, 30), BlockAssertions.createBooleanSequenceBlock(15, 25));
    Optional<Integer> hashChannel = Optional.empty();
    ImmutableList<List<Block>> channels = ImmutableList.of(extraChannel, varcharChannel, longChannel, doubleChannel, booleanChannel, extraUnusedChannel);
    List<Block> precomputedHash = ImmutableList.of();
    if (hashEnabled) {
        ImmutableList.Builder<Block> hashChannelBuilder = ImmutableList.builder();
        for (int i = 0; i < 3; i++) {
            hashChannelBuilder.add(TypeUtils.getHashBlock(joinTypes, varcharChannel.get(i), longChannel.get(i), doubleChannel.get(i), booleanChannel.get(i)));
        }
        hashChannel = Optional.of(6);
        precomputedHash = hashChannelBuilder.build();
        channels = ImmutableList.of(extraChannel, varcharChannel, longChannel, doubleChannel, booleanChannel, extraUnusedChannel, precomputedHash);
        types = ImmutableList.of(VARCHAR, VARCHAR, BIGINT, DOUBLE, BOOLEAN, VARCHAR, BIGINT);
        outputTypes = ImmutableList.of(VARCHAR, BIGINT, DOUBLE, BOOLEAN, VARCHAR, BIGINT);
        outputChannels = Ints.asList(1, 2, 3, 4, 0, 6);
    }
    PagesHashStrategyFactory pagesHashStrategyFactory = joinCompiler.compilePagesHashStrategyFactory(types, joinChannels, Optional.of(outputChannels));
    PagesHashStrategy hashStrategy = pagesHashStrategyFactory.createPagesHashStrategy(channels, hashChannel);
    // todo add tests for filter function
    PagesHashStrategy expectedHashStrategy = new SimplePagesHashStrategy(types, outputChannels, channels, joinChannels, hashChannel);
    // verify channel count
    assertEquals(hashStrategy.getChannelCount(), outputChannels.size());
    // verify size
    long sizeInBytes = channels.stream().flatMap(List::stream).mapToLong(Block::getRetainedSizeInBytes).sum();
    assertEquals(hashStrategy.getSizeInBytes(), sizeInBytes);
    // verify hashStrategy is consistent with equals and hash code from block
    for (int leftBlockIndex = 0; leftBlockIndex < varcharChannel.size(); leftBlockIndex++) {
        PageBuilder pageBuilder = new PageBuilder(outputTypes);
        Block[] leftBlocks = new Block[4];
        leftBlocks[0] = varcharChannel.get(leftBlockIndex);
        leftBlocks[1] = longChannel.get(leftBlockIndex);
        leftBlocks[2] = doubleChannel.get(leftBlockIndex);
        leftBlocks[3] = booleanChannel.get(leftBlockIndex);
        int leftPositionCount = varcharChannel.get(leftBlockIndex).getPositionCount();
        for (int leftBlockPosition = 0; leftBlockPosition < leftPositionCount; leftBlockPosition++) {
            // hash code of position must match block hash
            assertEquals(hashStrategy.hashPosition(leftBlockIndex, leftBlockPosition), expectedHashStrategy.hashPosition(leftBlockIndex, leftBlockPosition));
            // position must be equal to itself
            assertTrue(hashStrategy.positionEqualsPositionIgnoreNulls(leftBlockIndex, leftBlockPosition, leftBlockIndex, leftBlockPosition));
            assertTrue(hashStrategy.positionEqualsPosition(leftBlockIndex, leftBlockPosition, leftBlockIndex, leftBlockPosition));
            // check equality of every position against every other position in the block
            for (int rightBlockIndex = 0; rightBlockIndex < varcharChannel.size(); rightBlockIndex++) {
                Block rightBlock = varcharChannel.get(rightBlockIndex);
                for (int rightBlockPosition = 0; rightBlockPosition < rightBlock.getPositionCount(); rightBlockPosition++) {
                    assertEquals(hashStrategy.positionEqualsPositionIgnoreNulls(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expectedHashStrategy.positionEqualsPositionIgnoreNulls(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition));
                    assertEquals(hashStrategy.positionEqualsPosition(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expectedHashStrategy.positionEqualsPosition(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition));
                }
            }
            // check equality of every position against every other position in the block cursor
            for (int rightBlockIndex = 0; rightBlockIndex < varcharChannel.size(); rightBlockIndex++) {
                Block[] rightBlocks = new Block[4];
                rightBlocks[0] = varcharChannel.get(rightBlockIndex);
                rightBlocks[1] = longChannel.get(rightBlockIndex);
                rightBlocks[2] = doubleChannel.get(rightBlockIndex);
                rightBlocks[3] = booleanChannel.get(rightBlockIndex);
                int rightPositionCount = varcharChannel.get(rightBlockIndex).getPositionCount();
                for (int rightPosition = 0; rightPosition < rightPositionCount; rightPosition++) {
                    boolean expected = expectedHashStrategy.positionEqualsRow(leftBlockIndex, leftBlockPosition, rightPosition, new Page(rightBlocks));
                    assertEquals(hashStrategy.positionEqualsRow(leftBlockIndex, leftBlockPosition, rightPosition, new Page(rightBlocks)), expected);
                    assertEquals(hashStrategy.rowEqualsRow(leftBlockPosition, new Page(leftBlocks), rightPosition, new Page(rightBlocks)), expected);
                    assertEquals(hashStrategy.positionEqualsRowIgnoreNulls(leftBlockIndex, leftBlockPosition, rightPosition, new Page(rightBlocks)), expected);
                }
            }
            // write position to output block
            pageBuilder.declarePosition();
            hashStrategy.appendTo(leftBlockIndex, leftBlockPosition, pageBuilder, 0);
        }
        // verify output block matches
        Page page = pageBuilder.build();
        if (hashEnabled) {
            assertPageEquals(outputTypes, page, new Page(varcharChannel.get(leftBlockIndex), longChannel.get(leftBlockIndex), doubleChannel.get(leftBlockIndex), booleanChannel.get(leftBlockIndex), extraChannel.get(leftBlockIndex), precomputedHash.get(leftBlockIndex)));
        } else {
            assertPageEquals(outputTypes, page, new Page(varcharChannel.get(leftBlockIndex), longChannel.get(leftBlockIndex), doubleChannel.get(leftBlockIndex), booleanChannel.get(leftBlockIndex), extraChannel.get(leftBlockIndex)));
        }
    }
}

Example 18 with PageBuilder

use of com.facebook.presto.spi.PageBuilder in project presto by prestodb.

the class TestJoinProbeCompiler method testSingleChannel.

@Test(dataProvider = "hashEnabledValues")
public void testSingleChannel(boolean hashEnabled) throws Exception {
    taskContext.addPipelineContext(0, true, true).addDriverContext();
    ImmutableList<Type> types = ImmutableList.of(VARCHAR, DOUBLE);
    ImmutableList<Type> outputTypes = ImmutableList.of(VARCHAR);
    List<Integer> outputChannels = ImmutableList.of(0);
    LookupSourceSupplierFactory lookupSourceSupplierFactory = joinCompiler.compileLookupSourceFactory(types, Ints.asList(0));
    // crate hash strategy with a single channel blocks -- make sure there is some overlap in values
    List<Block> varcharChannel = ImmutableList.of(BlockAssertions.createStringSequenceBlock(10, 20), BlockAssertions.createStringSequenceBlock(20, 30), BlockAssertions.createStringSequenceBlock(15, 25));
    List<Block> extraUnusedDoubleChannel = ImmutableList.of(BlockAssertions.createDoubleSequenceBlock(10, 20), BlockAssertions.createDoubleSequenceBlock(20, 30), BlockAssertions.createDoubleSequenceBlock(15, 25));
    LongArrayList addresses = new LongArrayList();
    for (int blockIndex = 0; blockIndex < varcharChannel.size(); blockIndex++) {
        Block block = varcharChannel.get(blockIndex);
        for (int positionIndex = 0; positionIndex < block.getPositionCount(); positionIndex++) {
            addresses.add(encodeSyntheticAddress(blockIndex, positionIndex));
        }
    }
    Optional<Integer> hashChannel = Optional.empty();
    List<List<Block>> channels = ImmutableList.of(varcharChannel, extraUnusedDoubleChannel);
    if (hashEnabled) {
        ImmutableList.Builder<Block> hashChannelBuilder = ImmutableList.builder();
        for (Block block : varcharChannel) {
            hashChannelBuilder.add(TypeUtils.getHashBlock(ImmutableList.<Type>of(VARCHAR), block));
        }
        types = ImmutableList.of(VARCHAR, DOUBLE, BigintType.BIGINT);
        hashChannel = Optional.of(2);
        channels = ImmutableList.of(varcharChannel, extraUnusedDoubleChannel, hashChannelBuilder.build());
        outputChannels = ImmutableList.of(0, 2);
        outputTypes = ImmutableList.of(VARCHAR, BigintType.BIGINT);
    }
    LookupSource lookupSource = lookupSourceSupplierFactory.createLookupSourceSupplier(taskContext.getSession().toConnectorSession(), addresses, channels, hashChannel, Optional.empty()).get();
    JoinProbeCompiler joinProbeCompiler = new JoinProbeCompiler();
    JoinProbeFactory probeFactory = joinProbeCompiler.internalCompileJoinProbe(types, outputChannels, Ints.asList(0), hashChannel);
    Page page = SequencePageBuilder.createSequencePage(types, 10, 10, 10);
    Page outputPage = new Page(page.getBlock(0));
    if (hashEnabled) {
        page = new Page(page.getBlock(0), page.getBlock(1), TypeUtils.getHashBlock(ImmutableList.of(VARCHAR), page.getBlock(0)));
        outputPage = new Page(page.getBlock(0), page.getBlock(2));
    }
    JoinProbe joinProbe = probeFactory.createJoinProbe(lookupSource, page);
    // verify channel count
    assertEquals(joinProbe.getOutputChannelCount(), outputChannels.size());
    PageBuilder pageBuilder = new PageBuilder(outputTypes);
    for (int position = 0; position < page.getPositionCount(); position++) {
        assertTrue(joinProbe.advanceNextPosition());
        pageBuilder.declarePosition();
        joinProbe.appendTo(pageBuilder);
        assertEquals(joinProbe.getCurrentJoinPosition(), lookupSource.getJoinPosition(position, page, page));
    }
    assertFalse(joinProbe.advanceNextPosition());
    assertPageEquals(outputTypes, pageBuilder.build(), outputPage);
}

Example 19 with PageBuilder

use of com.facebook.presto.spi.PageBuilder in project presto by prestodb.

the class BenchmarkPageProcessor method createInputPage.

public static Page createInputPage() {
    PageBuilder pageBuilder = new PageBuilder(ImmutableList.of(DOUBLE, DOUBLE, VARCHAR, DOUBLE));
    LineItemGenerator lineItemGenerator = new LineItemGenerator(1, 1, 1);
    Iterator<LineItem> iterator = lineItemGenerator.iterator();
    for (int i = 0; i < 10_000; i++) {
        pageBuilder.declarePosition();
        LineItem lineItem = iterator.next();
        DOUBLE.writeDouble(pageBuilder.getBlockBuilder(EXTENDED_PRICE), lineItem.getExtendedPrice());
        DOUBLE.writeDouble(pageBuilder.getBlockBuilder(DISCOUNT), lineItem.getDiscount());
        DATE.writeLong(pageBuilder.getBlockBuilder(SHIP_DATE), lineItem.getShipDate());
        DOUBLE.writeDouble(pageBuilder.getBlockBuilder(QUANTITY), lineItem.getQuantity());
    }
    return pageBuilder.build();
}

Example 20 with PageBuilder

use of com.facebook.presto.spi.PageBuilder in project presto by prestodb.

the class InCodeGeneratorBenchmark method setup.

@Setup
public void setup() {
    Random random = new Random();
    RowExpression[] arguments = new RowExpression[1 + inListCount];
    switch(type) {
        case StandardTypes.BIGINT:
            prestoType = BIGINT;
            for (int i = 1; i <= inListCount; i++) {
                arguments[i] = constant((long) random.nextInt(), BIGINT);
            }
            break;
        case StandardTypes.DOUBLE:
            prestoType = DOUBLE;
            for (int i = 1; i <= inListCount; i++) {
                arguments[i] = constant(random.nextDouble(), DOUBLE);
            }
            break;
        case StandardTypes.VARCHAR:
            prestoType = VARCHAR;
            for (int i = 1; i <= inListCount; i++) {
                arguments[i] = constant(Slices.utf8Slice(Long.toString(random.nextLong())), VARCHAR);
            }
            break;
        default:
            throw new IllegalStateException();
    }
    arguments[0] = field(0, prestoType);
    RowExpression project = field(0, prestoType);
    PageBuilder pageBuilder = new PageBuilder(ImmutableList.of(prestoType));
    for (int i = 0; i < 10_000; i++) {
        pageBuilder.declarePosition();
        switch(type) {
            case StandardTypes.BIGINT:
                BIGINT.writeLong(pageBuilder.getBlockBuilder(0), random.nextInt());
                break;
            case StandardTypes.DOUBLE:
                DOUBLE.writeDouble(pageBuilder.getBlockBuilder(0), random.nextDouble());
                break;
            case StandardTypes.VARCHAR:
                VARCHAR.writeSlice(pageBuilder.getBlockBuilder(0), Slices.utf8Slice(Long.toString(random.nextLong())));
                break;
        }
    }
    inputPage = pageBuilder.build();
    RowExpression filter = call(new Signature(IN, SCALAR, parseTypeSignature(StandardTypes.BOOLEAN)), BOOLEAN, arguments);
    processor = new ExpressionCompiler(MetadataManager.createTestMetadataManager()).compilePageProcessor(filter, ImmutableList.of(project)).get();
}