Examples with BlockBuilder com.facebook.presto.common.block.BlockBuilder used on opensource projects

Example 41 with BlockBuilder

use of com.facebook.presto.common.block.BlockBuilder in project presto by prestodb.

the class FloatBatchStreamReader method readBlock.

@Override
public Block readBlock() throws IOException {
    if (!rowGroupOpen) {
        openRowGroup();
    }
    if (readOffset > 0) {
        if (presentStream != null) {
            // skip ahead the present bit reader, but count the set bits
            // and use this as the skip size for the data reader
            readOffset = presentStream.countBitsSet(readOffset);
        }
        if (readOffset > 0) {
            if (dataStream == null) {
                throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but data stream is not present");
            }
            dataStream.skip(readOffset);
        }
    }
    if (dataStream == null && presentStream != null) {
        presentStream.skip(nextBatchSize);
        Block nullValueBlock = RunLengthEncodedBlock.create(REAL, null, nextBatchSize);
        readOffset = 0;
        nextBatchSize = 0;
        return nullValueBlock;
    }
    BlockBuilder builder = REAL.createBlockBuilder(null, nextBatchSize);
    if (presentStream == null) {
        if (dataStream == null) {
            throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but data stream is not present");
        }
        dataStream.nextVector(REAL, nextBatchSize, builder);
    } else {
        for (int i = 0; i < nextBatchSize; i++) {
            if (presentStream.nextBit()) {
                REAL.writeLong(builder, floatToRawIntBits(dataStream.next()));
            } else {
                builder.appendNull();
            }
        }
    }
    readOffset = 0;
    nextBatchSize = 0;
    return builder.build();
}

Example 42 with BlockBuilder

use of com.facebook.presto.common.block.BlockBuilder in project presto by prestodb.

the class MapFlatBatchStreamReader method getIntegerKeyBlockTemplate.

private Block getIntegerKeyBlockTemplate(Collection<DwrfSequenceEncoding> sequenceEncodings) {
    Type keyType;
    switch(keyOrcType) {
        case BYTE:
            keyType = TinyintType.TINYINT;
            break;
        case SHORT:
            keyType = SmallintType.SMALLINT;
            break;
        case INT:
            keyType = IntegerType.INTEGER;
            break;
        case LONG:
            keyType = BigintType.BIGINT;
            break;
        default:
            throw new IllegalArgumentException("Unsupported flat map key type: " + keyOrcType);
    }
    BlockBuilder blockBuilder = keyType.createBlockBuilder(null, sequenceEncodings.size());
    for (DwrfSequenceEncoding sequenceEncoding : sequenceEncodings) {
        keyType.writeLong(blockBuilder, sequenceEncoding.getKey().getIntKey());
    }
    return blockBuilder.build();
}

Example 43 with BlockBuilder

use of com.facebook.presto.common.block.BlockBuilder in project presto by prestodb.

the class TestPinotBrokerPageSourcePql method testPopulateFromPql.

@Test(dataProvider = "pqlResponses")
public void testPopulateFromPql(String pql, String pqlResponse, List<PinotColumnHandle> actualHandles, List<Integer> expectedColumnIndices, List<PinotColumnHandle> expectedColumnHandles, Optional<Class<? extends PrestoException>> expectedError) throws IOException {
    PqlParsedInfo pqlParsedInfo = getBasicInfoFromPql(pqlResponse);
    PinotQueryGenerator.GeneratedPinotQuery generatedPql = new PinotQueryGenerator.GeneratedPinotQuery(pinotTable.getTableName(), pql, PinotQueryGenerator.PinotQueryFormat.PQL, expectedColumnIndices, pqlParsedInfo.groupByColumns, false, false);
    PinotBrokerPageSourcePql pageSource = new PinotBrokerPageSourcePql(pinotConfig, new TestingConnectorSession(ImmutableList.of(booleanProperty("mark_data_fetch_exceptions_as_retriable", "Retry Pinot query on data fetch exceptions", pinotConfig.isMarkDataFetchExceptionsAsRetriable(), false))), generatedPql, actualHandles, expectedColumnHandles, new MockPinotClusterInfoFetcher(pinotConfig), objectMapper);
    PinotBrokerPageSourceBase.BlockAndTypeBuilder blockAndTypeBuilder = pageSource.buildBlockAndTypeBuilder(actualHandles, generatedPql);
    validateExpectedColumnIndices(expectedColumnIndices, expectedColumnHandles);
    List<BlockBuilder> columnBlockBuilders = blockAndTypeBuilder.getColumnBlockBuilders();
    List<Type> columnTypes = blockAndTypeBuilder.getColumnTypes();
    assertEquals(columnTypes.size(), columnBlockBuilders.size());
    int numNonNullTypes = 0;
    for (int i = 0; i < columnTypes.size(); i++) {
        Type type = columnTypes.get(i);
        BlockBuilder builder = columnBlockBuilders.get(i);
        assertEquals(type == null, builder == null);
        if (type != null) {
            numNonNullTypes++;
        }
    }
    assertEquals(numNonNullTypes, actualHandles.size());
    Optional<? extends PrestoException> thrown = Optional.empty();
    int rows = -1;
    try {
        rows = pageSource.populateFromQueryResults(generatedPql, columnBlockBuilders, columnTypes, pqlResponse);
    } catch (PrestoException e) {
        thrown = Optional.of(e);
    }
    Optional<? extends Class<? extends PrestoException>> thrownType = thrown.map(e -> e.getClass());
    Optional<String> errorString = thrown.map(e -> Throwables.getStackTraceAsString(e));
    assertEquals(thrownType, expectedError, String.format("Expected error %s, but got error of type %s: %s", expectedError, thrownType, errorString));
    if (!expectedError.isPresent()) {
        assertEquals(expectedColumnIndices.size(), pqlParsedInfo.columns);
        assertEquals(rows, pqlParsedInfo.rows);
    }
}

Example 44 with BlockBuilder

use of com.facebook.presto.common.block.BlockBuilder in project presto by prestodb.

the class PinotBrokerPageSourcePql method buildBlockAndTypeBuilder.

@Override
@VisibleForTesting
public BlockAndTypeBuilder buildBlockAndTypeBuilder(List<PinotColumnHandle> columnHandles, PinotQueryGenerator.GeneratedPinotQuery brokerPql) {
    // When we created the PQL, we came up with some column handles
    // however other optimizers post-pushdown can come in and prune/re-order the required column handles
    // so we need to map from the column handles the PQL corresponds to, to the actual column handles
    // needed in the scan.
    List<Type> expectedTypes = columnHandles.stream().map(PinotColumnHandle::getDataType).collect(Collectors.toList());
    PageBuilder pageBuilder = new PageBuilder(expectedTypes);
    checkState(brokerPql.getExpectedColumnIndices().size() >= expectedHandles.size());
    checkState(expectedHandles.size() >= columnHandles.size());
    // The expectedColumnHandles are the handles corresponding to the generated PQL
    // However, the engine could end up requesting only a permutation/subset of those handles
    // during the actual scan
    // Map the handles from planning time to the handles asked in the scan
    // so that we know which columns to discard.
    int[] handleMapping = new int[expectedHandles.size()];
    for (int i = 0; i < handleMapping.length; ++i) {
        handleMapping[i] = columnHandles.indexOf(expectedHandles.get(i));
    }
    ArrayList<BlockBuilder> columnBlockBuilders = new ArrayList<>();
    ArrayList<Type> columnTypes = new ArrayList<>();
    for (int expectedColumnIndex : brokerPql.getExpectedColumnIndices()) {
        // columnIndex is the index of this column in the current scan
        // It is obtained from the mapping and can be -ve, which means that the
        // expectedColumnIndex'th column returned by Pinot can be discarded.
        int columnIndex = -1;
        if (expectedColumnIndex >= 0) {
            columnIndex = handleMapping[expectedColumnIndex];
        }
        columnBlockBuilders.add(columnIndex >= 0 ? pageBuilder.getBlockBuilder(columnIndex) : null);
        columnTypes.add(columnIndex >= 0 ? expectedTypes.get(columnIndex) : null);
    }
    return new BlockAndTypeBuilder(pageBuilder, columnBlockBuilders, columnTypes);
}

Example 45 with BlockBuilder

use of com.facebook.presto.common.block.BlockBuilder in project presto by prestodb.

the class PinotBrokerPageSourceSql method buildBlockAndTypeBuilder.

@VisibleForTesting
@Override
public BlockAndTypeBuilder buildBlockAndTypeBuilder(List<PinotColumnHandle> columnHandles, GeneratedPinotQuery brokerSql) {
    // When we created the SQL, we came up with some column handles
    // however other optimizers post-pushdown can come in and prune/re-order the required column handles
    // so we need to map from the column handles the PQL corresponds to, to the actual column handles
    // needed in the scan.
    List<Type> expectedTypes = columnHandles.stream().map(PinotColumnHandle::getDataType).collect(Collectors.toList());
    PageBuilder pageBuilder = new PageBuilder(expectedTypes);
    // The expectedColumnHandles are the handles corresponding to the generated SQL
    // However, the engine could end up requesting only a permutation/subset of those handles
    // during the actual scan
    // Map the handles from planning time to the handles asked in the scan
    // so that we know which columns to discard.
    int[] handleMapping = new int[expectedHandles.size()];
    for (int i = 0; i < handleMapping.length; ++i) {
        handleMapping[i] = columnHandles.indexOf(expectedHandles.get(i));
    }
    ArrayList<BlockBuilder> columnBlockBuilders = new ArrayList<>();
    ArrayList<Type> columnTypes = new ArrayList<>();
    for (int expectedColumnIndex : brokerSql.getExpectedColumnIndices()) {
        // columnIndex is the index of this column in the current scan
        // It is obtained from the mapping and can be -ve, which means that the
        // expectedColumnIndex'th column returned by Pinot can be discarded.
        int columnIndex = -1;
        if (expectedColumnIndex >= 0) {
            columnIndex = handleMapping[expectedColumnIndex];
        }
        columnBlockBuilders.add(columnIndex >= 0 ? pageBuilder.getBlockBuilder(columnIndex) : null);
        columnTypes.add(columnIndex >= 0 ? expectedTypes.get(columnIndex) : null);
    }
    return new BlockAndTypeBuilder(pageBuilder, columnBlockBuilders, columnTypes);
}