Examples with ColumnReader io.prestosql.orc.reader.ColumnReader used on opensource projects

Example 1 with ColumnReader

use of io.prestosql.orc.reader.ColumnReader in project hetu-core by openlookeng.

the class OrcRecordReader method createColumnReaders.

private ColumnReader[] createColumnReaders(List<OrcColumn> columns, List<Type> readTypes, AggregatedMemoryContext systemMemoryContext, OrcBlockFactory blockFactory, OrcCacheStore orcCacheStore, OrcCacheProperties orcCacheProperties) throws OrcCorruptionException {
    ColumnReader[] columnReaders = new ColumnReader[columns.size()];
    for (int i = 0; i < columns.size(); i++) {
        int columnIndex = i;
        Type readType = readTypes.get(columnIndex);
        OrcColumn column = columns.get(columnIndex);
        ColumnReader columnReader = createColumnReader(readType, column, systemMemoryContext, blockFactory.createNestedBlockFactory(block -> blockLoaded(columnIndex, block)));
        if (orcCacheProperties.isRowDataCacheEnabled()) {
            columnReader = ColumnReaders.wrapWithCachingStreamReader(columnReader, column, orcCacheStore.getRowDataCache());
        }
        columnReaders[columnIndex] = columnReader;
    }
    return columnReaders;
}

Example 2 with ColumnReader

use of io.prestosql.orc.reader.ColumnReader in project hetu-core by openlookeng.

the class OrcRecordReader method validateWritePageChecksum.

private void validateWritePageChecksum(int batchSize) throws IOException {
    if (writeChecksumBuilder.isPresent()) {
        ColumnReader[] columnsReader = getColumnReaders();
        Block[] blocks = new Block[columnsReader.length];
        for (int columnIndex = 0; columnIndex < columnsReader.length; columnIndex++) {
            Block block = columnsReader[columnIndex].readBlock();
            blocks[columnIndex] = block;
            blockLoaded(columnIndex, block);
        }
        Page page = new Page(batchSize, blocks);
        validateWritePageChecksum(page);
    }
}

Example 3 with ColumnReader

use of io.prestosql.orc.reader.ColumnReader in project hetu-core by openlookeng.

the class OrcRecordReader method nextPage.

public Page nextPage() throws IOException {
    ColumnReader[] columnsReader = getColumnReaders();
    int batchSize = prepareNextBatch();
    if (batchSize < 0) {
        return null;
    }
    for (ColumnReader column : columnsReader) {
        if (column != null) {
            column.prepareNextRead(batchSize);
        }
    }
    batchRead(batchSize);
    matchingRowsInBatchArray = null;
    validateWritePageChecksum(batchSize);
    // create a lazy page
    blockFactory.nextPage();
    Arrays.fill(currentBytesPerCell, 0);
    Block[] blocks = new Block[columnsReader.length];
    for (int i = 0; i < columnsReader.length; i++) {
        int columnIndex = i;
        blocks[columnIndex] = blockFactory.createBlock(batchSize, () -> filterRows(columnsReader[columnIndex].readBlock()), block -> blockLoaded(columnIndex, block));
    }
    // only include page metadata if enabled
    if (pageMetadataEnabled) {
        Properties pageMetadata = new Properties();
        pageCount++;
        pageMetadata.setProperty(DATASOURCE_PAGE_NUMBER, String.valueOf(pageCount));
        if (isCurrentStripeFinished()) {
            // Only set the total page count when the current stripe has finished
            // Therefore whenever this property is available in pageMetaData,
            // it indicates that the stripe has finished and this is the last page
            pageMetadata.setProperty(DATASOURCE_TOTAL_PAGES, String.valueOf(pageCount));
            pageCount = 0;
        }
        pageMetadata.setProperty(DATASOURCE_STRIPE_NUMBER, String.valueOf(currentStripe));
        pageMetadata.setProperty(DATASOURCE_STRIPE_OFFSET, String.valueOf(stripes.get(currentStripe).getOffset()));
        pageMetadata.setProperty(DATASOURCE_STRIPE_LENGTH, String.valueOf(stripes.get(currentStripe).getTotalLength()));
        if (splitMetadata != null) {
            // Skip setting for testing (splitMetadata set as null)
            pageMetadata.setProperty(DATASOURCE_FILE_PATH, splitMetadata.getSplitIdentity());
            pageMetadata.setProperty(DATASOURCE_FILE_MODIFICATION, String.valueOf(splitMetadata.getLastModifiedTime()));
        }
        pageMetadata.setProperty(DATASOURCE_INDEX_LEVEL, "STRIPE");
        return new Page(batchSize, pageMetadata, blocks);
    } else {
        return new Page(batchSize, blocks);
    }
}

Example 4 with ColumnReader

use of io.prestosql.orc.reader.ColumnReader in project hetu-core by openlookeng.

the class TestAbstractNumbericColumnReader method testTypeCoercionDate.

@Test
public void testTypeCoercionDate() throws OrcCorruptionException {
    OrcColumn column = new OrcColumn("hdfs://hacluster/user/hive/warehouse/tpcds_orc_hive_1000.db/catalog_sales/cs_sold_date_sk=2452268/000896_0", new OrcColumnId(3), "cs_order_number", OrcType.OrcTypeKind.DATE, new OrcDataSourceId("hdfs://hacluster/user/hive/warehouse/tpcds_orc_hive_1000.db/catalog_sales/cs_sold_date_sk=2452268/000896_0"), ImmutableList.of());
    ColumnReader actualDateColumnReader = ColumnReaders.createColumnReader(type, column, AggregatedMemoryContext.newSimpleAggregatedMemoryContext(), null);
    DateColumnReader expectedDateColumnReader = new DateColumnReader(type, column, AggregatedMemoryContext.newSimpleAggregatedMemoryContext().newLocalMemoryContext(ColumnReaders.class.getSimpleName()));
    assertEquals(actualDateColumnReader.toString(), expectedDateColumnReader.toString());
}

Example 5 with ColumnReader

use of io.prestosql.orc.reader.ColumnReader in project hetu-core by openlookeng.

the class TestAbstractNumbericColumnReader method testTypeCoercionBigInt.

@Test
public void testTypeCoercionBigInt() throws OrcCorruptionException {
    OrcColumn column = new OrcColumn("hdfs://hacluster/user/hive/warehouse/tpcds_orc_hive_1000.db/catalog_sales/cs_sold_date_sk=2452268/000896_0", new OrcColumnId(3), "cs_order_number", OrcType.OrcTypeKind.LONG, new OrcDataSourceId("hdfs://hacluster/user/hive/warehouse/tpcds_orc_hive_1000.db/catalog_sales/cs_sold_date_sk=2452268/000896_0"), ImmutableList.of());
    ColumnReader actualLongColumnReader = ColumnReaders.createColumnReader(type, column, AggregatedMemoryContext.newSimpleAggregatedMemoryContext(), null);
    LongColumnReader expectedLongColumnReader = new LongColumnReader(type, column, AggregatedMemoryContext.newSimpleAggregatedMemoryContext().newLocalMemoryContext(ColumnReaders.class.getSimpleName()));
    assertEquals(actualLongColumnReader.toString(), expectedLongColumnReader.toString());
}