Examples with StreamReader com.facebook.presto.orc.reader.StreamReader used on opensource projects

Example 1 with StreamReader

use of com.facebook.presto.orc.reader.StreamReader in project presto by prestodb.

the class OrcRecordReader method nextBatch.

public int nextBatch() throws IOException {
    // update position for current row group (advancing resets them)
    filePosition += currentBatchSize;
    currentPosition += currentBatchSize;
    // if next row is within the current group return
    if (nextRowInGroup >= currentGroupRowCount) {
        // attempt to advance to next row group
        if (!advanceToNextRowGroup()) {
            filePosition = fileRowCount;
            currentPosition = totalRowCount;
            return -1;
        }
    }
    currentBatchSize = toIntExact(min(MAX_BATCH_SIZE, currentGroupRowCount - nextRowInGroup));
    for (StreamReader column : streamReaders) {
        if (column != null) {
            column.prepareNextRead(currentBatchSize);
        }
    }
    nextRowInGroup += currentBatchSize;
    return currentBatchSize;
}

Example 2 with StreamReader

use of com.facebook.presto.orc.reader.StreamReader in project presto by prestodb.

the class OrcRecordReader method createStreamReaders.

private static StreamReader[] createStreamReaders(OrcDataSource orcDataSource, List<OrcType> types, DateTimeZone hiveStorageTimeZone, Map<Integer, Type> includedColumns) {
    List<StreamDescriptor> streamDescriptors = createStreamDescriptor("", "", 0, types, orcDataSource).getNestedStreams();
    OrcType rowType = types.get(0);
    StreamReader[] streamReaders = new StreamReader[rowType.getFieldCount()];
    for (int columnId = 0; columnId < rowType.getFieldCount(); columnId++) {
        if (includedColumns.containsKey(columnId)) {
            StreamDescriptor streamDescriptor = streamDescriptors.get(columnId);
            streamReaders[columnId] = StreamReaders.createStreamReader(streamDescriptor, hiveStorageTimeZone);
        }
    }
    return streamReaders;
}

Example 3 with StreamReader

use of com.facebook.presto.orc.reader.StreamReader in project presto by prestodb.

the class OrcRecordReader method advanceToNextStripe.

private void advanceToNextStripe() throws IOException {
    currentStripeSystemMemoryContext.close();
    currentStripeSystemMemoryContext = systemMemoryUsage.newAggregatedMemoryContext();
    rowGroups = ImmutableList.<RowGroup>of().iterator();
    currentStripe++;
    if (currentStripe >= stripes.size()) {
        return;
    }
    if (currentStripe > 0) {
        currentStripePosition += stripes.get(currentStripe - 1).getNumberOfRows();
    }
    StripeInformation stripeInformation = stripes.get(currentStripe);
    Stripe stripe = stripeReader.readStripe(stripeInformation, currentStripeSystemMemoryContext);
    if (stripe != null) {
        // Give readers access to dictionary streams
        StreamSources dictionaryStreamSources = stripe.getDictionaryStreamSources();
        List<ColumnEncoding> columnEncodings = stripe.getColumnEncodings();
        for (StreamReader column : streamReaders) {
            if (column != null) {
                column.startStripe(dictionaryStreamSources, columnEncodings);
            }
        }
        rowGroups = stripe.getRowGroups().iterator();
    }
}

Example 4 with StreamReader

use of com.facebook.presto.orc.reader.StreamReader in project presto by prestodb.

the class OrcRecordReader method advanceToNextRowGroup.

private boolean advanceToNextRowGroup() throws IOException {
    nextRowInGroup = 0;
    while (!rowGroups.hasNext() && currentStripe < stripes.size()) {
        advanceToNextStripe();
    }
    if (!rowGroups.hasNext()) {
        currentGroupRowCount = 0;
        return false;
    }
    RowGroup currentRowGroup = rowGroups.next();
    currentGroupRowCount = currentRowGroup.getRowCount();
    currentPosition = currentStripePosition + currentRowGroup.getRowOffset();
    filePosition = stripeFilePositions.get(currentStripe) + currentRowGroup.getRowOffset();
    // give reader data streams from row group
    StreamSources rowGroupStreamSources = currentRowGroup.getStreamSources();
    for (StreamReader column : streamReaders) {
        if (column != null) {
            column.startRowGroup(rowGroupStreamSources);
        }
    }
    return true;
}