Examples with PageReadStore org.apache.parquet.column.page.PageReadStore used on opensource projects

Example 31 with PageReadStore

use of org.apache.parquet.column.page.PageReadStore in project parquet-mr by apache.

the class InternalParquetRecordReader method checkRead.

private void checkRead() throws IOException {
    if (current == totalCountLoadedSoFar) {
        if (current != 0) {
            totalTimeSpentProcessingRecords += (System.currentTimeMillis() - startedAssemblingCurrentBlockAt);
            if (LOG.isInfoEnabled()) {
                LOG.info("Assembled and processed " + totalCountLoadedSoFar + " records from " + columnCount + " columns in " + totalTimeSpentProcessingRecords + " ms: " + ((float) totalCountLoadedSoFar / totalTimeSpentProcessingRecords) + " rec/ms, " + ((float) totalCountLoadedSoFar * columnCount / totalTimeSpentProcessingRecords) + " cell/ms");
                final long totalTime = totalTimeSpentProcessingRecords + totalTimeSpentReadingBytes;
                if (totalTime != 0) {
                    final long percentReading = 100 * totalTimeSpentReadingBytes / totalTime;
                    final long percentProcessing = 100 * totalTimeSpentProcessingRecords / totalTime;
                    LOG.info("time spent so far " + percentReading + "% reading (" + totalTimeSpentReadingBytes + " ms) and " + percentProcessing + "% processing (" + totalTimeSpentProcessingRecords + " ms)");
                }
            }
        }
        LOG.info("at row " + current + ". reading next block");
        long t0 = System.currentTimeMillis();
        PageReadStore pages = reader.readNextFilteredRowGroup();
        if (pages == null) {
            throw new IOException("expecting more rows but reached last block. Read " + current + " out of " + total);
        }
        long timeSpentReading = System.currentTimeMillis() - t0;
        totalTimeSpentReadingBytes += timeSpentReading;
        BenchmarkCounter.incrementTime(timeSpentReading);
        if (LOG.isInfoEnabled())
            LOG.info("block read in memory in {} ms. row count = {}", timeSpentReading, pages.getRowCount());
        LOG.debug("initializing Record assembly with requested schema {}", requestedSchema);
        MessageColumnIO columnIO = columnIOFactory.getColumnIO(requestedSchema, fileSchema, strictTypeChecking);
        recordReader = columnIO.getRecordReader(pages, recordConverter, filterRecords ? filter : FilterCompat.NOOP);
        startedAssemblingCurrentBlockAt = System.currentTimeMillis();
        totalCountLoadedSoFar += pages.getRowCount();
        ++currentBlock;
    }
}