use of com.facebook.presto.orc.FileOrcDataSource in project presto by prestodb.
the class TestWriterBlockRawSize method testFileMetadataRawSize.
@Test
public void testFileMetadataRawSize() throws IOException {
Type type = INTEGER;
List<Type> types = ImmutableList.of(type);
int numBlocksPerRowGroup = 3;
int numBlocksPerStripe = numBlocksPerRowGroup * 5;
int numStripes = 4;
int numBlocksPerFile = numBlocksPerStripe * numStripes + 1;
BlockBuilder blockBuilder = type.createBlockBuilder(null, NUM_ELEMENTS * 2);
for (int i = 0; i < NUM_ELEMENTS; i++) {
blockBuilder.appendNull();
type.writeLong(blockBuilder, i);
}
long blockRawSize = ((FixedWidthType) type).getFixedSize() * NUM_ELEMENTS + NUM_ELEMENTS;
Block block = blockBuilder.build();
Block[] blocks = new Block[] { block };
OrcWriterOptions writerOptions = OrcWriterOptions.builder().withRowGroupMaxRowCount(block.getPositionCount() * numBlocksPerRowGroup).withFlushPolicy(DefaultOrcWriterFlushPolicy.builder().withStripeMaxRowCount(block.getPositionCount() * numBlocksPerStripe).build()).build();
for (OrcEncoding encoding : OrcEncoding.values()) {
try (TempFile tempFile = new TempFile()) {
OrcWriter writer = createOrcWriter(tempFile.getFile(), encoding, ZSTD, Optional.empty(), types, writerOptions, new OrcWriterStats());
for (int i = 0; i < numBlocksPerFile; i++) {
writer.write(new Page(blocks));
}
writer.close();
writer.validate(new FileOrcDataSource(tempFile.getFile(), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true));
Footer footer = OrcTester.getFileMetadata(tempFile.getFile(), encoding).getFooter();
verifyValue(encoding, footer.getRawSize(), blockRawSize * numBlocksPerFile);
assertEquals(footer.getStripes().size(), numStripes + 1);
int numBlocksRemaining = numBlocksPerFile;
for (StripeInformation stripeInfo : footer.getStripes()) {
int numBlocksInStripe = Math.min(numBlocksRemaining, numBlocksPerStripe);
verifyValue(encoding, stripeInfo.getRawDataSize(), blockRawSize * numBlocksInStripe);
numBlocksRemaining -= numBlocksInStripe;
}
}
}
}
use of com.facebook.presto.orc.FileOrcDataSource in project presto by prestodb.
the class OrcStorageManager method computeShardStats.
private List<ColumnStats> computeShardStats(File file) {
try (OrcDataSource dataSource = fileOrcDataSource(defaultReaderAttributes, file)) {
OrcReader reader = new OrcReader(dataSource, new OrcMetadataReader(), defaultReaderAttributes.getMaxMergeDistance(), defaultReaderAttributes.getMaxReadSize());
ImmutableList.Builder<ColumnStats> list = ImmutableList.builder();
for (ColumnInfo info : getColumnInfo(reader)) {
computeColumnStats(reader, info.getColumnId(), info.getType()).ifPresent(list::add);
}
return list.build();
} catch (IOException e) {
throw new PrestoException(RAPTOR_ERROR, "Failed to read file: " + file, e);
}
}
Aggregations