Search in sources :

Example 1 with OrcWriteValidationMode

use of io.trino.orc.OrcWriteValidation.OrcWriteValidationMode in project trino by trinodb.

the class TestOrcWriter method testWriteOutputStreamsInOrder.

@Test
public void testWriteOutputStreamsInOrder() throws IOException {
    for (OrcWriteValidationMode validationMode : OrcWriteValidationMode.values()) {
        TempFile tempFile = new TempFile();
        List<String> columnNames = ImmutableList.of("test1", "test2", "test3", "test4", "test5");
        List<Type> types = ImmutableList.of(VARCHAR, VARCHAR, VARCHAR, VARCHAR, VARCHAR);
        OrcWriter writer = new OrcWriter(new OutputStreamOrcDataSink(new FileOutputStream(tempFile.getFile())), ImmutableList.of("test1", "test2", "test3", "test4", "test5"), types, OrcType.createRootOrcType(columnNames, types), NONE, new OrcWriterOptions().withStripeMinSize(DataSize.of(0, MEGABYTE)).withStripeMaxSize(DataSize.of(32, MEGABYTE)).withStripeMaxRowCount(ORC_STRIPE_SIZE).withRowGroupMaxRowCount(ORC_ROW_GROUP_SIZE).withDictionaryMaxMemory(DataSize.of(32, MEGABYTE)).withBloomFilterColumns(ImmutableSet.copyOf(columnNames)), ImmutableMap.of(), true, validationMode, new OrcWriterStats());
        // write down some data with unsorted streams
        String[] data = new String[] { "a", "bbbbb", "ccc", "dd", "eeee" };
        Block[] blocks = new Block[data.length];
        int entries = 65536;
        BlockBuilder blockBuilder = VARCHAR.createBlockBuilder(null, entries);
        for (int i = 0; i < data.length; i++) {
            byte[] bytes = data[i].getBytes(UTF_8);
            for (int j = 0; j < entries; j++) {
                // force to write different data
                bytes[0] = (byte) ((bytes[0] + 1) % 128);
                blockBuilder.writeBytes(Slices.wrappedBuffer(bytes, 0, bytes.length), 0, bytes.length);
                blockBuilder.closeEntry();
            }
            blocks[i] = blockBuilder.build();
            blockBuilder = blockBuilder.newBlockBuilderLike(null);
        }
        writer.write(new Page(blocks));
        writer.close();
        // read the footer and verify the streams are ordered by size
        OrcDataSource orcDataSource = new FileOrcDataSource(tempFile.getFile(), READER_OPTIONS);
        Footer footer = OrcReader.createOrcReader(orcDataSource, READER_OPTIONS).orElseThrow(() -> new RuntimeException("File is empty")).getFooter();
        // OrcReader closes the original data source because it buffers the full file, so we need to reopen
        orcDataSource = new FileOrcDataSource(tempFile.getFile(), READER_OPTIONS);
        for (StripeInformation stripe : footer.getStripes()) {
            // read the footer
            Slice tailBuffer = orcDataSource.readFully(stripe.getOffset() + stripe.getIndexLength() + stripe.getDataLength(), toIntExact(stripe.getFooterLength()));
            try (InputStream inputStream = new OrcInputStream(OrcChunkLoader.create(orcDataSource.getId(), tailBuffer, Optional.empty(), newSimpleAggregatedMemoryContext()))) {
                StripeFooter stripeFooter = new OrcMetadataReader().readStripeFooter(footer.getTypes(), inputStream, ZoneId.of("UTC"));
                int size = 0;
                boolean dataStreamStarted = false;
                for (Stream stream : stripeFooter.getStreams()) {
                    if (isIndexStream(stream)) {
                        assertFalse(dataStreamStarted);
                        continue;
                    }
                    dataStreamStarted = true;
                    // verify sizes in order
                    assertGreaterThanOrEqual(stream.getLength(), size);
                    size = stream.getLength();
                }
            }
        }
    }
}
Also used : Page(io.trino.spi.Page) OrcWriteValidationMode(io.trino.orc.OrcWriteValidation.OrcWriteValidationMode) StripeReader.isIndexStream(io.trino.orc.StripeReader.isIndexStream) Stream(io.trino.orc.metadata.Stream) OrcInputStream(io.trino.orc.stream.OrcInputStream) FileOutputStream(java.io.FileOutputStream) InputStream(java.io.InputStream) BlockBuilder(io.trino.spi.block.BlockBuilder) OrcInputStream(io.trino.orc.stream.OrcInputStream) OrcInputStream(io.trino.orc.stream.OrcInputStream) InputStream(java.io.InputStream) OrcMetadataReader(io.trino.orc.metadata.OrcMetadataReader) Type(io.trino.spi.type.Type) OrcType(io.trino.orc.metadata.OrcType) StripeFooter(io.trino.orc.metadata.StripeFooter) Slice(io.airlift.slice.Slice) FileOutputStream(java.io.FileOutputStream) StripeFooter(io.trino.orc.metadata.StripeFooter) Footer(io.trino.orc.metadata.Footer) Block(io.trino.spi.block.Block) StripeInformation(io.trino.orc.metadata.StripeInformation) Test(org.testng.annotations.Test)

Aggregations

Slice (io.airlift.slice.Slice)1 OrcWriteValidationMode (io.trino.orc.OrcWriteValidation.OrcWriteValidationMode)1 StripeReader.isIndexStream (io.trino.orc.StripeReader.isIndexStream)1 Footer (io.trino.orc.metadata.Footer)1 OrcMetadataReader (io.trino.orc.metadata.OrcMetadataReader)1 OrcType (io.trino.orc.metadata.OrcType)1 Stream (io.trino.orc.metadata.Stream)1 StripeFooter (io.trino.orc.metadata.StripeFooter)1 StripeInformation (io.trino.orc.metadata.StripeInformation)1 OrcInputStream (io.trino.orc.stream.OrcInputStream)1 Page (io.trino.spi.Page)1 Block (io.trino.spi.block.Block)1 BlockBuilder (io.trino.spi.block.BlockBuilder)1 Type (io.trino.spi.type.Type)1 FileOutputStream (java.io.FileOutputStream)1 InputStream (java.io.InputStream)1 Test (org.testng.annotations.Test)1