Examples with LocalOrcDataEnvironment com.facebook.presto.raptor.filesystem.LocalOrcDataEnvironment used on opensource projects

Example 6 with LocalOrcDataEnvironment

use of com.facebook.presto.raptor.filesystem.LocalOrcDataEnvironment in project presto by prestodb.

the class TestLocalFileStorageService method setup.

@BeforeMethod
public void setup() {
    temporary = createTempDir();
    store = new LocalFileStorageService(new LocalOrcDataEnvironment(), temporary.toURI());
    store.start();
}

Example 7 with LocalOrcDataEnvironment

use of com.facebook.presto.raptor.filesystem.LocalOrcDataEnvironment in project presto by prestodb.

the class TestOrcFileRewriter method testRewriteAllRowsDeleted.

@Test
public void testRewriteAllRowsDeleted() throws Exception {
    List<Long> columnIds = ImmutableList.of(3L);
    List<Type> columnTypes = ImmutableList.of(BIGINT);
    File file = new File(temporary, randomUUID().toString());
    try (FileWriter writer = OrcTestingUtil.createFileWriter(columnIds, columnTypes, file)) {
        writer.appendPages(rowPagesBuilder(columnTypes).row(123L).row(456L).build());
    }
    BitSet rowsToDelete = new BitSet();
    rowsToDelete.set(0);
    rowsToDelete.set(1);
    File newFile = new File(temporary, randomUUID().toString());
    FileSystem fileSystem = new LocalOrcDataEnvironment().getFileSystem(DEFAULT_RAPTOR_CONTEXT);
    OrcFileInfo info = createFileRewriter().rewrite(fileSystem, getColumnTypes(columnIds, columnTypes), path(file), path(newFile), rowsToDelete);
    assertEquals(info.getRowCount(), 0);
    assertEquals(info.getUncompressedSize(), 0);
    assertFalse(newFile.exists());
}

Example 8 with LocalOrcDataEnvironment

use of com.facebook.presto.raptor.filesystem.LocalOrcDataEnvironment in project presto by prestodb.

the class TestOrcFileRewriter method testRewriteNoRowsDeleted.

@Test
public void testRewriteNoRowsDeleted() throws Exception {
    List<Long> columnIds = ImmutableList.of(3L);
    List<Type> columnTypes = ImmutableList.of(BIGINT);
    File file = new File(temporary, randomUUID().toString());
    try (FileWriter writer = OrcTestingUtil.createFileWriter(columnIds, columnTypes, file)) {
        writer.appendPages(rowPagesBuilder(columnTypes).row(123L).row(456L).build());
    }
    BitSet rowsToDelete = new BitSet();
    File newFile = new File(temporary, randomUUID().toString());
    FileSystem fileSystem = new LocalOrcDataEnvironment().getFileSystem(DEFAULT_RAPTOR_CONTEXT);
    OrcFileInfo info = createFileRewriter().rewrite(fileSystem, getColumnTypes(columnIds, columnTypes), path(file), path(newFile), rowsToDelete);
    assertEquals(info.getRowCount(), 2);
    assertBetweenInclusive(info.getUncompressedSize(), 16L, 16L * 2);
    assertEquals(readAllBytes(newFile.toPath()), readAllBytes(file.toPath()));
}

Example 9 with LocalOrcDataEnvironment

use of com.facebook.presto.raptor.filesystem.LocalOrcDataEnvironment in project presto by prestodb.

the class TestOrcFileRewriter method testUncompressedSize.

@Test
public void testUncompressedSize() throws Exception {
    List<Long> columnIds = ImmutableList.of(1L, 2L, 3L, 4L, 5L);
    List<Type> columnTypes = ImmutableList.of(BOOLEAN, BIGINT, DOUBLE, createVarcharType(10), VARBINARY);
    File file = new File(temporary, randomUUID().toString());
    try (FileWriter writer = OrcTestingUtil.createFileWriter(columnIds, columnTypes, file)) {
        List<Page> pages = rowPagesBuilder(columnTypes).row(true, 123L, 98.7, "hello", utf8Slice("abc")).row(false, 456L, 65.4, "world", utf8Slice("xyz")).row(null, null, null, null, null).build();
        writer.appendPages(pages);
    }
    File newFile = new File(temporary, randomUUID().toString());
    FileSystem fileSystem = new LocalOrcDataEnvironment().getFileSystem(DEFAULT_RAPTOR_CONTEXT);
    OrcFileInfo info = createFileRewriter().rewrite(fileSystem, getColumnTypes(columnIds, columnTypes), path(file), path(newFile), new BitSet());
    assertEquals(info.getRowCount(), 3);
    assertBetweenInclusive(info.getUncompressedSize(), 55L, 55L * 2);
}

Example 10 with LocalOrcDataEnvironment

use of com.facebook.presto.raptor.filesystem.LocalOrcDataEnvironment in project presto by prestodb.

the class TestOrcFileRewriter method testRewriteWithoutMetadata.

@Test
public void testRewriteWithoutMetadata() throws Exception {
    List<Long> columnIds = ImmutableList.of(3L, 7L);
    List<Type> columnTypes = ImmutableList.of(BIGINT, createVarcharType(20));
    File file = new File(temporary, randomUUID().toString());
    try (FileWriter writer = createFileWriter(columnIds, columnTypes, file, false)) {
        List<Page> pages = rowPagesBuilder(columnTypes).row(123L, "hello").row(777L, "sky").build();
        writer.appendPages(pages);
    }
    try (OrcDataSource dataSource = fileOrcDataSource(file)) {
        OrcBatchRecordReader reader = createReader(dataSource, columnIds, columnTypes);
        assertEquals(reader.getReaderRowCount(), 2);
        assertEquals(reader.getFileRowCount(), 2);
        assertEquals(reader.getSplitLength(), file.length());
        assertEquals(reader.nextBatch(), 2);
        Block column0 = reader.readBlock(0);
        assertEquals(column0.getPositionCount(), 2);
        for (int i = 0; i < 2; i++) {
            assertEquals(column0.isNull(i), false);
        }
        assertEquals(BIGINT.getLong(column0, 0), 123L);
        assertEquals(BIGINT.getLong(column0, 1), 777L);
        Block column1 = reader.readBlock(1);
        assertEquals(column1.getPositionCount(), 2);
        for (int i = 0; i < 2; i++) {
            assertEquals(column1.isNull(i), false);
        }
        assertEquals(createVarcharType(20).getSlice(column1, 0), utf8Slice("hello"));
        assertEquals(createVarcharType(20).getSlice(column1, 1), utf8Slice("sky"));
        assertFalse(reader.getUserMetadata().containsKey(OrcFileMetadata.KEY));
    }
    BitSet rowsToDelete = new BitSet(5);
    rowsToDelete.set(1);
    File newFile = new File(temporary, randomUUID().toString());
    FileSystem fileSystem = new LocalOrcDataEnvironment().getFileSystem(DEFAULT_RAPTOR_CONTEXT);
    OrcFileInfo info = createFileRewriter().rewrite(fileSystem, getColumnTypes(columnIds, columnTypes), path(file), path(newFile), rowsToDelete);
    assertEquals(info.getRowCount(), 1);
    assertBetweenInclusive(info.getUncompressedSize(), 13L, 13L * 2);
    try (OrcDataSource dataSource = fileOrcDataSource(newFile)) {
        OrcBatchRecordReader reader = createReader(dataSource, columnIds, columnTypes);
        assertEquals(reader.getReaderRowCount(), 1);
        assertEquals(reader.getFileRowCount(), 1);
        assertEquals(reader.getSplitLength(), newFile.length());
        assertEquals(reader.nextBatch(), 1);
        Block column0 = reader.readBlock(0);
        assertEquals(column0.getPositionCount(), 1);
        assertEquals(column0.isNull(0), false);
        assertEquals(BIGINT.getLong(column0, 0), 123L);
        Block column1 = reader.readBlock(1);
        assertEquals(column1.getPositionCount(), 1);
        assertEquals(column1.isNull(0), false);
        assertEquals(createVarcharType(20).getSlice(column1, 0), utf8Slice("hello"));
        assertFalse(reader.getUserMetadata().containsKey(OrcFileMetadata.KEY));
    }
}