use of com.facebook.presto.common.Page in project presto by prestodb.
the class TestOrcStorageManager method testReader.
@Test
public void testReader() throws Exception {
OrcStorageManager manager = createOrcStorageManager();
List<Long> columnIds = ImmutableList.of(2L, 4L, 6L, 7L, 8L, 9L);
List<Type> columnTypes = ImmutableList.of(BIGINT, createVarcharType(10), VARBINARY, DATE, BOOLEAN, DOUBLE);
byte[] bytes1 = octets(0x00, 0xFE, 0xFF);
byte[] bytes3 = octets(0x01, 0x02, 0x19, 0x80);
StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes);
Object[][] doubles = { { 881L, "-inf", null, null, null, Double.NEGATIVE_INFINITY }, { 882L, "+inf", null, null, null, Double.POSITIVE_INFINITY }, { 883L, "nan", null, null, null, Double.NaN }, { 884L, "min", null, null, null, Double.MIN_VALUE }, { 885L, "max", null, null, null, Double.MAX_VALUE }, { 886L, "pzero", null, null, null, 0.0 }, { 887L, "nzero", null, null, null, -0.0 } };
List<Page> pages = rowPagesBuilder(columnTypes).row(123L, "hello", wrappedBuffer(bytes1), sqlDate(2001, 8, 22).getDays(), true, 123.45).row(null, null, null, null, null, null).row(456L, "bye", wrappedBuffer(bytes3), sqlDate(2005, 4, 22).getDays(), false, 987.65).rows(doubles).build();
sink.appendPages(pages);
List<ShardInfo> shards = getFutureValue(sink.commit());
assertEquals(shards.size(), 1);
UUID uuid = Iterables.getOnlyElement(shards).getShardUuid();
MaterializedResult expected = resultBuilder(SESSION, columnTypes).row(123L, "hello", sqlBinary(bytes1), sqlDate(2001, 8, 22), true, 123.45).row(null, null, null, null, null, null).row(456L, "bye", sqlBinary(bytes3), sqlDate(2005, 4, 22), false, 987.65).rows(doubles).build();
// no tuple domain (all)
TupleDomain<RaptorColumnHandle> tupleDomain = TupleDomain.all();
try (ConnectorPageSource pageSource = getPageSource(manager, columnIds, columnTypes, uuid, tupleDomain)) {
MaterializedResult result = materializeSourceDataStream(SESSION, pageSource, columnTypes);
assertEquals(result.getRowCount(), expected.getRowCount());
assertEquals(result, expected);
}
// tuple domain within the column range
tupleDomain = TupleDomain.fromFixedValues(ImmutableMap.<RaptorColumnHandle, NullableValue>builder().put(new RaptorColumnHandle("test", "c1", 2, BIGINT), NullableValue.of(BIGINT, 124L)).build());
try (ConnectorPageSource pageSource = getPageSource(manager, columnIds, columnTypes, uuid, tupleDomain)) {
MaterializedResult result = materializeSourceDataStream(SESSION, pageSource, columnTypes);
assertEquals(result.getRowCount(), expected.getRowCount());
}
// tuple domain outside the column range
tupleDomain = TupleDomain.fromFixedValues(ImmutableMap.<RaptorColumnHandle, NullableValue>builder().put(new RaptorColumnHandle("test", "c1", 2, BIGINT), NullableValue.of(BIGINT, 122L)).build());
try (ConnectorPageSource pageSource = getPageSource(manager, columnIds, columnTypes, uuid, tupleDomain)) {
MaterializedResult result = materializeSourceDataStream(SESSION, pageSource, columnTypes);
assertEquals(result.getRowCount(), 0);
}
}
use of com.facebook.presto.common.Page in project presto by prestodb.
the class TestOrcStorageManager method testWriterRollback.
public void testWriterRollback() {
// verify staging directory is empty
File staging = new File(new File(temporary, "data"), "staging");
assertDirectory(staging);
assertEquals(staging.list(), new String[] {});
// create a shard in staging
OrcStorageManager manager = createOrcStorageManager();
List<Long> columnIds = ImmutableList.of(3L, 7L);
List<Type> columnTypes = ImmutableList.of(BIGINT, createVarcharType(10));
StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes);
List<Page> pages = rowPagesBuilder(columnTypes).row(123L, "hello").row(456L, "bye").build();
sink.appendPages(pages);
sink.flush();
// verify shard exists in staging
String[] files = staging.list();
assertNotNull(files);
String stagingFile = Arrays.stream(files).filter(file -> file.endsWith(".orc")).findFirst().orElseThrow(() -> new AssertionError("file not found in staging"));
// rollback should cleanup staging files
sink.rollback();
files = staging.list();
assertNotNull(files);
assertTrue(Arrays.stream(files).noneMatch(stagingFile::equals));
}
use of com.facebook.presto.common.Page in project presto by prestodb.
the class TestRaptorConnector method assertSplitShard.
private void assertSplitShard(Type temporalType, String min, String max, String userTimeZone, int expectedSplits) throws Exception {
ConnectorSession session = new TestingConnectorSession("user", Optional.of("test"), Optional.empty(), getTimeZoneKey(userTimeZone), ENGLISH, System.currentTimeMillis(), new RaptorSessionProperties(new StorageManagerConfig()).getSessionProperties(), ImmutableMap.of(), true, Optional.empty(), ImmutableSet.of(), Optional.empty(), ImmutableMap.of());
ConnectorTransactionHandle transaction = connector.beginTransaction(READ_COMMITTED, false);
connector.getMetadata(transaction).createTable(SESSION, new ConnectorTableMetadata(new SchemaTableName("test", "test"), ImmutableList.of(new ColumnMetadata("id", BIGINT), new ColumnMetadata("time", temporalType)), ImmutableMap.of(TEMPORAL_COLUMN_PROPERTY, "time", TABLE_SUPPORTS_DELTA_DELETE, false)), false);
connector.commit(transaction);
ConnectorTransactionHandle txn1 = connector.beginTransaction(READ_COMMITTED, false);
ConnectorTableHandle handle1 = getTableHandle(connector.getMetadata(txn1), "test");
ConnectorInsertTableHandle insertTableHandle = connector.getMetadata(txn1).beginInsert(session, handle1);
ConnectorPageSink raptorPageSink = connector.getPageSinkProvider().createPageSink(txn1, session, insertTableHandle, PageSinkContext.defaultContext());
Object timestamp1 = null;
Object timestamp2 = null;
if (temporalType.equals(TIMESTAMP)) {
timestamp1 = new SqlTimestamp(parseTimestampLiteral(getTimeZoneKey(userTimeZone), min), getTimeZoneKey(userTimeZone));
timestamp2 = new SqlTimestamp(parseTimestampLiteral(getTimeZoneKey(userTimeZone), max), getTimeZoneKey(userTimeZone));
} else if (temporalType.equals(DATE)) {
timestamp1 = new SqlDate(parseDate(min));
timestamp2 = new SqlDate(parseDate(max));
}
Page inputPage = MaterializedResult.resultBuilder(session, ImmutableList.of(BIGINT, temporalType)).row(1L, timestamp1).row(2L, timestamp2).build().toPage();
raptorPageSink.appendPage(inputPage);
Collection<Slice> shards = raptorPageSink.finish().get();
assertEquals(shards.size(), expectedSplits);
connector.getMetadata(txn1).dropTable(session, handle1);
connector.commit(txn1);
}
use of com.facebook.presto.common.Page in project presto by prestodb.
the class TestOrcFileRewriter method testUncompressedSize.
@Test
public void testUncompressedSize() throws Exception {
List<Long> columnIds = ImmutableList.of(1L, 2L, 3L, 4L, 5L);
List<Type> columnTypes = ImmutableList.of(BOOLEAN, BIGINT, DOUBLE, createVarcharType(10), VARBINARY);
File file = new File(temporary, randomUUID().toString());
try (FileWriter writer = OrcTestingUtil.createFileWriter(columnIds, columnTypes, file)) {
List<Page> pages = rowPagesBuilder(columnTypes).row(true, 123L, 98.7, "hello", utf8Slice("abc")).row(false, 456L, 65.4, "world", utf8Slice("xyz")).row(null, null, null, null, null).build();
writer.appendPages(pages);
}
File newFile = new File(temporary, randomUUID().toString());
FileSystem fileSystem = new LocalOrcDataEnvironment().getFileSystem(DEFAULT_RAPTOR_CONTEXT);
OrcFileInfo info = createFileRewriter().rewrite(fileSystem, getColumnTypes(columnIds, columnTypes), path(file), path(newFile), new BitSet());
assertEquals(info.getRowCount(), 3);
assertBetweenInclusive(info.getUncompressedSize(), 55L, 55L * 2);
}
use of com.facebook.presto.common.Page in project presto by prestodb.
the class TestOrcFileRewriter method testRewriteWithoutMetadata.
@Test
public void testRewriteWithoutMetadata() throws Exception {
List<Long> columnIds = ImmutableList.of(3L, 7L);
List<Type> columnTypes = ImmutableList.of(BIGINT, createVarcharType(20));
File file = new File(temporary, randomUUID().toString());
try (FileWriter writer = createFileWriter(columnIds, columnTypes, file, false)) {
List<Page> pages = rowPagesBuilder(columnTypes).row(123L, "hello").row(777L, "sky").build();
writer.appendPages(pages);
}
try (OrcDataSource dataSource = fileOrcDataSource(file)) {
OrcBatchRecordReader reader = createReader(dataSource, columnIds, columnTypes);
assertEquals(reader.getReaderRowCount(), 2);
assertEquals(reader.getFileRowCount(), 2);
assertEquals(reader.getSplitLength(), file.length());
assertEquals(reader.nextBatch(), 2);
Block column0 = reader.readBlock(0);
assertEquals(column0.getPositionCount(), 2);
for (int i = 0; i < 2; i++) {
assertEquals(column0.isNull(i), false);
}
assertEquals(BIGINT.getLong(column0, 0), 123L);
assertEquals(BIGINT.getLong(column0, 1), 777L);
Block column1 = reader.readBlock(1);
assertEquals(column1.getPositionCount(), 2);
for (int i = 0; i < 2; i++) {
assertEquals(column1.isNull(i), false);
}
assertEquals(createVarcharType(20).getSlice(column1, 0), utf8Slice("hello"));
assertEquals(createVarcharType(20).getSlice(column1, 1), utf8Slice("sky"));
assertFalse(reader.getUserMetadata().containsKey(OrcFileMetadata.KEY));
}
BitSet rowsToDelete = new BitSet(5);
rowsToDelete.set(1);
File newFile = new File(temporary, randomUUID().toString());
FileSystem fileSystem = new LocalOrcDataEnvironment().getFileSystem(DEFAULT_RAPTOR_CONTEXT);
OrcFileInfo info = createFileRewriter().rewrite(fileSystem, getColumnTypes(columnIds, columnTypes), path(file), path(newFile), rowsToDelete);
assertEquals(info.getRowCount(), 1);
assertBetweenInclusive(info.getUncompressedSize(), 13L, 13L * 2);
try (OrcDataSource dataSource = fileOrcDataSource(newFile)) {
OrcBatchRecordReader reader = createReader(dataSource, columnIds, columnTypes);
assertEquals(reader.getReaderRowCount(), 1);
assertEquals(reader.getFileRowCount(), 1);
assertEquals(reader.getSplitLength(), newFile.length());
assertEquals(reader.nextBatch(), 1);
Block column0 = reader.readBlock(0);
assertEquals(column0.getPositionCount(), 1);
assertEquals(column0.isNull(0), false);
assertEquals(BIGINT.getLong(column0, 0), 123L);
Block column1 = reader.readBlock(1);
assertEquals(column1.getPositionCount(), 1);
assertEquals(column1.isNull(0), false);
assertEquals(createVarcharType(20).getSlice(column1, 0), utf8Slice("hello"));
assertFalse(reader.getUserMetadata().containsKey(OrcFileMetadata.KEY));
}
}
Aggregations