use of com.facebook.presto.spi.PageBuilder in project presto by prestodb.
the class TestTupleFilterProcessor method testFullPageFilter.
@Test
public void testFullPageFilter() throws Exception {
Page tuplePage = Iterables.getOnlyElement(rowPagesBuilder(BIGINT, VARCHAR, DOUBLE).row(1L, "a", 0.1).build());
List<Type> outputTypes = ImmutableList.of(VARCHAR, BIGINT, BOOLEAN, DOUBLE, DOUBLE);
TupleFilterProcessor tupleFilterProcessor = new TupleFilterProcessor(tuplePage, outputTypes, new int[] { 1, 0, 3 });
Page inputPage = Iterables.getOnlyElement(rowPagesBuilder(outputTypes).row("a", 1L, true, 0.1, 0.0).row("b", 1L, true, 0.1, 2.0).row("a", 1L, false, 0.1, 2.0).row("a", 0L, false, 0.2, 0.2).build());
PageBuilder pageBuilder = new PageBuilder(outputTypes);
int end = tupleFilterProcessor.process(SESSION, inputPage, 0, inputPage.getPositionCount(), pageBuilder);
Page actualPage = pageBuilder.build();
Page expectedPage = Iterables.getOnlyElement(rowPagesBuilder(outputTypes).row("a", 1L, true, 0.1, 0.0).row("a", 1L, false, 0.1, 2.0).build());
assertEquals(end, inputPage.getPositionCount());
assertPageEquals(outputTypes, actualPage, expectedPage);
}
use of com.facebook.presto.spi.PageBuilder in project presto by prestodb.
the class AbstractTestHiveFileFormats method createTestFile.
public static FileSplit createTestFile(String filePath, HiveStorageFormat storageFormat, HiveCompressionCodec compressionCodec, List<TestColumn> testColumns, ConnectorSession session, int numRows, HiveFileWriterFactory fileWriterFactory) throws Exception {
// filter out partition keys, which are not written to the file
testColumns = ImmutableList.copyOf(filter(testColumns, not(TestColumn::isPartitionKey)));
List<Type> types = testColumns.stream().map(TestColumn::getType).map(HiveType::valueOf).map(type -> type.getType(TYPE_MANAGER)).collect(toList());
PageBuilder pageBuilder = new PageBuilder(types);
for (int rowNumber = 0; rowNumber < numRows; rowNumber++) {
pageBuilder.declarePosition();
for (int columnNumber = 0; columnNumber < testColumns.size(); columnNumber++) {
serializeObject(types.get(columnNumber), pageBuilder.getBlockBuilder(columnNumber), testColumns.get(columnNumber).getWriteValue(), testColumns.get(columnNumber).getObjectInspector(), false);
}
}
Page page = pageBuilder.build();
JobConf jobConf = new JobConf();
configureCompression(jobConf, compressionCodec);
Properties tableProperties = new Properties();
tableProperties.setProperty("columns", Joiner.on(',').join(transform(testColumns, TestColumn::getName)));
tableProperties.setProperty("columns.types", Joiner.on(',').join(transform(testColumns, TestColumn::getType)));
Optional<HiveFileWriter> fileWriter = fileWriterFactory.createFileWriter(new Path(filePath), testColumns.stream().map(TestColumn::getName).collect(toList()), StorageFormat.fromHiveStorageFormat(storageFormat), tableProperties, jobConf, session);
HiveFileWriter hiveFileWriter = fileWriter.orElseThrow(() -> new IllegalArgumentException("fileWriterFactory"));
hiveFileWriter.appendRows(page);
hiveFileWriter.commit();
return new FileSplit(new Path(filePath), 0, new File(filePath).length(), new String[0]);
}
use of com.facebook.presto.spi.PageBuilder in project presto by prestodb.
the class BenchmarkPageProcessor method execute.
public static Page execute(Page inputPage, PageProcessor processor) {
PageBuilder pageBuilder = new PageBuilder(ImmutableList.of(DOUBLE));
int count = processor.process(null, inputPage, 0, inputPage.getPositionCount(), pageBuilder);
checkState(count == inputPage.getPositionCount());
return pageBuilder.build();
}
use of com.facebook.presto.spi.PageBuilder in project presto by prestodb.
the class InCodeGeneratorBenchmark method benchmark.
@Benchmark
public Page benchmark() {
PageBuilder pageBuilder = new PageBuilder(ImmutableList.of(prestoType));
int count = processor.process(null, inputPage, 0, inputPage.getPositionCount(), pageBuilder);
checkState(count == inputPage.getPositionCount());
return pageBuilder.build();
}
use of com.facebook.presto.spi.PageBuilder in project presto by prestodb.
the class HiveFileFormatBenchmark method createTpchDataSet.
private static <E extends TpchEntity> TestData createTpchDataSet(FileFormat format, TpchTable<E> tpchTable, List<TpchColumn<E>> columns) {
List<String> columnNames = columns.stream().map(TpchColumn::getColumnName).collect(toList());
List<Type> columnTypes = columns.stream().map(HiveFileFormatBenchmark::getColumnType).map(type -> format.supportsDate() || !DATE.equals(type) ? type : createUnboundedVarcharType()).collect(toList());
PageBuilder pageBuilder = new PageBuilder(columnTypes);
ImmutableList.Builder<Page> pages = ImmutableList.builder();
long dataSize = 0;
for (E row : tpchTable.createGenerator(10, 1, 1)) {
pageBuilder.declarePosition();
for (int i = 0; i < columns.size(); i++) {
TpchColumn<E> column = columns.get(i);
BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(i);
switch(column.getType().getBase()) {
case IDENTIFIER:
BIGINT.writeLong(blockBuilder, column.getIdentifier(row));
break;
case INTEGER:
INTEGER.writeLong(blockBuilder, column.getInteger(row));
break;
case DATE:
if (format.supportsDate()) {
DATE.writeLong(blockBuilder, column.getDate(row));
} else {
createUnboundedVarcharType().writeString(blockBuilder, column.getString(row));
}
break;
case DOUBLE:
DOUBLE.writeDouble(blockBuilder, column.getDouble(row));
break;
case VARCHAR:
createUnboundedVarcharType().writeSlice(blockBuilder, Slices.utf8Slice(column.getString(row)));
break;
default:
throw new IllegalArgumentException("Unsupported type " + column.getType());
}
}
if (pageBuilder.isFull()) {
Page page = pageBuilder.build();
pages.add(page);
pageBuilder.reset();
dataSize += page.getSizeInBytes();
if (dataSize >= MIN_DATA_SIZE) {
break;
}
}
}
return new TestData(columnNames, columnTypes, pages.build());
}
Aggregations