use of com.facebook.presto.common.Page in project presto by prestodb.
the class BenchmarkHashAndStreamingAggregationOperators method benchmark.
@Benchmark
public List<Page> benchmark(Context context) {
DriverContext driverContext = context.createTaskContext().addPipelineContext(0, true, true, false).addDriverContext();
Operator operator = context.getOperatorFactory().createOperator(driverContext);
Iterator<Page> input = context.getPages().iterator();
ImmutableList.Builder<Page> outputPages = ImmutableList.builder();
boolean finishing = false;
for (int loops = 0; !operator.isFinished() && loops < 1_000_000; loops++) {
if (operator.needsInput()) {
if (input.hasNext()) {
Page inputPage = input.next();
operator.addInput(inputPage);
} else if (!finishing) {
operator.finish();
finishing = true;
}
}
Page outputPage = operator.getOutput();
if (outputPage != null) {
outputPages.add(outputPage);
}
}
return outputPages.build();
}
use of com.facebook.presto.common.Page in project presto by prestodb.
the class TestJoinCompiler method testMultiChannel.
@Test(dataProvider = "hashEnabledValues")
public void testMultiChannel(boolean hashEnabled) {
// compile a single channel hash strategy
JoinCompiler joinCompiler = new JoinCompiler(MetadataManager.createTestMetadataManager(), new FeaturesConfig());
List<Type> types = ImmutableList.of(VARCHAR, VARCHAR, BIGINT, DOUBLE, BOOLEAN, VARCHAR);
List<Type> joinTypes = ImmutableList.of(VARCHAR, BIGINT, DOUBLE, BOOLEAN);
List<Type> outputTypes = ImmutableList.of(VARCHAR, BIGINT, DOUBLE, BOOLEAN, VARCHAR);
List<Integer> joinChannels = Ints.asList(1, 2, 3, 4);
List<Integer> outputChannels = Ints.asList(1, 2, 3, 4, 0);
// crate hash strategy with a single channel blocks -- make sure there is some overlap in values
List<Block> extraChannel = ImmutableList.of(BlockAssertions.createStringSequenceBlock(10, 20), BlockAssertions.createStringSequenceBlock(20, 30), BlockAssertions.createStringSequenceBlock(15, 25));
List<Block> varcharChannel = ImmutableList.of(BlockAssertions.createStringSequenceBlock(10, 20), BlockAssertions.createStringSequenceBlock(20, 30), BlockAssertions.createStringSequenceBlock(15, 25));
List<Block> longChannel = ImmutableList.of(BlockAssertions.createLongSequenceBlock(10, 20), BlockAssertions.createLongSequenceBlock(20, 30), BlockAssertions.createLongSequenceBlock(15, 25));
List<Block> doubleChannel = ImmutableList.of(BlockAssertions.createDoubleSequenceBlock(10, 20), BlockAssertions.createDoubleSequenceBlock(20, 30), BlockAssertions.createDoubleSequenceBlock(15, 25));
List<Block> booleanChannel = ImmutableList.of(BlockAssertions.createBooleanSequenceBlock(10, 20), BlockAssertions.createBooleanSequenceBlock(20, 30), BlockAssertions.createBooleanSequenceBlock(15, 25));
List<Block> extraUnusedChannel = ImmutableList.of(BlockAssertions.createBooleanSequenceBlock(10, 20), BlockAssertions.createBooleanSequenceBlock(20, 30), BlockAssertions.createBooleanSequenceBlock(15, 25));
OptionalInt hashChannel = OptionalInt.empty();
ImmutableList<List<Block>> channels = ImmutableList.of(extraChannel, varcharChannel, longChannel, doubleChannel, booleanChannel, extraUnusedChannel);
List<Block> precomputedHash = ImmutableList.of();
if (hashEnabled) {
ImmutableList.Builder<Block> hashChannelBuilder = ImmutableList.builder();
for (int i = 0; i < 3; i++) {
hashChannelBuilder.add(TypeUtils.getHashBlock(joinTypes, varcharChannel.get(i), longChannel.get(i), doubleChannel.get(i), booleanChannel.get(i)));
}
hashChannel = OptionalInt.of(6);
precomputedHash = hashChannelBuilder.build();
channels = ImmutableList.of(extraChannel, varcharChannel, longChannel, doubleChannel, booleanChannel, extraUnusedChannel, precomputedHash);
types = ImmutableList.of(VARCHAR, VARCHAR, BIGINT, DOUBLE, BOOLEAN, VARCHAR, BIGINT);
outputTypes = ImmutableList.of(VARCHAR, BIGINT, DOUBLE, BOOLEAN, VARCHAR, BIGINT);
outputChannels = Ints.asList(1, 2, 3, 4, 0, 6);
}
PagesHashStrategyFactory pagesHashStrategyFactory = joinCompiler.compilePagesHashStrategyFactory(types, joinChannels, Optional.of(outputChannels));
PagesHashStrategy hashStrategy = pagesHashStrategyFactory.createPagesHashStrategy(channels, hashChannel);
// todo add tests for filter function
PagesHashStrategy expectedHashStrategy = new SimplePagesHashStrategy(types, outputChannels, channels, joinChannels, hashChannel, Optional.empty(), FUNCTION_MANAGER, groupByUsesEqualTo);
// verify channel count
assertEquals(hashStrategy.getChannelCount(), outputChannels.size());
// verify size
int instanceSize = ClassLayout.parseClass(hashStrategy.getClass()).instanceSize();
long sizeInBytes = instanceSize + channels.stream().flatMap(List::stream).mapToLong(Block::getRetainedSizeInBytes).sum();
assertEquals(hashStrategy.getSizeInBytes(), sizeInBytes);
// verify hashStrategy is consistent with equals and hash code from block
for (int leftBlockIndex = 0; leftBlockIndex < varcharChannel.size(); leftBlockIndex++) {
PageBuilder pageBuilder = new PageBuilder(outputTypes);
Block[] leftBlocks = new Block[4];
leftBlocks[0] = varcharChannel.get(leftBlockIndex);
leftBlocks[1] = longChannel.get(leftBlockIndex);
leftBlocks[2] = doubleChannel.get(leftBlockIndex);
leftBlocks[3] = booleanChannel.get(leftBlockIndex);
int leftPositionCount = varcharChannel.get(leftBlockIndex).getPositionCount();
for (int leftBlockPosition = 0; leftBlockPosition < leftPositionCount; leftBlockPosition++) {
// hash code of position must match block hash
assertEquals(hashStrategy.hashPosition(leftBlockIndex, leftBlockPosition), expectedHashStrategy.hashPosition(leftBlockIndex, leftBlockPosition));
// position must be equal to itself
assertTrue(hashStrategy.positionEqualsPositionIgnoreNulls(leftBlockIndex, leftBlockPosition, leftBlockIndex, leftBlockPosition));
assertTrue(hashStrategy.positionEqualsPosition(leftBlockIndex, leftBlockPosition, leftBlockIndex, leftBlockPosition));
// check equality of every position against every other position in the block
for (int rightBlockIndex = 0; rightBlockIndex < varcharChannel.size(); rightBlockIndex++) {
Block rightBlock = varcharChannel.get(rightBlockIndex);
for (int rightBlockPosition = 0; rightBlockPosition < rightBlock.getPositionCount(); rightBlockPosition++) {
assertEquals(hashStrategy.positionEqualsPositionIgnoreNulls(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expectedHashStrategy.positionEqualsPositionIgnoreNulls(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition));
assertEquals(hashStrategy.positionEqualsPosition(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expectedHashStrategy.positionEqualsPosition(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition));
}
}
// check equality of every position against every other position in the block cursor
for (int rightBlockIndex = 0; rightBlockIndex < varcharChannel.size(); rightBlockIndex++) {
Block[] rightBlocks = new Block[4];
rightBlocks[0] = varcharChannel.get(rightBlockIndex);
rightBlocks[1] = longChannel.get(rightBlockIndex);
rightBlocks[2] = doubleChannel.get(rightBlockIndex);
rightBlocks[3] = booleanChannel.get(rightBlockIndex);
int rightPositionCount = varcharChannel.get(rightBlockIndex).getPositionCount();
for (int rightPosition = 0; rightPosition < rightPositionCount; rightPosition++) {
boolean expected = expectedHashStrategy.positionEqualsRow(leftBlockIndex, leftBlockPosition, rightPosition, new Page(rightBlocks));
assertEquals(hashStrategy.positionEqualsRow(leftBlockIndex, leftBlockPosition, rightPosition, new Page(rightBlocks)), expected);
assertEquals(hashStrategy.rowEqualsRow(leftBlockPosition, new Page(leftBlocks), rightPosition, new Page(rightBlocks)), expected);
assertEquals(hashStrategy.positionEqualsRowIgnoreNulls(leftBlockIndex, leftBlockPosition, rightPosition, new Page(rightBlocks)), expected);
}
}
// write position to output block
pageBuilder.declarePosition();
hashStrategy.appendTo(leftBlockIndex, leftBlockPosition, pageBuilder, 0);
}
// verify output block matches
Page page = pageBuilder.build();
if (hashEnabled) {
assertPageEquals(outputTypes, page, new Page(varcharChannel.get(leftBlockIndex), longChannel.get(leftBlockIndex), doubleChannel.get(leftBlockIndex), booleanChannel.get(leftBlockIndex), extraChannel.get(leftBlockIndex), precomputedHash.get(leftBlockIndex)));
} else {
assertPageEquals(outputTypes, page, new Page(varcharChannel.get(leftBlockIndex), longChannel.get(leftBlockIndex), doubleChannel.get(leftBlockIndex), booleanChannel.get(leftBlockIndex), extraChannel.get(leftBlockIndex)));
}
}
}
use of com.facebook.presto.common.Page in project presto by prestodb.
the class TestJoinCompiler method testSingleChannel.
@Test(dataProvider = "hashEnabledValues")
public void testSingleChannel(boolean hashEnabled) {
List<Type> joinTypes = ImmutableList.of(VARCHAR);
List<Integer> joinChannels = Ints.asList(0);
// compile a single channel hash strategy
PagesHashStrategyFactory pagesHashStrategyFactory = joinCompiler.compilePagesHashStrategyFactory(joinTypes, joinChannels);
// create hash strategy with a single channel blocks -- make sure there is some overlap in values
List<Block> channel = ImmutableList.of(BlockAssertions.createStringSequenceBlock(10, 20), BlockAssertions.createStringSequenceBlock(20, 30), BlockAssertions.createStringSequenceBlock(15, 25));
OptionalInt hashChannel = OptionalInt.empty();
List<List<Block>> channels = ImmutableList.of(channel);
if (hashEnabled) {
ImmutableList.Builder<Block> hashChannelBuilder = ImmutableList.builder();
for (Block block : channel) {
hashChannelBuilder.add(TypeUtils.getHashBlock(joinTypes, block));
}
hashChannel = OptionalInt.of(1);
channels = ImmutableList.of(channel, hashChannelBuilder.build());
}
PagesHashStrategy hashStrategy = pagesHashStrategyFactory.createPagesHashStrategy(channels, hashChannel);
// verify channel count
assertEquals(hashStrategy.getChannelCount(), 1);
// verify hashStrategy is consistent with equals and hash code from block
for (int leftBlockIndex = 0; leftBlockIndex < channel.size(); leftBlockIndex++) {
Block leftBlock = channel.get(leftBlockIndex);
PageBuilder pageBuilder = new PageBuilder(ImmutableList.of(VARCHAR));
for (int leftBlockPosition = 0; leftBlockPosition < leftBlock.getPositionCount(); leftBlockPosition++) {
// hash code of position must match block hash
assertEquals(hashStrategy.hashPosition(leftBlockIndex, leftBlockPosition), hashPosition(VARCHAR, leftBlock, leftBlockPosition));
// position must be equal to itself
assertTrue(hashStrategy.positionEqualsPositionIgnoreNulls(leftBlockIndex, leftBlockPosition, leftBlockIndex, leftBlockPosition));
// check equality of every position against every other position in the block
for (int rightBlockIndex = 0; rightBlockIndex < channel.size(); rightBlockIndex++) {
Block rightBlock = channel.get(rightBlockIndex);
for (int rightBlockPosition = 0; rightBlockPosition < rightBlock.getPositionCount(); rightBlockPosition++) {
boolean expected = positionEqualsPosition(VARCHAR, leftBlock, leftBlockPosition, rightBlock, rightBlockPosition);
assertEquals(hashStrategy.positionEqualsRow(leftBlockIndex, leftBlockPosition, rightBlockPosition, new Page(rightBlock)), expected);
assertEquals(hashStrategy.rowEqualsRow(leftBlockPosition, new Page(leftBlock), rightBlockPosition, new Page(rightBlock)), expected);
assertEquals(hashStrategy.positionEqualsRowIgnoreNulls(leftBlockIndex, leftBlockPosition, rightBlockPosition, new Page(rightBlock)), expected);
assertEquals(hashStrategy.positionEqualsPositionIgnoreNulls(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expected);
assertEquals(hashStrategy.positionEqualsPosition(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expected);
}
}
// check equality of every position against every other position in the block cursor
for (int rightBlockIndex = 0; rightBlockIndex < channel.size(); rightBlockIndex++) {
Block rightBlock = channel.get(rightBlockIndex);
for (int rightBlockPosition = 0; rightBlockPosition < rightBlock.getPositionCount(); rightBlockPosition++) {
boolean expected = positionEqualsPosition(VARCHAR, leftBlock, leftBlockPosition, rightBlock, rightBlockPosition);
assertEquals(hashStrategy.positionEqualsRow(leftBlockIndex, leftBlockPosition, rightBlockPosition, new Page(rightBlock)), expected);
assertEquals(hashStrategy.rowEqualsRow(leftBlockPosition, new Page(leftBlock), rightBlockPosition, new Page(rightBlock)), expected);
assertEquals(hashStrategy.positionEqualsRowIgnoreNulls(leftBlockIndex, leftBlockPosition, rightBlockPosition, new Page(rightBlock)), expected);
assertEquals(hashStrategy.positionEqualsPositionIgnoreNulls(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expected);
assertEquals(hashStrategy.positionEqualsPosition(leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition), expected);
}
}
// write position to output block
pageBuilder.declarePosition();
hashStrategy.appendTo(leftBlockIndex, leftBlockPosition, pageBuilder, 0);
}
// verify output block matches
assertBlockEquals(VARCHAR, pageBuilder.build().getBlock(0), leftBlock);
}
}
use of com.facebook.presto.common.Page in project presto by prestodb.
the class TestRowExpressionPredicateCompiler method test.
@Test
public void test() {
InputReferenceExpression a = new InputReferenceExpression(Optional.empty(), 0, BIGINT);
InputReferenceExpression b = new InputReferenceExpression(Optional.empty(), 1, BIGINT);
Block aBlock = createLongBlock(5, 5, 5, 5, 5);
Block bBlock = createLongBlock(1, 3, 5, 7, 0);
// b - a >= 0
RowExpression sum = call("<", functionResolution.comparisonFunction(GREATER_THAN_OR_EQUAL, BIGINT, BIGINT), BOOLEAN, call("b - a", functionResolution.arithmeticFunction(SUBTRACT, BIGINT, BIGINT), BIGINT, b, a), constant(0L, BIGINT));
PredicateCompiler compiler = new RowExpressionPredicateCompiler(metadata, 10_000);
Predicate compiledSum = compiler.compilePredicate(SESSION.getSqlFunctionProperties(), SESSION.getSessionFunctions(), sum).get();
assertEquals(Arrays.asList(1, 0), Ints.asList(compiledSum.getInputChannels()));
Page page = new Page(bBlock, aBlock);
assertFalse(compiledSum.evaluate(SESSION.getSqlFunctionProperties(), page, 0));
assertFalse(compiledSum.evaluate(SESSION.getSqlFunctionProperties(), page, 1));
assertTrue(compiledSum.evaluate(SESSION.getSqlFunctionProperties(), page, 2));
assertTrue(compiledSum.evaluate(SESSION.getSqlFunctionProperties(), page, 3));
assertFalse(compiledSum.evaluate(SESSION.getSqlFunctionProperties(), page, 4));
// b * 2 < 10
RowExpression timesTwo = call("=", functionResolution.comparisonFunction(LESS_THAN, BIGINT, BIGINT), BOOLEAN, call("b * 2", functionResolution.arithmeticFunction(MULTIPLY, BIGINT, BIGINT), BIGINT, b, constant(2L, BIGINT)), constant(10L, BIGINT));
Predicate compiledTimesTwo = compiler.compilePredicate(SESSION.getSqlFunctionProperties(), SESSION.getSessionFunctions(), timesTwo).get();
assertEquals(Arrays.asList(1), Ints.asList(compiledTimesTwo.getInputChannels()));
page = new Page(bBlock);
assertTrue(compiledTimesTwo.evaluate(SESSION.getSqlFunctionProperties(), page, 0));
assertTrue(compiledTimesTwo.evaluate(SESSION.getSqlFunctionProperties(), page, 1));
assertFalse(compiledTimesTwo.evaluate(SESSION.getSqlFunctionProperties(), page, 2));
assertFalse(compiledTimesTwo.evaluate(SESSION.getSqlFunctionProperties(), page, 3));
assertTrue(compiledTimesTwo.evaluate(SESSION.getSqlFunctionProperties(), page, 4));
}
use of com.facebook.presto.common.Page in project presto by prestodb.
the class TestPageFunctionCompiler method testCommonSubExpressionInFilter.
@Test
public void testCommonSubExpressionInFilter() {
PageFunctionCompiler functionCompiler = new PageFunctionCompiler(createTestMetadataManager(), 0);
Supplier<PageFilter> pageFilter = functionCompiler.compileFilter(SESSION.getSqlFunctionProperties(), new SpecialFormExpression(AND, BIGINT, ADD_X_Y_GREATER_THAN_2, ADD_X_Y_LESS_THAN_10), true, Optional.empty());
Page input = createLongBlockPage(2, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
SelectedPositions positions = filter(pageFilter.get(), input);
assertEquals(positions.size(), 3);
assertEquals(positions.getPositions(), new int[] { 2, 3, 4 });
}
Aggregations