Search in sources :

Example 1 with PredicateCompiler

use of com.facebook.presto.spi.relation.PredicateCompiler in project presto by prestodb.

the class TestRowExpressionPredicateCompiler method test.

@Test
public void test() {
    InputReferenceExpression a = new InputReferenceExpression(Optional.empty(), 0, BIGINT);
    InputReferenceExpression b = new InputReferenceExpression(Optional.empty(), 1, BIGINT);
    Block aBlock = createLongBlock(5, 5, 5, 5, 5);
    Block bBlock = createLongBlock(1, 3, 5, 7, 0);
    // b - a >= 0
    RowExpression sum = call("<", functionResolution.comparisonFunction(GREATER_THAN_OR_EQUAL, BIGINT, BIGINT), BOOLEAN, call("b - a", functionResolution.arithmeticFunction(SUBTRACT, BIGINT, BIGINT), BIGINT, b, a), constant(0L, BIGINT));
    PredicateCompiler compiler = new RowExpressionPredicateCompiler(metadata, 10_000);
    Predicate compiledSum = compiler.compilePredicate(SESSION.getSqlFunctionProperties(), SESSION.getSessionFunctions(), sum).get();
    assertEquals(Arrays.asList(1, 0), Ints.asList(compiledSum.getInputChannels()));
    Page page = new Page(bBlock, aBlock);
    assertFalse(compiledSum.evaluate(SESSION.getSqlFunctionProperties(), page, 0));
    assertFalse(compiledSum.evaluate(SESSION.getSqlFunctionProperties(), page, 1));
    assertTrue(compiledSum.evaluate(SESSION.getSqlFunctionProperties(), page, 2));
    assertTrue(compiledSum.evaluate(SESSION.getSqlFunctionProperties(), page, 3));
    assertFalse(compiledSum.evaluate(SESSION.getSqlFunctionProperties(), page, 4));
    // b * 2 < 10
    RowExpression timesTwo = call("=", functionResolution.comparisonFunction(LESS_THAN, BIGINT, BIGINT), BOOLEAN, call("b * 2", functionResolution.arithmeticFunction(MULTIPLY, BIGINT, BIGINT), BIGINT, b, constant(2L, BIGINT)), constant(10L, BIGINT));
    Predicate compiledTimesTwo = compiler.compilePredicate(SESSION.getSqlFunctionProperties(), SESSION.getSessionFunctions(), timesTwo).get();
    assertEquals(Arrays.asList(1), Ints.asList(compiledTimesTwo.getInputChannels()));
    page = new Page(bBlock);
    assertTrue(compiledTimesTwo.evaluate(SESSION.getSqlFunctionProperties(), page, 0));
    assertTrue(compiledTimesTwo.evaluate(SESSION.getSqlFunctionProperties(), page, 1));
    assertFalse(compiledTimesTwo.evaluate(SESSION.getSqlFunctionProperties(), page, 2));
    assertFalse(compiledTimesTwo.evaluate(SESSION.getSqlFunctionProperties(), page, 3));
    assertTrue(compiledTimesTwo.evaluate(SESSION.getSqlFunctionProperties(), page, 4));
}
Also used : InputReferenceExpression(com.facebook.presto.spi.relation.InputReferenceExpression) Block(com.facebook.presto.common.block.Block) RowExpression(com.facebook.presto.spi.relation.RowExpression) Page(com.facebook.presto.common.Page) PredicateCompiler(com.facebook.presto.spi.relation.PredicateCompiler) Predicate(com.facebook.presto.common.relation.Predicate) Test(org.testng.annotations.Test)

Example 2 with PredicateCompiler

use of com.facebook.presto.spi.relation.PredicateCompiler in project presto by prestodb.

the class OrcSelectivePageSourceFactory method toFilterFunctions.

/**
 * Split filter expression into groups of conjuncts that depend on the same set of inputs,
 * then compile each group into FilterFunction.
 */
private static List<FilterFunction> toFilterFunctions(RowExpression filter, Optional<BucketAdapter> bucketAdapter, ConnectorSession session, DeterminismEvaluator determinismEvaluator, PredicateCompiler predicateCompiler) {
    ImmutableList.Builder<FilterFunction> filterFunctions = ImmutableList.builder();
    bucketAdapter.map(predicate -> new FilterFunction(session.getSqlFunctionProperties(), true, predicate)).ifPresent(filterFunctions::add);
    if (TRUE_CONSTANT.equals(filter)) {
        return filterFunctions.build();
    }
    DynamicFilterExtractResult extractDynamicFilterResult = extractDynamicFilters(filter);
    // dynamic filter will be added through subfield pushdown
    filter = and(extractDynamicFilterResult.getStaticConjuncts());
    if (!isAdaptiveFilterReorderingEnabled(session)) {
        filterFunctions.add(new FilterFunction(session.getSqlFunctionProperties(), determinismEvaluator.isDeterministic(filter), predicateCompiler.compilePredicate(session.getSqlFunctionProperties(), session.getSessionFunctions(), filter).get()));
        return filterFunctions.build();
    }
    List<RowExpression> conjuncts = extractConjuncts(filter);
    if (conjuncts.size() == 1) {
        filterFunctions.add(new FilterFunction(session.getSqlFunctionProperties(), determinismEvaluator.isDeterministic(filter), predicateCompiler.compilePredicate(session.getSqlFunctionProperties(), session.getSessionFunctions(), filter).get()));
        return filterFunctions.build();
    }
    // Use LinkedHashMap to preserve user-specified order of conjuncts. This will be the initial order in which filters are applied.
    Map<Set<Integer>, List<RowExpression>> inputsToConjuncts = new LinkedHashMap<>();
    for (RowExpression conjunct : conjuncts) {
        inputsToConjuncts.computeIfAbsent(extractInputs(conjunct), k -> new ArrayList<>()).add(conjunct);
    }
    inputsToConjuncts.values().stream().map(expressions -> binaryExpression(AND, expressions)).map(predicate -> new FilterFunction(session.getSqlFunctionProperties(), determinismEvaluator.isDeterministic(predicate), predicateCompiler.compilePredicate(session.getSqlFunctionProperties(), session.getSessionFunctions(), predicate).get())).forEach(filterFunctions::add);
    return filterFunctions.build();
}
Also used : LogicalRowExpressions.extractConjuncts(com.facebook.presto.expressions.LogicalRowExpressions.extractConjuncts) Page(com.facebook.presto.common.Page) HiveCoercer(com.facebook.presto.hive.HiveCoercer) DateTimeZone(org.joda.time.DateTimeZone) Arrays(java.util.Arrays) Maps.uniqueIndex(com.google.common.collect.Maps.uniqueIndex) FixedPageSource(com.facebook.presto.spi.FixedPageSource) AGGREGATED(com.facebook.presto.hive.HiveColumnHandle.ColumnType.AGGREGATED) Configuration(org.apache.hadoop.conf.Configuration) Map(java.util.Map) Predicate(com.facebook.presto.common.relation.Predicate) HiveSessionProperties.isOrcBloomFiltersEnabled(com.facebook.presto.hive.HiveSessionProperties.isOrcBloomFiltersEnabled) OrcDataSource(com.facebook.presto.orc.OrcDataSource) FileFormatDataSourceStats(com.facebook.presto.hive.FileFormatDataSourceStats) FSDataInputStream(org.apache.hadoop.fs.FSDataInputStream) SqlFunctionProperties(com.facebook.presto.common.function.SqlFunctionProperties) HiveSessionProperties.isOrcZstdJniDecompressionEnabled(com.facebook.presto.hive.HiveSessionProperties.isOrcZstdJniDecompressionEnabled) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) HiveFileContext(com.facebook.presto.hive.HiveFileContext) ConnectorSession(com.facebook.presto.spi.ConnectorSession) ORC(com.facebook.presto.orc.OrcEncoding.ORC) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) DefaultRowExpressionTraversalVisitor(com.facebook.presto.expressions.DefaultRowExpressionTraversalVisitor) REGULAR(com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR) DwrfKeyProvider(com.facebook.presto.orc.DwrfKeyProvider) OrcReaderOptions(com.facebook.presto.orc.OrcReaderOptions) BucketAdaptation(com.facebook.presto.hive.BucketAdaptation) ArrayList(java.util.ArrayList) LinkedHashMap(java.util.LinkedHashMap) DwrfEncryptionProvider(com.facebook.presto.orc.DwrfEncryptionProvider) OrcDataSourceId(com.facebook.presto.orc.OrcDataSourceId) HIVE_INVALID_BUCKET_FILES(com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_BUCKET_FILES) OrcSelectiveRecordReader(com.facebook.presto.orc.OrcSelectiveRecordReader) SubfieldExtractor(com.facebook.presto.hive.SubfieldExtractor) LogicalRowExpressions.and(com.facebook.presto.expressions.LogicalRowExpressions.and) IOException(java.io.IOException) HiveBucketing.getHiveBucket(com.facebook.presto.hive.HiveBucketing.getHiveBucket) OrcReader(com.facebook.presto.orc.OrcReader) DynamicFilters.extractDynamicFilters(com.facebook.presto.expressions.DynamicFilters.extractDynamicFilters) HdfsEnvironment(com.facebook.presto.hive.HdfsEnvironment) TupleDomainOrcPredicate(com.facebook.presto.orc.TupleDomainOrcPredicate) VariableReferenceExpression(com.facebook.presto.spi.relation.VariableReferenceExpression) HiveSessionProperties.getOrcTinyStripeThreshold(com.facebook.presto.hive.HiveSessionProperties.getOrcTinyStripeThreshold) StandardFunctionResolution(com.facebook.presto.spi.function.StandardFunctionResolution) OrcSerde(org.apache.hadoop.hive.ql.io.orc.OrcSerde) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) HiveSessionProperties.getOrcLazyReadSmallRanges(com.facebook.presto.hive.HiveSessionProperties.getOrcLazyReadSmallRanges) AND(com.facebook.presto.spi.relation.SpecialFormExpression.Form.AND) Path(org.apache.hadoop.fs.Path) EncryptionInformation(com.facebook.presto.hive.EncryptionInformation) CallExpression(com.facebook.presto.spi.relation.CallExpression) SpecialFormExpression(com.facebook.presto.spi.relation.SpecialFormExpression) BiMap(com.google.common.collect.BiMap) HiveClientConfig(com.facebook.presto.hive.HiveClientConfig) StripeMetadataSourceFactory(com.facebook.presto.orc.StripeMetadataSourceFactory) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) Collection(java.util.Collection) INITIAL_BATCH_SIZE(com.facebook.presto.orc.OrcReader.INITIAL_BATCH_SIZE) OrcPredicate(com.facebook.presto.orc.OrcPredicate) FileNotFoundException(java.io.FileNotFoundException) TRUE_CONSTANT(com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT) HiveSessionProperties.isAdaptiveFilterReorderingEnabled(com.facebook.presto.hive.HiveSessionProperties.isAdaptiveFilterReorderingEnabled) String.format(java.lang.String.format) DataSize(io.airlift.units.DataSize) List(java.util.List) RowExpressionService(com.facebook.presto.spi.relation.RowExpressionService) HiveOrcAggregatedMemoryContext(com.facebook.presto.hive.HiveOrcAggregatedMemoryContext) HiveSelectivePageSourceFactory(com.facebook.presto.hive.HiveSelectivePageSourceFactory) Optional(java.util.Optional) LogicalRowExpressions.binaryExpression(com.facebook.presto.expressions.LogicalRowExpressions.binaryExpression) HiveColumnHandle(com.facebook.presto.hive.HiveColumnHandle) PredicateCompiler(com.facebook.presto.spi.relation.PredicateCompiler) InputReferenceExpression(com.facebook.presto.spi.relation.InputReferenceExpression) IntStream(java.util.stream.IntStream) DynamicFilterExtractResult(com.facebook.presto.expressions.DynamicFilters.DynamicFilterExtractResult) DeterminismEvaluator(com.facebook.presto.spi.relation.DeterminismEvaluator) HiveSessionProperties.getOrcMaxMergeDistance(com.facebook.presto.hive.HiveSessionProperties.getOrcMaxMergeDistance) Strings.nullToEmpty(com.google.common.base.Strings.nullToEmpty) HiveType(com.facebook.presto.hive.HiveType) HashMap(java.util.HashMap) PrestoException(com.facebook.presto.spi.PrestoException) Function(java.util.function.Function) HIVE_CANNOT_OPEN_SPLIT(com.facebook.presto.hive.HiveErrorCode.HIVE_CANNOT_OPEN_SPLIT) HIVE_MISSING_DATA(com.facebook.presto.hive.HiveErrorCode.HIVE_MISSING_DATA) HiveSessionProperties.getOrcMaxReadBlockSize(com.facebook.presto.hive.HiveSessionProperties.getOrcMaxReadBlockSize) ImmutableBiMap(com.google.common.collect.ImmutableBiMap) Inject(javax.inject.Inject) HashSet(java.util.HashSet) Subfield(com.facebook.presto.common.Subfield) ImmutableList(com.google.common.collect.ImmutableList) Verify.verify(com.google.common.base.Verify.verify) UncheckedExecutionException(com.google.common.util.concurrent.UncheckedExecutionException) FilterFunction(com.facebook.presto.common.predicate.FilterFunction) TypeManager(com.facebook.presto.common.type.TypeManager) HiveSessionProperties.getOrcMaxBufferSize(com.facebook.presto.hive.HiveSessionProperties.getOrcMaxBufferSize) Objects.requireNonNull(java.util.Objects.requireNonNull) OrcFileTailSource(com.facebook.presto.orc.cache.OrcFileTailSource) Type(com.facebook.presto.common.type.Type) RowExpression(com.facebook.presto.spi.relation.RowExpression) Storage(com.facebook.presto.hive.metastore.Storage) OrcAggregatedMemoryContext(com.facebook.presto.orc.OrcAggregatedMemoryContext) OrcEncoding(com.facebook.presto.orc.OrcEncoding) HiveSessionProperties.getOrcStreamBufferSize(com.facebook.presto.hive.HiveSessionProperties.getOrcStreamBufferSize) NO_ENCRYPTION(com.facebook.presto.orc.DwrfEncryptionProvider.NO_ENCRYPTION) Maps(com.google.common.collect.Maps) TupleDomain(com.facebook.presto.common.predicate.TupleDomain) TupleDomainFilter(com.facebook.presto.common.predicate.TupleDomainFilter) TypeInfo(org.apache.hadoop.hive.serde2.typeinfo.TypeInfo) Consumer(java.util.function.Consumer) HiveUtil.typedPartitionKey(com.facebook.presto.hive.HiveUtil.typedPartitionKey) ConnectorPageSource(com.facebook.presto.spi.ConnectorPageSource) HiveUtil.getPhysicalHiveColumnHandles(com.facebook.presto.hive.HiveUtil.getPhysicalHiveColumnHandles) ImmutableBiMap.toImmutableBiMap(com.google.common.collect.ImmutableBiMap.toImmutableBiMap) RowExpressionNodeInliner.replaceExpression(com.facebook.presto.expressions.RowExpressionNodeInliner.replaceExpression) FilterFunction(com.facebook.presto.common.predicate.FilterFunction) Set(java.util.Set) ImmutableSet(com.google.common.collect.ImmutableSet) HashSet(java.util.HashSet) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) ImmutableList(com.google.common.collect.ImmutableList) ArrayList(java.util.ArrayList) RowExpression(com.facebook.presto.spi.relation.RowExpression) LinkedHashMap(java.util.LinkedHashMap) DynamicFilterExtractResult(com.facebook.presto.expressions.DynamicFilters.DynamicFilterExtractResult) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) ArrayList(java.util.ArrayList) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList)

Example 3 with PredicateCompiler

use of com.facebook.presto.spi.relation.PredicateCompiler in project presto by prestodb.

the class TestRowExpressionPredicateCompiler method testCache.

@Test
public void testCache() {
    // a * 2 < 10
    RowExpression predicate = call("=", functionResolution.comparisonFunction(LESS_THAN, BIGINT, BIGINT), BOOLEAN, call("a * 2", functionResolution.arithmeticFunction(MULTIPLY, BIGINT, BIGINT), BIGINT, new InputReferenceExpression(Optional.empty(), 1, BIGINT), constant(2L, BIGINT)), constant(10L, BIGINT));
    PredicateCompiler compiler = new RowExpressionPredicateCompiler(metadata, 10_000);
    assertSame(compiler.compilePredicate(SESSION.getSqlFunctionProperties(), SESSION.getSessionFunctions(), predicate), compiler.compilePredicate(SESSION.getSqlFunctionProperties(), SESSION.getSessionFunctions(), predicate));
    PredicateCompiler noCacheCompiler = new RowExpressionPredicateCompiler(metadata, 0);
    assertNotSame(noCacheCompiler.compilePredicate(SESSION.getSqlFunctionProperties(), SESSION.getSessionFunctions(), predicate), noCacheCompiler.compilePredicate(SESSION.getSqlFunctionProperties(), SESSION.getSessionFunctions(), predicate));
}
Also used : InputReferenceExpression(com.facebook.presto.spi.relation.InputReferenceExpression) RowExpression(com.facebook.presto.spi.relation.RowExpression) PredicateCompiler(com.facebook.presto.spi.relation.PredicateCompiler) Test(org.testng.annotations.Test)

Aggregations

InputReferenceExpression (com.facebook.presto.spi.relation.InputReferenceExpression)3 PredicateCompiler (com.facebook.presto.spi.relation.PredicateCompiler)3 RowExpression (com.facebook.presto.spi.relation.RowExpression)3 Page (com.facebook.presto.common.Page)2 Predicate (com.facebook.presto.common.relation.Predicate)2 Subfield (com.facebook.presto.common.Subfield)1 Block (com.facebook.presto.common.block.Block)1 SqlFunctionProperties (com.facebook.presto.common.function.SqlFunctionProperties)1 FilterFunction (com.facebook.presto.common.predicate.FilterFunction)1 TupleDomain (com.facebook.presto.common.predicate.TupleDomain)1 TupleDomainFilter (com.facebook.presto.common.predicate.TupleDomainFilter)1 Type (com.facebook.presto.common.type.Type)1 TypeManager (com.facebook.presto.common.type.TypeManager)1 DefaultRowExpressionTraversalVisitor (com.facebook.presto.expressions.DefaultRowExpressionTraversalVisitor)1 DynamicFilterExtractResult (com.facebook.presto.expressions.DynamicFilters.DynamicFilterExtractResult)1 DynamicFilters.extractDynamicFilters (com.facebook.presto.expressions.DynamicFilters.extractDynamicFilters)1 TRUE_CONSTANT (com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT)1 LogicalRowExpressions.and (com.facebook.presto.expressions.LogicalRowExpressions.and)1 LogicalRowExpressions.binaryExpression (com.facebook.presto.expressions.LogicalRowExpressions.binaryExpression)1 LogicalRowExpressions.extractConjuncts (com.facebook.presto.expressions.LogicalRowExpressions.extractConjuncts)1