Examples with Metadata com.facebook.presto.metadata.Metadata used on opensource projects

Example 46 with Metadata

use of com.facebook.presto.metadata.Metadata in project presto by prestodb.

the class TestCursorProcessorCompiler method testRewriteRowExpressionWithCSE.

@Test
public void testRewriteRowExpressionWithCSE() {
    CursorProcessorCompiler cseCursorCompiler = new CursorProcessorCompiler(METADATA, true, emptyMap());
    ClassDefinition cursorProcessorClassDefinition = new ClassDefinition(a(PUBLIC, FINAL), makeClassName(CursorProcessor.class.getSimpleName()), type(Object.class), type(CursorProcessor.class));
    RowExpression filter = new SpecialFormExpression(AND, BIGINT, ADD_X_Y_GREATER_THAN_2);
    List<RowExpression> projections = ImmutableList.of(ADD_X_Y_Z);
    List<RowExpression> rowExpressions = ImmutableList.<RowExpression>builder().addAll(projections).add(filter).build();
    Map<Integer, Map<RowExpression, VariableReferenceExpression>> commonSubExpressionsByLevel = collectCSEByLevel(rowExpressions);
    Map<VariableReferenceExpression, CommonSubExpressionRewriter.CommonSubExpressionFields> cseFields = declareCommonSubExpressionFields(cursorProcessorClassDefinition, commonSubExpressionsByLevel);
    Map<RowExpression, VariableReferenceExpression> commonSubExpressions = commonSubExpressionsByLevel.values().stream().flatMap(m -> m.entrySet().stream()).collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue));
    // X+Y as CSE
    assertEquals(1, cseFields.size());
    VariableReferenceExpression cseVariable = cseFields.keySet().iterator().next();
    RowExpression rewrittenFilter = cseCursorCompiler.rewriteRowExpressionsWithCSE(ImmutableList.of(filter), commonSubExpressions).get(0);
    List<RowExpression> rewrittenProjections = cseCursorCompiler.rewriteRowExpressionsWithCSE(projections, commonSubExpressions);
    // X+Y+Z contains CSE X+Y
    assertTrue(((CallExpression) rewrittenProjections.get(0)).getArguments().contains(cseVariable));
    // X+Y > 2 consists CSE X+Y
    assertTrue(((CallExpression) ((SpecialFormExpression) rewrittenFilter).getArguments().get(0)).getArguments().contains(cseVariable));
}

Example 47 with Metadata

use of com.facebook.presto.metadata.Metadata in project presto by prestodb.

the class SectionExecutionFactory method createSectionExecutions.

/**
 * returns a List of SectionExecutions in a postorder representation of the tree
 */
public SectionExecution createSectionExecutions(Session session, StreamingPlanSection section, ExchangeLocationsConsumer locationsConsumer, Optional<int[]> bucketToPartition, OutputBuffers outputBuffers, boolean summarizeTaskInfo, RemoteTaskFactory remoteTaskFactory, SplitSourceFactory splitSourceFactory, int attemptId) {
    // Only fetch a distribution once per section to ensure all stages see the same machine assignments
    Map<PartitioningHandle, NodePartitionMap> partitioningCache = new HashMap<>();
    TableWriteInfo tableWriteInfo = createTableWriteInfo(section.getPlan(), metadata, session);
    List<StageExecutionAndScheduler> sectionStages = createStreamingLinkedStageExecutions(session, locationsConsumer, section.getPlan().withBucketToPartition(bucketToPartition), partitioningHandle -> partitioningCache.computeIfAbsent(partitioningHandle, handle -> nodePartitioningManager.getNodePartitioningMap(session, handle)), tableWriteInfo, Optional.empty(), summarizeTaskInfo, remoteTaskFactory, splitSourceFactory, attemptId);
    StageExecutionAndScheduler rootStage = getLast(sectionStages);
    rootStage.getStageExecution().setOutputBuffers(outputBuffers);
    return new SectionExecution(rootStage, sectionStages);
}

Example 48 with Metadata

use of com.facebook.presto.metadata.Metadata in project presto by prestodb.

the class PageFunctionCompiler method defineFilterClass.

private ClassDefinition defineFilterClass(SqlFunctionProperties sqlFunctionProperties, Map<SqlFunctionId, SqlInvokedFunction> sessionFunctions, RowExpression filter, InputChannels inputChannels, CallSiteBinder callSiteBinder, boolean isOptimizeCommonSubExpression, Optional<String> classNameSuffix) {
    ClassDefinition classDefinition = new ClassDefinition(a(PUBLIC, FINAL), generateFilterClassName(classNameSuffix), type(Object.class), type(PageFilter.class));
    CachedInstanceBinder cachedInstanceBinder = new CachedInstanceBinder(classDefinition, callSiteBinder);
    Map<LambdaDefinitionExpression, CompiledLambda> compiledLambdaMap = generateMethodsForLambda(classDefinition, callSiteBinder, cachedInstanceBinder, filter, metadata, sqlFunctionProperties, sessionFunctions);
    // cse
    Map<VariableReferenceExpression, CommonSubExpressionFields> cseFields = ImmutableMap.of();
    RowExpressionCompiler compiler = new RowExpressionCompiler(classDefinition, callSiteBinder, cachedInstanceBinder, new FieldAndVariableReferenceCompiler(callSiteBinder, cseFields), metadata, sqlFunctionProperties, sessionFunctions, compiledLambdaMap);
    if (isOptimizeCommonSubExpression) {
        Map<Integer, Map<RowExpression, VariableReferenceExpression>> commonSubExpressionsByLevel = collectCSEByLevel(filter);
        if (!commonSubExpressionsByLevel.isEmpty()) {
            cseFields = declareCommonSubExpressionFields(classDefinition, commonSubExpressionsByLevel);
            compiler = new RowExpressionCompiler(classDefinition, callSiteBinder, cachedInstanceBinder, new FieldAndVariableReferenceCompiler(callSiteBinder, cseFields), metadata, sqlFunctionProperties, sessionFunctions, compiledLambdaMap);
            generateCommonSubExpressionMethods(classDefinition, compiler, commonSubExpressionsByLevel, cseFields);
            Map<RowExpression, VariableReferenceExpression> commonSubExpressions = commonSubExpressionsByLevel.values().stream().flatMap(m -> m.entrySet().stream()).collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue));
            filter = rewriteExpressionWithCSE(filter, commonSubExpressions);
            if (log.isDebugEnabled()) {
                log.debug("Extracted %d common sub-expressions", commonSubExpressions.size());
                commonSubExpressions.entrySet().forEach(entry -> log.debug("\t%s = %s", entry.getValue(), entry.getKey()));
                log.debug("Rewrote filter: %s", filter);
            }
        }
    }
    generateFilterMethod(classDefinition, compiler, filter, cseFields);
    FieldDefinition selectedPositions = classDefinition.declareField(a(PRIVATE), "selectedPositions", boolean[].class);
    generatePageFilterMethod(classDefinition, selectedPositions);
    // isDeterministic
    classDefinition.declareMethod(a(PUBLIC), "isDeterministic", type(boolean.class)).getBody().append(constantBoolean(determinismEvaluator.isDeterministic(filter))).retBoolean();
    // getInputChannels
    classDefinition.declareMethod(a(PUBLIC), "getInputChannels", type(InputChannels.class)).getBody().append(invoke(callSiteBinder.bind(inputChannels, InputChannels.class), "getInputChannels")).retObject();
    // toString
    String toStringResult = toStringHelper(classDefinition.getType().getJavaClassName()).add("filter", filter).toString();
    classDefinition.declareMethod(a(PUBLIC), "toString", type(String.class)).getBody().append(invoke(callSiteBinder.bind(toStringResult, String.class), "toString")).retObject();
    // constructor
    MethodDefinition constructorDefinition = classDefinition.declareConstructor(a(PUBLIC));
    BytecodeBlock body = constructorDefinition.getBody();
    Variable thisVariable = constructorDefinition.getThis();
    body.comment("super();").append(thisVariable).invokeConstructor(Object.class).append(thisVariable.setField(selectedPositions, newArray(type(boolean[].class), 0)));
    initializeCommonSubExpressionFields(cseFields.values(), thisVariable, body);
    cachedInstanceBinder.generateInitializations(thisVariable, body);
    body.ret();
    return classDefinition;
}

Example 49 with Metadata

use of com.facebook.presto.metadata.Metadata in project presto by prestodb.

the class LambdaBytecodeGenerator method generateMethodsForLambda.

private static Map<LambdaDefinitionExpression, CompiledLambda> generateMethodsForLambda(ClassDefinition containerClassDefinition, CallSiteBinder callSiteBinder, CachedInstanceBinder cachedInstanceBinder, List<RowExpression> expressions, Metadata metadata, SqlFunctionProperties sqlFunctionProperties, Map<SqlFunctionId, SqlInvokedFunction> sessionFunctions, String methodNamePrefix, Set<LambdaDefinitionExpression> existingCompiledLambdas) {
    Set<LambdaDefinitionExpression> lambdaExpressions = expressions.stream().map(LambdaExpressionExtractor::extractLambdaExpressions).flatMap(List::stream).filter(lambda -> !existingCompiledLambdas.contains(lambda)).collect(toImmutableSet());
    ImmutableMap.Builder<LambdaDefinitionExpression, CompiledLambda> compiledLambdaMap = ImmutableMap.builder();
    int counter = existingCompiledLambdas.size();
    for (LambdaDefinitionExpression lambdaExpression : lambdaExpressions) {
        CompiledLambda compiledLambda = LambdaBytecodeGenerator.preGenerateLambdaExpression(lambdaExpression, methodNamePrefix + "lambda_" + counter, containerClassDefinition, compiledLambdaMap.build(), callSiteBinder, cachedInstanceBinder, metadata, sqlFunctionProperties, sessionFunctions);
        compiledLambdaMap.put(lambdaExpression, compiledLambda);
        counter++;
    }
    return compiledLambdaMap.build();
}

Example 50 with Metadata

use of com.facebook.presto.metadata.Metadata in project presto by prestodb.

the class DynamicFilterMatcher method match.

private MatchResult match(FilterNode filterNode, Session session, Metadata metadata, SymbolAliases symbolAliases) {
    checkState(this.filterNode == null, "filterNode must be null at this point");
    this.filterNode = filterNode;
    this.symbolAliases = symbolAliases;
    LogicalRowExpressions logicalRowExpressions = new LogicalRowExpressions(new RowExpressionDeterminismEvaluator(metadata.getFunctionAndTypeManager()), new FunctionResolution(metadata.getFunctionAndTypeManager()), metadata.getFunctionAndTypeManager());
    boolean staticFilterMatches = expectedStaticFilter.map(filter -> {
        RowExpressionVerifier verifier = new RowExpressionVerifier(symbolAliases, metadata, session);
        RowExpression staticFilter = logicalRowExpressions.combineConjuncts(extractDynamicFilters(filterNode.getPredicate()).getStaticConjuncts());
        return verifier.process(filter, staticFilter);
    }).orElse(true);
    return new MatchResult(match() && staticFilterMatches);
}