Examples with ILogicalPlan org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan used on opensource projects

Example 11 with ILogicalPlan

use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan in project asterixdb by apache.

the class SubstituteVariableVisitor method visitGroupByOperator.

@Override
public Void visitGroupByOperator(GroupByOperator op, Pair<LogicalVariable, LogicalVariable> pair) throws AlgebricksException {
    subst(pair.first, pair.second, op.getGroupByList());
    subst(pair.first, pair.second, op.getDecorList());
    for (ILogicalPlan p : op.getNestedPlans()) {
        for (Mutable<ILogicalOperator> r : p.getRoots()) {
            VariableUtilities.substituteVariablesInDescendantsAndSelf(r.getValue(), pair.first, pair.second, ctx);
        }
    }
    substVarTypes(op, pair);
    return null;
}

Example 12 with ILogicalPlan

use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan in project asterixdb by apache.

the class ProducedVariableVisitor method visitSubplanOperator.

@Override
public Void visitSubplanOperator(SubplanOperator op, Void arg) throws AlgebricksException {
    Set<LogicalVariable> producedVars = new HashSet<>();
    Set<LogicalVariable> liveVars = new HashSet<>();
    for (ILogicalPlan p : op.getNestedPlans()) {
        for (Mutable<ILogicalOperator> r : p.getRoots()) {
            VariableUtilities.getProducedVariablesInDescendantsAndSelf(r.getValue(), producedVars);
            VariableUtilities.getSubplanLocalLiveVariables(r.getValue(), liveVars);
        }
    }
    producedVars.retainAll(liveVars);
    producedVariables.addAll(producedVars);
    return null;
}

Example 13 with ILogicalPlan

use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan in project asterixdb by apache.

the class APIFramework method compileQuery.

public JobSpecification compileQuery(IClusterInfoCollector clusterInfoCollector, MetadataProvider metadataProvider, Query rwQ, int varCounter, String outputDatasetName, SessionOutput output, ICompiledDmlStatement statement) throws AlgebricksException, RemoteException, ACIDException {
    SessionConfig conf = output.config();
    if (!conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS) && conf.is(SessionConfig.OOB_REWRITTEN_EXPR_TREE)) {
        output.out().println();
        printPlanPrefix(output, "Rewritten expression tree");
        if (rwQ != null) {
            rwQ.accept(astPrintVisitorFactory.createLangVisitor(output.out()), 0);
        }
        printPlanPostfix(output);
    }
    org.apache.asterix.common.transactions.JobId asterixJobId = JobIdFactory.generateJobId();
    metadataProvider.setJobId(asterixJobId);
    ILangExpressionToPlanTranslator t = translatorFactory.createExpressionToPlanTranslator(metadataProvider, varCounter);
    ILogicalPlan plan;
    // statement = null when it's a query
    if (statement == null || statement.getKind() != Statement.Kind.LOAD) {
        plan = t.translate(rwQ, outputDatasetName, statement);
    } else {
        plan = t.translateLoad(statement);
    }
    if (!conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS) && conf.is(SessionConfig.OOB_LOGICAL_PLAN)) {
        output.out().println();
        printPlanPrefix(output, "Logical plan");
        if (rwQ != null || (statement != null && statement.getKind() == Statement.Kind.LOAD)) {
            LogicalOperatorPrettyPrintVisitor pvisitor = new LogicalOperatorPrettyPrintVisitor(output.out());
            PlanPrettyPrinter.printPlan(plan, pvisitor, 0);
        }
        printPlanPostfix(output);
    }
    CompilerProperties compilerProperties = metadataProvider.getApplicationContext().getCompilerProperties();
    int frameSize = compilerProperties.getFrameSize();
    Map<String, String> querySpecificConfig = metadataProvider.getConfig();
    // Validates the user-overridden query parameters.
    validateConfig(querySpecificConfig);
    int sortFrameLimit = getFrameLimit(CompilerProperties.COMPILER_SORTMEMORY_KEY, querySpecificConfig.get(CompilerProperties.COMPILER_SORTMEMORY_KEY), compilerProperties.getSortMemorySize(), frameSize, MIN_FRAME_LIMIT_FOR_SORT);
    int groupFrameLimit = getFrameLimit(CompilerProperties.COMPILER_GROUPMEMORY_KEY, querySpecificConfig.get(CompilerProperties.COMPILER_GROUPMEMORY_KEY), compilerProperties.getGroupMemorySize(), frameSize, MIN_FRAME_LIMIT_FOR_GROUP_BY);
    int joinFrameLimit = getFrameLimit(CompilerProperties.COMPILER_JOINMEMORY_KEY, querySpecificConfig.get(CompilerProperties.COMPILER_JOINMEMORY_KEY), compilerProperties.getJoinMemorySize(), frameSize, MIN_FRAME_LIMIT_FOR_JOIN);
    OptimizationConfUtil.getPhysicalOptimizationConfig().setFrameSize(frameSize);
    OptimizationConfUtil.getPhysicalOptimizationConfig().setMaxFramesExternalSort(sortFrameLimit);
    OptimizationConfUtil.getPhysicalOptimizationConfig().setMaxFramesExternalGroupBy(groupFrameLimit);
    OptimizationConfUtil.getPhysicalOptimizationConfig().setMaxFramesForJoin(joinFrameLimit);
    HeuristicCompilerFactoryBuilder builder = new HeuristicCompilerFactoryBuilder(OptimizationContextFactory.INSTANCE);
    builder.setPhysicalOptimizationConfig(OptimizationConfUtil.getPhysicalOptimizationConfig());
    builder.setLogicalRewrites(ruleSetFactory.getLogicalRewrites(metadataProvider.getApplicationContext()));
    builder.setPhysicalRewrites(ruleSetFactory.getPhysicalRewrites(metadataProvider.getApplicationContext()));
    IDataFormat format = metadataProvider.getFormat();
    ICompilerFactory compilerFactory = builder.create();
    builder.setExpressionEvalSizeComputer(format.getExpressionEvalSizeComputer());
    builder.setIMergeAggregationExpressionFactory(new MergeAggregationExpressionFactory());
    builder.setPartialAggregationTypeComputer(new PartialAggregationTypeComputer());
    builder.setExpressionTypeComputer(ExpressionTypeComputer.INSTANCE);
    builder.setMissableTypeComputer(MissableTypeComputer.INSTANCE);
    builder.setConflictingTypeResolver(ConflictingTypeResolver.INSTANCE);
    int parallelism = getParallelism(querySpecificConfig.get(CompilerProperties.COMPILER_PARALLELISM_KEY), compilerProperties.getParallelism());
    AlgebricksAbsolutePartitionConstraint computationLocations = chooseLocations(clusterInfoCollector, parallelism, metadataProvider.getClusterLocations());
    builder.setClusterLocations(computationLocations);
    ICompiler compiler = compilerFactory.createCompiler(plan, metadataProvider, t.getVarCounter());
    if (conf.isOptimize()) {
        compiler.optimize();
        if (conf.is(SessionConfig.OOB_OPTIMIZED_LOGICAL_PLAN)) {
            if (conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS)) {
                // For Optimizer tests.
                AlgebricksAppendable buffer = new AlgebricksAppendable(output.out());
                PlanPrettyPrinter.printPhysicalOps(plan, buffer, 0);
            } else {
                printPlanPrefix(output, "Optimized logical plan");
                if (rwQ != null || (statement != null && statement.getKind() == Statement.Kind.LOAD)) {
                    LogicalOperatorPrettyPrintVisitor pvisitor = new LogicalOperatorPrettyPrintVisitor(output.out());
                    PlanPrettyPrinter.printPlan(plan, pvisitor, 0);
                }
                printPlanPostfix(output);
            }
        }
    }
    if (rwQ != null && rwQ.isExplain()) {
        try {
            LogicalOperatorPrettyPrintVisitor pvisitor = new LogicalOperatorPrettyPrintVisitor();
            PlanPrettyPrinter.printPlan(plan, pvisitor, 0);
            ResultUtil.printResults(metadataProvider.getApplicationContext(), pvisitor.get().toString(), output, new Stats(), null);
            return null;
        } catch (IOException e) {
            throw new AlgebricksException(e);
        }
    }
    if (!conf.isGenerateJobSpec()) {
        return null;
    }
    builder.setBinaryBooleanInspectorFactory(format.getBinaryBooleanInspectorFactory());
    builder.setBinaryIntegerInspectorFactory(format.getBinaryIntegerInspectorFactory());
    builder.setComparatorFactoryProvider(format.getBinaryComparatorFactoryProvider());
    builder.setExpressionRuntimeProvider(new ExpressionRuntimeProvider(QueryLogicalExpressionJobGen.INSTANCE));
    builder.setHashFunctionFactoryProvider(format.getBinaryHashFunctionFactoryProvider());
    builder.setHashFunctionFamilyProvider(format.getBinaryHashFunctionFamilyProvider());
    builder.setMissingWriterFactory(format.getMissingWriterFactory());
    builder.setPredicateEvaluatorFactoryProvider(format.getPredicateEvaluatorFactoryProvider());
    final SessionConfig.OutputFormat outputFormat = conf.fmt();
    switch(outputFormat) {
        case LOSSLESS_JSON:
            builder.setPrinterProvider(format.getLosslessJSONPrinterFactoryProvider());
            break;
        case CSV:
            builder.setPrinterProvider(format.getCSVPrinterFactoryProvider());
            break;
        case ADM:
            builder.setPrinterProvider(format.getADMPrinterFactoryProvider());
            break;
        case CLEAN_JSON:
            builder.setPrinterProvider(format.getCleanJSONPrinterFactoryProvider());
            break;
        default:
            throw new AlgebricksException("Unexpected OutputFormat: " + outputFormat);
    }
    builder.setSerializerDeserializerProvider(format.getSerdeProvider());
    builder.setTypeTraitProvider(format.getTypeTraitProvider());
    builder.setNormalizedKeyComputerFactoryProvider(format.getNormalizedKeyComputerFactoryProvider());
    JobEventListenerFactory jobEventListenerFactory = new JobEventListenerFactory(asterixJobId, metadataProvider.isWriteTransaction());
    JobSpecification spec = compiler.createJob(metadataProvider.getApplicationContext(), jobEventListenerFactory);
    // When the top-level statement is a query, the statement parameter is null.
    if (statement == null) {
        // Sets a required capacity, only for read-only queries.
        // DDLs and DMLs are considered not that frequent.
        spec.setRequiredClusterCapacity(ResourceUtils.getRequiredCompacity(plan, computationLocations, sortFrameLimit, groupFrameLimit, joinFrameLimit, frameSize));
    }
    if (conf.is(SessionConfig.OOB_HYRACKS_JOB)) {
        printPlanPrefix(output, "Hyracks job");
        if (rwQ != null) {
            try {
                output.out().println(new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(spec.toJSON()));
            } catch (IOException e) {
                throw new AlgebricksException(e);
            }
            output.out().println(spec.getUserConstraints());
        }
        printPlanPostfix(output);
    }
    return spec;
}

Example 14 with ILogicalPlan

use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan in project asterixdb by apache.

the class SortGroupByPOperator method computeDeliveredProperties.

@Override
public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
    List<ILocalStructuralProperty> propsLocal = new LinkedList<ILocalStructuralProperty>();
    GroupByOperator gOp = (GroupByOperator) op;
    Set<LogicalVariable> columnSet = new ListSet<LogicalVariable>();
    List<OrderColumn> ocs = new ArrayList<OrderColumn>();
    if (!columnSet.isEmpty()) {
        propsLocal.add(new LocalGroupingProperty(columnSet));
    }
    for (OrderColumn oc : orderColumns) {
        ocs.add(oc);
    }
    propsLocal.add(new LocalOrderProperty(ocs));
    for (ILogicalPlan p : gOp.getNestedPlans()) {
        for (Mutable<ILogicalOperator> r : p.getRoots()) {
            ILogicalOperator rOp = r.getValue();
            propsLocal.addAll(rOp.getDeliveredPhysicalProperties().getLocalProperties());
        }
    }
    ILogicalOperator op2 = op.getInputs().get(0).getValue();
    IPhysicalPropertiesVector childProp = op2.getDeliveredPhysicalProperties();
    deliveredProperties = new StructuralPropertiesVector(childProp.getPartitioningProperty(), propsLocal);
}

Example 15 with ILogicalPlan

use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan in project asterixdb by apache.

the class LogicalOperatorPrettyPrintVisitor method printNestedPlans.

protected void printNestedPlans(AbstractOperatorWithNestedPlans op, Integer indent) throws AlgebricksException {
    boolean first = true;
    if (op.getNestedPlans().isEmpty()) {
        buffer.append("}");
    } else {
        for (ILogicalPlan p : op.getNestedPlans()) {
            // PrettyPrintUtil.indent(buffer, level + 10).append("var " +
            // p.first + ":\n");
            buffer.append("\n");
            if (first) {
                first = false;
            } else {
                addIndent(indent).append("       {\n");
            }
            PlanPrettyPrinter.printPlan(p, this, indent + 10);
            addIndent(indent).append("       }");
        }
    }
}