Examples with AlgebricksException org.apache.hyracks.algebricks.common.exceptions.AlgebricksException used on opensource projects

Example 16 with AlgebricksException

use of org.apache.hyracks.algebricks.common.exceptions.AlgebricksException in project asterixdb by apache.

the class AbstractPreclusteredGroupByPOperator method getRequiredPropertiesForChildren.

@Override
public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op, IPhysicalPropertiesVector reqdByParent, IOptimizationContext context) {
    GroupByOperator gby = (GroupByOperator) op;
    StructuralPropertiesVector[] pv = new StructuralPropertiesVector[1];
    if (gby.isGroupAll() && gby.isGlobal()) {
        if (op.getExecutionMode() == ExecutionMode.UNPARTITIONED) {
            pv[0] = new StructuralPropertiesVector(IPartitioningProperty.UNPARTITIONED, null);
            return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
        } else {
            return emptyUnaryRequirements();
        }
    }
    List<ILocalStructuralProperty> localProps = new ArrayList<>();
    Set<LogicalVariable> gbvars = new ListSet<>(columnList);
    LocalGroupingProperty groupProp = new LocalGroupingProperty(gbvars, new ArrayList<>(columnList));
    boolean goon = true;
    for (ILogicalPlan p : gby.getNestedPlans()) {
        // groupings
        for (Mutable<ILogicalOperator> r : p.getRoots()) {
            AbstractLogicalOperator op1 = (AbstractLogicalOperator) r.getValue();
            if (op1.getOperatorTag() == LogicalOperatorTag.AGGREGATE) {
                AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
                IPhysicalOperator pop2 = op2.getPhysicalOperator();
                if (pop2 instanceof AbstractPreclusteredGroupByPOperator) {
                    List<LogicalVariable> gbyColumns = ((AbstractPreclusteredGroupByPOperator) pop2).getGbyColumns();
                    List<LogicalVariable> sndOrder = new ArrayList<>();
                    sndOrder.addAll(gbyColumns);
                    Set<LogicalVariable> freeVars = new HashSet<>();
                    try {
                        OperatorPropertiesUtil.getFreeVariablesInSelfOrDesc(op2, freeVars);
                    } catch (AlgebricksException e) {
                        throw new IllegalStateException(e);
                    }
                    // Only considers group key variables defined out-side the outer-most group-by operator.
                    sndOrder.retainAll(freeVars);
                    groupProp.getColumnSet().addAll(sndOrder);
                    groupProp.getPreferredOrderEnforcer().addAll(sndOrder);
                    goon = false;
                    break;
                }
            }
        }
        if (!goon) {
            break;
        }
    }
    localProps.add(groupProp);
    if (reqdByParent != null) {
        // propagate parent requirements
        List<ILocalStructuralProperty> lpPar = reqdByParent.getLocalProperties();
        if (lpPar != null) {
            boolean allOk = true;
            List<ILocalStructuralProperty> props = new ArrayList<>(lpPar.size());
            for (ILocalStructuralProperty prop : lpPar) {
                if (prop.getPropertyType() != PropertyType.LOCAL_ORDER_PROPERTY) {
                    allOk = false;
                    break;
                }
                LocalOrderProperty lop = (LocalOrderProperty) prop;
                List<OrderColumn> orderColumns = new ArrayList<>();
                List<OrderColumn> ords = lop.getOrderColumns();
                for (OrderColumn ord : ords) {
                    Pair<LogicalVariable, Mutable<ILogicalExpression>> p = getGbyPairByRhsVar(gby, ord.getColumn());
                    if (p == null) {
                        p = getDecorPairByRhsVar(gby, ord.getColumn());
                        if (p == null) {
                            allOk = false;
                            break;
                        }
                    }
                    ILogicalExpression e = p.second.getValue();
                    if (e.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
                        throw new IllegalStateException("Right hand side of group-by assignment should have been normalized to a variable reference.");
                    }
                    LogicalVariable v = ((VariableReferenceExpression) e).getVariableReference();
                    orderColumns.add(new OrderColumn(v, ord.getOrder()));
                }
                props.add(new LocalOrderProperty(orderColumns));
            }
            List<FunctionalDependency> fdList = new ArrayList<>();
            for (Pair<LogicalVariable, Mutable<ILogicalExpression>> decorPair : gby.getDecorList()) {
                List<LogicalVariable> hd = gby.getGbyVarList();
                List<LogicalVariable> tl = new ArrayList<>();
                tl.add(((VariableReferenceExpression) decorPair.second.getValue()).getVariableReference());
                fdList.add(new FunctionalDependency(hd, tl));
            }
            if (allOk && PropertiesUtil.matchLocalProperties(localProps, props, new HashMap<>(), fdList)) {
                localProps = props;
            }
        }
    }
    IPartitioningProperty pp = null;
    AbstractLogicalOperator aop = (AbstractLogicalOperator) op;
    if (aop.getExecutionMode() == ExecutionMode.PARTITIONED) {
        pp = new UnorderedPartitionedProperty(new ListSet<>(columnList), context.getComputationNodeDomain());
    }
    pv[0] = new StructuralPropertiesVector(pp, localProps);
    return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
}

Example 17 with AlgebricksException

use of org.apache.hyracks.algebricks.common.exceptions.AlgebricksException in project asterixdb by apache.

the class AbstractHashJoinPOperator method getRequiredPropertiesForChildren.

@Override
public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op, IPhysicalPropertiesVector reqdByParent, IOptimizationContext context) {
    // In a cost-based optimizer, we would also try to propagate the
    // parent's partitioning requirements.
    IPartitioningProperty pp1;
    IPartitioningProperty pp2;
    switch(partitioningType) {
        case PAIRWISE:
            pp1 = new UnorderedPartitionedProperty(new ListSet<>(keysLeftBranch), context.getComputationNodeDomain());
            pp2 = new UnorderedPartitionedProperty(new ListSet<>(keysRightBranch), context.getComputationNodeDomain());
            break;
        case BROADCAST:
            pp1 = new RandomPartitioningProperty(context.getComputationNodeDomain());
            pp2 = new BroadcastPartitioningProperty(context.getComputationNodeDomain());
            break;
        default:
            throw new IllegalStateException();
    }
    StructuralPropertiesVector[] pv = new StructuralPropertiesVector[2];
    pv[0] = OperatorPropertiesUtil.checkUnpartitionedAndGetPropertiesVector(op, new StructuralPropertiesVector(pp1, null));
    pv[1] = OperatorPropertiesUtil.checkUnpartitionedAndGetPropertiesVector(op, new StructuralPropertiesVector(pp2, null));
    IPartitioningRequirementsCoordinator prc;
    switch(kind) {
        case INNER:
            {
                prc = IPartitioningRequirementsCoordinator.EQCLASS_PARTITIONING_COORDINATOR;
                break;
            }
        case LEFT_OUTER:
            {
                prc = new IPartitioningRequirementsCoordinator() {

                    @Override
                    public Pair<Boolean, IPartitioningProperty> coordinateRequirements(IPartitioningProperty requirements, IPartitioningProperty firstDeliveredPartitioning, ILogicalOperator op, IOptimizationContext context) throws AlgebricksException {
                        if (firstDeliveredPartitioning != null && requirements != null && firstDeliveredPartitioning.getPartitioningType() == requirements.getPartitioningType()) {
                            switch(requirements.getPartitioningType()) {
                                case UNORDERED_PARTITIONED:
                                    {
                                        UnorderedPartitionedProperty upp1 = (UnorderedPartitionedProperty) firstDeliveredPartitioning;
                                        Set<LogicalVariable> set1 = upp1.getColumnSet();
                                        UnorderedPartitionedProperty uppreq = (UnorderedPartitionedProperty) requirements;
                                        Set<LogicalVariable> modifuppreq = new ListSet<LogicalVariable>();
                                        Map<LogicalVariable, EquivalenceClass> eqmap = context.getEquivalenceClassMap(op);
                                        Set<LogicalVariable> covered = new ListSet<LogicalVariable>();
                                        Set<LogicalVariable> keysCurrent = uppreq.getColumnSet();
                                        List<LogicalVariable> keysFirst = (keysRightBranch.containsAll(keysCurrent)) ? keysRightBranch : keysLeftBranch;
                                        List<LogicalVariable> keysSecond = keysFirst == keysRightBranch ? keysLeftBranch : keysRightBranch;
                                        for (LogicalVariable r : uppreq.getColumnSet()) {
                                            EquivalenceClass ecSnd = eqmap.get(r);
                                            boolean found = false;
                                            int j = 0;
                                            for (LogicalVariable rvar : keysFirst) {
                                                if (rvar == r || ecSnd != null && eqmap.get(rvar) == ecSnd) {
                                                    found = true;
                                                    break;
                                                }
                                                j++;
                                            }
                                            if (!found) {
                                                throw new IllegalStateException("Did not find a variable equivalent to " + r + " among " + keysFirst);
                                            }
                                            LogicalVariable v2 = keysSecond.get(j);
                                            EquivalenceClass ecFst = eqmap.get(v2);
                                            for (LogicalVariable vset1 : set1) {
                                                if (vset1 == v2 || ecFst != null && eqmap.get(vset1) == ecFst) {
                                                    covered.add(vset1);
                                                    modifuppreq.add(r);
                                                    break;
                                                }
                                            }
                                            if (covered.equals(set1)) {
                                                break;
                                            }
                                        }
                                        if (!covered.equals(set1)) {
                                            throw new AlgebricksException("Could not modify " + requirements + " to agree with partitioning property " + firstDeliveredPartitioning + " delivered by previous input operator.");
                                        }
                                        UnorderedPartitionedProperty upp2 = new UnorderedPartitionedProperty(modifuppreq, requirements.getNodeDomain());
                                        return new Pair<>(false, upp2);
                                    }
                                case ORDERED_PARTITIONED:
                                    {
                                        throw new NotImplementedException();
                                    }
                            }
                        }
                        return new Pair<>(true, requirements);
                    }
                };
                break;
            }
        default:
            {
                throw new IllegalStateException();
            }
    }
    return new PhysicalRequirements(pv, prc);
}

Example 18 with AlgebricksException

use of org.apache.hyracks.algebricks.common.exceptions.AlgebricksException in project asterixdb by apache.

the class APIFramework method compileQuery.

public JobSpecification compileQuery(IClusterInfoCollector clusterInfoCollector, MetadataProvider metadataProvider, Query rwQ, int varCounter, String outputDatasetName, SessionOutput output, ICompiledDmlStatement statement) throws AlgebricksException, RemoteException, ACIDException {
    SessionConfig conf = output.config();
    if (!conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS) && conf.is(SessionConfig.OOB_REWRITTEN_EXPR_TREE)) {
        output.out().println();
        printPlanPrefix(output, "Rewritten expression tree");
        if (rwQ != null) {
            rwQ.accept(astPrintVisitorFactory.createLangVisitor(output.out()), 0);
        }
        printPlanPostfix(output);
    }
    org.apache.asterix.common.transactions.JobId asterixJobId = JobIdFactory.generateJobId();
    metadataProvider.setJobId(asterixJobId);
    ILangExpressionToPlanTranslator t = translatorFactory.createExpressionToPlanTranslator(metadataProvider, varCounter);
    ILogicalPlan plan;
    // statement = null when it's a query
    if (statement == null || statement.getKind() != Statement.Kind.LOAD) {
        plan = t.translate(rwQ, outputDatasetName, statement);
    } else {
        plan = t.translateLoad(statement);
    }
    if (!conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS) && conf.is(SessionConfig.OOB_LOGICAL_PLAN)) {
        output.out().println();
        printPlanPrefix(output, "Logical plan");
        if (rwQ != null || (statement != null && statement.getKind() == Statement.Kind.LOAD)) {
            LogicalOperatorPrettyPrintVisitor pvisitor = new LogicalOperatorPrettyPrintVisitor(output.out());
            PlanPrettyPrinter.printPlan(plan, pvisitor, 0);
        }
        printPlanPostfix(output);
    }
    CompilerProperties compilerProperties = metadataProvider.getApplicationContext().getCompilerProperties();
    int frameSize = compilerProperties.getFrameSize();
    Map<String, String> querySpecificConfig = metadataProvider.getConfig();
    // Validates the user-overridden query parameters.
    validateConfig(querySpecificConfig);
    int sortFrameLimit = getFrameLimit(CompilerProperties.COMPILER_SORTMEMORY_KEY, querySpecificConfig.get(CompilerProperties.COMPILER_SORTMEMORY_KEY), compilerProperties.getSortMemorySize(), frameSize, MIN_FRAME_LIMIT_FOR_SORT);
    int groupFrameLimit = getFrameLimit(CompilerProperties.COMPILER_GROUPMEMORY_KEY, querySpecificConfig.get(CompilerProperties.COMPILER_GROUPMEMORY_KEY), compilerProperties.getGroupMemorySize(), frameSize, MIN_FRAME_LIMIT_FOR_GROUP_BY);
    int joinFrameLimit = getFrameLimit(CompilerProperties.COMPILER_JOINMEMORY_KEY, querySpecificConfig.get(CompilerProperties.COMPILER_JOINMEMORY_KEY), compilerProperties.getJoinMemorySize(), frameSize, MIN_FRAME_LIMIT_FOR_JOIN);
    OptimizationConfUtil.getPhysicalOptimizationConfig().setFrameSize(frameSize);
    OptimizationConfUtil.getPhysicalOptimizationConfig().setMaxFramesExternalSort(sortFrameLimit);
    OptimizationConfUtil.getPhysicalOptimizationConfig().setMaxFramesExternalGroupBy(groupFrameLimit);
    OptimizationConfUtil.getPhysicalOptimizationConfig().setMaxFramesForJoin(joinFrameLimit);
    HeuristicCompilerFactoryBuilder builder = new HeuristicCompilerFactoryBuilder(OptimizationContextFactory.INSTANCE);
    builder.setPhysicalOptimizationConfig(OptimizationConfUtil.getPhysicalOptimizationConfig());
    builder.setLogicalRewrites(ruleSetFactory.getLogicalRewrites(metadataProvider.getApplicationContext()));
    builder.setPhysicalRewrites(ruleSetFactory.getPhysicalRewrites(metadataProvider.getApplicationContext()));
    IDataFormat format = metadataProvider.getFormat();
    ICompilerFactory compilerFactory = builder.create();
    builder.setExpressionEvalSizeComputer(format.getExpressionEvalSizeComputer());
    builder.setIMergeAggregationExpressionFactory(new MergeAggregationExpressionFactory());
    builder.setPartialAggregationTypeComputer(new PartialAggregationTypeComputer());
    builder.setExpressionTypeComputer(ExpressionTypeComputer.INSTANCE);
    builder.setMissableTypeComputer(MissableTypeComputer.INSTANCE);
    builder.setConflictingTypeResolver(ConflictingTypeResolver.INSTANCE);
    int parallelism = getParallelism(querySpecificConfig.get(CompilerProperties.COMPILER_PARALLELISM_KEY), compilerProperties.getParallelism());
    AlgebricksAbsolutePartitionConstraint computationLocations = chooseLocations(clusterInfoCollector, parallelism, metadataProvider.getClusterLocations());
    builder.setClusterLocations(computationLocations);
    ICompiler compiler = compilerFactory.createCompiler(plan, metadataProvider, t.getVarCounter());
    if (conf.isOptimize()) {
        compiler.optimize();
        if (conf.is(SessionConfig.OOB_OPTIMIZED_LOGICAL_PLAN)) {
            if (conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS)) {
                // For Optimizer tests.
                AlgebricksAppendable buffer = new AlgebricksAppendable(output.out());
                PlanPrettyPrinter.printPhysicalOps(plan, buffer, 0);
            } else {
                printPlanPrefix(output, "Optimized logical plan");
                if (rwQ != null || (statement != null && statement.getKind() == Statement.Kind.LOAD)) {
                    LogicalOperatorPrettyPrintVisitor pvisitor = new LogicalOperatorPrettyPrintVisitor(output.out());
                    PlanPrettyPrinter.printPlan(plan, pvisitor, 0);
                }
                printPlanPostfix(output);
            }
        }
    }
    if (rwQ != null && rwQ.isExplain()) {
        try {
            LogicalOperatorPrettyPrintVisitor pvisitor = new LogicalOperatorPrettyPrintVisitor();
            PlanPrettyPrinter.printPlan(plan, pvisitor, 0);
            ResultUtil.printResults(metadataProvider.getApplicationContext(), pvisitor.get().toString(), output, new Stats(), null);
            return null;
        } catch (IOException e) {
            throw new AlgebricksException(e);
        }
    }
    if (!conf.isGenerateJobSpec()) {
        return null;
    }
    builder.setBinaryBooleanInspectorFactory(format.getBinaryBooleanInspectorFactory());
    builder.setBinaryIntegerInspectorFactory(format.getBinaryIntegerInspectorFactory());
    builder.setComparatorFactoryProvider(format.getBinaryComparatorFactoryProvider());
    builder.setExpressionRuntimeProvider(new ExpressionRuntimeProvider(QueryLogicalExpressionJobGen.INSTANCE));
    builder.setHashFunctionFactoryProvider(format.getBinaryHashFunctionFactoryProvider());
    builder.setHashFunctionFamilyProvider(format.getBinaryHashFunctionFamilyProvider());
    builder.setMissingWriterFactory(format.getMissingWriterFactory());
    builder.setPredicateEvaluatorFactoryProvider(format.getPredicateEvaluatorFactoryProvider());
    final SessionConfig.OutputFormat outputFormat = conf.fmt();
    switch(outputFormat) {
        case LOSSLESS_JSON:
            builder.setPrinterProvider(format.getLosslessJSONPrinterFactoryProvider());
            break;
        case CSV:
            builder.setPrinterProvider(format.getCSVPrinterFactoryProvider());
            break;
        case ADM:
            builder.setPrinterProvider(format.getADMPrinterFactoryProvider());
            break;
        case CLEAN_JSON:
            builder.setPrinterProvider(format.getCleanJSONPrinterFactoryProvider());
            break;
        default:
            throw new AlgebricksException("Unexpected OutputFormat: " + outputFormat);
    }
    builder.setSerializerDeserializerProvider(format.getSerdeProvider());
    builder.setTypeTraitProvider(format.getTypeTraitProvider());
    builder.setNormalizedKeyComputerFactoryProvider(format.getNormalizedKeyComputerFactoryProvider());
    JobEventListenerFactory jobEventListenerFactory = new JobEventListenerFactory(asterixJobId, metadataProvider.isWriteTransaction());
    JobSpecification spec = compiler.createJob(metadataProvider.getApplicationContext(), jobEventListenerFactory);
    // When the top-level statement is a query, the statement parameter is null.
    if (statement == null) {
        // Sets a required capacity, only for read-only queries.
        // DDLs and DMLs are considered not that frequent.
        spec.setRequiredClusterCapacity(ResourceUtils.getRequiredCompacity(plan, computationLocations, sortFrameLimit, groupFrameLimit, joinFrameLimit, frameSize));
    }
    if (conf.is(SessionConfig.OOB_HYRACKS_JOB)) {
        printPlanPrefix(output, "Hyracks job");
        if (rwQ != null) {
            try {
                output.out().println(new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(spec.toJSON()));
            } catch (IOException e) {
                throw new AlgebricksException(e);
            }
            output.out().println(spec.getUserConstraints());
        }
        printPlanPostfix(output);
    }
    return spec;
}

Example 19 with AlgebricksException

use of org.apache.hyracks.algebricks.common.exceptions.AlgebricksException in project asterixdb by apache.

the class ResultUtil method printResultHandle.

public static void printResultHandle(SessionOutput output, ResultHandle handle) throws HyracksDataException {
    try {
        final AlgebricksAppendable app = new AlgebricksAppendable(output.out());
        output.appendHandle(app, handle.toString());
    } catch (AlgebricksException e) {
        LOGGER.warn("error printing handle", e);
    }
}

Example 20 with AlgebricksException

use of org.apache.hyracks.algebricks.common.exceptions.AlgebricksException in project asterixdb by apache.

the class QueryServiceServlet method handleRequest.

private void handleRequest(RequestParameters param, IServletResponse response) throws IOException {
    LOGGER.info(param.toString());
    long elapsedStart = System.nanoTime();
    final StringWriter stringWriter = new StringWriter();
    final PrintWriter resultWriter = new PrintWriter(stringWriter);
    ResultDelivery delivery = parseResultDelivery(param.mode);
    String handleUrl = getHandleUrl(param.host, param.path, delivery);
    SessionOutput sessionOutput = createSessionOutput(param, handleUrl, resultWriter);
    SessionConfig sessionConfig = sessionOutput.config();
    HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, HttpUtil.Encoding.UTF8);
    HttpResponseStatus status = HttpResponseStatus.OK;
    Stats stats = new Stats();
    long[] execStartEnd = new long[] { -1, -1 };
    resultWriter.print("{\n");
    printRequestId(resultWriter);
    printClientContextID(resultWriter, param);
    printSignature(resultWriter);
    printType(resultWriter, sessionConfig);
    try {
        if (param.statement == null || param.statement.isEmpty()) {
            throw new AsterixException("Empty request, no statement provided");
        }
        String statementsText = param.statement + ";";
        executeStatement(statementsText, sessionOutput, delivery, stats, param, handleUrl, execStartEnd);
        if (ResultDelivery.IMMEDIATE == delivery || ResultDelivery.DEFERRED == delivery) {
            ResultUtil.printStatus(sessionOutput, ResultStatus.SUCCESS);
        }
    } catch (AlgebricksException | TokenMgrError | org.apache.asterix.aqlplus.parser.TokenMgrError pe) {
        GlobalConfig.ASTERIX_LOGGER.log(Level.INFO, pe.getMessage(), pe);
        ResultUtil.printError(resultWriter, pe);
        ResultUtil.printStatus(sessionOutput, ResultStatus.FATAL);
        status = HttpResponseStatus.BAD_REQUEST;
    } catch (HyracksException pe) {
        GlobalConfig.ASTERIX_LOGGER.log(Level.WARNING, pe.getMessage(), pe);
        ResultUtil.printError(resultWriter, pe);
        ResultUtil.printStatus(sessionOutput, ResultStatus.FATAL);
        status = HttpResponseStatus.INTERNAL_SERVER_ERROR;
    } catch (Exception e) {
        GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, "Unexpected exception", e);
        ResultUtil.printError(resultWriter, e);
        ResultUtil.printStatus(sessionOutput, ResultStatus.FATAL);
        status = HttpResponseStatus.INTERNAL_SERVER_ERROR;
    } finally {
        if (execStartEnd[0] == -1) {
            execStartEnd[1] = -1;
        } else if (execStartEnd[1] == -1) {
            execStartEnd[1] = System.nanoTime();
        }
    }
    printMetrics(resultWriter, System.nanoTime() - elapsedStart, execStartEnd[1] - execStartEnd[0], stats.getCount(), stats.getSize());
    resultWriter.print("}\n");
    resultWriter.flush();
    String result = stringWriter.toString();
    GlobalConfig.ASTERIX_LOGGER.log(Level.FINE, result);
    response.setStatus(status);
    response.writer().print(result);
    if (response.writer().checkError()) {
        LOGGER.warning("Error flushing output writer");
    }
}