Examples with Pair org.apache.hyracks.algebricks.common.utils.Pair used on opensource projects

Example 16 with Pair

use of org.apache.hyracks.algebricks.common.utils.Pair in project asterixdb by apache.

the class PigletMetadataProvider method getScannerRuntime.

@Override
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getScannerRuntime(IDataSource<String> dataSource, List<LogicalVariable> scanVariables, List<LogicalVariable> projectVariables, boolean projectPushed, List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars, IOperatorSchema opSchema, IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig) throws AlgebricksException {
    PigletFileDataSource ds = (PigletFileDataSource) dataSource;
    FileSplit[] fileSplits = ds.getFileSplits();
    String[] locations = new String[fileSplits.length];
    for (int i = 0; i < fileSplits.length; ++i) {
        locations[i] = fileSplits[i].getNodeName();
    }
    IFileSplitProvider fsp = new ConstantFileSplitProvider(fileSplits);
    Object[] colTypes = ds.getSchemaTypes();
    IValueParserFactory[] vpfs = new IValueParserFactory[colTypes.length];
    ISerializerDeserializer[] serDesers = new ISerializerDeserializer[colTypes.length];
    for (int i = 0; i < colTypes.length; ++i) {
        Type colType = (Type) colTypes[i];
        IValueParserFactory vpf;
        ISerializerDeserializer serDeser;
        switch(colType.getTag()) {
            case INTEGER:
                vpf = IntegerParserFactory.INSTANCE;
                serDeser = IntegerSerializerDeserializer.INSTANCE;
                break;
            case CHAR_ARRAY:
                vpf = UTF8StringParserFactory.INSTANCE;
                serDeser = new UTF8StringSerializerDeserializer();
                break;
            case FLOAT:
                vpf = FloatParserFactory.INSTANCE;
                serDeser = FloatSerializerDeserializer.INSTANCE;
                break;
            default:
                throw new UnsupportedOperationException();
        }
        vpfs[i] = vpf;
        serDesers[i] = serDeser;
    }
    ITupleParserFactory tpf = new DelimitedDataTupleParserFactory(vpfs, ',');
    RecordDescriptor rDesc = new RecordDescriptor(serDesers);
    IOperatorDescriptor scanner = new FileScanOperatorDescriptor(jobSpec, fsp, tpf, rDesc);
    AlgebricksAbsolutePartitionConstraint constraint = new AlgebricksAbsolutePartitionConstraint(locations);
    return new Pair<>(scanner, constraint);
}

Example 17 with Pair

use of org.apache.hyracks.algebricks.common.utils.Pair in project asterixdb by apache.

the class JobBuilder method addMicroOpToMetaRuntimeOp.

private void addMicroOpToMetaRuntimeOp(ILogicalOperator aop) {
    Integer k = algebraicOpBelongingToMetaAsterixOp.get(aop);
    if (k == null) {
        k = createNewMetaOpInfo(aop);
    }
    ArrayList<ILogicalOperator> destList = outEdges.get(aop);
    if (destList == null || destList.size() != 1) {
        // for now, we only support linear plans inside meta-ops.
        return;
    }
    ILogicalOperator dest = destList.get(0);
    Integer j = algebraicOpBelongingToMetaAsterixOp.get(dest);
    if (j == null && microOps.get(dest) != null) {
        algebraicOpBelongingToMetaAsterixOp.put(dest, k);
        List<Pair<IPushRuntimeFactory, RecordDescriptor>> aodContent1 = metaAsterixOpSkeletons.get(k);
        aodContent1.add(microOps.get(dest));
    } else if (j != null && j.intValue() != k.intValue()) {
        // merge the j component into the k component
        List<Pair<IPushRuntimeFactory, RecordDescriptor>> aodContent1 = metaAsterixOpSkeletons.get(k);
        List<Pair<IPushRuntimeFactory, RecordDescriptor>> aodContent2 = metaAsterixOpSkeletons.get(j);
        aodContent1.addAll(aodContent2);
        metaAsterixOpSkeletons.remove(j);
        for (ILogicalOperator m : algebraicOpBelongingToMetaAsterixOp.keySet()) {
            Integer g = algebraicOpBelongingToMetaAsterixOp.get(m);
            if (g.intValue() == j.intValue()) {
                algebraicOpBelongingToMetaAsterixOp.put(m, k);
            }
        }
    }
}

Example 18 with Pair

use of org.apache.hyracks.algebricks.common.utils.Pair in project asterixdb by apache.

the class JobBuilder method createNewMetaOpInfo.

private int createNewMetaOpInfo(ILogicalOperator aop) {
    int n = aodCounter;
    aodCounter++;
    List<Pair<IPushRuntimeFactory, RecordDescriptor>> metaOpContents = new ArrayList<>();
    metaOpContents.add(microOps.get(aop));
    metaAsterixOpSkeletons.put(n, metaOpContents);
    algebraicOpBelongingToMetaAsterixOp.put(aop, n);
    return n;
}

Example 19 with Pair

use of org.apache.hyracks.algebricks.common.utils.Pair in project asterixdb by apache.

the class PushAggregateIntoNestedSubplanRule method extractAggFunctionsFromExpression.

/**
     * @param exprRef
     * @param nspWithAgg
     * @param context
     * @return a pair whose first member is a boolean which is true iff
     *         something was changed in the expression tree rooted at expr. The
     *         second member is the result of transforming expr.
     * @throws AlgebricksException
     */
private Pair<Boolean, ILogicalExpression> extractAggFunctionsFromExpression(Mutable<ILogicalExpression> exprRef, Map<LogicalVariable, AbstractOperatorWithNestedPlans> nspWithAgg, Map<ILogicalExpression, ILogicalExpression> aggregateExprToVarExpr, IOptimizationContext context) throws AlgebricksException {
    ILogicalExpression expr = exprRef.getValue();
    switch(expr.getExpressionTag()) {
        case FUNCTION_CALL:
            AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) expr;
            FunctionIdentifier fi = BuiltinFunctions.getAggregateFunction(fce.getFunctionIdentifier());
            if (fi != null) {
                ILogicalExpression a1 = fce.getArguments().get(0).getValue();
                if (a1.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
                    LogicalVariable argVar = ((VariableReferenceExpression) a1).getVariableReference();
                    AbstractOperatorWithNestedPlans nspOp = nspWithAgg.get(argVar);
                    if (nspOp != null) {
                        if (!aggregateExprToVarExpr.containsKey(expr)) {
                            LogicalVariable newVar = context.newVar();
                            AggregateFunctionCallExpression aggFun = BuiltinFunctions.makeAggregateFunctionExpression(fi, fce.getArguments());
                            rewriteAggregateInNestedSubplan(argVar, nspOp, aggFun, newVar, context);
                            ILogicalExpression newVarExpr = new VariableReferenceExpression(newVar);
                            aggregateExprToVarExpr.put(expr, newVarExpr);
                            return new Pair<>(Boolean.TRUE, newVarExpr);
                        } else {
                            ILogicalExpression varExpr = aggregateExprToVarExpr.get(expr);
                            return new Pair<>(Boolean.TRUE, varExpr);
                        }
                    }
                }
            }
            boolean change = false;
            for (Mutable<ILogicalExpression> a : fce.getArguments()) {
                Pair<Boolean, ILogicalExpression> aggArg = extractAggFunctionsFromExpression(a, nspWithAgg, aggregateExprToVarExpr, context);
                if (aggArg.first.booleanValue()) {
                    a.setValue(aggArg.second);
                    change = true;
                }
            }
            return new Pair<>(change, fce);
        case VARIABLE:
        case CONSTANT:
            return new Pair<>(Boolean.FALSE, expr);
        default:
            throw new IllegalArgumentException();
    }
}

Example 20 with Pair

use of org.apache.hyracks.algebricks.common.utils.Pair in project asterixdb by apache.

the class PushAggregateIntoNestedSubplanRule method removeRedundantListifies.

private void removeRedundantListifies(Map<LogicalVariable, Integer> nspAggVars, Map<LogicalVariable, AbstractOperatorWithNestedPlans> nspWithAgg, Map<LogicalVariable, Integer> nspAggVarToPlanIndex) throws AlgebricksException {
    List<Pair<AbstractOperatorWithNestedPlans, Integer>> removeList = new ArrayList<>();
    for (Map.Entry<LogicalVariable, Integer> aggVarEntry : nspAggVars.entrySet()) {
        LogicalVariable aggVar = aggVarEntry.getKey();
        int occurs = aggVarEntry.getValue();
        if (occurs == 0) {
            AbstractOperatorWithNestedPlans nspOp = nspWithAgg.get(aggVar);
            AggregateOperator aggOp = (AggregateOperator) nspOp.getNestedPlans().get(nspAggVarToPlanIndex.get(aggVar)).getRoots().get(0).getValue();
            int pos = aggOp.getVariables().indexOf(aggVar);
            if (pos >= 0) {
                aggOp.getVariables().remove(pos);
                aggOp.getExpressions().remove(pos);
                List<LogicalVariable> producedVarsAtAgg = new ArrayList<>();
                VariableUtilities.getProducedVariablesInDescendantsAndSelf(aggOp, producedVarsAtAgg);
                if (producedVarsAtAgg.isEmpty()) {
                    removeList.add(new Pair<>(nspOp, nspAggVarToPlanIndex.get(aggVar)));
                }
            }
        }
    }
    // Collects subplans that is to be removed.
    Map<AbstractOperatorWithNestedPlans, List<ILogicalPlan>> nspToSubplanListMap = new HashMap<>();
    for (Pair<AbstractOperatorWithNestedPlans, Integer> remove : removeList) {
        AbstractOperatorWithNestedPlans groupByOperator = remove.first;
        ILogicalPlan subplan = remove.first.getNestedPlans().get(remove.second);
        if (nspToSubplanListMap.containsKey(groupByOperator)) {
            List<ILogicalPlan> subplans = nspToSubplanListMap.get(groupByOperator);
            subplans.add(subplan);
        } else {
            List<ILogicalPlan> subplans = new ArrayList<>();
            subplans.add(subplan);
            nspToSubplanListMap.put(groupByOperator, subplans);
        }
    }
    // Removes subplans.
    for (Map.Entry<AbstractOperatorWithNestedPlans, List<ILogicalPlan>> entry : nspToSubplanListMap.entrySet()) {
        entry.getKey().getNestedPlans().removeAll(entry.getValue());
    }
}