Examples with ParameterizedBuiltinOp org.apache.sysml.hops.ParameterizedBuiltinOp used on opensource projects

Example 16 with ParameterizedBuiltinOp

use of org.apache.sysml.hops.ParameterizedBuiltinOp in project incubator-systemml by apache.

the class RewriteAlgebraicSimplificationStatic method simplifyGroupedAggregate.

private static Hop simplifyGroupedAggregate(Hop hi) {
    if (// aggregate
    hi instanceof ParameterizedBuiltinOp && ((ParameterizedBuiltinOp) hi).getOp() == ParamBuiltinOp.GROUPEDAGG) {
        ParameterizedBuiltinOp phi = (ParameterizedBuiltinOp) hi;
        if (// aggregate(fn="count")
        phi.isCountFunction() && // only for vector
        phi.getTargetHop().getDim2() == 1) {
            HashMap<String, Integer> params = phi.getParamIndexMap();
            int ix1 = params.get(Statement.GAGG_TARGET);
            int ix2 = params.get(Statement.GAGG_GROUPS);
            // check for unnecessary memory consumption for "count"
            if (ix1 != ix2 && phi.getInput().get(ix1) != phi.getInput().get(ix2)) {
                Hop th = phi.getInput().get(ix1);
                Hop gh = phi.getInput().get(ix2);
                HopRewriteUtils.replaceChildReference(hi, th, gh, ix1);
                LOG.debug("Applied simplifyGroupedAggregateCount");
            }
        }
    }
    return hi;
}

Example 17 with ParameterizedBuiltinOp

use of org.apache.sysml.hops.ParameterizedBuiltinOp in project incubator-systemml by apache.

the class OptimizerRuleBased method rAssignRemainingParallelism.

protected void rAssignRemainingParallelism(OptNode n, int parforK, int opsK) {
    ArrayList<OptNode> childs = n.getChilds();
    if (childs != null) {
        boolean recompileSB = false;
        for (OptNode c : childs) {
            if (c.getNodeType() == NodeType.PARFOR) {
                // constrain max parfor parallelism by problem size
                int tmpN = Integer.parseInt(c.getParam(ParamType.NUM_ITERATIONS));
                int tmpK = (tmpN < parforK) ? tmpN : parforK;
                // set parfor degree of parallelism
                long id = c.getID();
                c.setK(tmpK);
                ParForProgramBlock pfpb = (ParForProgramBlock) OptTreeConverter.getAbstractPlanMapping().getMappedProg(id)[1];
                pfpb.setDegreeOfParallelism(tmpK);
                // distribute remaining parallelism
                int remainParforK = getRemainingParallelismParFor(parforK, tmpK);
                int remainOpsK = getRemainingParallelismOps(opsK, tmpK);
                rAssignRemainingParallelism(c, remainParforK, remainOpsK);
            } else if (c.getNodeType() == NodeType.HOP) {
                // set degree of parallelism for multi-threaded leaf nodes
                Hop h = OptTreeConverter.getAbstractPlanMapping().getMappedHop(c.getID());
                if (ConfigurationManager.isParallelMatrixOperations() && // abop, datagenop, qop, paramop
                h instanceof MultiThreadedHop && !(// only paramop-grpagg
                h instanceof ParameterizedBuiltinOp && !HopRewriteUtils.isValidOp(((ParameterizedBuiltinOp) h).getOp(), ParamBuiltinOp.GROUPEDAGG, ParamBuiltinOp.REXPAND)) && !(// only unaryop-cumulativeagg
                h instanceof UnaryOp && !((UnaryOp) h).isCumulativeUnaryOperation()) && !(// only reorgop-transpose
                h instanceof ReorgOp && ((ReorgOp) h).getOp() != ReOrgOp.TRANSPOSE)) {
                    MultiThreadedHop mhop = (MultiThreadedHop) h;
                    // set max constraint in hop
                    mhop.setMaxNumThreads(opsK);
                    // set optnode k (for explain)
                    c.setK(opsK);
                    // need to recompile SB, if changed constraint
                    recompileSB = true;
                } else // for all other multi-threaded hops set k=1 to simply debugging
                if (h instanceof MultiThreadedHop) {
                    MultiThreadedHop mhop = (MultiThreadedHop) h;
                    // set max constraint in hop
                    mhop.setMaxNumThreads(1);
                    // set optnode k (for explain)
                    c.setK(1);
                }
            } else
                rAssignRemainingParallelism(c, parforK, opsK);
        }
        // recompile statement block if required
        if (recompileSB) {
            try {
                // guaranteed to be a last-level block (see hop change)
                ProgramBlock pb = (ProgramBlock) OptTreeConverter.getAbstractPlanMapping().getMappedProg(n.getID())[1];
                Recompiler.recompileProgramBlockInstructions(pb);
            } catch (Exception ex) {
                throw new DMLRuntimeException(ex);
            }
        }
    }
}

Example 18 with ParameterizedBuiltinOp

use of org.apache.sysml.hops.ParameterizedBuiltinOp in project systemml by apache.

the class OptimizerRuleBased method rAssignRemainingParallelism.

protected void rAssignRemainingParallelism(OptNode n, int parforK, int opsK) {
    ArrayList<OptNode> childs = n.getChilds();
    if (childs != null) {
        boolean recompileSB = false;
        for (OptNode c : childs) {
            if (c.getNodeType() == NodeType.PARFOR) {
                // constrain max parfor parallelism by problem size
                int tmpN = Integer.parseInt(c.getParam(ParamType.NUM_ITERATIONS));
                int tmpK = (tmpN < parforK) ? tmpN : parforK;
                // set parfor degree of parallelism
                long id = c.getID();
                c.setK(tmpK);
                ParForProgramBlock pfpb = (ParForProgramBlock) OptTreeConverter.getAbstractPlanMapping().getMappedProg(id)[1];
                pfpb.setDegreeOfParallelism(tmpK);
                // distribute remaining parallelism
                int remainParforK = getRemainingParallelismParFor(parforK, tmpK);
                int remainOpsK = getRemainingParallelismOps(opsK, tmpK);
                rAssignRemainingParallelism(c, remainParforK, remainOpsK);
            } else if (c.getNodeType() == NodeType.HOP) {
                // set degree of parallelism for multi-threaded leaf nodes
                Hop h = OptTreeConverter.getAbstractPlanMapping().getMappedHop(c.getID());
                if (ConfigurationManager.isParallelMatrixOperations() && // abop, datagenop, qop, paramop
                h instanceof MultiThreadedHop && !(// only paramop-grpagg
                h instanceof ParameterizedBuiltinOp && !HopRewriteUtils.isValidOp(((ParameterizedBuiltinOp) h).getOp(), ParamBuiltinOp.GROUPEDAGG, ParamBuiltinOp.REXPAND)) && !(// only unaryop-cumulativeagg
                h instanceof UnaryOp && !((UnaryOp) h).isCumulativeUnaryOperation()) && !(// only reorgop-transpose
                h instanceof ReorgOp && ((ReorgOp) h).getOp() != ReOrgOp.TRANSPOSE)) {
                    MultiThreadedHop mhop = (MultiThreadedHop) h;
                    // set max constraint in hop
                    mhop.setMaxNumThreads(opsK);
                    // set optnode k (for explain)
                    c.setK(opsK);
                    // need to recompile SB, if changed constraint
                    recompileSB = true;
                } else // for all other multi-threaded hops set k=1 to simply debugging
                if (h instanceof MultiThreadedHop) {
                    MultiThreadedHop mhop = (MultiThreadedHop) h;
                    // set max constraint in hop
                    mhop.setMaxNumThreads(1);
                    // set optnode k (for explain)
                    c.setK(1);
                }
            } else
                rAssignRemainingParallelism(c, parforK, opsK);
        }
        // recompile statement block if required
        if (recompileSB) {
            try {
                // guaranteed to be a last-level block (see hop change)
                ProgramBlock pb = (ProgramBlock) OptTreeConverter.getAbstractPlanMapping().getMappedProg(n.getID())[1];
                Recompiler.recompileProgramBlockInstructions(pb);
            } catch (Exception ex) {
                throw new DMLRuntimeException(ex);
            }
        }
    }
}

Example 19 with ParameterizedBuiltinOp

use of org.apache.sysml.hops.ParameterizedBuiltinOp in project systemml by apache.

the class DMLTranslator method processParameterizedBuiltinFunctionExpression.

/**
 * Construct Hops from parse tree : Process ParameterizedBuiltinFunction Expression in an
 * assignment statement
 *
 * @param source parameterized built-in function
 * @param target data identifier
 * @param hops map of high-level operators
 * @return high-level operator
 */
private Hop processParameterizedBuiltinFunctionExpression(ParameterizedBuiltinFunctionExpression source, DataIdentifier target, HashMap<String, Hop> hops) {
    // this expression has multiple "named" parameters
    HashMap<String, Hop> paramHops = new HashMap<>();
    // -- construct hops for all input parameters
    // -- store them in hashmap so that their "name"s are maintained
    Hop pHop = null;
    for (String paramName : source.getVarParams().keySet()) {
        pHop = processExpression(source.getVarParam(paramName), null, hops);
        paramHops.put(paramName, pHop);
    }
    Hop currBuiltinOp = null;
    if (target == null) {
        target = createTarget(source);
    }
    // construct hop based on opcode
    switch(source.getOpCode()) {
        case CDF:
        case INVCDF:
        case QNORM:
        case QT:
        case QF:
        case QCHISQ:
        case QEXP:
        case PNORM:
        case PT:
        case PF:
        case PCHISQ:
        case PEXP:
            currBuiltinOp = constructDfHop(target.getName(), target.getDataType(), target.getValueType(), source.getOpCode(), paramHops);
            break;
        case GROUPEDAGG:
            currBuiltinOp = new ParameterizedBuiltinOp(target.getName(), target.getDataType(), target.getValueType(), ParamBuiltinOp.GROUPEDAGG, paramHops);
            break;
        case RMEMPTY:
            currBuiltinOp = new ParameterizedBuiltinOp(target.getName(), target.getDataType(), target.getValueType(), ParamBuiltinOp.RMEMPTY, paramHops);
            break;
        case REPLACE:
            currBuiltinOp = new ParameterizedBuiltinOp(target.getName(), target.getDataType(), target.getValueType(), ParamBuiltinOp.REPLACE, paramHops);
            break;
        case LOWER_TRI:
            currBuiltinOp = new ParameterizedBuiltinOp(target.getName(), target.getDataType(), target.getValueType(), ParamBuiltinOp.LOWER_TRI, paramHops);
            break;
        case UPPER_TRI:
            currBuiltinOp = new ParameterizedBuiltinOp(target.getName(), target.getDataType(), target.getValueType(), ParamBuiltinOp.UPPER_TRI, paramHops);
            break;
        case ORDER:
            ArrayList<Hop> inputs = new ArrayList<>();
            inputs.add(paramHops.get("target"));
            inputs.add(paramHops.get("by"));
            inputs.add(paramHops.get("decreasing"));
            inputs.add(paramHops.get("index.return"));
            currBuiltinOp = new ReorgOp(target.getName(), target.getDataType(), target.getValueType(), ReOrgOp.SORT, inputs);
            break;
        case TRANSFORMAPPLY:
            currBuiltinOp = new ParameterizedBuiltinOp(target.getName(), target.getDataType(), target.getValueType(), ParamBuiltinOp.TRANSFORMAPPLY, paramHops);
            break;
        case TRANSFORMDECODE:
            currBuiltinOp = new ParameterizedBuiltinOp(target.getName(), target.getDataType(), target.getValueType(), ParamBuiltinOp.TRANSFORMDECODE, paramHops);
            break;
        case TRANSFORMCOLMAP:
            currBuiltinOp = new ParameterizedBuiltinOp(target.getName(), target.getDataType(), target.getValueType(), ParamBuiltinOp.TRANSFORMCOLMAP, paramHops);
            break;
        case TRANSFORMMETA:
            currBuiltinOp = new ParameterizedBuiltinOp(target.getName(), target.getDataType(), target.getValueType(), ParamBuiltinOp.TRANSFORMMETA, paramHops);
            break;
        case TOSTRING:
            // check for input data type and only compile toString Hop for matrices/frames,
            // for scalars, we compile (s + "") to ensure consistent string output value types
            currBuiltinOp = !paramHops.get("target").getDataType().isScalar() ? new ParameterizedBuiltinOp(target.getName(), target.getDataType(), target.getValueType(), ParamBuiltinOp.TOSTRING, paramHops) : HopRewriteUtils.createBinary(paramHops.get("target"), new LiteralOp(""), OpOp2.PLUS);
            break;
        default:
            throw new ParseException(source.printErrorLocation() + "processParameterizedBuiltinFunctionExpression() -- Unknown operation: " + source.getOpCode());
    }
    setIdentifierParams(currBuiltinOp, source.getOutput());
    currBuiltinOp.setParseInfo(source);
    return currBuiltinOp;
}

Example 20 with ParameterizedBuiltinOp

use of org.apache.sysml.hops.ParameterizedBuiltinOp in project systemml by apache.

the class TemplateCell method isValidOperation.

protected static boolean isValidOperation(Hop hop) {
    // prepare indicators for binary operations
    boolean isBinaryMatrixScalar = false;
    boolean isBinaryMatrixVector = false;
    boolean isBinaryMatrixMatrix = false;
    if (hop instanceof BinaryOp && hop.getDataType().isMatrix()) {
        Hop left = hop.getInput().get(0);
        Hop right = hop.getInput().get(1);
        DataType ldt = left.getDataType();
        DataType rdt = right.getDataType();
        isBinaryMatrixScalar = (ldt.isScalar() || rdt.isScalar());
        isBinaryMatrixVector = hop.dimsKnown() && ((ldt.isMatrix() && TemplateUtils.isVectorOrScalar(right)) || (rdt.isMatrix() && TemplateUtils.isVectorOrScalar(left)));
        isBinaryMatrixMatrix = hop.dimsKnown() && HopRewriteUtils.isEqualSize(left, right) && ldt.isMatrix() && rdt.isMatrix();
    }
    // prepare indicators for ternary operations
    boolean isTernaryVectorScalarVector = false;
    boolean isTernaryMatrixScalarMatrixDense = false;
    boolean isTernaryIfElse = (HopRewriteUtils.isTernary(hop, OpOp3.IFELSE) && hop.getDataType().isMatrix());
    if (hop instanceof TernaryOp && hop.getInput().size() == 3 && hop.dimsKnown() && HopRewriteUtils.checkInputDataTypes(hop, DataType.MATRIX, DataType.SCALAR, DataType.MATRIX)) {
        Hop left = hop.getInput().get(0);
        Hop right = hop.getInput().get(2);
        isTernaryVectorScalarVector = TemplateUtils.isVector(left) && TemplateUtils.isVector(right);
        isTernaryMatrixScalarMatrixDense = HopRewriteUtils.isEqualSize(left, right) && !HopRewriteUtils.isSparse(left) && !HopRewriteUtils.isSparse(right);
    }
    // check supported unary, binary, ternary operations
    return hop.getDataType() == DataType.MATRIX && TemplateUtils.isOperationSupported(hop) && (hop instanceof UnaryOp || isBinaryMatrixScalar || isBinaryMatrixVector || isBinaryMatrixMatrix || isTernaryVectorScalarVector || isTernaryMatrixScalarMatrixDense || isTernaryIfElse || (hop instanceof ParameterizedBuiltinOp && ((ParameterizedBuiltinOp) hop).getOp() == ParamBuiltinOp.REPLACE));
}