Examples with SortKeys org.apache.sysml.lops.SortKeys used on opensource projects

Example 1 with SortKeys

use of org.apache.sysml.lops.SortKeys in project incubator-systemml by apache.

the class TernaryOp method constructLopsQuantile.

/**
	 * Method to construct LOPs when op = QUANTILE | INTERQUANTILE.
	 * 
	 * @throws HopsException if HopsException occurs
	 * @throws LopsException if LopsException occurs
	 */
private void constructLopsQuantile() throws HopsException, LopsException {
    if (_op != OpOp3.QUANTILE && _op != OpOp3.INTERQUANTILE)
        throw new HopsException("Unexpected operation: " + _op + ", expecting " + OpOp3.QUANTILE + " or " + OpOp3.INTERQUANTILE);
    ExecType et = optFindExecType();
    if (et == ExecType.MR) {
        CombineBinary combine = CombineBinary.constructCombineLop(OperationTypes.PreSort, getInput().get(0).constructLops(), getInput().get(1).constructLops(), DataType.MATRIX, getValueType());
        SortKeys sort = SortKeys.constructSortByValueLop(combine, SortKeys.OperationTypes.WithWeights, DataType.MATRIX, getValueType(), et);
        // If only a single quantile is computed, then "pick" operation executes in CP.
        ExecType et_pick = (getInput().get(2).getDataType() == DataType.SCALAR ? ExecType.CP : ExecType.MR);
        PickByCount pick = new PickByCount(sort, getInput().get(2).constructLops(), getDataType(), getValueType(), (_op == Hop.OpOp3.QUANTILE) ? PickByCount.OperationTypes.VALUEPICK : PickByCount.OperationTypes.RANGEPICK, et_pick, false);
        combine.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getInput().get(0).getRowsInBlock(), getInput().get(0).getColsInBlock(), getInput().get(0).getNnz());
        sort.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getInput().get(0).getRowsInBlock(), getInput().get(0).getColsInBlock(), getInput().get(0).getNnz());
        setOutputDimensions(pick);
        setLineNumbers(pick);
        setLops(pick);
    } else //CP/Spark 
    {
        SortKeys sort = SortKeys.constructSortByValueLop(getInput().get(0).constructLops(), getInput().get(1).constructLops(), SortKeys.OperationTypes.WithWeights, getInput().get(0).getDataType(), getInput().get(0).getValueType(), et);
        PickByCount pick = new PickByCount(sort, getInput().get(2).constructLops(), getDataType(), getValueType(), (_op == Hop.OpOp3.QUANTILE) ? PickByCount.OperationTypes.VALUEPICK : PickByCount.OperationTypes.RANGEPICK, et, true);
        sort.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getInput().get(0).getRowsInBlock(), getInput().get(0).getColsInBlock(), getInput().get(0).getNnz());
        setOutputDimensions(pick);
        setLineNumbers(pick);
        setLops(pick);
    }
}

Example 2 with SortKeys

use of org.apache.sysml.lops.SortKeys in project incubator-systemml by apache.

the class BinaryOp method constructLopsMedian.

private void constructLopsMedian(ExecType et) throws HopsException, LopsException {
    if (et == ExecType.MR) {
        CombineBinary combine = CombineBinary.constructCombineLop(OperationTypes.PreSort, getInput().get(0).constructLops(), getInput().get(1).constructLops(), DataType.MATRIX, getValueType());
        SortKeys sort = SortKeys.constructSortByValueLop(combine, SortKeys.OperationTypes.WithWeights, DataType.MATRIX, getValueType(), et);
        combine.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
        // Sort dimensions are same as the first input
        sort.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getInput().get(0).getRowsInBlock(), getInput().get(0).getColsInBlock(), getInput().get(0).getNnz());
        ExecType et_pick = ExecType.CP;
        PickByCount pick = new PickByCount(sort, Data.createLiteralLop(ValueType.DOUBLE, Double.toString(0.5)), getDataType(), getValueType(), PickByCount.OperationTypes.MEDIAN, et_pick, false);
        pick.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
        pick.setAllPositions(this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
        setLops(pick);
    } else {
        SortKeys sort = SortKeys.constructSortByValueLop(getInput().get(0).constructLops(), getInput().get(1).constructLops(), SortKeys.OperationTypes.WithWeights, getInput().get(0).getDataType(), getInput().get(0).getValueType(), et);
        sort.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getInput().get(0).getRowsInBlock(), getInput().get(0).getColsInBlock(), getInput().get(0).getNnz());
        PickByCount pick = new PickByCount(sort, Data.createLiteralLop(ValueType.DOUBLE, Double.toString(0.5)), getDataType(), getValueType(), PickByCount.OperationTypes.MEDIAN, et, true);
        pick.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
        pick.setAllPositions(this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
        setLops(pick);
    }
}

Example 3 with SortKeys

use of org.apache.sysml.lops.SortKeys in project incubator-systemml by apache.

the class ReorgOp method constructLops.

@Override
public Lop constructLops() throws HopsException, LopsException {
    //return already created lops
    if (getLops() != null)
        return getLops();
    ExecType et = optFindExecType();
    switch(op) {
        case TRANSPOSE:
            {
                Lop lin = getInput().get(0).constructLops();
                if (lin instanceof Transform && ((Transform) lin).getOperationType() == OperationTypes.Transpose)
                    //if input is already a transpose, avoid redundant transpose ops
                    setLops(lin.getInputs().get(0));
                else if (getDim1() == 1 && getDim2() == 1)
                    //if input of size 1x1, avoid unnecessary transpose
                    setLops(lin);
                else {
                    //general case
                    int k = OptimizerUtils.getConstrainedNumThreads(_maxNumThreads);
                    if (DMLScript.USE_ACCELERATOR && (DMLScript.FORCE_ACCELERATOR || getMemEstimate() < OptimizerUtils.GPU_MEMORY_BUDGET)) {
                        et = ExecType.GPU;
                    }
                    Transform transform1 = new Transform(lin, HopsTransf2Lops.get(op), getDataType(), getValueType(), et, k);
                    setOutputDimensions(transform1);
                    setLineNumbers(transform1);
                    setLops(transform1);
                }
                break;
            }
        case DIAG:
            {
                Transform transform1 = new Transform(getInput().get(0).constructLops(), HopsTransf2Lops.get(op), getDataType(), getValueType(), et);
                setOutputDimensions(transform1);
                setLineNumbers(transform1);
                setLops(transform1);
                break;
            }
        case REV:
            {
                Lop rev = null;
                if (et == ExecType.MR) {
                    Lop tmp = new Transform(getInput().get(0).constructLops(), HopsTransf2Lops.get(op), getDataType(), getValueType(), et);
                    setOutputDimensions(tmp);
                    setLineNumbers(tmp);
                    Group group1 = new Group(tmp, Group.OperationTypes.Sort, DataType.MATRIX, getValueType());
                    setOutputDimensions(group1);
                    setLineNumbers(group1);
                    rev = new Aggregate(group1, Aggregate.OperationTypes.Sum, DataType.MATRIX, getValueType(), et);
                } else {
                    //CP/SPARK
                    rev = new Transform(getInput().get(0).constructLops(), HopsTransf2Lops.get(op), getDataType(), getValueType(), et);
                }
                setOutputDimensions(rev);
                setLineNumbers(rev);
                setLops(rev);
                break;
            }
        case RESHAPE:
            {
                if (et == ExecType.MR) {
                    Transform transform1 = new Transform(getInput().get(0).constructLops(), HopsTransf2Lops.get(op), getDataType(), getValueType(), et);
                    setOutputDimensions(transform1);
                    setLineNumbers(transform1);
                    for (//rows, cols, byrow
                    int i = 1; //rows, cols, byrow
                    i <= 3; //rows, cols, byrow
                    i++) {
                        Lop ltmp = getInput().get(i).constructLops();
                        transform1.addInput(ltmp);
                        ltmp.addOutput(transform1);
                    }
                    //force order of added lops
                    transform1.setLevel();
                    Group group1 = new Group(transform1, Group.OperationTypes.Sort, DataType.MATRIX, getValueType());
                    setOutputDimensions(group1);
                    setLineNumbers(group1);
                    Aggregate agg1 = new Aggregate(group1, Aggregate.OperationTypes.Sum, DataType.MATRIX, getValueType(), et);
                    setOutputDimensions(agg1);
                    setLineNumbers(agg1);
                    setLops(agg1);
                } else //CP/SPARK
                {
                    Transform transform1 = new Transform(getInput().get(0).constructLops(), HopsTransf2Lops.get(op), getDataType(), getValueType(), et);
                    setOutputDimensions(transform1);
                    setLineNumbers(transform1);
                    for (//rows, cols, byrow
                    int i = 1; //rows, cols, byrow
                    i <= 3; //rows, cols, byrow
                    i++) {
                        Lop ltmp = getInput().get(i).constructLops();
                        transform1.addInput(ltmp);
                        ltmp.addOutput(transform1);
                    }
                    //force order of added lops
                    transform1.setLevel();
                    setLops(transform1);
                }
                break;
            }
        case SORT:
            {
                Hop input = getInput().get(0);
                Hop by = getInput().get(1);
                Hop desc = getInput().get(2);
                Hop ixret = getInput().get(3);
                if (et == ExecType.MR) {
                    if (!(desc instanceof LiteralOp && ixret instanceof LiteralOp)) {
                        LOG.warn("Unsupported non-constant ordering parameters, using defaults and mark for recompilation.");
                        setRequiresRecompile();
                        desc = new LiteralOp(false);
                        ixret = new LiteralOp(false);
                    }
                    //Step 1: extraction (if unknown ncol or multiple columns)
                    Hop vinput = input;
                    if (input.getDim2() != 1) {
                        vinput = new IndexingOp("tmp1", getDataType(), getValueType(), input, new LiteralOp(1L), HopRewriteUtils.createValueHop(input, true), by, by, false, true);
                        vinput.refreshSizeInformation();
                        vinput.setOutputBlocksizes(getRowsInBlock(), getColsInBlock());
                        HopRewriteUtils.copyLineNumbers(this, vinput);
                    }
                    //Step 2: Index vector sort 
                    Hop voutput = null;
                    if (2 * OptimizerUtils.estimateSize(vinput.getDim1(), vinput.getDim2()) > OptimizerUtils.getLocalMemBudget() || FORCE_DIST_SORT_INDEXES) {
                        //large vector, fallback to MR sort
                        //sort indexes according to given values
                        SortKeys sort = new SortKeys(vinput.constructLops(), HopRewriteUtils.getBooleanValueSafe((LiteralOp) desc), SortKeys.OperationTypes.Indexes, vinput.getDataType(), vinput.getValueType(), ExecType.MR);
                        sort.getOutputParameters().setDimensions(vinput.getDim1(), 1, vinput.getRowsInBlock(), vinput.getColsInBlock(), vinput.getNnz());
                        setLineNumbers(sort);
                        //note: this sortindexes includes also the shift by offsets and 
                        //final aggregate because sideways passing of offsets would
                        //not nicely fit the current instruction model
                        setLops(sort);
                        voutput = this;
                    } else {
                        //small vector, use in-memory sort
                        ArrayList<Hop> sinputs = new ArrayList<Hop>();
                        sinputs.add(vinput);
                        //by (always vector)
                        sinputs.add(new LiteralOp(1));
                        sinputs.add(desc);
                        //indexreturn (always indexes)
                        sinputs.add(new LiteralOp(true));
                        voutput = new ReorgOp("tmp3", getDataType(), getValueType(), ReOrgOp.SORT, sinputs);
                        HopRewriteUtils.copyLineNumbers(this, voutput);
                        //explicitly construct CP lop; otherwise there is danger of infinite recursion if forced runtime platform.
                        voutput.setLops(constructCPOrSparkSortLop(vinput, sinputs.get(1), sinputs.get(2), sinputs.get(3), ExecType.CP, false));
                        voutput.getLops().getOutputParameters().setDimensions(vinput.getDim1(), vinput.getDim2(), vinput.getRowsInBlock(), vinput.getColsInBlock(), vinput.getNnz());
                        setLops(voutput.constructLops());
                    }
                    // -- done via X' = table(seq(), IX') %*% X;
                    if (!HopRewriteUtils.getBooleanValueSafe((LiteralOp) ixret)) {
                        //generate seq 
                        DataGenOp seq = HopRewriteUtils.createSeqDataGenOp(voutput);
                        seq.setName("tmp4");
                        seq.refreshSizeInformation();
                        //select exec type
                        seq.computeMemEstimate(new MemoTable());
                        HopRewriteUtils.copyLineNumbers(this, seq);
                        //generate table
                        TernaryOp table = new TernaryOp("tmp5", DataType.MATRIX, ValueType.DOUBLE, OpOp3.CTABLE, seq, voutput, new LiteralOp(1L));
                        table.setOutputBlocksizes(getRowsInBlock(), getColsInBlock());
                        table.refreshSizeInformation();
                        //force MR 
                        table.setForcedExecType(ExecType.MR);
                        HopRewriteUtils.copyLineNumbers(this, table);
                        table.setDisjointInputs(true);
                        table.setOutputEmptyBlocks(false);
                        //generate matrix mult
                        AggBinaryOp mmult = HopRewriteUtils.createMatrixMultiply(table, input);
                        //force MR 
                        mmult.setForcedExecType(ExecType.MR);
                        setLops(mmult.constructLops());
                        //cleanups
                        HopRewriteUtils.removeChildReference(table, input);
                    }
                } else //CP or Spark
                {
                    if (et == ExecType.SPARK && !FORCE_DIST_SORT_INDEXES)
                        bSortSPRewriteApplicable = isSortSPRewriteApplicable();
                    Lop transform1 = constructCPOrSparkSortLop(input, by, desc, ixret, et, bSortSPRewriteApplicable);
                    setOutputDimensions(transform1);
                    setLineNumbers(transform1);
                    setLops(transform1);
                }
                break;
            }
        default:
            throw new HopsException("Unsupported lops construction for operation type '" + op + "'.");
    }
    //add reblock/checkpoint lops if necessary
    constructAndSetLopsDataFlowProperties();
    return getLops();
}

Example 4 with SortKeys

use of org.apache.sysml.lops.SortKeys in project incubator-systemml by apache.

the class UnaryOp method constructLopsMedian.

private Lop constructLopsMedian() throws HopsException, LopsException {
    ExecType et = optFindExecType();
    if (et == ExecType.MR) {
        CombineUnary combine = CombineUnary.constructCombineLop(getInput().get(0).constructLops(), getDataType(), getValueType());
        SortKeys sort = SortKeys.constructSortByValueLop(combine, SortKeys.OperationTypes.WithoutWeights, DataType.MATRIX, ValueType.DOUBLE, et);
        combine.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
        // Sort dimensions are same as the first input
        sort.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getInput().get(0).getRowsInBlock(), getInput().get(0).getColsInBlock(), getInput().get(0).getNnz());
        // If only a single quantile is computed, then "pick" operation executes in CP.
        ExecType et_pick = ExecType.CP;
        PickByCount pick = new PickByCount(sort, Data.createLiteralLop(ValueType.DOUBLE, Double.toString(0.5)), getDataType(), getValueType(), PickByCount.OperationTypes.MEDIAN, et_pick, false);
        pick.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
        setLineNumbers(pick);
        return pick;
    } else {
        SortKeys sort = SortKeys.constructSortByValueLop(getInput().get(0).constructLops(), SortKeys.OperationTypes.WithoutWeights, DataType.MATRIX, ValueType.DOUBLE, et);
        sort.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getInput().get(0).getRowsInBlock(), getInput().get(0).getColsInBlock(), getInput().get(0).getNnz());
        PickByCount pick = new PickByCount(sort, Data.createLiteralLop(ValueType.DOUBLE, Double.toString(0.5)), getDataType(), getValueType(), PickByCount.OperationTypes.MEDIAN, et, true);
        pick.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
        setLineNumbers(pick);
        setLops(pick);
        return pick;
    }
}

Example 5 with SortKeys

use of org.apache.sysml.lops.SortKeys in project incubator-systemml by apache.

the class UnaryOp method constructLopsIQM.

private Lop constructLopsIQM() throws HopsException, LopsException {
    ExecType et = optFindExecType();
    Hop input = getInput().get(0);
    if (et == ExecType.MR) {
        CombineUnary combine = CombineUnary.constructCombineLop(input.constructLops(), DataType.MATRIX, getValueType());
        combine.getOutputParameters().setDimensions(input.getDim1(), input.getDim2(), input.getRowsInBlock(), input.getColsInBlock(), input.getNnz());
        SortKeys sort = SortKeys.constructSortByValueLop(combine, SortKeys.OperationTypes.WithoutWeights, DataType.MATRIX, ValueType.DOUBLE, ExecType.MR);
        // Sort dimensions are same as the first input
        sort.getOutputParameters().setDimensions(input.getDim1(), input.getDim2(), input.getRowsInBlock(), input.getColsInBlock(), input.getNnz());
        Data lit = Data.createLiteralLop(ValueType.DOUBLE, Double.toString(0.25));
        lit.setAllPositions(this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
        PickByCount pick = new PickByCount(sort, lit, DataType.MATRIX, getValueType(), PickByCount.OperationTypes.RANGEPICK);
        pick.getOutputParameters().setDimensions(-1, -1, getRowsInBlock(), getColsInBlock(), -1);
        setLineNumbers(pick);
        PartialAggregate pagg = new PartialAggregate(pick, HopsAgg2Lops.get(Hop.AggOp.SUM), HopsDirection2Lops.get(Hop.Direction.RowCol), DataType.MATRIX, getValueType());
        setLineNumbers(pagg);
        // Set the dimensions of PartialAggregate LOP based on the
        // direction in which aggregation is performed
        pagg.setDimensionsBasedOnDirection(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock());
        Group group1 = new Group(pagg, Group.OperationTypes.Sort, DataType.MATRIX, getValueType());
        group1.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
        setLineNumbers(group1);
        Aggregate agg1 = new Aggregate(group1, HopsAgg2Lops.get(Hop.AggOp.SUM), DataType.MATRIX, getValueType(), ExecType.MR);
        agg1.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
        agg1.setupCorrectionLocation(pagg.getCorrectionLocation());
        setLineNumbers(agg1);
        UnaryCP unary1 = new UnaryCP(agg1, HopsOpOp1LopsUS.get(OpOp1.CAST_AS_SCALAR), getDataType(), getValueType());
        unary1.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
        setLineNumbers(unary1);
        Unary iqm = new Unary(sort, unary1, Unary.OperationTypes.MR_IQM, DataType.SCALAR, ValueType.DOUBLE, ExecType.CP);
        iqm.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
        setLineNumbers(iqm);
        return iqm;
    } else {
        SortKeys sort = SortKeys.constructSortByValueLop(input.constructLops(), SortKeys.OperationTypes.WithoutWeights, DataType.MATRIX, ValueType.DOUBLE, et);
        sort.getOutputParameters().setDimensions(input.getDim1(), input.getDim2(), input.getRowsInBlock(), input.getColsInBlock(), input.getNnz());
        PickByCount pick = new PickByCount(sort, null, getDataType(), getValueType(), PickByCount.OperationTypes.IQM, et, true);
        pick.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
        setLineNumbers(pick);
        return pick;
    }
}