Examples with CombineBinary org.apache.sysml.lops.CombineBinary used on opensource projects

Example 1 with CombineBinary

use of org.apache.sysml.lops.CombineBinary in project incubator-systemml by apache.

the class TernaryOp method constructLopsQuantile.

/**
	 * Method to construct LOPs when op = QUANTILE | INTERQUANTILE.
	 * 
	 * @throws HopsException if HopsException occurs
	 * @throws LopsException if LopsException occurs
	 */
private void constructLopsQuantile() throws HopsException, LopsException {
    if (_op != OpOp3.QUANTILE && _op != OpOp3.INTERQUANTILE)
        throw new HopsException("Unexpected operation: " + _op + ", expecting " + OpOp3.QUANTILE + " or " + OpOp3.INTERQUANTILE);
    ExecType et = optFindExecType();
    if (et == ExecType.MR) {
        CombineBinary combine = CombineBinary.constructCombineLop(OperationTypes.PreSort, getInput().get(0).constructLops(), getInput().get(1).constructLops(), DataType.MATRIX, getValueType());
        SortKeys sort = SortKeys.constructSortByValueLop(combine, SortKeys.OperationTypes.WithWeights, DataType.MATRIX, getValueType(), et);
        // If only a single quantile is computed, then "pick" operation executes in CP.
        ExecType et_pick = (getInput().get(2).getDataType() == DataType.SCALAR ? ExecType.CP : ExecType.MR);
        PickByCount pick = new PickByCount(sort, getInput().get(2).constructLops(), getDataType(), getValueType(), (_op == Hop.OpOp3.QUANTILE) ? PickByCount.OperationTypes.VALUEPICK : PickByCount.OperationTypes.RANGEPICK, et_pick, false);
        combine.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getInput().get(0).getRowsInBlock(), getInput().get(0).getColsInBlock(), getInput().get(0).getNnz());
        sort.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getInput().get(0).getRowsInBlock(), getInput().get(0).getColsInBlock(), getInput().get(0).getNnz());
        setOutputDimensions(pick);
        setLineNumbers(pick);
        setLops(pick);
    } else //CP/Spark 
    {
        SortKeys sort = SortKeys.constructSortByValueLop(getInput().get(0).constructLops(), getInput().get(1).constructLops(), SortKeys.OperationTypes.WithWeights, getInput().get(0).getDataType(), getInput().get(0).getValueType(), et);
        PickByCount pick = new PickByCount(sort, getInput().get(2).constructLops(), getDataType(), getValueType(), (_op == Hop.OpOp3.QUANTILE) ? PickByCount.OperationTypes.VALUEPICK : PickByCount.OperationTypes.RANGEPICK, et, true);
        sort.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getInput().get(0).getRowsInBlock(), getInput().get(0).getColsInBlock(), getInput().get(0).getNnz());
        setOutputDimensions(pick);
        setLineNumbers(pick);
        setLops(pick);
    }
}

Example 2 with CombineBinary

use of org.apache.sysml.lops.CombineBinary in project incubator-systemml by apache.

the class Dag method getInputPathsAndParameters.

// Method to populate inputs and also populates node index mapping.
private static void getInputPathsAndParameters(Lop node, ArrayList<Lop> execNodes, ArrayList<String> inputStrings, ArrayList<InputInfo> inputInfos, ArrayList<Long> numRows, ArrayList<Long> numCols, ArrayList<Long> numRowsPerBlock, ArrayList<Long> numColsPerBlock, HashMap<Lop, Integer> nodeIndexMapping, ArrayList<String> inputLabels, ArrayList<Lop> inputLops, ArrayList<Integer> MRJobLineNumbers) throws LopsException {
    // treat rand as an input.
    if (node.getType() == Type.DataGen && execNodes.contains(node) && !nodeIndexMapping.containsKey(node)) {
        numRows.add(node.getOutputParameters().getNumRows());
        numCols.add(node.getOutputParameters().getNumCols());
        numRowsPerBlock.add(node.getOutputParameters().getRowsInBlock());
        numColsPerBlock.add(node.getOutputParameters().getColsInBlock());
        inputStrings.add(node.getInstructions(inputStrings.size(), inputStrings.size()));
        if (DMLScript.ENABLE_DEBUG_MODE) {
            MRJobLineNumbers.add(node._beginLine);
        }
        inputInfos.add(InputInfo.TextCellInputInfo);
        nodeIndexMapping.put(node, inputStrings.size() - 1);
        return;
    }
    // get input file names
    if (!execNodes.contains(node) && !nodeIndexMapping.containsKey(node) && !(node.getExecLocation() == ExecLocation.Data) && (!(node.getExecLocation() == ExecLocation.ControlProgram && node.getDataType() == DataType.SCALAR)) || (!execNodes.contains(node) && node.getExecLocation() == ExecLocation.Data && ((Data) node).getOperationType() == Data.OperationTypes.READ && ((Data) node).getDataType() != DataType.SCALAR && !nodeIndexMapping.containsKey(node))) {
        if (node.getOutputParameters().getFile_name() != null) {
            inputStrings.add(node.getOutputParameters().getFile_name());
        } else {
            // use label name
            inputStrings.add(Lop.VARIABLE_NAME_PLACEHOLDER + node.getOutputParameters().getLabel() + Lop.VARIABLE_NAME_PLACEHOLDER);
        }
        inputLabels.add(node.getOutputParameters().getLabel());
        inputLops.add(node);
        numRows.add(node.getOutputParameters().getNumRows());
        numCols.add(node.getOutputParameters().getNumCols());
        numRowsPerBlock.add(node.getOutputParameters().getRowsInBlock());
        numColsPerBlock.add(node.getOutputParameters().getColsInBlock());
        InputInfo nodeInputInfo = null;
        // Check if file format type is binary or text and update infos
        if (node.getOutputParameters().isBlocked()) {
            if (node.getOutputParameters().getFormat() == Format.BINARY)
                nodeInputInfo = InputInfo.BinaryBlockInputInfo;
            else
                throw new LopsException("Invalid format (" + node.getOutputParameters().getFormat() + ") encountered for a node/lop (ID=" + node.getID() + ") with blocked output.");
        } else {
            if (node.getOutputParameters().getFormat() == Format.TEXT)
                nodeInputInfo = InputInfo.TextCellInputInfo;
            else
                nodeInputInfo = InputInfo.BinaryCellInputInfo;
        }
        // the information on key/value classes
        if (node.getType() == Type.SortKeys) {
            // SortKeys is the input to some other lop (say, L)
            // InputInfo of L is the ouputInfo of SortKeys, which is
            // (compactformat, doubleWriteable, IntWritable)
            nodeInputInfo = new InputInfo(PickFromCompactInputFormat.class, DoubleWritable.class, IntWritable.class);
        } else if (node.getType() == Type.CombineBinary) {
            // CombineBinary is the input to some other lop (say, L)
            // InputInfo of L is the ouputInfo of CombineBinary
            // And, the outputInfo of CombineBinary depends on the operation!
            CombineBinary combine = (CombineBinary) node;
            if (combine.getOperation() == org.apache.sysml.lops.CombineBinary.OperationTypes.PreSort) {
                nodeInputInfo = new InputInfo(SequenceFileInputFormat.class, DoubleWritable.class, IntWritable.class);
            } else if (combine.getOperation() == org.apache.sysml.lops.CombineBinary.OperationTypes.PreCentralMoment || combine.getOperation() == org.apache.sysml.lops.CombineBinary.OperationTypes.PreCovUnweighted || combine.getOperation() == org.apache.sysml.lops.CombineBinary.OperationTypes.PreGroupedAggUnweighted) {
                nodeInputInfo = InputInfo.WeightedPairInputInfo;
            }
        } else if (node.getType() == Type.CombineTernary) {
            nodeInputInfo = InputInfo.WeightedPairInputInfo;
        }
        inputInfos.add(nodeInputInfo);
        nodeIndexMapping.put(node, inputStrings.size() - 1);
        return;
    }
    // if exec nodes does not contain node at this point, return.
    if (!execNodes.contains(node))
        return;
    // process children recursively
    for (Lop lop : node.getInputs()) {
        getInputPathsAndParameters(lop, execNodes, inputStrings, inputInfos, numRows, numCols, numRowsPerBlock, numColsPerBlock, nodeIndexMapping, inputLabels, inputLops, MRJobLineNumbers);
    }
}

Example 3 with CombineBinary

use of org.apache.sysml.lops.CombineBinary in project incubator-systemml by apache.

the class BinaryOp method constructLopsMedian.

private void constructLopsMedian(ExecType et) throws HopsException, LopsException {
    if (et == ExecType.MR) {
        CombineBinary combine = CombineBinary.constructCombineLop(OperationTypes.PreSort, getInput().get(0).constructLops(), getInput().get(1).constructLops(), DataType.MATRIX, getValueType());
        SortKeys sort = SortKeys.constructSortByValueLop(combine, SortKeys.OperationTypes.WithWeights, DataType.MATRIX, getValueType(), et);
        combine.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
        // Sort dimensions are same as the first input
        sort.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getInput().get(0).getRowsInBlock(), getInput().get(0).getColsInBlock(), getInput().get(0).getNnz());
        ExecType et_pick = ExecType.CP;
        PickByCount pick = new PickByCount(sort, Data.createLiteralLop(ValueType.DOUBLE, Double.toString(0.5)), getDataType(), getValueType(), PickByCount.OperationTypes.MEDIAN, et_pick, false);
        pick.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
        pick.setAllPositions(this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
        setLops(pick);
    } else {
        SortKeys sort = SortKeys.constructSortByValueLop(getInput().get(0).constructLops(), getInput().get(1).constructLops(), SortKeys.OperationTypes.WithWeights, getInput().get(0).getDataType(), getInput().get(0).getValueType(), et);
        sort.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getInput().get(0).getRowsInBlock(), getInput().get(0).getColsInBlock(), getInput().get(0).getNnz());
        PickByCount pick = new PickByCount(sort, Data.createLiteralLop(ValueType.DOUBLE, Double.toString(0.5)), getDataType(), getValueType(), PickByCount.OperationTypes.MEDIAN, et, true);
        pick.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
        pick.setAllPositions(this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
        setLops(pick);
    }
}

Example 4 with CombineBinary

use of org.apache.sysml.lops.CombineBinary in project incubator-systemml by apache.

the class TernaryOp method constructLopsCentralMoment.

/**
	 * Method to construct LOPs when op = CENTRAILMOMENT.
	 * 
	 * @throws HopsException if HopsException occurs
	 * @throws LopsException if LopsException occurs
	 */
private void constructLopsCentralMoment() throws HopsException, LopsException {
    if (_op != OpOp3.CENTRALMOMENT)
        throw new HopsException("Unexpected operation: " + _op + ", expecting " + OpOp3.CENTRALMOMENT);
    ExecType et = optFindExecType();
    if (et == ExecType.MR) {
        CombineBinary combine = CombineBinary.constructCombineLop(OperationTypes.PreCentralMoment, getInput().get(0).constructLops(), getInput().get(1).constructLops(), DataType.MATRIX, getValueType());
        combine.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getInput().get(0).getRowsInBlock(), getInput().get(0).getColsInBlock(), getInput().get(0).getNnz());
        CentralMoment cm = new CentralMoment(combine, getInput().get(2).constructLops(), DataType.MATRIX, getValueType(), et);
        cm.getOutputParameters().setDimensions(1, 1, 0, 0, -1);
        setLineNumbers(cm);
        UnaryCP unary1 = new UnaryCP(cm, HopsOpOp1LopsUS.get(OpOp1.CAST_AS_SCALAR), getDataType(), getValueType());
        unary1.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
        setLineNumbers(unary1);
        setLops(unary1);
    } else //CP / SPARK
    {
        CentralMoment cm = new CentralMoment(getInput().get(0).constructLops(), getInput().get(1).constructLops(), getInput().get(2).constructLops(), getDataType(), getValueType(), et);
        cm.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
        setLineNumbers(cm);
        setLops(cm);
    }
}

Example 5 with CombineBinary

use of org.apache.sysml.lops.CombineBinary in project incubator-systemml by apache.

the class BinaryOp method constructLopsIQM.

private void constructLopsIQM(ExecType et) throws HopsException, LopsException {
    if (et == ExecType.MR) {
        CombineBinary combine = CombineBinary.constructCombineLop(OperationTypes.PreSort, (Lop) getInput().get(0).constructLops(), (Lop) getInput().get(1).constructLops(), DataType.MATRIX, getValueType());
        combine.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getInput().get(0).getRowsInBlock(), getInput().get(0).getColsInBlock(), getInput().get(0).getNnz());
        SortKeys sort = SortKeys.constructSortByValueLop(combine, SortKeys.OperationTypes.WithWeights, DataType.MATRIX, ValueType.DOUBLE, ExecType.MR);
        // Sort dimensions are same as the first input
        sort.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getInput().get(0).getRowsInBlock(), getInput().get(0).getColsInBlock(), getInput().get(0).getNnz());
        Data lit = Data.createLiteralLop(ValueType.DOUBLE, Double.toString(0.25));
        setLineNumbers(lit);
        PickByCount pick = new PickByCount(sort, lit, DataType.MATRIX, getValueType(), PickByCount.OperationTypes.RANGEPICK);
        pick.getOutputParameters().setDimensions(-1, -1, getRowsInBlock(), getColsInBlock(), -1);
        setLineNumbers(pick);
        PartialAggregate pagg = new PartialAggregate(pick, HopsAgg2Lops.get(Hop.AggOp.SUM), HopsDirection2Lops.get(Hop.Direction.RowCol), DataType.MATRIX, getValueType());
        setLineNumbers(pagg);
        // Set the dimensions of PartialAggregate LOP based on the
        // direction in which aggregation is performed
        pagg.setDimensionsBasedOnDirection(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock());
        Group group1 = new Group(pagg, Group.OperationTypes.Sort, DataType.MATRIX, getValueType());
        setOutputDimensions(group1);
        setLineNumbers(group1);
        Aggregate agg1 = new Aggregate(group1, HopsAgg2Lops.get(Hop.AggOp.SUM), DataType.MATRIX, getValueType(), ExecType.MR);
        setOutputDimensions(agg1);
        agg1.setupCorrectionLocation(pagg.getCorrectionLocation());
        setLineNumbers(agg1);
        UnaryCP unary1 = new UnaryCP(agg1, HopsOpOp1LopsUS.get(OpOp1.CAST_AS_SCALAR), DataType.SCALAR, getValueType());
        unary1.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
        setLineNumbers(unary1);
        Unary iqm = new Unary(sort, unary1, Unary.OperationTypes.MR_IQM, DataType.SCALAR, ValueType.DOUBLE, ExecType.CP);
        iqm.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
        setLineNumbers(iqm);
        setLops(iqm);
    } else {
        SortKeys sort = SortKeys.constructSortByValueLop(getInput().get(0).constructLops(), getInput().get(1).constructLops(), SortKeys.OperationTypes.WithWeights, getInput().get(0).getDataType(), getInput().get(0).getValueType(), et);
        sort.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getInput().get(0).getRowsInBlock(), getInput().get(0).getColsInBlock(), getInput().get(0).getNnz());
        PickByCount pick = new PickByCount(sort, null, getDataType(), getValueType(), PickByCount.OperationTypes.IQM, et, true);
        setOutputDimensions(pick);
        setLineNumbers(pick);
        setLops(pick);
    }
}