Examples with Lop org.apache.sysml.lops.Lop used on opensource projects

Example 31 with Lop

use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.

the class BinaryOp method constructLopsAppend.

private void constructLopsAppend(ExecType et) {
    DataType dt1 = getInput().get(0).getDataType();
    DataType dt2 = getInput().get(1).getDataType();
    ValueType vt1 = getInput().get(0).getValueType();
    ValueType vt2 = getInput().get(1).getValueType();
    boolean cbind = op == OpOp2.CBIND;
    // sanity check for input data types
    if (!((dt1 == DataType.MATRIX && dt2 == DataType.MATRIX) || (dt1 == DataType.FRAME && dt2 == DataType.FRAME) || (dt1 == DataType.SCALAR && dt2 == DataType.SCALAR && vt1 == ValueType.STRING && vt2 == ValueType.STRING))) {
        throw new HopsException("Append can only apply to two matrices, two frames, or two scalar strings!");
    }
    Lop append = null;
    if (dt1 == DataType.MATRIX || dt1 == DataType.FRAME) {
        long rlen = cbind ? getInput().get(0).getDim1() : (getInput().get(0).dimsKnown() && getInput().get(1).dimsKnown()) ? getInput().get(0).getDim1() + getInput().get(1).getDim1() : -1;
        long clen = cbind ? ((getInput().get(0).dimsKnown() && getInput().get(1).dimsKnown()) ? getInput().get(0).getDim2() + getInput().get(1).getDim2() : -1) : getInput().get(0).getDim2();
        if (et == ExecType.MR) {
            append = constructMRAppendLop(getInput().get(0), getInput().get(1), getDataType(), getValueType(), cbind, this);
        } else if (et == ExecType.SPARK) {
            append = constructSPAppendLop(getInput().get(0), getInput().get(1), getDataType(), getValueType(), cbind, this);
            append.getOutputParameters().setDimensions(rlen, clen, getRowsInBlock(), getColsInBlock(), getNnz());
        } else // CP
        {
            // offset 1st input
            Lop offset = createOffsetLop(getInput().get(0), cbind);
            append = new Append(getInput().get(0).constructLops(), getInput().get(1).constructLops(), offset, getDataType(), getValueType(), cbind, et);
            append.getOutputParameters().setDimensions(rlen, clen, getRowsInBlock(), getColsInBlock(), getNnz());
        }
    } else // SCALAR-STRING and SCALAR-STRING (always CP)
    {
        append = new Append(getInput().get(0).constructLops(), getInput().get(1).constructLops(), Data.createLiteralLop(ValueType.INT, "-1"), getDataType(), getValueType(), cbind, ExecType.CP);
        append.getOutputParameters().setDimensions(0, 0, -1, -1, -1);
    }
    setLineNumbers(append);
    setLops(append);
}

Example 32 with Lop

use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.

the class BinaryOp method constructSPAppendLop.

public static Lop constructSPAppendLop(Hop left, Hop right, DataType dt, ValueType vt, boolean cbind, Hop current) {
    Lop ret = null;
    // offset 1st input
    Lop offset = createOffsetLop(left, cbind);
    AppendMethod am = optFindAppendSPMethod(left.getDim1(), left.getDim2(), right.getDim1(), right.getDim2(), right.getRowsInBlock(), right.getColsInBlock(), right.getNnz(), cbind, dt);
    switch(am) {
        case // special case map-only append
        MR_MAPPEND:
            {
                ret = new AppendM(left.constructLops(), right.constructLops(), offset, current.getDataType(), current.getValueType(), cbind, false, ExecType.SPARK);
                break;
            }
        case // special case reduce append w/ one column block
        MR_RAPPEND:
            {
                ret = new AppendR(left.constructLops(), right.constructLops(), current.getDataType(), current.getValueType(), cbind, ExecType.SPARK);
                break;
            }
        case MR_GAPPEND:
            {
                // offset second input
                Lop offset2 = createOffsetLop(right, cbind);
                ret = new AppendG(left.constructLops(), right.constructLops(), offset, offset2, current.getDataType(), current.getValueType(), cbind, ExecType.SPARK);
                break;
            }
        case SP_GAlignedAppend:
            {
                ret = new AppendGAlignedSP(left.constructLops(), right.constructLops(), offset, current.getDataType(), current.getValueType(), cbind);
                break;
            }
        default:
            throw new HopsException("Invalid SP append method: " + am);
    }
    ret.setAllPositions(current.getFilename(), current.getBeginLine(), current.getBeginColumn(), current.getEndLine(), current.getEndColumn());
    return ret;
}

Example 33 with Lop

use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.

the class DataGenOp method constructLops.

@Override
public Lop constructLops() {
    // return already created lops
    if (getLops() != null)
        return getLops();
    ExecType et = optFindExecType();
    HashMap<String, Lop> inputLops = new HashMap<>();
    for (Entry<String, Integer> cur : _paramIndexMap.entrySet()) {
        if (cur.getKey().equals(DataExpression.RAND_ROWS) && rowsKnown())
            inputLops.put(cur.getKey(), new LiteralOp(_dim1).constructLops());
        else if (cur.getKey().equals(DataExpression.RAND_COLS) && colsKnown())
            inputLops.put(cur.getKey(), new LiteralOp(_dim2).constructLops());
        else
            inputLops.put(cur.getKey(), getInput().get(cur.getValue()).constructLops());
    }
    DataGen rnd = new DataGen(_op, _id, inputLops, _baseDir, getDataType(), getValueType(), et);
    int k = OptimizerUtils.getConstrainedNumThreads(_maxNumThreads);
    rnd.setNumThreads(k);
    rnd.getOutputParameters().setDimensions(getDim1(), getDim2(), // robust handling for blocksize (important for -exec singlenode; otherwise incorrect results)
    (getRowsInBlock() > 0) ? getRowsInBlock() : ConfigurationManager.getBlocksize(), (getColsInBlock() > 0) ? getColsInBlock() : ConfigurationManager.getBlocksize(), // actual rand nnz might differ (in cp/mr they are corrected after execution)
    (_op == DataGenMethod.RAND && et == ExecType.SPARK && getNnz() != 0) ? -1 : getNnz(), getUpdateType());
    setLineNumbers(rnd);
    setLops(rnd);
    // add reblock/checkpoint lops if necessary
    constructAndSetLopsDataFlowProperties();
    return getLops();
}

Example 34 with Lop

use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.

the class DataOp method constructLops.

@Override
public Lop constructLops() {
    // return already created lops
    if (getLops() != null)
        return getLops();
    ExecType et = optFindExecType();
    Lop l = null;
    // construct lops for all input parameters
    HashMap<String, Lop> inputLops = new HashMap<>();
    for (Entry<String, Integer> cur : _paramIndexMap.entrySet()) {
        inputLops.put(cur.getKey(), getInput().get(cur.getValue()).constructLops());
    }
    // Create the lop
    switch(_dataop) {
        case TRANSIENTREAD:
            l = new Data(HopsData2Lops.get(_dataop), null, inputLops, getName(), null, getDataType(), getValueType(), true, getInputFormatType());
            setOutputDimensions(l);
            break;
        case PERSISTENTREAD:
            l = new Data(HopsData2Lops.get(_dataop), null, inputLops, getName(), null, getDataType(), getValueType(), false, getInputFormatType());
            l.getOutputParameters().setDimensions(getDim1(), getDim2(), _inRowsInBlock, _inColsInBlock, getNnz(), getUpdateType());
            break;
        case PERSISTENTWRITE:
            l = new Data(HopsData2Lops.get(_dataop), getInput().get(0).constructLops(), inputLops, getName(), null, getDataType(), getValueType(), false, getInputFormatType());
            ((Data) l).setExecType(et);
            setOutputDimensions(l);
            break;
        case TRANSIENTWRITE:
            l = new Data(HopsData2Lops.get(_dataop), getInput().get(0).constructLops(), inputLops, getName(), null, getDataType(), getValueType(), true, getInputFormatType());
            setOutputDimensions(l);
            break;
        case FUNCTIONOUTPUT:
            l = new Data(HopsData2Lops.get(_dataop), getInput().get(0).constructLops(), inputLops, getName(), null, getDataType(), getValueType(), true, getInputFormatType());
            ((Data) l).setExecType(et);
            setOutputDimensions(l);
            break;
        default:
            throw new LopsException("Invalid operation type for Data LOP: " + _dataop);
    }
    setLineNumbers(l);
    setLops(l);
    // add reblock/checkpoint lops if necessary
    constructAndSetLopsDataFlowProperties();
    return getLops();
}

Example 35 with Lop

use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.

the class Hop method constructAndSetCompressionLopIfRequired.

private void constructAndSetCompressionLopIfRequired() {
    // determine execution type
    ExecType et = ExecType.CP;
    if (OptimizerUtils.isSparkExecutionMode() && getDataType() != DataType.SCALAR) {
        // persist and unpersist calls (4x the memory budget is conservative)
        if (OptimizerUtils.isHybridExecutionMode() && 2 * _outputMemEstimate < OptimizerUtils.getLocalMemBudget() || _etypeForced == ExecType.CP) {
            et = ExecType.CP;
        } else // default case
        {
            et = ExecType.SPARK;
        }
    }
    // add reblock lop to output if required
    if (_requiresCompression) {
        try {
            Lop compress = new Compression(getLops(), getDataType(), getValueType(), et);
            setOutputDimensions(compress);
            setLineNumbers(compress);
            setLops(compress);
        } catch (LopsException ex) {
            throw new HopsException(ex);
        }
    }
}