Examples with DataType org.apache.sysml.parser.Expression.DataType used on opensource projects

Example 26 with DataType

use of org.apache.sysml.parser.Expression.DataType in project systemml by apache.

the class ProgramConverter method parseDataIdentifier.

private static DataIdentifier parseDataIdentifier(String in) {
    StringTokenizer st = new StringTokenizer(in, DATA_FIELD_DELIM);
    String name = st.nextToken();
    DataType dt = DataType.valueOf(st.nextToken());
    ValueType vt = ValueType.valueOf(st.nextToken());
    DataIdentifier dat = new DataIdentifier(name);
    dat.setDataType(dt);
    dat.setValueType(vt);
    return dat;
}

Example 27 with DataType

use of org.apache.sysml.parser.Expression.DataType in project systemml by apache.

the class BinaryOp method constructLopsAppend.

private void constructLopsAppend(ExecType et) {
    DataType dt1 = getInput().get(0).getDataType();
    DataType dt2 = getInput().get(1).getDataType();
    ValueType vt1 = getInput().get(0).getValueType();
    ValueType vt2 = getInput().get(1).getValueType();
    boolean cbind = op == OpOp2.CBIND;
    // sanity check for input data types
    if (!((dt1 == DataType.MATRIX && dt2 == DataType.MATRIX) || (dt1 == DataType.FRAME && dt2 == DataType.FRAME) || (dt1 == DataType.SCALAR && dt2 == DataType.SCALAR && vt1 == ValueType.STRING && vt2 == ValueType.STRING))) {
        throw new HopsException("Append can only apply to two matrices, two frames, or two scalar strings!");
    }
    Lop append = null;
    if (dt1 == DataType.MATRIX || dt1 == DataType.FRAME) {
        long rlen = cbind ? getInput().get(0).getDim1() : (getInput().get(0).dimsKnown() && getInput().get(1).dimsKnown()) ? getInput().get(0).getDim1() + getInput().get(1).getDim1() : -1;
        long clen = cbind ? ((getInput().get(0).dimsKnown() && getInput().get(1).dimsKnown()) ? getInput().get(0).getDim2() + getInput().get(1).getDim2() : -1) : getInput().get(0).getDim2();
        if (et == ExecType.MR) {
            append = constructMRAppendLop(getInput().get(0), getInput().get(1), getDataType(), getValueType(), cbind, this);
        } else if (et == ExecType.SPARK) {
            append = constructSPAppendLop(getInput().get(0), getInput().get(1), getDataType(), getValueType(), cbind, this);
            append.getOutputParameters().setDimensions(rlen, clen, getRowsInBlock(), getColsInBlock(), getNnz());
        } else // CP
        {
            // offset 1st input
            Lop offset = createOffsetLop(getInput().get(0), cbind);
            append = new Append(getInput().get(0).constructLops(), getInput().get(1).constructLops(), offset, getDataType(), getValueType(), cbind, et);
            append.getOutputParameters().setDimensions(rlen, clen, getRowsInBlock(), getColsInBlock(), getNnz());
        }
    } else // SCALAR-STRING and SCALAR-STRING (always CP)
    {
        append = new Append(getInput().get(0).constructLops(), getInput().get(1).constructLops(), Data.createLiteralLop(ValueType.INT, "-1"), getDataType(), getValueType(), cbind, ExecType.CP);
        append.getOutputParameters().setDimensions(0, 0, -1, -1, -1);
    }
    setLineNumbers(append);
    setLops(append);
}

Example 28 with DataType

use of org.apache.sysml.parser.Expression.DataType in project systemml by apache.

the class BinaryOp method refreshSizeInformation.

@Override
public void refreshSizeInformation() {
    Hop input1 = getInput().get(0);
    Hop input2 = getInput().get(1);
    DataType dt1 = input1.getDataType();
    DataType dt2 = input2.getDataType();
    if (getDataType() == DataType.SCALAR) {
        // do nothing always known
        setDim1(0);
        setDim2(0);
    } else // MATRIX OUTPUT
    {
        // TODO quantile
        if (op == OpOp2.CBIND) {
            setDim1(input1.rowsKnown() ? input1.getDim1() : input2.getDim1());
            // ensure both columns are known, otherwise dangerous underestimation due to +(-1)
            if (input1.colsKnown() && input2.colsKnown())
                setDim2(input1.getDim2() + input2.getDim2());
            else
                setDim2(-1);
            // ensure both nnz are known, otherwise dangerous underestimation due to +(-1)
            if (input1.getNnz() > 0 && input2.getNnz() > 0)
                setNnz(input1.getNnz() + input2.getNnz());
            else
                setNnz(-1);
        } else if (op == OpOp2.RBIND) {
            setDim2(colsKnown() ? input1.getDim2() : input2.getDim2());
            // ensure both rows are known, otherwise dangerous underestimation due to +(-1)
            if (input1.rowsKnown() && input2.rowsKnown())
                setDim1(input1.getDim1() + input2.getDim1());
            else
                setDim1(-1);
            // ensure both nnz are known, otherwise dangerous underestimation due to +(-1)
            if (input1.getNnz() > 0 && input2.getNnz() > 0)
                setNnz(input1.getNnz() + input2.getNnz());
            else
                setNnz(-1);
        } else if (op == OpOp2.SOLVE) {
            // normally the second input would be of equal size as the output
            // however, since we use qr internally, it also supports squared first inputs
            setDim1(input1.getDim2());
            setDim2(input2.getDim2());
        } else // general case
        {
            long ldim1, ldim2, lnnz1 = -1;
            if (dt1 == DataType.MATRIX && dt2 == DataType.SCALAR) {
                ldim1 = input1.getDim1();
                ldim2 = input1.getDim2();
                lnnz1 = input1.getNnz();
            } else if (dt1 == DataType.SCALAR && dt2 == DataType.MATRIX) {
                ldim1 = input2.getDim1();
                ldim2 = input2.getDim2();
            } else // MATRIX - MATRIX
            {
                // for cols we need to be careful with regard to matrix-vector operations
                if (// OUTER VECTOR OPERATION
                outer) {
                    ldim1 = input1.getDim1();
                    ldim2 = input2.getDim2();
                } else // GENERAL CASE
                {
                    ldim1 = (input1.rowsKnown()) ? input1.getDim1() : ((input2.getDim1() > 1) ? input2.getDim1() : -1);
                    ldim2 = (input1.colsKnown()) ? input1.getDim2() : ((input2.getDim2() > 1) ? input2.getDim2() : -1);
                    lnnz1 = input1.getNnz();
                }
            }
            setDim1(ldim1);
            setDim2(ldim2);
            // otherwise propagated via worst-case estimates
            if (op == OpOp2.POW || (input2 instanceof LiteralOp && OptimizerUtils.isBinaryOpConditionalSparseSafeExact(op, (LiteralOp) input2))) {
                setNnz(lnnz1);
            }
        }
    }
}

Example 29 with DataType

use of org.apache.sysml.parser.Expression.DataType in project systemml by apache.

the class BinaryOp method constructLopsCentralMoment.

private void constructLopsCentralMoment(ExecType et) {
    // The output data type is a SCALAR if central moment
    // gets computed in CP/SPARK, and it will be MATRIX otherwise.
    DataType dt = (et == ExecType.MR ? DataType.MATRIX : DataType.SCALAR);
    CentralMoment cm = new CentralMoment(getInput().get(0).constructLops(), getInput().get(1).constructLops(), dt, getValueType(), et);
    setLineNumbers(cm);
    if (et == ExecType.MR) {
        cm.getOutputParameters().setDimensions(1, 1, 0, 0, -1);
        UnaryCP unary1 = new UnaryCP(cm, HopsOpOp1LopsUS.get(OpOp1.CAST_AS_SCALAR), getDataType(), getValueType());
        unary1.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
        setLineNumbers(unary1);
        setLops(unary1);
    } else {
        cm.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
        setLops(cm);
    }
}

Example 30 with DataType

use of org.apache.sysml.parser.Expression.DataType in project systemml by apache.

the class BinaryOp method optFindExecType.

@Override
protected ExecType optFindExecType() {
    checkAndSetForcedPlatform();
    ExecType REMOTE = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
    DataType dt1 = getInput().get(0).getDataType();
    DataType dt2 = getInput().get(1).getDataType();
    if (_etypeForced != null) {
        _etype = _etypeForced;
    } else {
        if (OptimizerUtils.isMemoryBasedOptLevel()) {
            _etype = findExecTypeByMemEstimate();
        } else {
            _etype = null;
            if (dt1 == DataType.MATRIX && dt2 == DataType.MATRIX) {
                // OR if both are vectors
                if ((getInput().get(0).areDimsBelowThreshold() && getInput().get(1).areDimsBelowThreshold()) || (getInput().get(0).isVector() && getInput().get(1).isVector())) {
                    _etype = ExecType.CP;
                }
            } else if (dt1 == DataType.MATRIX && dt2 == DataType.SCALAR) {
                if (getInput().get(0).areDimsBelowThreshold() || getInput().get(0).isVector()) {
                    _etype = ExecType.CP;
                }
            } else if (dt1 == DataType.SCALAR && dt2 == DataType.MATRIX) {
                if (getInput().get(1).areDimsBelowThreshold() || getInput().get(1).isVector()) {
                    _etype = ExecType.CP;
                }
            } else {
                _etype = ExecType.CP;
            }
            // if no CP condition applied
            if (_etype == null)
                _etype = REMOTE;
        }
        // check for valid CP dimensions and matrix size
        checkAndSetInvalidCPDimsAndSize();
    }
    // single parent also in spark because it's likely cheap and reduces intermediates)
    if (_etype == ExecType.CP && _etypeForced != ExecType.CP && getDataType().isMatrix() && (dt1.isScalar() || dt2.isScalar()) && // scalar operations
    supportsMatrixScalarOperations() && // input is not checkpoint
    !(getInput().get(dt1.isScalar() ? 1 : 0) instanceof DataOp) && // unary scalar is only parent
    getInput().get(dt1.isScalar() ? 1 : 0).getParent().size() == 1 && // single block triggered exec
    !HopRewriteUtils.isSingleBlock(getInput().get(dt1.isScalar() ? 1 : 0)) && getInput().get(dt1.isScalar() ? 1 : 0).optFindExecType() == ExecType.SPARK) {
        // pull unary scalar operation into spark
        _etype = ExecType.SPARK;
    }
    // mark for recompile (forever)
    setRequiresRecompileIfNecessary();
    // ensure cp exec type for single-node operations
    if (op == OpOp2.SOLVE) {
        if (isGPUEnabled())
            _etype = ExecType.GPU;
        else
            _etype = ExecType.CP;
    }
    return _etype;
}