Examples with DataType org.apache.sysml.parser.Expression.DataType used on opensource projects

Example 46 with DataType

use of org.apache.sysml.parser.Expression.DataType in project incubator-systemml by apache.

the class VariableCPInstruction method parseInstruction.

public static VariableCPInstruction parseInstruction(String str) {
    String[] parts = InstructionUtils.getInstructionPartsWithValueType(str);
    String opcode = parts[0];
    VariableOperationCode voc = getVariableOperationCode(opcode);
    if (voc == VariableOperationCode.CreateVariable) {
        if (// && parts.length != 10 )
        parts.length < 5)
            throw new DMLRuntimeException("Invalid number of operands in createvar instruction: " + str);
    } else if (voc == VariableOperationCode.MoveVariable) {
        // mvvar tempA A; or mvvar mvar5 "data/out.mtx" "binary"
        if (parts.length != 3 && parts.length != 4)
            throw new DMLRuntimeException("Invalid number of operands in mvvar instruction: " + str);
    } else if (voc == VariableOperationCode.Write) {
        // Write instructions for csv files also include three additional parameters (hasHeader, delimiter, sparse)
        if (parts.length != 5 && parts.length != 8)
            throw new DMLRuntimeException("Invalid number of operands in write instruction: " + str);
    } else {
        if (voc != VariableOperationCode.RemoveVariable)
            // no output
            InstructionUtils.checkNumFields(parts, getArity(voc));
    }
    CPOperand in1 = null, in2 = null, in3 = null, in4 = null, out = null;
    switch(voc) {
        case CreateVariable:
            // variable name
            DataType dt = DataType.valueOf(parts[4]);
            ValueType vt = dt == DataType.MATRIX ? ValueType.DOUBLE : ValueType.STRING;
            int extSchema = (dt == DataType.FRAME && parts.length >= 13) ? 1 : 0;
            in1 = new CPOperand(parts[1], vt, dt);
            // file name
            in2 = new CPOperand(parts[2], ValueType.STRING, DataType.SCALAR);
            // file name override flag (always literal)
            in3 = new CPOperand(parts[3], ValueType.BOOLEAN, DataType.SCALAR);
            // format
            String fmt = parts[5];
            if (fmt.equalsIgnoreCase("csv")) {
                // 14 inputs: createvar corresponding to READ -- includes properties hasHeader, delim, fill, and fillValue
                if (parts.length < 15 + extSchema || parts.length > 17 + extSchema)
                    throw new DMLRuntimeException("Invalid number of operands in createvar instruction: " + str);
            } else {
                if (parts.length != 6 && parts.length != 12 + extSchema)
                    throw new DMLRuntimeException("Invalid number of operands in createvar instruction: " + str);
            }
            OutputInfo oi = OutputInfo.stringToOutputInfo(fmt);
            InputInfo ii = OutputInfo.getMatchingInputInfo(oi);
            MatrixCharacteristics mc = new MatrixCharacteristics();
            if (parts.length == 6) {
            // do nothing
            } else if (parts.length >= 11) {
                // matrix characteristics
                mc.setDimension(Long.parseLong(parts[6]), Long.parseLong(parts[7]));
                mc.setBlockSize(Integer.parseInt(parts[8]), Integer.parseInt(parts[9]));
                mc.setNonZeros(Long.parseLong(parts[10]));
            } else {
                throw new DMLRuntimeException("Invalid number of operands in createvar instruction: " + str);
            }
            MetaDataFormat iimd = new MetaDataFormat(mc, oi, ii);
            UpdateType updateType = UpdateType.COPY;
            if (parts.length >= 12)
                updateType = UpdateType.valueOf(parts[11].toUpperCase());
            // handle frame schema
            String schema = (dt == DataType.FRAME && parts.length >= 13) ? parts[parts.length - 1] : null;
            if (fmt.equalsIgnoreCase("csv")) {
                // Cretevar instructions for CSV format either has 13 or 14 inputs.
                // 13 inputs: createvar corresponding to WRITE -- includes properties hasHeader, delim, and sparse
                // 14 inputs: createvar corresponding to READ -- includes properties hasHeader, delim, fill, and fillValue
                FileFormatProperties fmtProperties = null;
                if (parts.length == 15 + extSchema) {
                    boolean hasHeader = Boolean.parseBoolean(parts[12]);
                    String delim = parts[13];
                    boolean sparse = Boolean.parseBoolean(parts[14]);
                    fmtProperties = new CSVFileFormatProperties(hasHeader, delim, sparse);
                } else {
                    boolean hasHeader = Boolean.parseBoolean(parts[12]);
                    String delim = parts[13];
                    boolean fill = Boolean.parseBoolean(parts[14]);
                    double fillValue = UtilFunctions.parseToDouble(parts[15]);
                    String naStrings = null;
                    if (parts.length == 17 + extSchema)
                        naStrings = parts[16];
                    fmtProperties = new CSVFileFormatProperties(hasHeader, delim, fill, fillValue, naStrings);
                }
                return new VariableCPInstruction(VariableOperationCode.CreateVariable, in1, in2, in3, iimd, updateType, fmtProperties, schema, opcode, str);
            } else {
                return new VariableCPInstruction(VariableOperationCode.CreateVariable, in1, in2, in3, iimd, updateType, schema, opcode, str);
            }
        case AssignVariable:
            in1 = new CPOperand(parts[1]);
            in2 = new CPOperand(parts[2]);
            break;
        case CopyVariable:
            // Value types are not given here
            in1 = new CPOperand(parts[1], ValueType.UNKNOWN, DataType.UNKNOWN);
            in2 = new CPOperand(parts[2], ValueType.UNKNOWN, DataType.UNKNOWN);
            break;
        case MoveVariable:
            in1 = new CPOperand(parts[1], ValueType.UNKNOWN, DataType.UNKNOWN);
            in2 = new CPOperand(parts[2], ValueType.UNKNOWN, DataType.UNKNOWN);
            if (parts.length > 3)
                in3 = new CPOperand(parts[3], ValueType.UNKNOWN, DataType.UNKNOWN);
            break;
        case RemoveVariable:
            VariableCPInstruction rminst = new VariableCPInstruction(getVariableOperationCode(opcode), null, null, null, out, opcode, str);
            for (int i = 1; i < parts.length; i++) rminst.addInput(new CPOperand(parts[i], ValueType.UNKNOWN, DataType.SCALAR));
            return rminst;
        case RemoveVariableAndFile:
            in1 = new CPOperand(parts[1]);
            in2 = new CPOperand(parts[2]);
            // second argument must be a boolean
            if (in2.getValueType() != ValueType.BOOLEAN)
                throw new DMLRuntimeException("Unexpected value type for second argument in: " + str);
            break;
        case CastAsScalarVariable:
        case CastAsMatrixVariable:
        case CastAsFrameVariable:
        case CastAsDoubleVariable:
        case CastAsIntegerVariable:
        case CastAsBooleanVariable:
            // first operand is a variable name => string value type
            in1 = new CPOperand(parts[1]);
            // output variable name
            out = new CPOperand(parts[2]);
            break;
        case Write:
            in1 = new CPOperand(parts[1]);
            in2 = new CPOperand(parts[2]);
            in3 = new CPOperand(parts[3]);
            FileFormatProperties fprops = null;
            if (in3.getName().equalsIgnoreCase("csv")) {
                boolean hasHeader = Boolean.parseBoolean(parts[4]);
                String delim = parts[5];
                boolean sparse = Boolean.parseBoolean(parts[6]);
                fprops = new CSVFileFormatProperties(hasHeader, delim, sparse);
                // description
                in4 = new CPOperand(parts[7]);
            } else {
                fprops = new FileFormatProperties();
                // description
                in4 = new CPOperand(parts[4]);
            }
            VariableCPInstruction inst = new VariableCPInstruction(getVariableOperationCode(opcode), in1, in2, in3, out, null, fprops, null, null, opcode, str);
            inst.addInput(in4);
            return inst;
        case Read:
            in1 = new CPOperand(parts[1]);
            in2 = new CPOperand(parts[2]);
            out = null;
            break;
        case SetFileName:
            // variable name
            in1 = new CPOperand(parts[1]);
            // file name
            in2 = new CPOperand(parts[2], ValueType.UNKNOWN, DataType.UNKNOWN);
            // option: remote or local
            in3 = new CPOperand(parts[3], ValueType.UNKNOWN, DataType.UNKNOWN);
            // return new VariableCPInstruction(getVariableOperationCode(opcode), in1, in2, in3, str);
            break;
    }
    return new VariableCPInstruction(getVariableOperationCode(opcode), in1, in2, in3, out, opcode, str);
}

Example 47 with DataType

use of org.apache.sysml.parser.Expression.DataType in project incubator-systemml by apache.

the class MatrixMatrixAxpyGPUInstruction method parseInstruction.

public static MatrixMatrixAxpyGPUInstruction parseInstruction(String str) {
    String[] parts = InstructionUtils.getInstructionPartsWithValueType(str);
    InstructionUtils.checkNumFields(parts, 4);
    String opcode = parts[0];
    int multiplier = 1;
    if (opcode.equals("-*"))
        multiplier = -1;
    CPOperand in1 = new CPOperand(parts[1]);
    CPOperand constant = new CPOperand(parts[2]);
    if (constant.getDataType() != DataType.SCALAR)
        throw new DMLRuntimeException("Expected second operand to be a scalar");
    CPOperand in2 = new CPOperand(parts[3]);
    CPOperand out = new CPOperand(parts[4]);
    DataType dt1 = in1.getDataType();
    DataType dt2 = in2.getDataType();
    DataType dt3 = out.getDataType();
    Operator operator = (dt1 != dt2) ? InstructionUtils.parseScalarBinaryOperator(opcode, (dt1 == DataType.SCALAR)) : InstructionUtils.parseTernaryOperator(opcode);
    if (dt1 == DataType.MATRIX && dt2 == DataType.MATRIX && dt3 == DataType.MATRIX) {
        return new MatrixMatrixAxpyGPUInstruction(operator, in1, constant, multiplier, in2, out, opcode, str);
    } else if (dt3 == DataType.MATRIX && ((dt1 == DataType.SCALAR && dt2 == DataType.MATRIX) || (dt1 == DataType.MATRIX && dt2 == DataType.SCALAR))) {
        throw new DMLRuntimeException("Unsupported GPU PlusMult/MinusMult ArithmeticInstruction.");
    // return new ScalarMatrixArithmeticGPUInstruction(operator, in1, in2, out, opcode, str);
    } else
        throw new DMLRuntimeException("Unsupported GPU ArithmeticInstruction.");
}

Example 48 with DataType

use of org.apache.sysml.parser.Expression.DataType in project incubator-systemml by apache.

the class BinarySPInstruction method parseInstruction.

public static BinarySPInstruction parseInstruction(String str) {
    CPOperand in1 = new CPOperand("", ValueType.UNKNOWN, DataType.UNKNOWN);
    CPOperand in2 = new CPOperand("", ValueType.UNKNOWN, DataType.UNKNOWN);
    CPOperand out = new CPOperand("", ValueType.UNKNOWN, DataType.UNKNOWN);
    String opcode = null;
    boolean isBroadcast = false;
    VectorType vtype = null;
    if (str.startsWith("SPARK" + Lop.OPERAND_DELIMITOR + "map")) {
        String[] parts = InstructionUtils.getInstructionPartsWithValueType(str);
        InstructionUtils.checkNumFields(parts, 5);
        opcode = parts[0];
        in1.split(parts[1]);
        in2.split(parts[2]);
        out.split(parts[3]);
        vtype = VectorType.valueOf(parts[5]);
        isBroadcast = true;
    } else {
        opcode = parseBinaryInstruction(str, in1, in2, out);
    }
    DataType dt1 = in1.getDataType();
    DataType dt2 = in2.getDataType();
    Operator operator = InstructionUtils.parseExtendedBinaryOrBuiltinOperator(opcode, in1, in2);
    if (dt1 == DataType.MATRIX || dt2 == DataType.MATRIX) {
        if (dt1 == DataType.MATRIX && dt2 == DataType.MATRIX) {
            if (isBroadcast)
                return new BinaryMatrixBVectorSPInstruction(operator, in1, in2, out, vtype, opcode, str);
            else
                return new BinaryMatrixMatrixSPInstruction(operator, in1, in2, out, opcode, str);
        } else
            return new BinaryMatrixScalarSPInstruction(operator, in1, in2, out, opcode, str);
    }
    return null;
}

Example 49 with DataType

use of org.apache.sysml.parser.Expression.DataType in project incubator-systemml by apache.

the class CtableCPInstruction method findCtableOperation.

private Ctable.OperationTypes findCtableOperation() {
    DataType dt1 = input1.getDataType();
    DataType dt2 = input2.getDataType();
    DataType dt3 = input3.getDataType();
    return Ctable.findCtableOperationByInputDataTypes(dt1, dt2, dt3);
}

Example 50 with DataType

use of org.apache.sysml.parser.Expression.DataType in project incubator-systemml by apache.

the class GenerateClassesForMLContext method generateFunctionCallMethod.

/**
 * Obtain method for invoking a script function.
 *
 * @param scriptFilePath
 *            the path to a script file
 * @param fs
 *            a SystemML function statement
 * @param dmlFunctionCall
 *            a string representing the invocation of a script function
 * @return string representation of a method that performs a function call
 */
public static String generateFunctionCallMethod(String scriptFilePath, FunctionStatement fs, String dmlFunctionCall) {
    createFunctionOutputClass(scriptFilePath, fs);
    StringBuilder sb = new StringBuilder();
    sb.append("public ");
    // begin return type
    ArrayList<DataIdentifier> oparams = fs.getOutputParams();
    if (oparams.size() == 0) {
        sb.append("void");
    } else if (oparams.size() == 1) {
        // if 1 output, no need to encapsulate it, so return the output
        // directly
        DataIdentifier oparam = oparams.get(0);
        String type = getParamTypeAsString(oparam);
        sb.append(type);
    } else {
        String fullFunctionOutputClassName = getFullFunctionOutputClassName(scriptFilePath, fs);
        sb.append(fullFunctionOutputClassName);
    }
    sb.append(" ");
    // end return type
    sb.append(fs.getName());
    sb.append("(");
    ArrayList<DataIdentifier> inputParams = fs.getInputParams();
    for (int i = 0; i < inputParams.size(); i++) {
        if (i > 0) {
            sb.append(", ");
        }
        DataIdentifier inputParam = inputParams.get(i);
        /*
			 * Note: Using Object is currently preferrable to using
			 * datatype/valuetype to explicitly set the input type to
			 * Integer/Double/Boolean/String since Object allows the automatic
			 * handling of things such as automatic conversions from longs to
			 * ints.
			 */
        sb.append("Object ");
        sb.append(inputParam.getName());
    }
    sb.append(") {\n");
    sb.append("String scriptString = \"" + dmlFunctionCall + "\";\n");
    sb.append("org.apache.sysml.api.mlcontext.Script script = new org.apache.sysml.api.mlcontext.Script(scriptString);\n");
    if ((inputParams.size() > 0) || (oparams.size() > 0)) {
        sb.append("script");
    }
    for (int i = 0; i < inputParams.size(); i++) {
        DataIdentifier inputParam = inputParams.get(i);
        String name = inputParam.getName();
        sb.append(".in(\"" + name + "\", " + name + ")");
    }
    for (int i = 0; i < oparams.size(); i++) {
        DataIdentifier outputParam = oparams.get(i);
        String name = outputParam.getName();
        sb.append(".out(\"" + name + "\")");
    }
    if ((inputParams.size() > 0) || (oparams.size() > 0)) {
        sb.append(";\n");
    }
    sb.append("org.apache.sysml.api.mlcontext.MLResults results = script.execute();\n");
    if (oparams.size() == 0) {
        sb.append("return;\n");
    } else if (oparams.size() == 1) {
        DataIdentifier o = oparams.get(0);
        DataType dt = o.getDataType();
        ValueType vt = o.getValueType();
        if ((dt == DataType.SCALAR) && (vt == ValueType.INT)) {
            sb.append("long res = results.getLong(\"" + o.getName() + "\");\nreturn res;\n");
        } else if ((dt == DataType.SCALAR) && (vt == ValueType.DOUBLE)) {
            sb.append("double res = results.getDouble(\"" + o.getName() + "\");\nreturn res;\n");
        } else if ((dt == DataType.SCALAR) && (vt == ValueType.BOOLEAN)) {
            sb.append("boolean res = results.getBoolean(\"" + o.getName() + "\");\nreturn res;\n");
        } else if ((dt == DataType.SCALAR) && (vt == ValueType.STRING)) {
            sb.append("String res = results.getString(\"" + o.getName() + "\");\nreturn res;\n");
        } else if (dt == DataType.MATRIX) {
            sb.append("org.apache.sysml.api.mlcontext.Matrix res = results.getMatrix(\"" + o.getName() + "\");\nreturn res;\n");
        } else if (dt == DataType.FRAME) {
            sb.append("org.apache.sysml.api.mlcontext.Frame res = results.getFrame(\"" + o.getName() + "\");\nreturn res;\n");
        }
    } else {
        for (int i = 0; i < oparams.size(); i++) {
            DataIdentifier outputParam = oparams.get(i);
            String name = outputParam.getName().toLowerCase();
            String type = getParamTypeAsString(outputParam);
            DataType dt = outputParam.getDataType();
            ValueType vt = outputParam.getValueType();
            if ((dt == DataType.SCALAR) && (vt == ValueType.INT)) {
                sb.append(type + " " + name + " = results.getLong(\"" + outputParam.getName() + "\");\n");
            } else if ((dt == DataType.SCALAR) && (vt == ValueType.DOUBLE)) {
                sb.append(type + " " + name + " = results.getDouble(\"" + outputParam.getName() + "\");\n");
            } else if ((dt == DataType.SCALAR) && (vt == ValueType.BOOLEAN)) {
                sb.append(type + " " + name + " = results.getBoolean(\"" + outputParam.getName() + "\");\n");
            } else if ((dt == DataType.SCALAR) && (vt == ValueType.STRING)) {
                sb.append(type + " " + name + " = results.getString(\"" + outputParam.getName() + "\");\n");
            } else if (dt == DataType.MATRIX) {
                sb.append(type + " " + name + " = results.getMatrix(\"" + outputParam.getName() + "\");\n");
            } else if (dt == DataType.FRAME) {
                sb.append(type + " " + name + " = results.getFrame(\"" + outputParam.getName() + "\");\n");
            }
        }
        String ffocn = getFullFunctionOutputClassName(scriptFilePath, fs);
        sb.append(ffocn + " res = new " + ffocn + "(");
        for (int i = 0; i < oparams.size(); i++) {
            if (i > 0) {
                sb.append(", ");
            }
            DataIdentifier outputParam = oparams.get(i);
            String name = outputParam.getName().toLowerCase();
            sb.append(name);
        }
        sb.append(");\nreturn res;\n");
    }
    sb.append("}\n");
    return sb.toString();
}