Examples with FileFormatProperties org.apache.sysml.runtime.matrix.data.FileFormatProperties used on opensource projects

Example 1 with FileFormatProperties

use of org.apache.sysml.runtime.matrix.data.FileFormatProperties in project incubator-systemml by apache.

the class WriteSPInstruction method parseInstruction.

public static WriteSPInstruction parseInstruction(String str) throws DMLRuntimeException {
    String[] parts = InstructionUtils.getInstructionPartsWithValueType(str);
    String opcode = parts[0];
    if (!opcode.equals("write")) {
        throw new DMLRuntimeException("Unsupported opcode");
    }
    // Write instructions for csv files also include three additional parameters (hasHeader, delimiter, sparse)
    if (parts.length != 5 && parts.length != 9) {
        throw new DMLRuntimeException("Invalid number of operands in write instruction: " + str);
    }
    //SPARK°write°_mVar2·MATRIX·DOUBLE°./src/test/scripts/functions/data/out/B·SCALAR·STRING·true°matrixmarket·SCALAR·STRING·true
    // _mVar2·MATRIX·DOUBLE
    CPOperand in1 = new CPOperand(parts[1]);
    CPOperand in2 = new CPOperand(parts[2]);
    CPOperand in3 = new CPOperand(parts[3]);
    WriteSPInstruction inst = new WriteSPInstruction(in1, in2, in3, opcode, str);
    if (in3.getName().equalsIgnoreCase("csv")) {
        boolean hasHeader = Boolean.parseBoolean(parts[4]);
        String delim = parts[5];
        boolean sparse = Boolean.parseBoolean(parts[6]);
        FileFormatProperties formatProperties = new CSVFileFormatProperties(hasHeader, delim, sparse);
        inst.setFormatProperties(formatProperties);
        boolean isInputMB = Boolean.parseBoolean(parts[7]);
        inst.setInputMatrixBlock(isInputMB);
        CPOperand in4 = new CPOperand(parts[8]);
        inst.input4 = in4;
    } else {
        FileFormatProperties ffp = new FileFormatProperties();
        CPOperand in4 = new CPOperand(parts[4]);
        inst.input4 = in4;
        inst.setFormatProperties(ffp);
    }
    return inst;
}

Example 2 with FileFormatProperties

use of org.apache.sysml.runtime.matrix.data.FileFormatProperties in project incubator-systemml by apache.

the class VariableCPInstruction method parseInstruction.

public static VariableCPInstruction parseInstruction(String str) throws DMLRuntimeException {
    String[] parts = InstructionUtils.getInstructionPartsWithValueType(str);
    String opcode = parts[0];
    VariableOperationCode voc = getVariableOperationCode(opcode);
    int _arity = -1;
    if (voc == VariableOperationCode.CreateVariable) {
        if (//&& parts.length != 10 )
        parts.length < 5)
            throw new DMLRuntimeException("Invalid number of operands in createvar instruction: " + str);
    } else if (voc == VariableOperationCode.MoveVariable) {
        // mvvar tempA A; or mvvar mvar5 "data/out.mtx" "binary"
        if (parts.length != 3 && parts.length != 4)
            throw new DMLRuntimeException("Invalid number of operands in mvvar instruction: " + str);
    } else if (voc == VariableOperationCode.Write) {
        // Write instructions for csv files also include three additional parameters (hasHeader, delimiter, sparse)
        if (parts.length != 5 && parts.length != 8)
            throw new DMLRuntimeException("Invalid number of operands in write instruction: " + str);
    } else {
        _arity = getArity(voc);
        // no output
        InstructionUtils.checkNumFields(parts, _arity);
    }
    CPOperand in1 = null, in2 = null, in3 = null, in4 = null, out = null;
    switch(voc) {
        case CreateVariable:
            // variable name 
            DataType dt = DataType.valueOf(parts[4]);
            ValueType vt = dt == DataType.MATRIX ? ValueType.DOUBLE : ValueType.STRING;
            int extSchema = (dt == DataType.FRAME && parts.length >= 13) ? 1 : 0;
            in1 = new CPOperand(parts[1], vt, dt);
            // file name
            in2 = new CPOperand(parts[2], ValueType.STRING, DataType.SCALAR);
            // file name override flag
            in3 = new CPOperand(parts[3], ValueType.BOOLEAN, DataType.SCALAR);
            // format 
            String fmt = parts[5];
            if (fmt.equalsIgnoreCase("csv")) {
                // 14 inputs: createvar corresponding to READ -- includes properties hasHeader, delim, fill, and fillValue
                if (parts.length < 15 + extSchema || parts.length > 17 + extSchema)
                    throw new DMLRuntimeException("Invalid number of operands in createvar instruction: " + str);
            } else {
                if (parts.length != 6 && parts.length != 12 + extSchema)
                    throw new DMLRuntimeException("Invalid number of operands in createvar instruction: " + str);
            }
            OutputInfo oi = OutputInfo.stringToOutputInfo(fmt);
            InputInfo ii = OutputInfo.getMatchingInputInfo(oi);
            MatrixCharacteristics mc = new MatrixCharacteristics();
            if (parts.length == 6) {
            // do nothing
            } else if (parts.length >= 11) {
                // matrix characteristics
                mc.setDimension(Long.parseLong(parts[6]), Long.parseLong(parts[7]));
                mc.setBlockSize(Integer.parseInt(parts[8]), Integer.parseInt(parts[9]));
                mc.setNonZeros(Long.parseLong(parts[10]));
            } else {
                throw new DMLRuntimeException("Invalid number of operands in createvar instruction: " + str);
            }
            MatrixFormatMetaData iimd = new MatrixFormatMetaData(mc, oi, ii);
            UpdateType updateType = UpdateType.COPY;
            if (parts.length >= 12)
                updateType = UpdateType.valueOf(parts[11].toUpperCase());
            //handle frame schema
            String schema = (dt == DataType.FRAME && parts.length >= 13) ? parts[parts.length - 1] : null;
            if (fmt.equalsIgnoreCase("csv")) {
                // Cretevar instructions for CSV format either has 13 or 14 inputs.
                // 13 inputs: createvar corresponding to WRITE -- includes properties hasHeader, delim, and sparse
                // 14 inputs: createvar corresponding to READ -- includes properties hasHeader, delim, fill, and fillValue
                FileFormatProperties fmtProperties = null;
                if (parts.length == 15 + extSchema) {
                    boolean hasHeader = Boolean.parseBoolean(parts[12]);
                    String delim = parts[13];
                    boolean sparse = Boolean.parseBoolean(parts[14]);
                    fmtProperties = new CSVFileFormatProperties(hasHeader, delim, sparse);
                } else {
                    boolean hasHeader = Boolean.parseBoolean(parts[12]);
                    String delim = parts[13];
                    boolean fill = Boolean.parseBoolean(parts[14]);
                    double fillValue = UtilFunctions.parseToDouble(parts[15]);
                    String naStrings = null;
                    if (parts.length == 17 + extSchema)
                        naStrings = parts[16];
                    fmtProperties = new CSVFileFormatProperties(hasHeader, delim, fill, fillValue, naStrings);
                }
                return new VariableCPInstruction(VariableOperationCode.CreateVariable, in1, in2, in3, iimd, updateType, parts.length, fmtProperties, schema, opcode, str);
            } else {
                return new VariableCPInstruction(VariableOperationCode.CreateVariable, in1, in2, in3, iimd, updateType, parts.length, schema, opcode, str);
            }
        case AssignVariable:
            in1 = new CPOperand(parts[1]);
            in2 = new CPOperand(parts[2]);
            break;
        case CopyVariable:
            // Value types are not given here
            in1 = new CPOperand(parts[1], ValueType.UNKNOWN, DataType.UNKNOWN);
            in2 = new CPOperand(parts[2], ValueType.UNKNOWN, DataType.UNKNOWN);
            break;
        case MoveVariable:
            in1 = new CPOperand(parts[1], ValueType.UNKNOWN, DataType.UNKNOWN);
            in2 = new CPOperand(parts[2], ValueType.UNKNOWN, DataType.UNKNOWN);
            if (parts.length > 3)
                in3 = new CPOperand(parts[3], ValueType.UNKNOWN, DataType.UNKNOWN);
            break;
        case RemoveVariable:
            in1 = new CPOperand(parts[1], ValueType.UNKNOWN, DataType.SCALAR);
            break;
        case RemoveVariableAndFile:
            in1 = new CPOperand(parts[1]);
            in2 = new CPOperand(parts[2]);
            // second argument must be a boolean
            if (in2.getValueType() != ValueType.BOOLEAN)
                throw new DMLRuntimeException("Unexpected value type for second argument in: " + str);
            break;
        case CastAsScalarVariable:
        case CastAsMatrixVariable:
        case CastAsFrameVariable:
        case CastAsDoubleVariable:
        case CastAsIntegerVariable:
        case CastAsBooleanVariable:
            // first operand is a variable name => string value type 
            in1 = new CPOperand(parts[1]);
            // output variable name 
            out = new CPOperand(parts[2]);
            break;
        case Write:
            in1 = new CPOperand(parts[1]);
            in2 = new CPOperand(parts[2]);
            in3 = new CPOperand(parts[3]);
            VariableCPInstruction inst = new VariableCPInstruction(getVariableOperationCode(opcode), in1, in2, in3, out, _arity, opcode, str);
            if (in3.getName().equalsIgnoreCase("csv")) {
                boolean hasHeader = Boolean.parseBoolean(parts[4]);
                String delim = parts[5];
                boolean sparse = Boolean.parseBoolean(parts[6]);
                FileFormatProperties formatProperties = new CSVFileFormatProperties(hasHeader, delim, sparse);
                inst.setFormatProperties(formatProperties);
                // description
                in4 = new CPOperand(parts[7]);
                inst.input4 = in4;
            } else {
                FileFormatProperties ffp = new FileFormatProperties();
                inst.setFormatProperties(ffp);
                // description
                in4 = new CPOperand(parts[4]);
                inst.input4 = in4;
            }
            return inst;
        case Read:
            in1 = new CPOperand(parts[1]);
            in2 = new CPOperand(parts[2]);
            out = null;
            break;
        case SetFileName:
            // variable name
            in1 = new CPOperand(parts[1]);
            // file name
            in2 = new CPOperand(parts[2], ValueType.UNKNOWN, DataType.UNKNOWN);
            // option: remote or local
            in3 = new CPOperand(parts[3], ValueType.UNKNOWN, DataType.UNKNOWN);
            //return new VariableCPInstruction(getVariableOperationCode(opcode), in1, in2, in3, str);
            break;
    }
    return new VariableCPInstruction(getVariableOperationCode(opcode), in1, in2, in3, out, _arity, opcode, str);
}