Examples with UpdateType org.apache.sysml.runtime.controlprogram.caching.MatrixObject.UpdateType used on opensource projects

Example 6 with UpdateType

use of org.apache.sysml.runtime.controlprogram.caching.MatrixObject.UpdateType in project incubator-systemml by apache.

the class ProgramBlock method prepareUpdateInPlaceVariables.

protected UpdateType[] prepareUpdateInPlaceVariables(ExecutionContext ec, long tid) {
    if (_sb == null || _sb.getUpdateInPlaceVars().isEmpty())
        return null;
    ArrayList<String> varnames = _sb.getUpdateInPlaceVars();
    UpdateType[] flags = new UpdateType[varnames.size()];
    for (int i = 0; i < flags.length; i++) {
        String varname = varnames.get(i);
        if (!ec.isMatrixObject(varname))
            continue;
        MatrixObject mo = ec.getMatrixObject(varname);
        flags[i] = mo.getUpdateType();
        // create deep copy if required and if it fits in thread-local mem budget
        if (flags[i] == UpdateType.COPY && OptimizerUtils.getLocalMemBudget() / 2 > OptimizerUtils.estimateSizeExactSparsity(mo.getMatrixCharacteristics())) {
            MatrixObject moNew = new MatrixObject(mo);
            MatrixBlock mbVar = mo.acquireRead();
            moNew.acquireModify(!mbVar.isInSparseFormat() ? new MatrixBlock(mbVar) : new MatrixBlock(mbVar, MatrixBlock.DEFAULT_INPLACE_SPARSEBLOCK, true));
            moNew.setFileName(mo.getFileName() + Lop.UPDATE_INPLACE_PREFIX + tid);
            mo.release();
            moNew.release();
            moNew.setUpdateType(UpdateType.INPLACE);
            ec.setVariable(varname, moNew);
        }
    }
    return flags;
}

Example 7 with UpdateType

use of org.apache.sysml.runtime.controlprogram.caching.MatrixObject.UpdateType in project incubator-systemml by apache.

the class VariableCPInstruction method parseInstruction.

public static VariableCPInstruction parseInstruction(String str) {
    String[] parts = InstructionUtils.getInstructionPartsWithValueType(str);
    String opcode = parts[0];
    VariableOperationCode voc = getVariableOperationCode(opcode);
    if (voc == VariableOperationCode.CreateVariable) {
        if (// && parts.length != 10 )
        parts.length < 5)
            throw new DMLRuntimeException("Invalid number of operands in createvar instruction: " + str);
    } else if (voc == VariableOperationCode.MoveVariable) {
        // mvvar tempA A; or mvvar mvar5 "data/out.mtx" "binary"
        if (parts.length != 3 && parts.length != 4)
            throw new DMLRuntimeException("Invalid number of operands in mvvar instruction: " + str);
    } else if (voc == VariableOperationCode.Write) {
        // Write instructions for csv files also include three additional parameters (hasHeader, delimiter, sparse)
        if (parts.length != 5 && parts.length != 8)
            throw new DMLRuntimeException("Invalid number of operands in write instruction: " + str);
    } else {
        if (voc != VariableOperationCode.RemoveVariable)
            // no output
            InstructionUtils.checkNumFields(parts, getArity(voc));
    }
    CPOperand in1 = null, in2 = null, in3 = null, in4 = null, out = null;
    switch(voc) {
        case CreateVariable:
            // variable name
            DataType dt = DataType.valueOf(parts[4]);
            ValueType vt = dt == DataType.MATRIX ? ValueType.DOUBLE : ValueType.STRING;
            int extSchema = (dt == DataType.FRAME && parts.length >= 13) ? 1 : 0;
            in1 = new CPOperand(parts[1], vt, dt);
            // file name
            in2 = new CPOperand(parts[2], ValueType.STRING, DataType.SCALAR);
            // file name override flag (always literal)
            in3 = new CPOperand(parts[3], ValueType.BOOLEAN, DataType.SCALAR);
            // format
            String fmt = parts[5];
            if (fmt.equalsIgnoreCase("csv")) {
                // 14 inputs: createvar corresponding to READ -- includes properties hasHeader, delim, fill, and fillValue
                if (parts.length < 15 + extSchema || parts.length > 17 + extSchema)
                    throw new DMLRuntimeException("Invalid number of operands in createvar instruction: " + str);
            } else {
                if (parts.length != 6 && parts.length != 12 + extSchema)
                    throw new DMLRuntimeException("Invalid number of operands in createvar instruction: " + str);
            }
            OutputInfo oi = OutputInfo.stringToOutputInfo(fmt);
            InputInfo ii = OutputInfo.getMatchingInputInfo(oi);
            MatrixCharacteristics mc = new MatrixCharacteristics();
            if (parts.length == 6) {
            // do nothing
            } else if (parts.length >= 11) {
                // matrix characteristics
                mc.setDimension(Long.parseLong(parts[6]), Long.parseLong(parts[7]));
                mc.setBlockSize(Integer.parseInt(parts[8]), Integer.parseInt(parts[9]));
                mc.setNonZeros(Long.parseLong(parts[10]));
            } else {
                throw new DMLRuntimeException("Invalid number of operands in createvar instruction: " + str);
            }
            MetaDataFormat iimd = new MetaDataFormat(mc, oi, ii);
            UpdateType updateType = UpdateType.COPY;
            if (parts.length >= 12)
                updateType = UpdateType.valueOf(parts[11].toUpperCase());
            // handle frame schema
            String schema = (dt == DataType.FRAME && parts.length >= 13) ? parts[parts.length - 1] : null;
            if (fmt.equalsIgnoreCase("csv")) {
                // Cretevar instructions for CSV format either has 13 or 14 inputs.
                // 13 inputs: createvar corresponding to WRITE -- includes properties hasHeader, delim, and sparse
                // 14 inputs: createvar corresponding to READ -- includes properties hasHeader, delim, fill, and fillValue
                FileFormatProperties fmtProperties = null;
                if (parts.length == 15 + extSchema) {
                    boolean hasHeader = Boolean.parseBoolean(parts[12]);
                    String delim = parts[13];
                    boolean sparse = Boolean.parseBoolean(parts[14]);
                    fmtProperties = new CSVFileFormatProperties(hasHeader, delim, sparse);
                } else {
                    boolean hasHeader = Boolean.parseBoolean(parts[12]);
                    String delim = parts[13];
                    boolean fill = Boolean.parseBoolean(parts[14]);
                    double fillValue = UtilFunctions.parseToDouble(parts[15]);
                    String naStrings = null;
                    if (parts.length == 17 + extSchema)
                        naStrings = parts[16];
                    fmtProperties = new CSVFileFormatProperties(hasHeader, delim, fill, fillValue, naStrings);
                }
                return new VariableCPInstruction(VariableOperationCode.CreateVariable, in1, in2, in3, iimd, updateType, fmtProperties, schema, opcode, str);
            } else {
                return new VariableCPInstruction(VariableOperationCode.CreateVariable, in1, in2, in3, iimd, updateType, schema, opcode, str);
            }
        case AssignVariable:
            in1 = new CPOperand(parts[1]);
            in2 = new CPOperand(parts[2]);
            break;
        case CopyVariable:
            // Value types are not given here
            in1 = new CPOperand(parts[1], ValueType.UNKNOWN, DataType.UNKNOWN);
            in2 = new CPOperand(parts[2], ValueType.UNKNOWN, DataType.UNKNOWN);
            break;
        case MoveVariable:
            in1 = new CPOperand(parts[1], ValueType.UNKNOWN, DataType.UNKNOWN);
            in2 = new CPOperand(parts[2], ValueType.UNKNOWN, DataType.UNKNOWN);
            if (parts.length > 3)
                in3 = new CPOperand(parts[3], ValueType.UNKNOWN, DataType.UNKNOWN);
            break;
        case RemoveVariable:
            VariableCPInstruction rminst = new VariableCPInstruction(getVariableOperationCode(opcode), null, null, null, out, opcode, str);
            for (int i = 1; i < parts.length; i++) rminst.addInput(new CPOperand(parts[i], ValueType.UNKNOWN, DataType.SCALAR));
            return rminst;
        case RemoveVariableAndFile:
            in1 = new CPOperand(parts[1]);
            in2 = new CPOperand(parts[2]);
            // second argument must be a boolean
            if (in2.getValueType() != ValueType.BOOLEAN)
                throw new DMLRuntimeException("Unexpected value type for second argument in: " + str);
            break;
        case CastAsScalarVariable:
        case CastAsMatrixVariable:
        case CastAsFrameVariable:
        case CastAsDoubleVariable:
        case CastAsIntegerVariable:
        case CastAsBooleanVariable:
            // first operand is a variable name => string value type
            in1 = new CPOperand(parts[1]);
            // output variable name
            out = new CPOperand(parts[2]);
            break;
        case Write:
            in1 = new CPOperand(parts[1]);
            in2 = new CPOperand(parts[2]);
            in3 = new CPOperand(parts[3]);
            FileFormatProperties fprops = null;
            if (in3.getName().equalsIgnoreCase("csv")) {
                boolean hasHeader = Boolean.parseBoolean(parts[4]);
                String delim = parts[5];
                boolean sparse = Boolean.parseBoolean(parts[6]);
                fprops = new CSVFileFormatProperties(hasHeader, delim, sparse);
                // description
                in4 = new CPOperand(parts[7]);
            } else {
                fprops = new FileFormatProperties();
                // description
                in4 = new CPOperand(parts[4]);
            }
            VariableCPInstruction inst = new VariableCPInstruction(getVariableOperationCode(opcode), in1, in2, in3, out, null, fprops, null, null, opcode, str);
            inst.addInput(in4);
            return inst;
        case Read:
            in1 = new CPOperand(parts[1]);
            in2 = new CPOperand(parts[2]);
            out = null;
            break;
        case SetFileName:
            // variable name
            in1 = new CPOperand(parts[1]);
            // file name
            in2 = new CPOperand(parts[2], ValueType.UNKNOWN, DataType.UNKNOWN);
            // option: remote or local
            in3 = new CPOperand(parts[3], ValueType.UNKNOWN, DataType.UNKNOWN);
            // return new VariableCPInstruction(getVariableOperationCode(opcode), in1, in2, in3, str);
            break;
    }
    return new VariableCPInstruction(getVariableOperationCode(opcode), in1, in2, in3, out, opcode, str);
}

Example 8 with UpdateType

use of org.apache.sysml.runtime.controlprogram.caching.MatrixObject.UpdateType in project incubator-systemml by apache.

the class MatrixIndexingCPInstruction method processInstruction.

@Override
public void processInstruction(ExecutionContext ec) {
    String opcode = getOpcode();
    IndexRange ixrange = getIndexRange(ec);
    // get original matrix
    MatrixObject mo = ec.getMatrixObject(input1.getName());
    // right indexing
    if (opcode.equalsIgnoreCase(RightIndex.OPCODE)) {
        MatrixBlock resultBlock = null;
        if (// via data partitioning
        mo.isPartitioned())
            resultBlock = mo.readMatrixPartition(ixrange.add(1));
        else // via slicing the in-memory matrix
        {
            // execute right indexing operation (with shallow row copies for range
            // of entire sparse rows, which is safe due to copy on update)
            MatrixBlock matBlock = ec.getMatrixInput(input1.getName(), getExtendedOpcode());
            resultBlock = matBlock.slice((int) ixrange.rowStart, (int) ixrange.rowEnd, (int) ixrange.colStart, (int) ixrange.colEnd, false, new MatrixBlock());
            // unpin rhs input
            ec.releaseMatrixInput(input1.getName(), getExtendedOpcode());
            // ensure correct sparse/dense output representation
            if (checkGuardedRepresentationChange(matBlock, resultBlock))
                resultBlock.examSparsity();
        }
        // unpin output
        ec.setMatrixOutput(output.getName(), resultBlock, getExtendedOpcode());
    } else // left indexing
    if (opcode.equalsIgnoreCase(LeftIndex.OPCODE)) {
        UpdateType updateType = mo.getUpdateType();
        if (DMLScript.STATISTICS) {
            if (updateType.isInPlace())
                Statistics.incrementTotalLixUIP();
            Statistics.incrementTotalLix();
        }
        MatrixBlock matBlock = ec.getMatrixInput(input1.getName(), getExtendedOpcode());
        MatrixBlock resultBlock = null;
        if (input2.getDataType() == DataType.MATRIX) {
            // MATRIX<-MATRIX
            MatrixBlock rhsMatBlock = ec.getMatrixInput(input2.getName(), getExtendedOpcode());
            resultBlock = matBlock.leftIndexingOperations(rhsMatBlock, ixrange, new MatrixBlock(), updateType);
            ec.releaseMatrixInput(input2.getName(), getExtendedOpcode());
        } else {
            // MATRIX<-SCALAR
            if (!ixrange.isScalar())
                throw new DMLRuntimeException("Invalid index range of scalar leftindexing: " + ixrange.toString() + ".");
            ScalarObject scalar = ec.getScalarInput(input2.getName(), ValueType.DOUBLE, input2.isLiteral());
            resultBlock = (MatrixBlock) matBlock.leftIndexingOperations(scalar, (int) ixrange.rowStart, (int) ixrange.colStart, new MatrixBlock(), updateType);
        }
        // unpin lhs input
        ec.releaseMatrixInput(input1.getName(), getExtendedOpcode());
        // ensure correct sparse/dense output representation
        // (memory guarded by release of input)
        resultBlock.examSparsity();
        // unpin output
        ec.setMatrixOutput(output.getName(), resultBlock, updateType, getExtendedOpcode());
    } else
        throw new DMLRuntimeException("Invalid opcode (" + opcode + ") encountered in MatrixIndexingCPInstruction.");
}