Examples with FrameObject org.apache.sysml.runtime.controlprogram.caching.FrameObject used on opensource projects

Example 21 with FrameObject

use of org.apache.sysml.runtime.controlprogram.caching.FrameObject in project incubator-systemml by apache.

the class ParameterizedBuiltinCPInstruction method processInstruction.

@Override
public void processInstruction(ExecutionContext ec) {
    String opcode = getOpcode();
    ScalarObject sores = null;
    if (opcode.equalsIgnoreCase("cdf")) {
        SimpleOperator op = (SimpleOperator) _optr;
        double result = op.fn.execute(params);
        sores = new DoubleObject(result);
        ec.setScalarOutput(output.getName(), sores);
    } else if (opcode.equalsIgnoreCase("invcdf")) {
        SimpleOperator op = (SimpleOperator) _optr;
        double result = op.fn.execute(params);
        sores = new DoubleObject(result);
        ec.setScalarOutput(output.getName(), sores);
    } else if (opcode.equalsIgnoreCase("groupedagg")) {
        // acquire locks
        MatrixBlock target = ec.getMatrixInput(params.get(Statement.GAGG_TARGET), getExtendedOpcode());
        MatrixBlock groups = ec.getMatrixInput(params.get(Statement.GAGG_GROUPS), getExtendedOpcode());
        MatrixBlock weights = null;
        if (params.get(Statement.GAGG_WEIGHTS) != null)
            weights = ec.getMatrixInput(params.get(Statement.GAGG_WEIGHTS), getExtendedOpcode());
        int ngroups = -1;
        if (params.get(Statement.GAGG_NUM_GROUPS) != null) {
            ngroups = (int) Double.parseDouble(params.get(Statement.GAGG_NUM_GROUPS));
        }
        // compute the result
        // num threads
        int k = Integer.parseInt(params.get("k"));
        MatrixBlock soresBlock = groups.groupedAggOperations(target, weights, new MatrixBlock(), ngroups, _optr, k);
        ec.setMatrixOutput(output.getName(), soresBlock, getExtendedOpcode());
        // release locks
        target = groups = weights = null;
        ec.releaseMatrixInput(params.get(Statement.GAGG_TARGET), getExtendedOpcode());
        ec.releaseMatrixInput(params.get(Statement.GAGG_GROUPS), getExtendedOpcode());
        if (params.get(Statement.GAGG_WEIGHTS) != null)
            ec.releaseMatrixInput(params.get(Statement.GAGG_WEIGHTS), getExtendedOpcode());
    } else if (opcode.equalsIgnoreCase("rmempty")) {
        String margin = params.get("margin");
        if (!(margin.equals("rows") || margin.equals("cols")))
            throw new DMLRuntimeException("Unspupported margin identifier '" + margin + "'.");
        // acquire locks
        MatrixBlock target = ec.getMatrixInput(params.get("target"), getExtendedOpcode());
        MatrixBlock select = params.containsKey("select") ? ec.getMatrixInput(params.get("select"), getExtendedOpcode()) : null;
        // compute the result
        boolean emptyReturn = Boolean.parseBoolean(params.get("empty.return").toLowerCase());
        MatrixBlock soresBlock = target.removeEmptyOperations(new MatrixBlock(), margin.equals("rows"), emptyReturn, select);
        // release locks
        ec.setMatrixOutput(output.getName(), soresBlock, getExtendedOpcode());
        ec.releaseMatrixInput(params.get("target"), getExtendedOpcode());
        if (params.containsKey("select"))
            ec.releaseMatrixInput(params.get("select"), getExtendedOpcode());
    } else if (opcode.equalsIgnoreCase("replace")) {
        // acquire locks
        MatrixBlock target = ec.getMatrixInput(params.get("target"), getExtendedOpcode());
        // compute the result
        double pattern = Double.parseDouble(params.get("pattern"));
        double replacement = Double.parseDouble(params.get("replacement"));
        MatrixBlock ret = (MatrixBlock) target.replaceOperations(new MatrixBlock(), pattern, replacement);
        // release locks
        ec.setMatrixOutput(output.getName(), ret, getExtendedOpcode());
        ec.releaseMatrixInput(params.get("target"), getExtendedOpcode());
    } else if (opcode.equalsIgnoreCase("rexpand")) {
        // acquire locks
        MatrixBlock target = ec.getMatrixInput(params.get("target"), getExtendedOpcode());
        // compute the result
        double maxVal = Double.parseDouble(params.get("max"));
        boolean dirVal = params.get("dir").equals("rows");
        boolean cast = Boolean.parseBoolean(params.get("cast"));
        boolean ignore = Boolean.parseBoolean(params.get("ignore"));
        int numThreads = Integer.parseInt(params.get("k"));
        MatrixBlock ret = (MatrixBlock) target.rexpandOperations(new MatrixBlock(), maxVal, dirVal, cast, ignore, numThreads);
        // release locks
        ec.setMatrixOutput(output.getName(), ret, getExtendedOpcode());
        ec.releaseMatrixInput(params.get("target"), getExtendedOpcode());
    } else if (opcode.equalsIgnoreCase("transformapply")) {
        // acquire locks
        FrameBlock data = ec.getFrameInput(params.get("target"));
        FrameBlock meta = ec.getFrameInput(params.get("meta"));
        String[] colNames = data.getColumnNames();
        // compute transformapply
        Encoder encoder = EncoderFactory.createEncoder(params.get("spec"), colNames, data.getNumColumns(), meta);
        MatrixBlock mbout = encoder.apply(data, new MatrixBlock(data.getNumRows(), data.getNumColumns(), false));
        // release locks
        ec.setMatrixOutput(output.getName(), mbout, getExtendedOpcode());
        ec.releaseFrameInput(params.get("target"));
        ec.releaseFrameInput(params.get("meta"));
    } else if (opcode.equalsIgnoreCase("transformdecode")) {
        // acquire locks
        MatrixBlock data = ec.getMatrixInput(params.get("target"), getExtendedOpcode());
        FrameBlock meta = ec.getFrameInput(params.get("meta"));
        String[] colnames = meta.getColumnNames();
        // compute transformdecode
        Decoder decoder = DecoderFactory.createDecoder(getParameterMap().get("spec"), colnames, null, meta);
        FrameBlock fbout = decoder.decode(data, new FrameBlock(decoder.getSchema()));
        fbout.setColumnNames(Arrays.copyOfRange(colnames, 0, fbout.getNumColumns()));
        // release locks
        ec.setFrameOutput(output.getName(), fbout);
        ec.releaseMatrixInput(params.get("target"), getExtendedOpcode());
        ec.releaseFrameInput(params.get("meta"));
    } else if (opcode.equalsIgnoreCase("transformcolmap")) {
        // acquire locks
        FrameBlock meta = ec.getFrameInput(params.get("target"));
        String[] colNames = meta.getColumnNames();
        // compute transformapply
        Encoder encoder = EncoderFactory.createEncoder(params.get("spec"), colNames, meta.getNumColumns(), null);
        MatrixBlock mbout = encoder.getColMapping(meta, new MatrixBlock(meta.getNumColumns(), 3, false));
        // release locks
        ec.setMatrixOutput(output.getName(), mbout, getExtendedOpcode());
        ec.releaseFrameInput(params.get("target"));
    } else if (opcode.equalsIgnoreCase("transformmeta")) {
        // get input spec and path
        String spec = getParameterMap().get("spec");
        String path = getParameterMap().get(ParameterizedBuiltinFunctionExpression.TF_FN_PARAM_MTD);
        String delim = getParameterMap().containsKey("sep") ? getParameterMap().get("sep") : TfUtils.TXMTD_SEP;
        // execute transform meta data read
        FrameBlock meta = null;
        try {
            meta = TfMetaUtils.readTransformMetaDataFromFile(spec, path, delim);
        } catch (Exception ex) {
            throw new DMLRuntimeException(ex);
        }
        // release locks
        ec.setFrameOutput(output.getName(), meta);
    } else if (opcode.equalsIgnoreCase("toString")) {
        // handle input parameters
        int rows = (getParam("rows") != null) ? Integer.parseInt(getParam("rows")) : TOSTRING_MAXROWS;
        int cols = (getParam("cols") != null) ? Integer.parseInt(getParam("cols")) : TOSTRING_MAXCOLS;
        int decimal = (getParam("decimal") != null) ? Integer.parseInt(getParam("decimal")) : TOSTRING_DECIMAL;
        boolean sparse = (getParam("sparse") != null) ? Boolean.parseBoolean(getParam("sparse")) : TOSTRING_SPARSE;
        String separator = (getParam("sep") != null) ? getParam("sep") : TOSTRING_SEPARATOR;
        String lineseparator = (getParam("linesep") != null) ? getParam("linesep") : TOSTRING_LINESEPARATOR;
        // get input matrix/frame and convert to string
        CacheableData<?> data = ec.getCacheableData(getParam("target"));
        String out = null;
        if (data instanceof MatrixObject) {
            MatrixBlock matrix = (MatrixBlock) data.acquireRead();
            warnOnTrunction(matrix, rows, cols);
            out = DataConverter.toString(matrix, sparse, separator, lineseparator, rows, cols, decimal);
        } else if (data instanceof FrameObject) {
            FrameBlock frame = (FrameBlock) data.acquireRead();
            warnOnTrunction(frame, rows, cols);
            out = DataConverter.toString(frame, sparse, separator, lineseparator, rows, cols, decimal);
        } else {
            throw new DMLRuntimeException("toString only converts matrix or frames to string");
        }
        ec.releaseCacheableData(getParam("target"));
        ec.setScalarOutput(output.getName(), new StringObject(out));
    } else {
        throw new DMLRuntimeException("Unknown opcode : " + opcode);
    }
}

Example 22 with FrameObject

use of org.apache.sysml.runtime.controlprogram.caching.FrameObject in project incubator-systemml by apache.

the class VariableCPInstruction method processInstruction.

@Override
public void processInstruction(ExecutionContext ec) {
    switch(opcode) {
        case CreateVariable:
            if (getInput1().getDataType() == DataType.MATRIX) {
                // create new variable for symbol table and cache
                // (existing objects gets cleared through rmvar instructions)
                String fname = getInput2().getName();
                // check if unique filename needs to be generated
                if (Boolean.parseBoolean(getInput3().getName())) {
                    fname = new StringBuilder(fname.length() + 16).append(fname).append('_').append(_uniqueVarID.getNextID()).toString();
                }
                MatrixObject mobj = new MatrixObject(getInput1().getValueType(), fname);
                // clone meta data because it is updated on copy-on-write, otherwise there
                // is potential for hidden side effects between variables.
                mobj.setMetaData((MetaData) metadata.clone());
                mobj.setFileFormatProperties(_formatProperties);
                mobj.setUpdateType(_updateType);
                ec.setVariable(getInput1().getName(), mobj);
                if (DMLScript.STATISTICS && _updateType.isInPlace())
                    Statistics.incrementTotalUIPVar();
            } else if (getInput1().getDataType() == DataType.FRAME) {
                String fname = getInput2().getName();
                FrameObject fobj = new FrameObject(fname);
                fobj.setMetaData((MetaData) metadata.clone());
                fobj.setFileFormatProperties(_formatProperties);
                if (_schema != null)
                    // after metadata
                    fobj.setSchema(_schema);
                ec.setVariable(getInput1().getName(), fobj);
            } else if (getInput1().getDataType() == DataType.SCALAR) {
                // created variable not called for scalars
                ec.setScalarOutput(getInput1().getName(), null);
            } else {
                throw new DMLRuntimeException("Unexpected data type: " + getInput1().getDataType());
            }
            break;
        case AssignVariable:
            // assign value of variable to the other
            ec.setScalarOutput(getInput2().getName(), ec.getScalarInput(getInput1()));
            break;
        case CopyVariable:
            processCopyInstruction(ec);
            break;
        case MoveVariable:
            processMoveInstruction(ec);
            break;
        case RemoveVariable:
            for (CPOperand input : inputs) processRemoveVariableInstruction(ec, input.getName());
            break;
        case RemoveVariableAndFile:
            // Remove the variable from HashMap _variables, and possibly delete the data on disk.
            boolean del = ((BooleanObject) ec.getScalarInput(getInput2().getName(), getInput2().getValueType(), true)).getBooleanValue();
            MatrixObject m = (MatrixObject) ec.removeVariable(getInput1().getName());
            if (!del) {
                // therefore data must be exported if dirty flag is set
                if (m.isDirty())
                    m.exportData();
            } else {
                // throw new DMLRuntimeException("rmfilevar w/ true is not expected! " + instString);
                // cleanDataOnHDFS(pb, input1.getName());
                cleanDataOnHDFS(m);
            }
            // check if in-memory object can be cleaned up
            if (!ec.getVariables().hasReferences(m)) {
                // no other variable in the symbol table points to the same Data object as that of input1.getName()
                // remove matrix object from cache
                m.clearData();
            }
            break;
        case // castAsScalarVariable
        CastAsScalarVariable:
            if (getInput1().getDataType() == DataType.FRAME) {
                FrameBlock fBlock = ec.getFrameInput(getInput1().getName());
                if (fBlock.getNumRows() != 1 || fBlock.getNumColumns() != 1)
                    throw new DMLRuntimeException("Dimension mismatch - unable to cast frame '" + getInput1().getName() + "' of dimension (" + fBlock.getNumRows() + " x " + fBlock.getNumColumns() + ") to scalar.");
                Object value = fBlock.get(0, 0);
                ec.releaseFrameInput(getInput1().getName());
                ec.setScalarOutput(output.getName(), ScalarObjectFactory.createScalarObject(fBlock.getSchema()[0], value));
            } else {
                // assume DataType.MATRIX otherwise
                MatrixBlock mBlock = ec.getMatrixInput(getInput1().getName(), getExtendedOpcode());
                if (mBlock.getNumRows() != 1 || mBlock.getNumColumns() != 1)
                    throw new DMLRuntimeException("Dimension mismatch - unable to cast matrix '" + getInput1().getName() + "' of dimension (" + mBlock.getNumRows() + " x " + mBlock.getNumColumns() + ") to scalar.");
                double value = mBlock.getValue(0, 0);
                ec.releaseMatrixInput(getInput1().getName(), getExtendedOpcode());
                ec.setScalarOutput(output.getName(), new DoubleObject(value));
            }
            break;
        case CastAsMatrixVariable:
            {
                MatrixBlock out = null;
                if (getInput1().getDataType() == DataType.FRAME) {
                    FrameBlock fin = ec.getFrameInput(getInput1().getName());
                    out = DataConverter.convertToMatrixBlock(fin);
                    ec.releaseFrameInput(getInput1().getName());
                } else {
                    // assume DataType.SCALAR otherwise
                    ScalarObject scalarInput = ec.getScalarInput(getInput1().getName(), getInput1().getValueType(), getInput1().isLiteral());
                    out = new MatrixBlock(1, 1, false);
                    out.quickSetValue(0, 0, scalarInput.getDoubleValue());
                }
                ec.setMatrixOutput(output.getName(), out, getExtendedOpcode());
                break;
            }
        case CastAsFrameVariable:
            {
                FrameBlock out = null;
                if (getInput1().getDataType() == DataType.SCALAR) {
                    ScalarObject scalarInput = ec.getScalarInput(getInput1());
                    out = new FrameBlock(1, getInput1().getValueType());
                    out.ensureAllocatedColumns(1);
                    out.set(0, 0, scalarInput.getStringValue());
                } else {
                    // DataType.FRAME
                    MatrixBlock min = ec.getMatrixInput(getInput1().getName(), getExtendedOpcode());
                    out = DataConverter.convertToFrameBlock(min);
                    ec.releaseMatrixInput(getInput1().getName(), getExtendedOpcode());
                }
                ec.setFrameOutput(output.getName(), out);
                break;
            }
        case CastAsDoubleVariable:
            {
                ScalarObject scalarInput = ec.getScalarInput(getInput1());
                ec.setScalarOutput(output.getName(), new DoubleObject(scalarInput.getDoubleValue()));
                break;
            }
        case CastAsIntegerVariable:
            {
                ScalarObject scalarInput = ec.getScalarInput(getInput1());
                ec.setScalarOutput(output.getName(), new IntObject(scalarInput.getLongValue()));
                break;
            }
        case CastAsBooleanVariable:
            {
                ScalarObject scalarInput = ec.getScalarInput(getInput1());
                ec.setScalarOutput(output.getName(), new BooleanObject(scalarInput.getBooleanValue()));
                break;
            }
        case Read:
            ScalarObject res = null;
            try {
                switch(getInput1().getValueType()) {
                    case DOUBLE:
                        double d = MapReduceTool.readDoubleFromHDFSFile(getInput2().getName());
                        res = (ScalarObject) new DoubleObject(d);
                        break;
                    case INT:
                        long i = MapReduceTool.readIntegerFromHDFSFile(getInput2().getName());
                        res = (ScalarObject) new IntObject(i);
                        break;
                    case BOOLEAN:
                        boolean b = MapReduceTool.readBooleanFromHDFSFile(getInput2().getName());
                        res = (ScalarObject) new BooleanObject(b);
                        break;
                    case STRING:
                        String s = MapReduceTool.readStringFromHDFSFile(getInput2().getName());
                        res = (ScalarObject) new StringObject(s);
                        break;
                    default:
                        throw new DMLRuntimeException("Invalid value type (" + getInput1().getValueType() + ") while processing readScalar instruction.");
                }
            } catch (IOException e) {
                throw new DMLRuntimeException(e);
            }
            ec.setScalarOutput(getInput1().getName(), res);
            break;
        case Write:
            processWriteInstruction(ec);
            break;
        case SetFileName:
            Data data = ec.getVariable(getInput1().getName());
            if (data.getDataType() == DataType.MATRIX) {
                if (getInput3().getName().equalsIgnoreCase("remote")) {
                    ((MatrixObject) data).setFileName(getInput2().getName());
                } else {
                    throw new DMLRuntimeException("Invalid location (" + getInput3().getName() + ") in SetFileName instruction: " + instString);
                }
            } else {
                throw new DMLRuntimeException("Invalid data type (" + getInput1().getDataType() + ") in SetFileName instruction: " + instString);
            }
            break;
        default:
            throw new DMLRuntimeException("Unknown opcode: " + opcode);
    }
}

Example 23 with FrameObject

use of org.apache.sysml.runtime.controlprogram.caching.FrameObject in project incubator-systemml by apache.

the class ReblockSPInstruction method processFrameReblockInstruction.

@SuppressWarnings("unchecked")
protected void processFrameReblockInstruction(SparkExecutionContext sec, InputInfo iinfo) {
    FrameObject fo = sec.getFrameObject(input1.getName());
    MatrixCharacteristics mcOut = sec.getMatrixCharacteristics(output.getName());
    if (iinfo == InputInfo.TextCellInputInfo) {
        // get the input textcell rdd
        JavaPairRDD<LongWritable, Text> lines = (JavaPairRDD<LongWritable, Text>) sec.getRDDHandleForVariable(input1.getName(), iinfo);
        // convert textcell to binary block
        JavaPairRDD<Long, FrameBlock> out = FrameRDDConverterUtils.textCellToBinaryBlock(sec.getSparkContext(), lines, mcOut, fo.getSchema());
        // put output RDD handle into symbol table
        sec.setRDDHandleForVariable(output.getName(), out);
        sec.addLineageRDD(output.getName(), input1.getName());
    } else if (iinfo == InputInfo.CSVInputInfo) {
        // HACK ALERT: Until we introduces the rewrite to insert csvrblock for non-persistent read
        // throw new DMLRuntimeException("CSVInputInfo is not supported for ReblockSPInstruction");
        CSVReblockSPInstruction csvInstruction = null;
        boolean hasHeader = false;
        String delim = ",";
        boolean fill = false;
        double fillValue = 0;
        if (fo.getFileFormatProperties() instanceof CSVFileFormatProperties && fo.getFileFormatProperties() != null) {
            CSVFileFormatProperties props = (CSVFileFormatProperties) fo.getFileFormatProperties();
            hasHeader = props.hasHeader();
            delim = props.getDelim();
            fill = props.isFill();
            fillValue = props.getFillValue();
        }
        csvInstruction = new CSVReblockSPInstruction(null, input1, output, mcOut.getRowsPerBlock(), mcOut.getColsPerBlock(), hasHeader, delim, fill, fillValue, "csvrblk", instString);
        csvInstruction.processInstruction(sec);
    } else {
        throw new DMLRuntimeException("The given InputInfo is not implemented " + "for ReblockSPInstruction: " + InputInfo.inputInfoToString(iinfo));
    }
}

Example 24 with FrameObject

use of org.apache.sysml.runtime.controlprogram.caching.FrameObject in project incubator-systemml by apache.

the class MRJobInstruction method extractInputMatrices.

/**
 * Extracts input variables with MATRIX data type, and stores references to
 * corresponding matrix objects in <code>inputMatrices</code>. Also, stores
 * the data types in <code>inputDataTypes</code>.
 *
 * @param ec execution context
 * @return array of matrix objects
 */
public MatrixObject[] extractInputMatrices(ExecutionContext ec) {
    ArrayList<MatrixObject> inputmat = new ArrayList<>();
    inputDataTypes = new DataType[inputVars.length];
    for (int i = 0; i < inputVars.length; i++) {
        Data d = ec.getVariable(inputVars[i]);
        inputDataTypes[i] = d.getDataType();
        if (d.getDataType() == DataType.MATRIX) {
            inputmat.add((MatrixObject) d);
        } else if (d.getDataType() == DataType.FRAME) {
            // FIXME conversion from frame to matrix object (meta data only) to adhere to
            // the given matrix-based mr job submission framework
            FrameObject fo = (FrameObject) d;
            MatrixObject mo = new MatrixObject(fo.getValueType(), fo.getFileName(), fo.getMetaData());
            mo.setFileFormatProperties(fo.getFileFormatProperties());
            inputmat.add(mo);
        }
    }
    inputMatrices = inputmat.toArray(new MatrixObject[inputmat.size()]);
    // populate auxiliary data structures
    populateInputs();
    return inputMatrices;
}

Example 25 with FrameObject

use of org.apache.sysml.runtime.controlprogram.caching.FrameObject in project incubator-systemml by apache.

the class EvalNaryCPInstruction method processInstruction.

@Override
public void processInstruction(ExecutionContext ec) {
    // 1. get the namespace and func
    String funcName = ec.getScalarInput(inputs[0]).getStringValue();
    if (funcName.contains(Program.KEY_DELIM))
        throw new DMLRuntimeException("Eval calls to '" + funcName + "', i.e., a function outside " + "the default " + "namespace, are not supported yet. Please call the function directly.");
    // bound the inputs to avoiding being deleted after the function call
    CPOperand[] boundInputs = Arrays.copyOfRange(inputs, 1, inputs.length);
    ArrayList<String> boundOutputNames = new ArrayList<>();
    boundOutputNames.add(output.getName());
    ArrayList<String> boundInputNames = new ArrayList<>();
    for (CPOperand input : boundInputs) {
        boundInputNames.add(input.getName());
    }
    // 2. copy the created output matrix
    MatrixObject outputMO = new MatrixObject(ec.getMatrixObject(output.getName()));
    // 3. call the function
    FunctionCallCPInstruction fcpi = new FunctionCallCPInstruction(null, funcName, boundInputs, boundInputNames, boundOutputNames, "eval func");
    fcpi.processInstruction(ec);
    // 4. convert the result to matrix
    Data newOutput = ec.getVariable(output);
    if (newOutput instanceof MatrixObject) {
        return;
    }
    MatrixBlock mb = null;
    if (newOutput instanceof ScalarObject) {
        // convert scalar to matrix
        mb = new MatrixBlock(((ScalarObject) newOutput).getDoubleValue());
    } else if (newOutput instanceof FrameObject) {
        // convert frame to matrix
        mb = DataConverter.convertToMatrixBlock(((FrameObject) newOutput).acquireRead());
        ec.cleanupCacheableData((FrameObject) newOutput);
    }
    outputMO.acquireModify(mb);
    outputMO.release();
    ec.setVariable(output.getName(), outputMO);
}