Examples with MetaData org.apache.sysml.runtime.matrix.MetaData used on opensource projects

Example 1 with MetaData

use of org.apache.sysml.runtime.matrix.MetaData in project incubator-systemml by apache.

the class ExecutionContext method setMetaData.

public void setMetaData(String varName, long nrows, long ncols) {
    MatrixObject mo = getMatrixObject(varName);
    if (mo.getNumRows() == nrows && mo.getNumColumns() == ncols)
        return;
    MetaData oldMetaData = mo.getMetaData();
    if (oldMetaData == null || !(oldMetaData instanceof MetaDataFormat))
        throw new DMLRuntimeException("Metadata not available");
    MatrixCharacteristics mc = new MatrixCharacteristics(nrows, ncols, (int) mo.getNumRowsPerBlock(), (int) mo.getNumColumnsPerBlock());
    mo.setMetaData(new MetaDataFormat(mc, ((MetaDataFormat) oldMetaData).getOutputInfo(), ((MetaDataFormat) oldMetaData).getInputInfo()));
}

Example 2 with MetaData

use of org.apache.sysml.runtime.matrix.MetaData in project systemml by apache.

the class VariableCPInstruction method processInstruction.

@Override
public void processInstruction(ExecutionContext ec) {
    switch(opcode) {
        case CreateVariable:
            if (getInput1().getDataType() == DataType.MATRIX) {
                // create new variable for symbol table and cache
                // (existing objects gets cleared through rmvar instructions)
                String fname = getInput2().getName();
                // check if unique filename needs to be generated
                if (Boolean.parseBoolean(getInput3().getName())) {
                    fname = new StringBuilder(fname.length() + 16).append(fname).append('_').append(_uniqueVarID.getNextID()).toString();
                }
                MatrixObject mobj = new MatrixObject(getInput1().getValueType(), fname);
                // clone meta data because it is updated on copy-on-write, otherwise there
                // is potential for hidden side effects between variables.
                mobj.setMetaData((MetaData) metadata.clone());
                mobj.setFileFormatProperties(_formatProperties);
                mobj.setUpdateType(_updateType);
                ec.setVariable(getInput1().getName(), mobj);
                if (DMLScript.STATISTICS && _updateType.isInPlace())
                    Statistics.incrementTotalUIPVar();
            } else if (getInput1().getDataType() == DataType.FRAME) {
                String fname = getInput2().getName();
                FrameObject fobj = new FrameObject(fname);
                fobj.setMetaData((MetaData) metadata.clone());
                fobj.setFileFormatProperties(_formatProperties);
                if (_schema != null)
                    // after metadata
                    fobj.setSchema(_schema);
                ec.setVariable(getInput1().getName(), fobj);
            } else if (getInput1().getDataType() == DataType.SCALAR) {
                // created variable not called for scalars
                ec.setScalarOutput(getInput1().getName(), null);
            } else {
                throw new DMLRuntimeException("Unexpected data type: " + getInput1().getDataType());
            }
            break;
        case AssignVariable:
            // assign value of variable to the other
            ec.setScalarOutput(getInput2().getName(), ec.getScalarInput(getInput1()));
            break;
        case CopyVariable:
            processCopyInstruction(ec);
            break;
        case MoveVariable:
            processMoveInstruction(ec);
            break;
        case RemoveVariable:
            for (CPOperand input : inputs) processRemoveVariableInstruction(ec, input.getName());
            break;
        case RemoveVariableAndFile:
            // Remove the variable from HashMap _variables, and possibly delete the data on disk.
            boolean del = ((BooleanObject) ec.getScalarInput(getInput2().getName(), getInput2().getValueType(), true)).getBooleanValue();
            MatrixObject m = (MatrixObject) ec.removeVariable(getInput1().getName());
            if (!del) {
                // therefore data must be exported if dirty flag is set
                if (m.isDirty())
                    m.exportData();
            } else {
                // throw new DMLRuntimeException("rmfilevar w/ true is not expected! " + instString);
                // cleanDataOnHDFS(pb, input1.getName());
                cleanDataOnHDFS(m);
            }
            // check if in-memory object can be cleaned up
            if (!ec.getVariables().hasReferences(m)) {
                // no other variable in the symbol table points to the same Data object as that of input1.getName()
                // remove matrix object from cache
                m.clearData();
            }
            break;
        case // castAsScalarVariable
        CastAsScalarVariable:
            if (getInput1().getDataType() == DataType.FRAME) {
                FrameBlock fBlock = ec.getFrameInput(getInput1().getName());
                if (fBlock.getNumRows() != 1 || fBlock.getNumColumns() != 1)
                    throw new DMLRuntimeException("Dimension mismatch - unable to cast frame '" + getInput1().getName() + "' of dimension (" + fBlock.getNumRows() + " x " + fBlock.getNumColumns() + ") to scalar.");
                Object value = fBlock.get(0, 0);
                ec.releaseFrameInput(getInput1().getName());
                ec.setScalarOutput(output.getName(), ScalarObjectFactory.createScalarObject(fBlock.getSchema()[0], value));
            } else {
                // assume DataType.MATRIX otherwise
                MatrixBlock mBlock = ec.getMatrixInput(getInput1().getName(), getExtendedOpcode());
                if (mBlock.getNumRows() != 1 || mBlock.getNumColumns() != 1)
                    throw new DMLRuntimeException("Dimension mismatch - unable to cast matrix '" + getInput1().getName() + "' of dimension (" + mBlock.getNumRows() + " x " + mBlock.getNumColumns() + ") to scalar.");
                double value = mBlock.getValue(0, 0);
                ec.releaseMatrixInput(getInput1().getName(), getExtendedOpcode());
                ec.setScalarOutput(output.getName(), new DoubleObject(value));
            }
            break;
        case CastAsMatrixVariable:
            {
                MatrixBlock out = null;
                if (getInput1().getDataType() == DataType.FRAME) {
                    FrameBlock fin = ec.getFrameInput(getInput1().getName());
                    out = DataConverter.convertToMatrixBlock(fin);
                    ec.releaseFrameInput(getInput1().getName());
                } else {
                    // assume DataType.SCALAR otherwise
                    ScalarObject scalarInput = ec.getScalarInput(getInput1().getName(), getInput1().getValueType(), getInput1().isLiteral());
                    out = new MatrixBlock(1, 1, false);
                    out.quickSetValue(0, 0, scalarInput.getDoubleValue());
                }
                ec.setMatrixOutput(output.getName(), out, getExtendedOpcode());
                break;
            }
        case CastAsFrameVariable:
            {
                FrameBlock out = null;
                if (getInput1().getDataType() == DataType.SCALAR) {
                    ScalarObject scalarInput = ec.getScalarInput(getInput1());
                    out = new FrameBlock(1, getInput1().getValueType());
                    out.ensureAllocatedColumns(1);
                    out.set(0, 0, scalarInput.getStringValue());
                } else {
                    // DataType.FRAME
                    MatrixBlock min = ec.getMatrixInput(getInput1().getName(), getExtendedOpcode());
                    out = DataConverter.convertToFrameBlock(min);
                    ec.releaseMatrixInput(getInput1().getName(), getExtendedOpcode());
                }
                ec.setFrameOutput(output.getName(), out);
                break;
            }
        case CastAsDoubleVariable:
            {
                ScalarObject scalarInput = ec.getScalarInput(getInput1());
                ec.setScalarOutput(output.getName(), new DoubleObject(scalarInput.getDoubleValue()));
                break;
            }
        case CastAsIntegerVariable:
            {
                ScalarObject scalarInput = ec.getScalarInput(getInput1());
                ec.setScalarOutput(output.getName(), new IntObject(scalarInput.getLongValue()));
                break;
            }
        case CastAsBooleanVariable:
            {
                ScalarObject scalarInput = ec.getScalarInput(getInput1());
                ec.setScalarOutput(output.getName(), new BooleanObject(scalarInput.getBooleanValue()));
                break;
            }
        case Read:
            ScalarObject res = null;
            try {
                switch(getInput1().getValueType()) {
                    case DOUBLE:
                        double d = MapReduceTool.readDoubleFromHDFSFile(getInput2().getName());
                        res = (ScalarObject) new DoubleObject(d);
                        break;
                    case INT:
                        long i = MapReduceTool.readIntegerFromHDFSFile(getInput2().getName());
                        res = (ScalarObject) new IntObject(i);
                        break;
                    case BOOLEAN:
                        boolean b = MapReduceTool.readBooleanFromHDFSFile(getInput2().getName());
                        res = (ScalarObject) new BooleanObject(b);
                        break;
                    case STRING:
                        String s = MapReduceTool.readStringFromHDFSFile(getInput2().getName());
                        res = (ScalarObject) new StringObject(s);
                        break;
                    default:
                        throw new DMLRuntimeException("Invalid value type (" + getInput1().getValueType() + ") while processing readScalar instruction.");
                }
            } catch (IOException e) {
                throw new DMLRuntimeException(e);
            }
            ec.setScalarOutput(getInput1().getName(), res);
            break;
        case Write:
            processWriteInstruction(ec);
            break;
        case SetFileName:
            Data data = ec.getVariable(getInput1().getName());
            if (data.getDataType() == DataType.MATRIX) {
                if (getInput3().getName().equalsIgnoreCase("remote")) {
                    ((MatrixObject) data).setFileName(getInput2().getName());
                } else {
                    throw new DMLRuntimeException("Invalid location (" + getInput3().getName() + ") in SetFileName instruction: " + instString);
                }
            } else {
                throw new DMLRuntimeException("Invalid data type (" + getInput1().getDataType() + ") in SetFileName instruction: " + instString);
            }
            break;
        default:
            throw new DMLRuntimeException("Unknown opcode: " + opcode);
    }
}

Example 3 with MetaData

use of org.apache.sysml.runtime.matrix.MetaData in project systemml by apache.

the class ExecutionContext method setMetaData.

public void setMetaData(String varName, long nrows, long ncols) {
    MatrixObject mo = getMatrixObject(varName);
    if (mo.getNumRows() == nrows && mo.getNumColumns() == ncols)
        return;
    MetaData oldMetaData = mo.getMetaData();
    if (oldMetaData == null || !(oldMetaData instanceof MetaDataFormat))
        throw new DMLRuntimeException("Metadata not available");
    MatrixCharacteristics mc = new MatrixCharacteristics(nrows, ncols, (int) mo.getNumRowsPerBlock(), (int) mo.getNumColumnsPerBlock());
    mo.setMetaData(new MetaDataFormat(mc, ((MetaDataFormat) oldMetaData).getOutputInfo(), ((MetaDataFormat) oldMetaData).getInputInfo()));
}

Example 4 with MetaData

use of org.apache.sysml.runtime.matrix.MetaData in project incubator-systemml by apache.

the class QuantilePickCPInstruction method processInstruction.

@Override
public void processInstruction(ExecutionContext ec) {
    switch(_type) {
        case VALUEPICK:
            if (// INMEM VALUEPICK
            _inmem) {
                MatrixBlock matBlock = ec.getMatrixInput(input1.getName(), getExtendedOpcode());
                if (input2.getDataType() == DataType.SCALAR) {
                    ScalarObject quantile = ec.getScalarInput(input2.getName(), input2.getValueType(), input2.isLiteral());
                    double picked = matBlock.pickValue(quantile.getDoubleValue());
                    ec.setScalarOutput(output.getName(), new DoubleObject(picked));
                } else {
                    MatrixBlock quantiles = ec.getMatrixInput(input2.getName(), getExtendedOpcode());
                    MatrixBlock resultBlock = (MatrixBlock) matBlock.pickValues(quantiles, new MatrixBlock());
                    quantiles = null;
                    ec.releaseMatrixInput(input2.getName(), getExtendedOpcode());
                    ec.setMatrixOutput(output.getName(), resultBlock, getExtendedOpcode());
                }
                ec.releaseMatrixInput(input1.getName(), getExtendedOpcode());
            } else // MR VALUEPICK
            {
                MatrixObject mat = ec.getMatrixObject(input1.getName());
                String fname = mat.getFileName();
                MetaData mdata = mat.getMetaData();
                ScalarObject pickindex = ec.getScalarInput(input2.getName(), input2.getValueType(), input2.isLiteral());
                if (mdata != null) {
                    try {
                        double picked = MapReduceTool.pickValue(fname, (MetaDataNumItemsByEachReducer) mdata, pickindex.getDoubleValue());
                        ec.setVariable(output.getName(), new DoubleObject(picked));
                    } catch (Exception e) {
                        throw new DMLRuntimeException(e);
                    }
                } else {
                    throw new DMLRuntimeException("Unexpected error while executing ValuePickCP: otherMetaData for file (" + fname + ") not found.");
                }
            }
            break;
        case MEDIAN:
            if (// INMEM MEDIAN
            _inmem) {
                double picked = ec.getMatrixInput(input1.getName(), getExtendedOpcode()).median();
                ec.setScalarOutput(output.getName(), new DoubleObject(picked));
                ec.releaseMatrixInput(input1.getName(), getExtendedOpcode());
                break;
            } else // MR MEDIAN
            {
                MatrixObject mat1 = (MatrixObject) ec.getVariable(input1.getName());
                String fname1 = mat1.getFileName();
                MetaData mdata1 = mat1.getMetaData();
                if (mdata1 != null) {
                    try {
                        double median = MapReduceTool.median(fname1, (MetaDataNumItemsByEachReducer) mdata1);
                        ec.setVariable(output.getName(), new DoubleObject(median));
                    } catch (Exception e) {
                        throw new DMLRuntimeException(e);
                    }
                } else {
                    throw new DMLRuntimeException("Unexpected error while executing ValuePickCP: otherMetaData for file (" + fname1 + ") not found.");
                }
            }
            break;
        case IQM:
            if (// INMEM IQM
            _inmem) {
                MatrixBlock matBlock1 = ec.getMatrixInput(input1.getName(), getExtendedOpcode());
                double iqm = matBlock1.interQuartileMean();
                ec.releaseMatrixInput(input1.getName(), getExtendedOpcode());
                ec.setScalarOutput(output.getName(), new DoubleObject(iqm));
            } else // MR IQM
            {
                MatrixObject inputMatrix = (MatrixObject) ec.getVariable(input1.getName());
                ScalarObject iqsum = ec.getScalarInput(input2.getName(), input2.getValueType(), input2.isLiteral());
                double[] q25 = null;
                double[] q75 = null;
                try {
                    q25 = MapReduceTool.pickValueWeight(inputMatrix.getFileName(), (MetaDataNumItemsByEachReducer) inputMatrix.getMetaData(), 0.25, false);
                    q75 = MapReduceTool.pickValueWeight(inputMatrix.getFileName(), (MetaDataNumItemsByEachReducer) inputMatrix.getMetaData(), 0.75, false);
                } catch (IOException e1) {
                    throw new DMLRuntimeException(e1);
                }
                double sumwt = UtilFunctions.getTotalLength((MetaDataNumItemsByEachReducer) ec.getMetaData(input1.getName()));
                double q25d = sumwt * 0.25;
                double q75d = sumwt * 0.75;
                // iqsum = interQuartileSum that includes complete portions of q25 and q75
                // . exclude top portion of q25 and bottom portion of q75
                double q25entry_weight = q25[0] * q25[1];
                double q25portion_include = (q25[2] - q25d) * q25[0];
                double q25portion_exclude = q25entry_weight - q25portion_include;
                double q75portion_exclude = (q75[2] - q75d) * q75[0];
                double mriqm = (iqsum.getDoubleValue() - q25portion_exclude - q75portion_exclude) / (sumwt * 0.5);
                ec.setScalarOutput(output.getName(), new DoubleObject(mriqm));
            }
            break;
        default:
            throw new DMLRuntimeException("Unsupported qpick operation type: " + _type);
    }
}

Example 5 with MetaData

use of org.apache.sysml.runtime.matrix.MetaData in project incubator-systemml by apache.

the class VariableCPInstruction method processInstruction.

@Override
public void processInstruction(ExecutionContext ec) {
    switch(opcode) {
        case CreateVariable:
            if (getInput1().getDataType() == DataType.MATRIX) {
                // create new variable for symbol table and cache
                // (existing objects gets cleared through rmvar instructions)
                String fname = getInput2().getName();
                // check if unique filename needs to be generated
                if (Boolean.parseBoolean(getInput3().getName())) {
                    fname = new StringBuilder(fname.length() + 16).append(fname).append('_').append(_uniqueVarID.getNextID()).toString();
                }
                MatrixObject mobj = new MatrixObject(getInput1().getValueType(), fname);
                // clone meta data because it is updated on copy-on-write, otherwise there
                // is potential for hidden side effects between variables.
                mobj.setMetaData((MetaData) metadata.clone());
                mobj.setFileFormatProperties(_formatProperties);
                mobj.setUpdateType(_updateType);
                ec.setVariable(getInput1().getName(), mobj);
                if (DMLScript.STATISTICS && _updateType.isInPlace())
                    Statistics.incrementTotalUIPVar();
            } else if (getInput1().getDataType() == DataType.FRAME) {
                String fname = getInput2().getName();
                FrameObject fobj = new FrameObject(fname);
                fobj.setMetaData((MetaData) metadata.clone());
                fobj.setFileFormatProperties(_formatProperties);
                if (_schema != null)
                    // after metadata
                    fobj.setSchema(_schema);
                ec.setVariable(getInput1().getName(), fobj);
            } else if (getInput1().getDataType() == DataType.SCALAR) {
                // created variable not called for scalars
                ec.setScalarOutput(getInput1().getName(), null);
            } else {
                throw new DMLRuntimeException("Unexpected data type: " + getInput1().getDataType());
            }
            break;
        case AssignVariable:
            // assign value of variable to the other
            ec.setScalarOutput(getInput2().getName(), ec.getScalarInput(getInput1()));
            break;
        case CopyVariable:
            processCopyInstruction(ec);
            break;
        case MoveVariable:
            processMoveInstruction(ec);
            break;
        case RemoveVariable:
            for (CPOperand input : inputs) processRemoveVariableInstruction(ec, input.getName());
            break;
        case RemoveVariableAndFile:
            // Remove the variable from HashMap _variables, and possibly delete the data on disk.
            boolean del = ((BooleanObject) ec.getScalarInput(getInput2().getName(), getInput2().getValueType(), true)).getBooleanValue();
            MatrixObject m = (MatrixObject) ec.removeVariable(getInput1().getName());
            if (!del) {
                // therefore data must be exported if dirty flag is set
                if (m.isDirty())
                    m.exportData();
            } else {
                // throw new DMLRuntimeException("rmfilevar w/ true is not expected! " + instString);
                // cleanDataOnHDFS(pb, input1.getName());
                cleanDataOnHDFS(m);
            }
            // check if in-memory object can be cleaned up
            if (!ec.getVariables().hasReferences(m)) {
                // no other variable in the symbol table points to the same Data object as that of input1.getName()
                // remove matrix object from cache
                m.clearData();
            }
            break;
        case // castAsScalarVariable
        CastAsScalarVariable:
            if (getInput1().getDataType() == DataType.FRAME) {
                FrameBlock fBlock = ec.getFrameInput(getInput1().getName());
                if (fBlock.getNumRows() != 1 || fBlock.getNumColumns() != 1)
                    throw new DMLRuntimeException("Dimension mismatch - unable to cast frame '" + getInput1().getName() + "' of dimension (" + fBlock.getNumRows() + " x " + fBlock.getNumColumns() + ") to scalar.");
                Object value = fBlock.get(0, 0);
                ec.releaseFrameInput(getInput1().getName());
                ec.setScalarOutput(output.getName(), ScalarObjectFactory.createScalarObject(fBlock.getSchema()[0], value));
            } else {
                // assume DataType.MATRIX otherwise
                MatrixBlock mBlock = ec.getMatrixInput(getInput1().getName(), getExtendedOpcode());
                if (mBlock.getNumRows() != 1 || mBlock.getNumColumns() != 1)
                    throw new DMLRuntimeException("Dimension mismatch - unable to cast matrix '" + getInput1().getName() + "' of dimension (" + mBlock.getNumRows() + " x " + mBlock.getNumColumns() + ") to scalar.");
                double value = mBlock.getValue(0, 0);
                ec.releaseMatrixInput(getInput1().getName(), getExtendedOpcode());
                ec.setScalarOutput(output.getName(), new DoubleObject(value));
            }
            break;
        case CastAsMatrixVariable:
            {
                MatrixBlock out = null;
                if (getInput1().getDataType() == DataType.FRAME) {
                    FrameBlock fin = ec.getFrameInput(getInput1().getName());
                    out = DataConverter.convertToMatrixBlock(fin);
                    ec.releaseFrameInput(getInput1().getName());
                } else {
                    // assume DataType.SCALAR otherwise
                    ScalarObject scalarInput = ec.getScalarInput(getInput1().getName(), getInput1().getValueType(), getInput1().isLiteral());
                    out = new MatrixBlock(1, 1, false);
                    out.quickSetValue(0, 0, scalarInput.getDoubleValue());
                }
                ec.setMatrixOutput(output.getName(), out, getExtendedOpcode());
                break;
            }
        case CastAsFrameVariable:
            {
                FrameBlock out = null;
                if (getInput1().getDataType() == DataType.SCALAR) {
                    ScalarObject scalarInput = ec.getScalarInput(getInput1());
                    out = new FrameBlock(1, getInput1().getValueType());
                    out.ensureAllocatedColumns(1);
                    out.set(0, 0, scalarInput.getStringValue());
                } else {
                    // DataType.FRAME
                    MatrixBlock min = ec.getMatrixInput(getInput1().getName(), getExtendedOpcode());
                    out = DataConverter.convertToFrameBlock(min);
                    ec.releaseMatrixInput(getInput1().getName(), getExtendedOpcode());
                }
                ec.setFrameOutput(output.getName(), out);
                break;
            }
        case CastAsDoubleVariable:
            {
                ScalarObject scalarInput = ec.getScalarInput(getInput1());
                ec.setScalarOutput(output.getName(), new DoubleObject(scalarInput.getDoubleValue()));
                break;
            }
        case CastAsIntegerVariable:
            {
                ScalarObject scalarInput = ec.getScalarInput(getInput1());
                ec.setScalarOutput(output.getName(), new IntObject(scalarInput.getLongValue()));
                break;
            }
        case CastAsBooleanVariable:
            {
                ScalarObject scalarInput = ec.getScalarInput(getInput1());
                ec.setScalarOutput(output.getName(), new BooleanObject(scalarInput.getBooleanValue()));
                break;
            }
        case Read:
            ScalarObject res = null;
            try {
                switch(getInput1().getValueType()) {
                    case DOUBLE:
                        double d = MapReduceTool.readDoubleFromHDFSFile(getInput2().getName());
                        res = (ScalarObject) new DoubleObject(d);
                        break;
                    case INT:
                        long i = MapReduceTool.readIntegerFromHDFSFile(getInput2().getName());
                        res = (ScalarObject) new IntObject(i);
                        break;
                    case BOOLEAN:
                        boolean b = MapReduceTool.readBooleanFromHDFSFile(getInput2().getName());
                        res = (ScalarObject) new BooleanObject(b);
                        break;
                    case STRING:
                        String s = MapReduceTool.readStringFromHDFSFile(getInput2().getName());
                        res = (ScalarObject) new StringObject(s);
                        break;
                    default:
                        throw new DMLRuntimeException("Invalid value type (" + getInput1().getValueType() + ") while processing readScalar instruction.");
                }
            } catch (IOException e) {
                throw new DMLRuntimeException(e);
            }
            ec.setScalarOutput(getInput1().getName(), res);
            break;
        case Write:
            processWriteInstruction(ec);
            break;
        case SetFileName:
            Data data = ec.getVariable(getInput1().getName());
            if (data.getDataType() == DataType.MATRIX) {
                if (getInput3().getName().equalsIgnoreCase("remote")) {
                    ((MatrixObject) data).setFileName(getInput2().getName());
                } else {
                    throw new DMLRuntimeException("Invalid location (" + getInput3().getName() + ") in SetFileName instruction: " + instString);
                }
            } else {
                throw new DMLRuntimeException("Invalid data type (" + getInput1().getDataType() + ") in SetFileName instruction: " + instString);
            }
            break;
        default:
            throw new DMLRuntimeException("Unknown opcode: " + opcode);
    }
}