Examples with MetaData org.apache.sysml.runtime.matrix.MetaData used on opensource projects

Example 6 with MetaData

use of org.apache.sysml.runtime.matrix.MetaData in project incubator-systemml by apache.

the class ResourceOptimizer method collectReadVariables.

private static void collectReadVariables(Hop hop, LocalVariableMap vars) {
    if (hop == null)
        return;
    // process childs
    for (Hop hi : hop.getInput()) collectReadVariables(hi, vars);
    // investigate hop exec type and known dimensions
    if (hop instanceof DataOp && hop.getDataType() == DataType.MATRIX && (((DataOp) hop).getDataOpType() == DataOpTypes.TRANSIENTREAD || ((DataOp) hop).getDataOpType() == DataOpTypes.PERSISTENTREAD)) {
        String varname = hop.getName();
        MatrixCharacteristics mc = new MatrixCharacteristics(hop.getDim1(), hop.getDim2(), (int) hop.getRowsInBlock(), (int) hop.getColsInBlock(), hop.getNnz());
        MatrixObject mo = new MatrixObject(ValueType.DOUBLE, "/tmp", new MetaData(mc));
        vars.put(varname, mo);
    }
    hop.setVisited();
}

Example 7 with MetaData

use of org.apache.sysml.runtime.matrix.MetaData in project systemml by apache.

the class QuantilePickCPInstruction method processInstruction.

@Override
public void processInstruction(ExecutionContext ec) {
    switch(_type) {
        case VALUEPICK:
            if (// INMEM VALUEPICK
            _inmem) {
                MatrixBlock matBlock = ec.getMatrixInput(input1.getName(), getExtendedOpcode());
                if (input2.getDataType() == DataType.SCALAR) {
                    ScalarObject quantile = ec.getScalarInput(input2.getName(), input2.getValueType(), input2.isLiteral());
                    double picked = matBlock.pickValue(quantile.getDoubleValue());
                    ec.setScalarOutput(output.getName(), new DoubleObject(picked));
                } else {
                    MatrixBlock quantiles = ec.getMatrixInput(input2.getName(), getExtendedOpcode());
                    MatrixBlock resultBlock = (MatrixBlock) matBlock.pickValues(quantiles, new MatrixBlock());
                    quantiles = null;
                    ec.releaseMatrixInput(input2.getName(), getExtendedOpcode());
                    ec.setMatrixOutput(output.getName(), resultBlock, getExtendedOpcode());
                }
                ec.releaseMatrixInput(input1.getName(), getExtendedOpcode());
            } else // MR VALUEPICK
            {
                MatrixObject mat = ec.getMatrixObject(input1.getName());
                String fname = mat.getFileName();
                MetaData mdata = mat.getMetaData();
                ScalarObject pickindex = ec.getScalarInput(input2.getName(), input2.getValueType(), input2.isLiteral());
                if (mdata != null) {
                    try {
                        double picked = MapReduceTool.pickValue(fname, (MetaDataNumItemsByEachReducer) mdata, pickindex.getDoubleValue());
                        ec.setVariable(output.getName(), new DoubleObject(picked));
                    } catch (Exception e) {
                        throw new DMLRuntimeException(e);
                    }
                } else {
                    throw new DMLRuntimeException("Unexpected error while executing ValuePickCP: otherMetaData for file (" + fname + ") not found.");
                }
            }
            break;
        case MEDIAN:
            if (// INMEM MEDIAN
            _inmem) {
                double picked = ec.getMatrixInput(input1.getName(), getExtendedOpcode()).median();
                ec.setScalarOutput(output.getName(), new DoubleObject(picked));
                ec.releaseMatrixInput(input1.getName(), getExtendedOpcode());
                break;
            } else // MR MEDIAN
            {
                MatrixObject mat1 = (MatrixObject) ec.getVariable(input1.getName());
                String fname1 = mat1.getFileName();
                MetaData mdata1 = mat1.getMetaData();
                if (mdata1 != null) {
                    try {
                        double median = MapReduceTool.median(fname1, (MetaDataNumItemsByEachReducer) mdata1);
                        ec.setVariable(output.getName(), new DoubleObject(median));
                    } catch (Exception e) {
                        throw new DMLRuntimeException(e);
                    }
                } else {
                    throw new DMLRuntimeException("Unexpected error while executing ValuePickCP: otherMetaData for file (" + fname1 + ") not found.");
                }
            }
            break;
        case IQM:
            if (// INMEM IQM
            _inmem) {
                MatrixBlock matBlock1 = ec.getMatrixInput(input1.getName(), getExtendedOpcode());
                double iqm = matBlock1.interQuartileMean();
                ec.releaseMatrixInput(input1.getName(), getExtendedOpcode());
                ec.setScalarOutput(output.getName(), new DoubleObject(iqm));
            } else // MR IQM
            {
                MatrixObject inputMatrix = (MatrixObject) ec.getVariable(input1.getName());
                ScalarObject iqsum = ec.getScalarInput(input2.getName(), input2.getValueType(), input2.isLiteral());
                double[] q25 = null;
                double[] q75 = null;
                try {
                    q25 = MapReduceTool.pickValueWeight(inputMatrix.getFileName(), (MetaDataNumItemsByEachReducer) inputMatrix.getMetaData(), 0.25, false);
                    q75 = MapReduceTool.pickValueWeight(inputMatrix.getFileName(), (MetaDataNumItemsByEachReducer) inputMatrix.getMetaData(), 0.75, false);
                } catch (IOException e1) {
                    throw new DMLRuntimeException(e1);
                }
                double sumwt = UtilFunctions.getTotalLength((MetaDataNumItemsByEachReducer) ec.getMetaData(input1.getName()));
                double q25d = sumwt * 0.25;
                double q75d = sumwt * 0.75;
                // iqsum = interQuartileSum that includes complete portions of q25 and q75
                // . exclude top portion of q25 and bottom portion of q75
                double q25entry_weight = q25[0] * q25[1];
                double q25portion_include = (q25[2] - q25d) * q25[0];
                double q25portion_exclude = q25entry_weight - q25portion_include;
                double q75portion_exclude = (q75[2] - q75d) * q75[0];
                double mriqm = (iqsum.getDoubleValue() - q25portion_exclude - q75portion_exclude) / (sumwt * 0.5);
                ec.setScalarOutput(output.getName(), new DoubleObject(mriqm));
            }
            break;
        default:
            throw new DMLRuntimeException("Unsupported qpick operation type: " + _type);
    }
}

Example 8 with MetaData

use of org.apache.sysml.runtime.matrix.MetaData in project systemml by apache.

the class ResourceOptimizer method collectReadVariables.

private static void collectReadVariables(Hop hop, LocalVariableMap vars) {
    if (hop == null)
        return;
    // process childs
    for (Hop hi : hop.getInput()) collectReadVariables(hi, vars);
    // investigate hop exec type and known dimensions
    if (hop instanceof DataOp && hop.getDataType() == DataType.MATRIX && (((DataOp) hop).getDataOpType() == DataOpTypes.TRANSIENTREAD || ((DataOp) hop).getDataOpType() == DataOpTypes.PERSISTENTREAD)) {
        String varname = hop.getName();
        MatrixCharacteristics mc = new MatrixCharacteristics(hop.getDim1(), hop.getDim2(), (int) hop.getRowsInBlock(), (int) hop.getColsInBlock(), hop.getNnz());
        MatrixObject mo = new MatrixObject(ValueType.DOUBLE, "/tmp", new MetaData(mc));
        vars.put(varname, mo);
    }
    hop.setVisited();
}