Examples with MetaDataNumItemsByEachReducer org.apache.sysml.runtime.matrix.MetaDataNumItemsByEachReducer used on opensource projects

Example 1 with MetaDataNumItemsByEachReducer

use of org.apache.sysml.runtime.matrix.MetaDataNumItemsByEachReducer in project incubator-systemml by apache.

the class CacheableData method toString.

@Override
public String toString() {
    StringBuilder str = new StringBuilder();
    str.append(getClass().getSimpleName());
    str.append(": ");
    str.append(_hdfsFileName + ", ");
    if (_metaData instanceof MetaDataNumItemsByEachReducer) {
        str.append("NumItemsByEachReducerMetaData");
    } else {
        try {
            MetaDataFormat md = (MetaDataFormat) _metaData;
            if (md != null) {
                MatrixCharacteristics mc = _metaData.getMatrixCharacteristics();
                str.append(mc.toString());
                InputInfo ii = md.getInputInfo();
                if (ii == null)
                    str.append("null");
                else {
                    str.append(", ");
                    str.append(InputInfo.inputInfoToString(ii));
                }
            } else {
                str.append("null, null");
            }
        } catch (Exception ex) {
            LOG.error(ex);
        }
    }
    str.append(", ");
    str.append(isDirty() ? "dirty" : "not-dirty");
    return str.toString();
}

Example 2 with MetaDataNumItemsByEachReducer

use of org.apache.sysml.runtime.matrix.MetaDataNumItemsByEachReducer in project systemml by apache.

the class CacheableData method toString.

@Override
public String toString() {
    StringBuilder str = new StringBuilder();
    str.append(getClass().getSimpleName());
    str.append(": ");
    str.append(_hdfsFileName + ", ");
    if (_metaData instanceof MetaDataNumItemsByEachReducer) {
        str.append("NumItemsByEachReducerMetaData");
    } else {
        try {
            MetaDataFormat md = (MetaDataFormat) _metaData;
            if (md != null) {
                MatrixCharacteristics mc = _metaData.getMatrixCharacteristics();
                str.append(mc.toString());
                InputInfo ii = md.getInputInfo();
                if (ii == null)
                    str.append("null");
                else {
                    str.append(", ");
                    str.append(InputInfo.inputInfoToString(ii));
                }
            } else {
                str.append("null, null");
            }
        } catch (Exception ex) {
            LOG.error(ex);
        }
    }
    str.append(", ");
    str.append(isDirty() ? "dirty" : "not-dirty");
    return str.toString();
}

Example 3 with MetaDataNumItemsByEachReducer

use of org.apache.sysml.runtime.matrix.MetaDataNumItemsByEachReducer in project incubator-systemml by apache.

the class QuantilePickCPInstruction method processInstruction.

@Override
public void processInstruction(ExecutionContext ec) {
    switch(_type) {
        case VALUEPICK:
            if (// INMEM VALUEPICK
            _inmem) {
                MatrixBlock matBlock = ec.getMatrixInput(input1.getName(), getExtendedOpcode());
                if (input2.getDataType() == DataType.SCALAR) {
                    ScalarObject quantile = ec.getScalarInput(input2.getName(), input2.getValueType(), input2.isLiteral());
                    double picked = matBlock.pickValue(quantile.getDoubleValue());
                    ec.setScalarOutput(output.getName(), new DoubleObject(picked));
                } else {
                    MatrixBlock quantiles = ec.getMatrixInput(input2.getName(), getExtendedOpcode());
                    MatrixBlock resultBlock = (MatrixBlock) matBlock.pickValues(quantiles, new MatrixBlock());
                    quantiles = null;
                    ec.releaseMatrixInput(input2.getName(), getExtendedOpcode());
                    ec.setMatrixOutput(output.getName(), resultBlock, getExtendedOpcode());
                }
                ec.releaseMatrixInput(input1.getName(), getExtendedOpcode());
            } else // MR VALUEPICK
            {
                MatrixObject mat = ec.getMatrixObject(input1.getName());
                String fname = mat.getFileName();
                MetaData mdata = mat.getMetaData();
                ScalarObject pickindex = ec.getScalarInput(input2.getName(), input2.getValueType(), input2.isLiteral());
                if (mdata != null) {
                    try {
                        double picked = MapReduceTool.pickValue(fname, (MetaDataNumItemsByEachReducer) mdata, pickindex.getDoubleValue());
                        ec.setVariable(output.getName(), new DoubleObject(picked));
                    } catch (Exception e) {
                        throw new DMLRuntimeException(e);
                    }
                } else {
                    throw new DMLRuntimeException("Unexpected error while executing ValuePickCP: otherMetaData for file (" + fname + ") not found.");
                }
            }
            break;
        case MEDIAN:
            if (// INMEM MEDIAN
            _inmem) {
                double picked = ec.getMatrixInput(input1.getName(), getExtendedOpcode()).median();
                ec.setScalarOutput(output.getName(), new DoubleObject(picked));
                ec.releaseMatrixInput(input1.getName(), getExtendedOpcode());
                break;
            } else // MR MEDIAN
            {
                MatrixObject mat1 = (MatrixObject) ec.getVariable(input1.getName());
                String fname1 = mat1.getFileName();
                MetaData mdata1 = mat1.getMetaData();
                if (mdata1 != null) {
                    try {
                        double median = MapReduceTool.median(fname1, (MetaDataNumItemsByEachReducer) mdata1);
                        ec.setVariable(output.getName(), new DoubleObject(median));
                    } catch (Exception e) {
                        throw new DMLRuntimeException(e);
                    }
                } else {
                    throw new DMLRuntimeException("Unexpected error while executing ValuePickCP: otherMetaData for file (" + fname1 + ") not found.");
                }
            }
            break;
        case IQM:
            if (// INMEM IQM
            _inmem) {
                MatrixBlock matBlock1 = ec.getMatrixInput(input1.getName(), getExtendedOpcode());
                double iqm = matBlock1.interQuartileMean();
                ec.releaseMatrixInput(input1.getName(), getExtendedOpcode());
                ec.setScalarOutput(output.getName(), new DoubleObject(iqm));
            } else // MR IQM
            {
                MatrixObject inputMatrix = (MatrixObject) ec.getVariable(input1.getName());
                ScalarObject iqsum = ec.getScalarInput(input2.getName(), input2.getValueType(), input2.isLiteral());
                double[] q25 = null;
                double[] q75 = null;
                try {
                    q25 = MapReduceTool.pickValueWeight(inputMatrix.getFileName(), (MetaDataNumItemsByEachReducer) inputMatrix.getMetaData(), 0.25, false);
                    q75 = MapReduceTool.pickValueWeight(inputMatrix.getFileName(), (MetaDataNumItemsByEachReducer) inputMatrix.getMetaData(), 0.75, false);
                } catch (IOException e1) {
                    throw new DMLRuntimeException(e1);
                }
                double sumwt = UtilFunctions.getTotalLength((MetaDataNumItemsByEachReducer) ec.getMetaData(input1.getName()));
                double q25d = sumwt * 0.25;
                double q75d = sumwt * 0.75;
                // iqsum = interQuartileSum that includes complete portions of q25 and q75
                // . exclude top portion of q25 and bottom portion of q75
                double q25entry_weight = q25[0] * q25[1];
                double q25portion_include = (q25[2] - q25d) * q25[0];
                double q25portion_exclude = q25entry_weight - q25portion_include;
                double q75portion_exclude = (q75[2] - q75d) * q75[0];
                double mriqm = (iqsum.getDoubleValue() - q25portion_exclude - q75portion_exclude) / (sumwt * 0.5);
                ec.setScalarOutput(output.getName(), new DoubleObject(mriqm));
            }
            break;
        default:
            throw new DMLRuntimeException("Unsupported qpick operation type: " + _type);
    }
}

Example 4 with MetaDataNumItemsByEachReducer

use of org.apache.sysml.runtime.matrix.MetaDataNumItemsByEachReducer in project incubator-systemml by apache.

the class MRJobInstruction method populateInputs.

/**
 * Auxiliary data structures that store information required to spawn MR jobs.
 * These data structures are populated by pulling out information from symbol
 * table. More specifically, from information stored in <code>inputMatrices</code>
 * and <code>outputMatrices</code>.
 */
private void populateInputs() {
    // Since inputVars can potentially contain scalar variables,
    // auxiliary data structures of size <code>inputMatrices.length</code>
    // are allocated instead of size <code>inputVars.length</code>
    // Allocate space
    inputs = new String[inputMatrices.length];
    inputInfos = new InputInfo[inputMatrices.length];
    rlens = new long[inputMatrices.length];
    clens = new long[inputMatrices.length];
    brlens = new int[inputMatrices.length];
    bclens = new int[inputMatrices.length];
    partitioned = new boolean[inputMatrices.length];
    pformats = new PDataPartitionFormat[inputMatrices.length];
    psizes = new int[inputMatrices.length];
    // populate information
    for (int i = 0; i < inputMatrices.length; i++) {
        inputs[i] = inputMatrices[i].getFileName();
        MatrixCharacteristics mc = inputMatrices[i].getMatrixCharacteristics();
        rlens[i] = mc.getRows();
        clens[i] = mc.getCols();
        brlens[i] = mc.getRowsPerBlock();
        bclens[i] = mc.getColsPerBlock();
        if (inputMatrices[i].getMetaData() instanceof MetaDataFormat) {
            inputInfos[i] = ((MetaDataFormat) inputMatrices[i].getMetaData()).getInputInfo();
        } else if (inputMatrices[i].getMetaData() instanceof MetaDataNumItemsByEachReducer) {
            inputInfos[i] = InputInfo.InputInfoForSortOutput;
            inputInfos[i].metadata = inputMatrices[i].getMetaData();
        }
        partitioned[i] = inputMatrices[i].isPartitioned();
        pformats[i] = inputMatrices[i].getPartitionFormat();
        psizes[i] = inputMatrices[i].getPartitionSize();
    }
}

Example 5 with MetaDataNumItemsByEachReducer

use of org.apache.sysml.runtime.matrix.MetaDataNumItemsByEachReducer in project systemml by apache.

the class MRJobInstruction method populateInputs.

/**
 * Auxiliary data structures that store information required to spawn MR jobs.
 * These data structures are populated by pulling out information from symbol
 * table. More specifically, from information stored in <code>inputMatrices</code>
 * and <code>outputMatrices</code>.
 */
private void populateInputs() {
    // Since inputVars can potentially contain scalar variables,
    // auxiliary data structures of size <code>inputMatrices.length</code>
    // are allocated instead of size <code>inputVars.length</code>
    // Allocate space
    inputs = new String[inputMatrices.length];
    inputInfos = new InputInfo[inputMatrices.length];
    rlens = new long[inputMatrices.length];
    clens = new long[inputMatrices.length];
    brlens = new int[inputMatrices.length];
    bclens = new int[inputMatrices.length];
    partitioned = new boolean[inputMatrices.length];
    pformats = new PDataPartitionFormat[inputMatrices.length];
    psizes = new int[inputMatrices.length];
    // populate information
    for (int i = 0; i < inputMatrices.length; i++) {
        inputs[i] = inputMatrices[i].getFileName();
        MatrixCharacteristics mc = inputMatrices[i].getMatrixCharacteristics();
        rlens[i] = mc.getRows();
        clens[i] = mc.getCols();
        brlens[i] = mc.getRowsPerBlock();
        bclens[i] = mc.getColsPerBlock();
        if (inputMatrices[i].getMetaData() instanceof MetaDataFormat) {
            inputInfos[i] = ((MetaDataFormat) inputMatrices[i].getMetaData()).getInputInfo();
        } else if (inputMatrices[i].getMetaData() instanceof MetaDataNumItemsByEachReducer) {
            inputInfos[i] = InputInfo.InputInfoForSortOutput;
            inputInfos[i].metadata = inputMatrices[i].getMetaData();
        }
        partitioned[i] = inputMatrices[i].isPartitioned();
        pformats[i] = inputMatrices[i].getPartitionFormat();
        psizes[i] = inputMatrices[i].getPartitionSize();
    }
}