Examples with MatrixCharacteristics org.apache.sysml.runtime.matrix.MatrixCharacteristics used on opensource projects

Example 91 with MatrixCharacteristics

use of org.apache.sysml.runtime.matrix.MatrixCharacteristics in project incubator-systemml by apache.

the class Recompiler method reconcileUpdatedCallVarsLoops.

public static boolean reconcileUpdatedCallVarsLoops(RecompileStatus oldCallStatus, RecompileStatus callStatus, StatementBlock sb) {
    boolean requiresRecompile = false;
    // handle matrices
    for (String varname : sb.variablesUpdated().getVariableNames()) {
        MatrixCharacteristics dat1 = oldCallStatus.getTWriteStats().get(varname);
        MatrixCharacteristics dat2 = callStatus.getTWriteStats().get(varname);
        if (dat1 != null && dat2 != null) {
            MatrixCharacteristics mcOld = dat1;
            MatrixCharacteristics mc = dat2;
            if (mcOld.getRows() != mc.getRows() || mcOld.getCols() != mc.getCols() || mcOld.getNonZeros() != mc.getNonZeros()) {
                long ldim1 = mc.getRows(), ldim2 = mc.getCols(), lnnz = mc.getNonZeros();
                // handle row dimension change in body
                if (mcOld.getRows() != mc.getRows()) {
                    ldim1 = -1;
                    requiresRecompile = true;
                }
                // handle column dimension change in body
                if (mcOld.getCols() != mc.getCols()) {
                    ldim2 = -1;
                    requiresRecompile = true;
                }
                // handle sparsity change
                if (mcOld.getNonZeros() != mc.getNonZeros()) {
                    lnnz = -1;
                    requiresRecompile = true;
                }
                MatrixCharacteristics moNew = new MatrixCharacteristics(ldim1, ldim2, -1, -1, lnnz);
                callStatus.getTWriteStats().put(varname, moNew);
            }
        }
    }
    return requiresRecompile;
}

Example 92 with MatrixCharacteristics

use of org.apache.sysml.runtime.matrix.MatrixCharacteristics in project incubator-systemml by apache.

the class RunMRJobs method executeInMemoryReblockOperations.

private static JobReturn executeInMemoryReblockOperations(MRJobInstruction inst, String shuffleInst, MatrixObject[] inputMatrices, MatrixObject[] outputMatrices) {
    MatrixCharacteristics[] mc = new MatrixCharacteristics[outputMatrices.length];
    ReblockInstruction[] rblkSet = MRInstructionParser.parseReblockInstructions(shuffleInst);
    byte[] results = inst.getIv_resultIndices();
    for (ReblockInstruction rblk : rblkSet) {
        // CP Reblock through caching framework (no copy required: same data, next op copies)
        MatrixBlock mb = inputMatrices[rblk.input].acquireRead();
        for (int i = 0; i < results.length; i++) if (rblk.output == results[i]) {
            outputMatrices[i].acquireModify(mb);
            outputMatrices[i].release();
            mc[i] = new MatrixCharacteristics(mb.getNumRows(), mb.getNumColumns(), rblk.brlen, rblk.bclen, mb.getNonZeros());
        }
        inputMatrices[rblk.input].release();
    }
    return new JobReturn(mc, inst.getOutputInfos(), true);
}

Example 93 with MatrixCharacteristics

use of org.apache.sysml.runtime.matrix.MatrixCharacteristics in project incubator-systemml by apache.

the class RelationalExpression method validateExpression.

/**
 * Validate parse tree : Process Relational Expression
 */
@Override
public void validateExpression(HashMap<String, DataIdentifier> ids, HashMap<String, ConstIdentifier> constVars, boolean conditional) {
    // check for functions calls in expression
    if (_left instanceof FunctionCallIdentifier) {
        raiseValidateError("user-defined function calls not supported in relational expressions", false, LanguageException.LanguageErrorCodes.UNSUPPORTED_EXPRESSION);
    }
    if (_right instanceof FunctionCallIdentifier) {
        raiseValidateError("user-defined function calls not supported in relational expressions", false, LanguageException.LanguageErrorCodes.UNSUPPORTED_EXPRESSION);
    }
    // handle <NUMERIC> == <BOOLEAN> --> convert <BOOLEAN> to numeric value
    if ((_left != null && _left instanceof BooleanIdentifier) || (_right != null && _right instanceof BooleanIdentifier)) {
        if ((_left instanceof IntIdentifier || _left instanceof DoubleIdentifier) || _right instanceof IntIdentifier || _right instanceof DoubleIdentifier) {
            if (_left instanceof BooleanIdentifier) {
                if (((BooleanIdentifier) _left).getValue())
                    this.setLeft(new IntIdentifier(1, _left));
                else
                    this.setLeft(new IntIdentifier(0, _left));
            } else if (_right instanceof BooleanIdentifier) {
                if (((BooleanIdentifier) _right).getValue())
                    this.setRight(new IntIdentifier(1, _right));
                else
                    this.setRight(new IntIdentifier(0, _right));
            }
        }
    }
    // recursive validate
    _left.validateExpression(ids, constVars, conditional);
    if (_right != null)
        _right.validateExpression(ids, constVars, conditional);
    // constant propagation (precondition for more complex constant folding rewrite)
    if (_left instanceof DataIdentifier && constVars.containsKey(((DataIdentifier) _left).getName()))
        _left = constVars.get(((DataIdentifier) _left).getName());
    if (_right instanceof DataIdentifier && constVars.containsKey(((DataIdentifier) _right).getName()))
        _right = constVars.get(((DataIdentifier) _right).getName());
    String outputName = getTempName();
    DataIdentifier output = new DataIdentifier(outputName);
    output.setParseInfo(this);
    boolean isLeftMatrix = (_left.getOutput() != null && _left.getOutput().getDataType() == DataType.MATRIX);
    boolean isRightMatrix = (_right.getOutput() != null && _right.getOutput().getDataType() == DataType.MATRIX);
    if (isLeftMatrix || isRightMatrix) {
        // Added to support matrix relational comparison
        if (isLeftMatrix && isRightMatrix) {
            checkMatchingDimensions(_left, _right, true);
        }
        MatrixCharacteristics dims = getBinaryMatrixCharacteristics(_left, _right);
        output.setDataType(DataType.MATRIX);
        output.setDimensions(dims.getRows(), dims.getCols());
        output.setBlockDimensions(dims.getRowsPerBlock(), dims.getColsPerBlock());
        // since SystemML only supports double matrices, the value type is forced to
        // double; once we support boolean matrices this needs to change
        output.setValueType(ValueType.DOUBLE);
    } else {
        output.setBooleanProperties();
    }
    this.setOutput(output);
}

Example 94 with MatrixCharacteristics

use of org.apache.sysml.runtime.matrix.MatrixCharacteristics in project incubator-systemml by apache.

the class CacheableData method toString.

@Override
public String toString() {
    StringBuilder str = new StringBuilder();
    str.append(getClass().getSimpleName());
    str.append(": ");
    str.append(_hdfsFileName + ", ");
    if (_metaData instanceof MetaDataNumItemsByEachReducer) {
        str.append("NumItemsByEachReducerMetaData");
    } else {
        try {
            MetaDataFormat md = (MetaDataFormat) _metaData;
            if (md != null) {
                MatrixCharacteristics mc = _metaData.getMatrixCharacteristics();
                str.append(mc.toString());
                InputInfo ii = md.getInputInfo();
                if (ii == null)
                    str.append("null");
                else {
                    str.append(", ");
                    str.append(InputInfo.inputInfoToString(ii));
                }
            } else {
                str.append("null, null");
            }
        } catch (Exception ex) {
            LOG.error(ex);
        }
    }
    str.append(", ");
    str.append(isDirty() ? "dirty" : "not-dirty");
    return str.toString();
}

Example 95 with MatrixCharacteristics

use of org.apache.sysml.runtime.matrix.MatrixCharacteristics in project incubator-systemml by apache.

the class CacheableData method writeMetaData.

protected void writeMetaData(String filePathAndName, String outputFormat, FileFormatProperties formatProperties) throws IOException {
    MetaDataFormat iimd = (MetaDataFormat) _metaData;
    if (iimd == null)
        throw new DMLRuntimeException("Unexpected error while writing mtd file (" + filePathAndName + ") -- metadata is null.");
    // Write the matrix to HDFS in requested format
    OutputInfo oinfo = (outputFormat != null ? OutputInfo.stringToOutputInfo(outputFormat) : InputInfo.getMatchingOutputInfo(iimd.getInputInfo()));
    if (oinfo != OutputInfo.MatrixMarketOutputInfo) {
        // Get the dimension information from the metadata stored within MatrixObject
        MatrixCharacteristics mc = iimd.getMatrixCharacteristics();
        // note: this is only required if singlenode (due to binarycell default)
        if (oinfo == OutputInfo.BinaryBlockOutputInfo && DMLScript.rtplatform == RUNTIME_PLATFORM.SINGLE_NODE && (mc.getRowsPerBlock() != ConfigurationManager.getBlocksize() || mc.getColsPerBlock() != ConfigurationManager.getBlocksize())) {
            mc = new MatrixCharacteristics(mc.getRows(), mc.getCols(), ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), mc.getNonZeros());
        }
        // write the actual meta data file
        MapReduceTool.writeMetaDataFile(filePathAndName + ".mtd", valueType, getSchema(), dataType, mc, oinfo, formatProperties);
    }
}