Examples with MatrixFormatMetaData org.apache.sysml.runtime.matrix.MatrixFormatMetaData used on opensource projects

Example 1 with MatrixFormatMetaData

use of org.apache.sysml.runtime.matrix.MatrixFormatMetaData in project incubator-systemml by apache.

the class InterProceduralAnalysis method populateLocalVariableMapForFunctionCall.

private void populateLocalVariableMapForFunctionCall(FunctionStatement fstmt, FunctionOp fop, LocalVariableMap callvars, LocalVariableMap vars, Set<Long> inputSafeNNZ, Integer numCalls) throws HopsException {
    ArrayList<DataIdentifier> inputVars = fstmt.getInputParams();
    ArrayList<Hop> inputOps = fop.getInput();
    for (int i = 0; i < inputVars.size(); i++) {
        //create mapping between input hops and vars
        DataIdentifier dat = inputVars.get(i);
        Hop input = inputOps.get(i);
        if (input.getDataType() == DataType.MATRIX) {
            //propagate matrix characteristics
            MatrixObject mo = new MatrixObject(ValueType.DOUBLE, null);
            MatrixCharacteristics mc = new MatrixCharacteristics(input.getDim1(), input.getDim2(), ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), inputSafeNNZ.contains(input.getHopID()) ? input.getNnz() : -1);
            MatrixFormatMetaData meta = new MatrixFormatMetaData(mc, null, null);
            mo.setMetaData(meta);
            vars.put(dat.getName(), mo);
        } else if (input.getDataType() == DataType.SCALAR) {
            //(for multiple calls, literal equivalence already checked)
            if (input instanceof LiteralOp) {
                vars.put(dat.getName(), ScalarObjectFactory.createScalarObject(input.getValueType(), (LiteralOp) input));
            } else //and input scalar is existing variable in symbol table
            if (PROPAGATE_SCALAR_VARS_INTO_FUN && numCalls != null && numCalls == 1 && input instanceof DataOp) {
                Data scalar = callvars.get(input.getName());
                if (scalar != null && scalar instanceof ScalarObject) {
                    vars.put(dat.getName(), scalar);
                }
            }
        }
    }
}

Example 2 with MatrixFormatMetaData

use of org.apache.sysml.runtime.matrix.MatrixFormatMetaData in project incubator-systemml by apache.

the class MLContextConversionUtil method binaryBlocksToFrameObject.

/**
	 * Convert a {@code JavaPairRDD<Long, FrameBlock>} to a {@code FrameObject}.
	 * 
	 * @param variableName
	 *            name of the variable associated with the frame
	 * @param binaryBlocks
	 *            {@code JavaPairRDD<Long, FrameBlock>} representation of a
	 *            binary-block frame
	 * @param frameMetadata
	 *            the frame metadata
	 * @return the {@code JavaPairRDD<Long, FrameBlock>} frame converted to a
	 *         {@code FrameObject}
	 */
public static FrameObject binaryBlocksToFrameObject(String variableName, JavaPairRDD<Long, FrameBlock> binaryBlocks, FrameMetadata frameMetadata) {
    MatrixCharacteristics mc = (frameMetadata != null) ? frameMetadata.asMatrixCharacteristics() : new MatrixCharacteristics();
    FrameObject frameObject = new FrameObject(OptimizerUtils.getUniqueTempFileName(), new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo), frameMetadata.getFrameSchema().getSchema().toArray(new ValueType[0]));
    frameObject.setRDDHandle(new RDDObject(binaryBlocks, variableName));
    return frameObject;
}

Example 3 with MatrixFormatMetaData

use of org.apache.sysml.runtime.matrix.MatrixFormatMetaData in project incubator-systemml by apache.

the class MLContextConversionUtil method javaRDDStringCSVToFrameObject.

/**
	 * Convert a {@code JavaRDD<String>} in CSV format to a {@code FrameObject}
	 * 
	 * @param variableName
	 *            name of the variable associated with the frame
	 * @param javaRDD
	 *            the Java RDD of strings
	 * @param frameMetadata
	 *            frame metadata
	 * @return the {@code JavaRDD<String>} converted to a {@code FrameObject}
	 */
public static FrameObject javaRDDStringCSVToFrameObject(String variableName, JavaRDD<String> javaRDD, FrameMetadata frameMetadata) {
    JavaPairRDD<LongWritable, Text> javaPairRDD = javaRDD.mapToPair(new ConvertStringToLongTextPair());
    MatrixCharacteristics mc = (frameMetadata != null) ? frameMetadata.asMatrixCharacteristics() : new MatrixCharacteristics();
    JavaPairRDD<LongWritable, Text> javaPairRDDText = javaPairRDD.mapToPair(new CopyTextInputFunction());
    FrameObject frameObject = new FrameObject(OptimizerUtils.getUniqueTempFileName(), new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo), frameMetadata.getFrameSchema().getSchema().toArray(new ValueType[0]));
    JavaPairRDD<Long, FrameBlock> rdd;
    try {
        rdd = FrameRDDConverterUtils.csvToBinaryBlock(jsc(), javaPairRDDText, mc, frameObject.getSchema(), false, ",", false, -1);
    } catch (DMLRuntimeException e) {
        e.printStackTrace();
        return null;
    }
    frameObject.setRDDHandle(new RDDObject(rdd, variableName));
    return frameObject;
}

Example 4 with MatrixFormatMetaData

use of org.apache.sysml.runtime.matrix.MatrixFormatMetaData in project incubator-systemml by apache.

the class MLContextConversionUtil method binaryBlocksToMatrixObject.

private static MatrixObject binaryBlocksToMatrixObject(String variableName, JavaPairRDD<MatrixIndexes, MatrixBlock> binaryBlocks, MatrixMetadata matrixMetadata, boolean copy) {
    MatrixCharacteristics mc = (matrixMetadata != null) ? matrixMetadata.asMatrixCharacteristics() : new MatrixCharacteristics();
    JavaPairRDD<MatrixIndexes, MatrixBlock> javaPairRdd = SparkUtils.copyBinaryBlockMatrix(binaryBlocks, copy);
    MatrixObject matrixObject = new MatrixObject(ValueType.DOUBLE, OptimizerUtils.getUniqueTempFileName(), new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo));
    matrixObject.setRDDHandle(new RDDObject(javaPairRdd, variableName));
    return matrixObject;
}

Example 5 with MatrixFormatMetaData

use of org.apache.sysml.runtime.matrix.MatrixFormatMetaData in project incubator-systemml by apache.

the class ExternalFunctionProgramBlockCP method createOutputMatrixObject.

@Override
protected MatrixObject createOutputMatrixObject(Matrix m) {
    MatrixObject ret = m.getMatrixObject();
    if (//otherwise, pass in-memory matrix from extfunct back to invoking program
    ret == null) {
        MatrixCharacteristics mc = new MatrixCharacteristics(m.getNumRows(), m.getNumCols(), ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
        MatrixFormatMetaData mfmd = new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
        ret = new MatrixObject(ValueType.DOUBLE, m.getFilePath(), mfmd);
    }
    //for allowing in-memory packagesupport matrices w/o filesnames
    if (ret.getFileName().equals(DEFAULT_FILENAME)) {
        ret.setFileName(createDefaultOutputFilePathAndName());
    }
    return ret;
}