Examples with MatrixObject org.apache.sysml.runtime.controlprogram.caching.MatrixObject used on opensource projects

Example 21 with MatrixObject

use of org.apache.sysml.runtime.controlprogram.caching.MatrixObject in project incubator-systemml by apache.

the class MLContextConversionUtil method javaRDDStringCSVToMatrixObject.

/**
 * Convert a {@code JavaRDD<String>} in CSV format to a {@code MatrixObject}
 *
 * @param javaRDD
 *            the Java RDD of strings
 * @param matrixMetadata
 *            matrix metadata
 * @return the {@code JavaRDD<String>} converted to a {@code MatrixObject}
 */
public static MatrixObject javaRDDStringCSVToMatrixObject(JavaRDD<String> javaRDD, MatrixMetadata matrixMetadata) {
    JavaPairRDD<LongWritable, Text> javaPairRDD = javaRDD.mapToPair(new ConvertStringToLongTextPair());
    MatrixCharacteristics mc = (matrixMetadata != null) ? matrixMetadata.asMatrixCharacteristics() : new MatrixCharacteristics();
    MatrixObject matrixObject = new MatrixObject(ValueType.DOUBLE, OptimizerUtils.getUniqueTempFileName(), new MetaDataFormat(mc, OutputInfo.CSVOutputInfo, InputInfo.CSVInputInfo));
    JavaPairRDD<LongWritable, Text> javaPairRDD2 = javaPairRDD.mapToPair(new CopyTextInputFunction());
    matrixObject.setRDDHandle(new RDDObject(javaPairRDD2));
    return matrixObject;
}

Example 22 with MatrixObject

use of org.apache.sysml.runtime.controlprogram.caching.MatrixObject in project incubator-systemml by apache.

the class MLContextConversionUtil method javaRDDStringIJVToMatrixObject.

/**
 * Convert a {@code JavaRDD<String>} in IJV format to a {@code MatrixObject}
 * . Note that metadata is required for IJV format.
 *
 * @param javaRDD
 *            the Java RDD of strings
 * @param matrixMetadata
 *            matrix metadata
 * @return the {@code JavaRDD<String>} converted to a {@code MatrixObject}
 */
public static MatrixObject javaRDDStringIJVToMatrixObject(JavaRDD<String> javaRDD, MatrixMetadata matrixMetadata) {
    JavaPairRDD<LongWritable, Text> javaPairRDD = javaRDD.mapToPair(new ConvertStringToLongTextPair());
    MatrixCharacteristics mc = (matrixMetadata != null) ? matrixMetadata.asMatrixCharacteristics() : new MatrixCharacteristics();
    MatrixObject matrixObject = new MatrixObject(ValueType.DOUBLE, OptimizerUtils.getUniqueTempFileName(), new MetaDataFormat(mc, OutputInfo.TextCellOutputInfo, InputInfo.TextCellInputInfo));
    JavaPairRDD<LongWritable, Text> javaPairRDD2 = javaPairRDD.mapToPair(new CopyTextInputFunction());
    matrixObject.setRDDHandle(new RDDObject(javaPairRDD2));
    return matrixObject;
}

Example 23 with MatrixObject

use of org.apache.sysml.runtime.controlprogram.caching.MatrixObject in project incubator-systemml by apache.

the class MLResults method getDataFrameDoubleNoIDColumn.

/**
 * Obtain an output as a {@code DataFrame} of doubles with no ID column.
 * <p>
 * The following matrix in DML:
 * </p>
 * <code>M = full('1 2 3 4', rows=2, cols=2);
 * </code>
 * <p>
 * is equivalent to the following {@code DataFrame} of doubles:
 * </p>
 * <code>[1.0,2.0]
 * <br>[3.0,4.0]
 * </code>
 *
 * @param outputName
 *            the name of the output
 * @return the output as a {@code DataFrame} of doubles with no ID column
 */
public Dataset<Row> getDataFrameDoubleNoIDColumn(String outputName) {
    if (isFrameObject(outputName)) {
        throw new MLContextException("This method currently supports only matrices");
    }
    MatrixObject mo = getMatrixObject(outputName);
    Dataset<Row> df = MLContextConversionUtil.matrixObjectToDataFrame(mo, sparkExecutionContext, false);
    return df.drop(RDDConverterUtils.DF_ID_COLUMN);
}

Example 24 with MatrixObject

use of org.apache.sysml.runtime.controlprogram.caching.MatrixObject in project incubator-systemml by apache.

the class MLResults method getDataFrameVectorNoIDColumn.

/**
 * Obtain an output as a {@code DataFrame} of vectors with no ID column.
 * <p>
 * The following matrix in DML:
 * </p>
 * <code>M = full('1 2 3 4', rows=2, cols=2);
 * </code>
 * <p>
 * is equivalent to the following {@code DataFrame} of vectors:
 * </p>
 * <code>[[1.0,2.0]]
 * <br>[[3.0,4.0]]
 * </code>
 *
 * @param outputName
 *            the name of the output
 * @return the output as a {@code DataFrame} of vectors with no ID column
 */
public Dataset<Row> getDataFrameVectorNoIDColumn(String outputName) {
    if (isFrameObject(outputName)) {
        throw new MLContextException("This method currently supports only matrices");
    }
    MatrixObject mo = getMatrixObject(outputName);
    Dataset<Row> df = MLContextConversionUtil.matrixObjectToDataFrame(mo, sparkExecutionContext, true);
    return df.drop(RDDConverterUtils.DF_ID_COLUMN);
}

Example 25 with MatrixObject

use of org.apache.sysml.runtime.controlprogram.caching.MatrixObject in project incubator-systemml by apache.

the class DMLDebuggerFunctions method printMatrixVariable.

/**
 * Print DML matrix variable in current frame (if existing)
 * @param variables Current frame variables
 * @param varname Variable name
 */
public void printMatrixVariable(LocalVariableMap variables, String varname) {
    if (varname == null) {
        System.err.println("No matrix variable name entered.");
        return;
    }
    if (variables != null && !variables.keySet().isEmpty()) {
        if (variables.get(varname) != null) {
            if (variables.get(varname).getDataType() == DataType.MATRIX) {
                try {
                    MatrixObject mo = (MatrixObject) variables.get(varname);
                    if (mo.getStatusAsString().equals("EMPTY") && (OptimizerUtils.estimateSizeExactSparsity(mo.getNumRows(), mo.getNumColumns(), mo.getSparsity()) > OptimizerUtils.getLocalMemBudget())) {
                        // TODO @jlugoma Need to add functionality to bring and display a block.
                        System.err.println("ERROR: DML matrix/vector dimensions are too large to fit in main memory.");
                        return;
                    }
                    MatrixBlock mb = mo.acquireRead();
                    prettyPrintMatrixBlock(mb, -1, -1);
                    mo.release();
                    if (mb.getNumRows() > DISPLAY_MAX_ROWS || mb.getNumColumns() > DISPLAY_MAX_COLUMNS) {
                        System.out.format("WARNING: DML matrix/vector is too large to display on the screen." + "\nOnly a snapshot of %d row(s) and %d column(s) is being displayed.\n", min(mb.getNumRows(), DISPLAY_MAX_ROWS), min(mb.getNumColumns(), DISPLAY_MAX_COLUMNS));
                    }
                    System.out.println("Metadata: " + variables.get(varname).getMetaData().toString());
                } catch (Exception e) {
                    System.err.println("Error processing display DML matrix command for variable " + varname + ".");
                    return;
                }
            } else
                System.out.println("Variable \"" + varname + "\" is not a matrix or vector variable.");
        } else
            System.out.println("DML matrix variable \"" + varname + "\" is not in the current frame scope. Try \"a\" to list all variables within current frame scope.");
    } else
        System.out.println("Symbol table for current frame is empty");
}