Examples with FrameBlock org.apache.sysml.runtime.matrix.data.FrameBlock used on opensource projects

Example 11 with FrameBlock

use of org.apache.sysml.runtime.matrix.data.FrameBlock in project incubator-systemml by apache.

the class ResultVariables method getFrame.

/**
	 * Obtain the frame represented by the given output variable.
	 * 
	 * @param varname output variable name
	 * @return frame as a two-dimensional string array
	 * @throws DMLException if DMLException occurs
	 */
public String[][] getFrame(String varname) throws DMLException {
    if (!_out.containsKey(varname))
        throw new DMLException("Non-existent output variable: " + varname);
    Data dat = _out.get(varname);
    //basic checks for data type	
    if (!(dat instanceof FrameObject))
        throw new DMLException("Expected frame result '" + varname + "' not a frame.");
    //convert output matrix to double array	
    FrameObject fo = (FrameObject) dat;
    FrameBlock frame = fo.acquireRead();
    String[][] ret = DataConverter.convertToStringFrame(frame);
    fo.release();
    return ret;
}

Example 12 with FrameBlock

use of org.apache.sysml.runtime.matrix.data.FrameBlock in project incubator-systemml by apache.

the class MLContextConversionUtil method javaRDDStringCSVToFrameObject.

/**
	 * Convert a {@code JavaRDD<String>} in CSV format to a {@code FrameObject}
	 * 
	 * @param variableName
	 *            name of the variable associated with the frame
	 * @param javaRDD
	 *            the Java RDD of strings
	 * @param frameMetadata
	 *            frame metadata
	 * @return the {@code JavaRDD<String>} converted to a {@code FrameObject}
	 */
public static FrameObject javaRDDStringCSVToFrameObject(String variableName, JavaRDD<String> javaRDD, FrameMetadata frameMetadata) {
    JavaPairRDD<LongWritable, Text> javaPairRDD = javaRDD.mapToPair(new ConvertStringToLongTextPair());
    MatrixCharacteristics mc = (frameMetadata != null) ? frameMetadata.asMatrixCharacteristics() : new MatrixCharacteristics();
    JavaPairRDD<LongWritable, Text> javaPairRDDText = javaPairRDD.mapToPair(new CopyTextInputFunction());
    FrameObject frameObject = new FrameObject(OptimizerUtils.getUniqueTempFileName(), new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo), frameMetadata.getFrameSchema().getSchema().toArray(new ValueType[0]));
    JavaPairRDD<Long, FrameBlock> rdd;
    try {
        rdd = FrameRDDConverterUtils.csvToBinaryBlock(jsc(), javaPairRDDText, mc, frameObject.getSchema(), false, ",", false, -1);
    } catch (DMLRuntimeException e) {
        e.printStackTrace();
        return null;
    }
    frameObject.setRDDHandle(new RDDObject(rdd, variableName));
    return frameObject;
}

Example 13 with FrameBlock

use of org.apache.sysml.runtime.matrix.data.FrameBlock in project incubator-systemml by apache.

the class MLContextConversionUtil method frameObjectToListStringIJV.

/**
	 * Convert a {@code FrameObject} to a {@code List<String>} in IJV format.
	 * 
	 * @param frameObject
	 *            the {@code FrameObject}
	 * @return the {@code FrameObject} converted to a {@code List<String>}
	 */
public static List<String> frameObjectToListStringIJV(FrameObject frameObject) {
    try {
        FrameBlock fb = frameObject.acquireRead();
        int rows = fb.getNumRows();
        int cols = fb.getNumColumns();
        List<String> list = new ArrayList<String>();
        StringBuilder sb = null;
        for (int i = 0; i < rows; i++) {
            sb = new StringBuilder();
            for (int j = 0; j < cols; j++) {
                if (fb.get(i, j) != null) {
                    sb = new StringBuilder();
                    sb.append(i + 1);
                    sb.append(" ");
                    sb.append(j + 1);
                    sb.append(" ");
                    sb.append(fb.get(i, j));
                    list.add(sb.toString());
                }
            }
        }
        frameObject.release();
        return list;
    } catch (CacheException e) {
        throw new MLContextException("Cache exception while converting frame object to List<String> IJV format", e);
    }
}

Example 14 with FrameBlock

use of org.apache.sysml.runtime.matrix.data.FrameBlock in project incubator-systemml by apache.

the class MLContextConversionUtil method frameObjectToBinaryBlockFrame.

/**
	 * Convert a {@code FrameObject} to a {@code BinaryBlockFrame}.
	 * 
	 * @param frameObject
	 *            the {@code FrameObject}
	 * @param sparkExecutionContext
	 *            the Spark execution context
	 * @return the {@code FrameObject} converted to a {@code BinaryBlockFrame}
	 */
public static BinaryBlockFrame frameObjectToBinaryBlockFrame(FrameObject frameObject, SparkExecutionContext sparkExecutionContext) {
    try {
        @SuppressWarnings("unchecked") JavaPairRDD<Long, FrameBlock> binaryBlock = (JavaPairRDD<Long, FrameBlock>) sparkExecutionContext.getRDDHandleForFrameObject(frameObject, InputInfo.BinaryBlockInputInfo);
        MatrixCharacteristics matrixCharacteristics = frameObject.getMatrixCharacteristics();
        FrameSchema fs = new FrameSchema(Arrays.asList(frameObject.getSchema()));
        FrameMetadata fm = new FrameMetadata(fs, matrixCharacteristics);
        return new BinaryBlockFrame(binaryBlock, fm);
    } catch (DMLRuntimeException e) {
        throw new MLContextException("DMLRuntimeException while converting frame object to BinaryBlockFrame", e);
    }
}

Example 15 with FrameBlock

use of org.apache.sysml.runtime.matrix.data.FrameBlock in project incubator-systemml by apache.

the class SparkExecutionContext method toFrameJavaPairRDD.

public static JavaPairRDD<Long, FrameBlock> toFrameJavaPairRDD(JavaSparkContext sc, FrameBlock src) throws DMLRuntimeException {
    long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
    LinkedList<Tuple2<Long, FrameBlock>> list = new LinkedList<Tuple2<Long, FrameBlock>>();
    //create and write subblocks of matrix
    int blksize = ConfigurationManager.getBlocksize();
    for (int blockRow = 0; blockRow < (int) Math.ceil(src.getNumRows() / (double) blksize); blockRow++) {
        int maxRow = (blockRow * blksize + blksize < src.getNumRows()) ? blksize : src.getNumRows() - blockRow * blksize;
        int roffset = blockRow * blksize;
        FrameBlock block = new FrameBlock(src.getSchema());
        //copy sub frame to block, incl meta data on first
        src.sliceOperations(roffset, roffset + maxRow - 1, 0, src.getNumColumns() - 1, block);
        if (roffset == 0)
            block.setColumnMetadata(src.getColumnMetadata());
        //append block to sequence file
        list.addLast(new Tuple2<Long, FrameBlock>((long) roffset + 1, block));
    }
    JavaPairRDD<Long, FrameBlock> result = sc.parallelizePairs(list);
    if (DMLScript.STATISTICS) {
        Statistics.accSparkParallelizeTime(System.nanoTime() - t0);
        Statistics.incSparkParallelizeCount(1);
    }
    return result;
}