Examples with FrameBlock org.apache.sysml.runtime.matrix.data.FrameBlock used on opensource projects

Example 21 with FrameBlock

use of org.apache.sysml.runtime.matrix.data.FrameBlock in project incubator-systemml by apache.

the class FrameSerializationTest method runFrameSerializeTest.

/**
	 * 
	 * @param sparseM1
	 * @param sparseM2
	 * @param instType
	 */
private void runFrameSerializeTest(ValueType[] schema, SerType stype) {
    try {
        //data generation
        double[][] A = getRandomMatrix(rows, schema.length, -10, 10, 0.9, 8234);
        //init data frame
        FrameBlock frame = new FrameBlock(schema);
        //init data frame 
        Object[] row = new Object[schema.length];
        for (int i = 0; i < rows; i++) {
            for (int j = 0; j < schema.length; j++) A[i][j] = UtilFunctions.objectToDouble(schema[j], row[j] = UtilFunctions.doubleToObject(schema[j], A[i][j]));
            frame.appendRow(row);
        }
        //core serialization and deserialization
        if (stype == SerType.WRITABLE_SER) {
            //serialization
            ByteArrayOutputStream bos = new ByteArrayOutputStream();
            DataOutputStream dos = new DataOutputStream(bos);
            frame.write(dos);
            //deserialization
            ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
            DataInputStream dis = new DataInputStream(bis);
            frame = new FrameBlock();
            frame.readFields(dis);
        } else if (stype == SerType.JAVA_SER) {
            //serialization
            ByteArrayOutputStream bos = new ByteArrayOutputStream();
            ObjectOutputStream oos = new ObjectOutputStream(bos);
            oos.writeObject(frame);
            //deserialization
            ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
            ObjectInputStream ois = new ObjectInputStream(bis);
            frame = (FrameBlock) ois.readObject();
        }
        //check basic meta data
        if (frame.getNumRows() != rows)
            Assert.fail("Wrong number of rows: " + frame.getNumRows() + ", expected: " + rows);
        //check correct values			
        for (int i = 0; i < rows; i++) for (int j = 0; j < schema.length; j++) {
            double tmp = UtilFunctions.objectToDouble(schema[j], frame.get(i, j));
            if (tmp != A[i][j])
                Assert.fail("Wrong get value for cell (" + i + "," + j + "): " + tmp + ", expected: " + A[i][j]);
        }
    } catch (Exception ex) {
        ex.printStackTrace();
        throw new RuntimeException(ex);
    }
}

Example 22 with FrameBlock

use of org.apache.sysml.runtime.matrix.data.FrameBlock in project incubator-systemml by apache.

the class FrameConverterTest method runMatrixConverterAndVerify.

/**
	 * 
	 * @param schema
	 * @param A
	 * @param type
	 * @param iinfo
	 * @param oinfo
	 * @param instType
	 */
private void runMatrixConverterAndVerify(ValueType[] schema, double[][] A, ConvType type, InputInfo iinfo, OutputInfo oinfo) throws IOException {
    try {
        MatrixCharacteristics mcMatrix = new MatrixCharacteristics(rows, schema.length, 1000, 1000, 0);
        MatrixCharacteristics mcFrame = new MatrixCharacteristics(rows, schema.length, -1, -1, -1);
        MatrixBlock matrixBlock1 = null;
        FrameBlock frame1 = null;
        if (type == ConvType.MAT2BIN) {
            //initialize the matrix (dense) data.
            matrixBlock1 = new MatrixBlock(rows, schema.length, false);
            matrixBlock1.init(A, rows, schema.length);
            //write matrix data to hdfs
            MatrixWriter matWriter = MatrixWriterFactory.createMatrixWriter(oinfo);
            matWriter.writeMatrixToHDFS(matrixBlock1, input("A"), rows, schema.length, mcMatrix.getRowsPerBlock(), mcMatrix.getColsPerBlock(), mcMatrix.getNonZeros());
        } else {
            //initialize the frame data.
            frame1 = new FrameBlock(schema);
            initFrameData(frame1, A, schema);
            //write frame data to hdfs
            FrameWriter writer = FrameWriterFactory.createFrameWriter(oinfo);
            writer.writeFrameToHDFS(frame1, input("A"), rows, schema.length);
        }
        //run converter under test
        runConverter(type, mcFrame, mcMatrix, Arrays.asList(schema), input("A"), output("B"));
        if (type == ConvType.MAT2BIN) {
            //read frame data from hdfs
            FrameReader reader = FrameReaderFactory.createFrameReader(iinfo);
            FrameBlock frame2 = reader.readFrameFromHDFS(output("B"), rows, schema.length);
            //verify input and output frame/matrix
            verifyFrameMatrixData(frame2, matrixBlock1);
        } else {
            //read matrix data from hdfs
            MatrixReader matReader = MatrixReaderFactory.createMatrixReader(iinfo);
            MatrixBlock matrixBlock2 = matReader.readMatrixFromHDFS(output("B"), rows, schema.length, mcMatrix.getRowsPerBlock(), mcMatrix.getColsPerBlock(), mcMatrix.getNonZeros());
            //verify input and output frame/matrix
            verifyFrameMatrixData(frame1, matrixBlock2);
        }
    } catch (Exception ex) {
        ex.printStackTrace();
        throw new RuntimeException(ex);
    } finally {
        MapReduceTool.deleteFileIfExistOnHDFS(input("A"));
        MapReduceTool.deleteFileIfExistOnHDFS(output("B"));
    }
}

Example 23 with FrameBlock

use of org.apache.sysml.runtime.matrix.data.FrameBlock in project incubator-systemml by apache.

the class FrameConverterTest method runConverterAndVerify.

/**
	 * 
	 * @param schema
	 * @param A
	 * @param type
	 * @param iinfo
	 * @param oinfo
	 * @param instType
	 */
private void runConverterAndVerify(ValueType[] schema, double[][] A, ConvType type, InputInfo iinfo, OutputInfo oinfo) throws IOException {
    try {
        //initialize the frame data.
        FrameBlock frame1 = new FrameBlock(schema);
        initFrameData(frame1, A, schema);
        //write frame data to hdfs
        FrameWriter writer = FrameWriterFactory.createFrameWriter(oinfo);
        writer.writeFrameToHDFS(frame1, input("A"), rows, schema.length);
        //run converter under test
        MatrixCharacteristics mc = new MatrixCharacteristics(rows, schema.length, -1, -1, -1);
        runConverter(type, mc, null, Arrays.asList(schema), input("A"), output("B"));
        //read frame data from hdfs
        FrameReader reader = FrameReaderFactory.createFrameReader(iinfo);
        FrameBlock frame2 = reader.readFrameFromHDFS(output("B"), rows, schema.length);
        //verify input and output frame
        verifyFrameData(frame1, frame2);
    } catch (Exception ex) {
        ex.printStackTrace();
        throw new RuntimeException(ex);
    } finally {
        MapReduceTool.deleteFileIfExistOnHDFS(input("A"));
        MapReduceTool.deleteFileIfExistOnHDFS(output("B"));
    }
}

Example 24 with FrameBlock

use of org.apache.sysml.runtime.matrix.data.FrameBlock in project incubator-systemml by apache.

the class FrameCopyTest method runFrameCopyTest.

/**
	 * 
	 * @param sparseM1
	 * @param sparseM2
	 * @param instType
	 */
private void runFrameCopyTest(ValueType[] schema1, ValueType[] schema2, AppendType atype) {
    try {
        //data generation
        double[][] A = getRandomMatrix(rows, schema1.length, -10, 10, 0.9, 2373);
        double[][] B = getRandomMatrix(rows, schema2.length, -10, 10, 0.9, 129);
        //Initialize the frame data.
        //init data frame 1
        FrameBlock frame1 = new FrameBlock(schema1);
        initFrameData(frame1, A, schema1);
        //init data frame 2
        FrameBlock frame2 = new FrameBlock(schema2);
        initFrameData(frame2, B, schema2);
        //copy from one frame to another.
        FrameBlock frame1Backup = new FrameBlock(frame1.getSchema(), frame1.getColumnNames());
        frame1Backup.copy(frame1);
        FrameBlock frame2Backup = new FrameBlock(frame2.getSchema(), frame2.getColumnNames());
        frame2Backup.copy(frame2);
        // Verify copied data.
        verifyFrameData(frame1, frame1Backup);
        verifyFrameData(frame2, frame2Backup);
        // update some data in original/backup frames
        int updateRow = rows / 2;
        updateFrameWithDummyData(frame1, updateRow);
        updateFrameWithDummyData(frame2, updateRow);
        // Verify that data modified only on target frames
        verifyFrameData(frame1, frame1Backup, updateRow, false);
        verifyFrameData(frame2, frame2Backup, updateRow, false);
    } catch (Exception ex) {
        ex.printStackTrace();
        throw new RuntimeException(ex);
    }
}

Example 25 with FrameBlock

use of org.apache.sysml.runtime.matrix.data.FrameBlock in project incubator-systemml by apache.

the class FrameEvictionTest method runFrameEvictionTest.

/**
	 * 
	 * @param schema
	 * @param sparse
	 * @param defaultMeta
	 * @param force
	 */
private void runFrameEvictionTest(ValueType[] schema, boolean sparse, boolean defaultMeta, boolean force) {
    try {
        //data generation
        double sparsity = sparse ? sparsity2 : sparsity1;
        double[][] A = getRandomMatrix(rows, schema.length, -10, 10, sparsity, 765);
        MatrixBlock mA = DataConverter.convertToMatrixBlock(A);
        FrameBlock fA = DataConverter.convertToFrameBlock(mA, schema);
        //create non-default column names
        if (!defaultMeta) {
            String[] colnames = new String[schema.length];
            for (int i = 0; i < schema.length; i++) colnames[i] = "Custom_name_" + i;
            fA.setColumnNames(colnames);
        }
        //setup caching
        CacheableData.initCaching("tmp_frame_eviction_test");
        //create frame object
        MatrixCharacteristics mc = new MatrixCharacteristics(rows, schema.length, -1, -1, -1);
        MatrixFormatMetaData meta = new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
        FrameObject fo = new FrameObject("fA", meta, schema);
        fo.acquireModify(fA);
        fo.release();
        //evict frame and clear in-memory reference
        if (force)
            LazyWriteBuffer.forceEviction();
        Method clearfo = CacheableData.class.getDeclaredMethod("clearCache", new Class[] {});
        //make method public
        clearfo.setAccessible(true);
        clearfo.invoke(fo, new Object[] {});
        //read frame through buffer pool (if forced, this is a read from disk
        //otherwise deserialization or simple reference depending on schema)
        FrameBlock fA2 = fo.acquireRead();
        fo.release();
        //compare frames
        String[][] sA = DataConverter.convertToStringFrame(fA);
        String[][] sA2 = DataConverter.convertToStringFrame(fA2);
        TestUtils.compareFrames(sA, sA2, rows, schema.length);
    } catch (Exception ex) {
        ex.printStackTrace();
        throw new RuntimeException(ex);
    }
}