Examples with FrameBlock org.apache.sysml.runtime.matrix.data.FrameBlock used on opensource projects

Example 26 with FrameBlock

use of org.apache.sysml.runtime.matrix.data.FrameBlock in project incubator-systemml by apache.

the class FrameReadWriteTest method runFrameReadWriteTest.

/**
	 * 
	 * @param sparseM1
	 * @param sparseM2
	 * @param instType
	 */
private void runFrameReadWriteTest(OutputInfo oinfo, ValueType[] schema1, ValueType[] schema2, boolean parallel) {
    boolean oldParText = CompilerConfig.FLAG_PARREADWRITE_TEXT;
    boolean oldParBin = CompilerConfig.FLAG_PARREADWRITE_BINARY;
    try {
        CompilerConfig.FLAG_PARREADWRITE_TEXT = parallel;
        CompilerConfig.FLAG_PARREADWRITE_BINARY = parallel;
        ConfigurationManager.setGlobalConfig(new CompilerConfig());
        //data generation
        double[][] A = getRandomMatrix(rows, schema1.length, -10, 10, 0.9, 2373);
        double[][] B = getRandomMatrix(rows, schema2.length, -10, 10, 0.9, 129);
        //Initialize the frame data.
        //init data frame 1
        FrameBlock frame1 = new FrameBlock(schema1);
        initFrameData(frame1, A, schema1);
        //init data frame 2
        FrameBlock frame2 = new FrameBlock(schema2);
        initFrameData(frame2, B, schema2);
        //Write frame data to disk
        CSVFileFormatProperties fprop = new CSVFileFormatProperties();
        fprop.setDelim(DELIMITER);
        fprop.setHeader(HEADER);
        writeAndVerifyData(oinfo, frame1, frame2, fprop);
    } catch (Exception ex) {
        ex.printStackTrace();
        throw new RuntimeException(ex);
    } finally {
        CompilerConfig.FLAG_PARREADWRITE_TEXT = oldParText;
        CompilerConfig.FLAG_PARREADWRITE_BINARY = oldParBin;
        ConfigurationManager.setGlobalConfig(new CompilerConfig());
    }
}

Example 27 with FrameBlock

use of org.apache.sysml.runtime.matrix.data.FrameBlock in project incubator-systemml by apache.

the class FrameCastingTest method runFrameCastingTest.

/**
	 * 
	 * @param sparseM1
	 * @param sparseM2
	 * @param instType
	 */
private void runFrameCastingTest(ValueType[] schema, CastType ctype) {
    try {
        //data generation
        double[][] A = getRandomMatrix(rows, schema.length, -10, 10, 0.9, 2412);
        for (int i = 0; i < rows; i++) {
            for (int j = 0; j < schema.length; j++) A[i][j] = UtilFunctions.objectToDouble(schema[j], UtilFunctions.doubleToObject(schema[j], A[i][j]));
        }
        //core casting operations
        FrameBlock frame = null;
        if (ctype == CastType.F2M) {
            //construct input schema
            FrameBlock frame1 = new FrameBlock(schema);
            Object[] row1 = new Object[schema.length];
            for (int i = 0; i < rows; i++) {
                for (int j = 0; j < schema.length; j++) row1[j] = UtilFunctions.doubleToObject(schema[j], A[i][j]);
                frame1.appendRow(row1);
            }
            MatrixBlock mb = DataConverter.convertToMatrixBlock(frame1);
            frame = DataConverter.convertToFrameBlock(mb);
        } else if (ctype == CastType.M2F_G) {
            MatrixBlock mb = DataConverter.convertToMatrixBlock(A);
            frame = DataConverter.convertToFrameBlock(mb);
        } else if (ctype == CastType.M2F_S) {
            MatrixBlock mb = DataConverter.convertToMatrixBlock(A);
            frame = DataConverter.convertToFrameBlock(mb, schema);
        }
        //check basic meta data
        if (frame.getNumRows() != rows)
            Assert.fail("Wrong number of rows: " + frame.getNumRows() + ", expected: " + rows);
        //check correct values
        ValueType[] lschema = frame.getSchema();
        for (int i = 0; i < rows; i++) for (int j = 0; j < lschema.length; j++) {
            double tmp = UtilFunctions.objectToDouble(lschema[j], frame.get(i, j));
            if (tmp != A[i][j])
                Assert.fail("Wrong get value for cell (" + i + "," + j + "): " + tmp + ", expected: " + A[i][j]);
        }
    } catch (Exception ex) {
        ex.printStackTrace();
        throw new RuntimeException(ex);
    }
}

Example 28 with FrameBlock

use of org.apache.sysml.runtime.matrix.data.FrameBlock in project incubator-systemml by apache.

the class FrameScalarCastingTest method runFrameCastingTest.

/**
	 * 
	 * @param testname
	 * @param schema
	 * @param wildcard
	 */
private void runFrameCastingTest(String testname, ValueType vt) {
    try {
        TestConfiguration config = getTestConfiguration(testname);
        loadTestConfiguration(config);
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + testname + ".dml";
        programArgs = new String[] { "-explain", "-args", input("A"), vt.toString(), output("B") };
        //input data and compare
        FrameBlock fb = new FrameBlock(1, vt);
        Object inval = UtilFunctions.objectToObject(vt, 7);
        fb.ensureAllocatedColumns(1);
        fb.set(0, 0, inval);
        //write inputs
        if (testname.equals(TEST_NAME1))
            FrameWriterFactory.createFrameWriter(OutputInfo.TextCellOutputInfo).writeFrameToHDFS(fb, input("A"), 1, 1);
        else
            MapReduceTool.writeObjectToHDFS(inval, input("A"));
        //run testcase
        runTest(true, false, null, -1);
        //read and compare scalars
        Object retval = null;
        if (testname.equals(TEST_NAME1)) {
            retval = MapReduceTool.readObjectFromHDFSFile(output("B"), vt);
        } else {
            retval = FrameReaderFactory.createFrameReader(InputInfo.TextCellInputInfo).readFrameFromHDFS(output("B"), new ValueType[] { vt }, 1, 1).get(0, 0);
        }
        Assert.assertEquals("Wrong output: " + retval + " (expected: " + inval + ")", inval, retval);
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    }
}

Example 29 with FrameBlock

use of org.apache.sysml.runtime.matrix.data.FrameBlock in project incubator-systemml by apache.

the class FrameSchemaReadTest method runFrameSchemaReadTest.

/**
	 * 
	 * @param testname
	 * @param schema
	 * @param wildcard
	 */
private void runFrameSchemaReadTest(String testname, ValueType[] schema, boolean wildcard) {
    try {
        TestConfiguration config = getTestConfiguration(testname);
        loadTestConfiguration(config);
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + testname + ".dml";
        programArgs = new String[] { "-explain", "-args", input("A"), getSchemaString(schema, wildcard), Integer.toString(rows), Integer.toString(schema.length), output("B") };
        //data generation
        double[][] A = getRandomMatrix(rows, schema.length, -10, 10, 0.9, 2373);
        //prepare input/output infos
        FrameBlock frame1 = new FrameBlock(schema);
        initFrameData(frame1, A, schema);
        //write frame data to hdfs
        FrameWriter writer = FrameWriterFactory.createFrameWriter(OutputInfo.CSVOutputInfo);
        writer.writeFrameToHDFS(frame1, input("A"), rows, schema.length);
        //run testcase
        runTest(true, false, null, -1);
        //read frame data from hdfs (not via readers to test physical schema)
        FrameReader reader = FrameReaderFactory.createFrameReader(InputInfo.BinaryBlockInputInfo);
        FrameBlock frame2 = ((FrameReaderBinaryBlock) reader).readFirstBlock(output("B"));
        //verify output schema
        ValueType[] schemaExpected = (testname.equals(TEST_NAME2) || wildcard) ? Collections.nCopies(schema.length, ValueType.STRING).toArray(new ValueType[0]) : schema;
        for (int i = 0; i < schemaExpected.length; i++) {
            Assert.assertEquals("Wrong result: " + frame2.getSchema()[i] + ".", schemaExpected[i], frame2.getSchema()[i]);
        }
    } catch (Exception ex) {
        ex.printStackTrace();
        throw new RuntimeException(ex);
    }
}

Example 30 with FrameBlock

use of org.apache.sysml.runtime.matrix.data.FrameBlock in project incubator-systemml by apache.

the class FrameReadMetaTest method runJMLCReadMetaTest.

/**
	 * 
	 * @param sparseM1
	 * @param sparseM2
	 * @param instType
	 * @throws IOException 
	 */
private void runJMLCReadMetaTest(String testname, boolean modelReuse, boolean readFrame, boolean useSpec) throws IOException {
    String TEST_NAME = testname;
    TestConfiguration config = getTestConfiguration(TEST_NAME);
    loadTestConfiguration(config);
    //establish connection to SystemML
    Connection conn = new Connection();
    //read meta data frame 
    String spec = MapReduceTool.readStringFromHDFSFile(SCRIPT_DIR + TEST_DIR + "tfmtd_example2/spec.json");
    FrameBlock M = readFrame ? DataConverter.convertToFrameBlock(conn.readStringFrame(SCRIPT_DIR + TEST_DIR + "tfmtd_frame_example/tfmtd_frame")) : conn.readTransformMetaDataFromFile(spec, SCRIPT_DIR + TEST_DIR + "tfmtd_example2/");
    try {
        //generate data based on recode maps
        HashMap<String, Long>[] RC = getRecodeMaps(spec, M);
        double[][] X = generateData(rows, cols, RC);
        String[][] F = null;
        //prepare input arguments
        HashMap<String, String> args = new HashMap<String, String>();
        args.put("$TRANSFORM_SPEC", spec);
        //read and precompile script
        String script = conn.readScript(SCRIPT_DIR + TEST_DIR + testname + ".dml");
        PreparedScript pstmt = conn.prepareScript(script, args, new String[] { "X", "M" }, new String[] { "F" }, false);
        if (modelReuse)
            pstmt.setFrame("M", M, true);
        //execute script multiple times (2 runs)
        for (int i = 0; i < 2; i++) {
            //bind input parameters
            if (!modelReuse)
                pstmt.setFrame("M", M, false);
            pstmt.setMatrix("X", X);
            //execute script
            ResultVariables rs = pstmt.executeScript();
            //get output parameter
            F = rs.getFrame("F");
        }
        //for all generated data, probe recode maps and compare versus output
        for (int i = 0; i < rows; i++) for (int j = 0; j < cols; j++) if (RC[j] != null) {
            Assert.assertEquals("Wrong result: " + F[i][j] + ".", Double.valueOf(X[i][j]), Double.valueOf(RC[j].get(F[i][j]).toString()));
        }
    } catch (Exception ex) {
        ex.printStackTrace();
        throw new IOException(ex);
    } finally {
        IOUtilFunctions.closeSilently(conn);
    }
}