Examples with FrameBlock org.apache.sysml.runtime.matrix.data.FrameBlock used on opensource projects

Example 31 with FrameBlock

use of org.apache.sysml.runtime.matrix.data.FrameBlock in project incubator-systemml by apache.

the class FrameScalarCastingTest method runFrameCastingTest.

/**
 * @param testname
 * @param schema
 * @param wildcard
 */
private void runFrameCastingTest(String testname, ValueType vt) {
    try {
        TestConfiguration config = getTestConfiguration(testname);
        loadTestConfiguration(config);
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + testname + ".dml";
        programArgs = new String[] { "-explain", "-args", input("A"), vt.toString(), output("B") };
        // input data and compare
        FrameBlock fb = new FrameBlock(1, vt);
        Object inval = UtilFunctions.objectToObject(vt, 7);
        fb.ensureAllocatedColumns(1);
        fb.set(0, 0, inval);
        // write inputs
        if (testname.equals(TEST_NAME1))
            FrameWriterFactory.createFrameWriter(OutputInfo.TextCellOutputInfo).writeFrameToHDFS(fb, input("A"), 1, 1);
        else
            MapReduceTool.writeObjectToHDFS(inval, input("A"));
        // run testcase
        runTest(true, false, null, -1);
        // read and compare scalars
        Object retval = null;
        if (testname.equals(TEST_NAME1)) {
            retval = MapReduceTool.readObjectFromHDFSFile(output("B"), vt);
        } else {
            retval = FrameReaderFactory.createFrameReader(InputInfo.TextCellInputInfo).readFrameFromHDFS(output("B"), new ValueType[] { vt }, 1, 1).get(0, 0);
        }
        Assert.assertEquals("Wrong output: " + retval + " (expected: " + inval + ")", inval, retval);
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    }
}

Example 32 with FrameBlock

use of org.apache.sysml.runtime.matrix.data.FrameBlock in project incubator-systemml by apache.

the class FrameSchemaReadTest method runFrameSchemaReadTest.

/**
 * @param testname
 * @param schema
 * @param wildcard
 */
private void runFrameSchemaReadTest(String testname, ValueType[] schema, boolean wildcard) {
    try {
        TestConfiguration config = getTestConfiguration(testname);
        loadTestConfiguration(config);
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + testname + ".dml";
        programArgs = new String[] { "-explain", "-args", input("A"), getSchemaString(schema, wildcard), Integer.toString(rows), Integer.toString(schema.length), output("B") };
        // data generation
        double[][] A = getRandomMatrix(rows, schema.length, -10, 10, 0.9, 2373);
        // prepare input/output infos
        FrameBlock frame1 = new FrameBlock(schema);
        initFrameData(frame1, A, schema);
        // write frame data to hdfs
        FrameWriter writer = FrameWriterFactory.createFrameWriter(OutputInfo.CSVOutputInfo);
        writer.writeFrameToHDFS(frame1, input("A"), rows, schema.length);
        // run testcase
        runTest(true, false, null, -1);
        // read frame data from hdfs (not via readers to test physical schema)
        FrameReader reader = FrameReaderFactory.createFrameReader(InputInfo.BinaryBlockInputInfo);
        FrameBlock frame2 = ((FrameReaderBinaryBlock) reader).readFirstBlock(output("B"));
        // verify output schema
        ValueType[] schemaExpected = (testname.equals(TEST_NAME2) || wildcard) ? Collections.nCopies(schema.length, ValueType.STRING).toArray(new ValueType[0]) : schema;
        for (int i = 0; i < schemaExpected.length; i++) {
            Assert.assertEquals("Wrong result: " + frame2.getSchema()[i] + ".", schemaExpected[i], frame2.getSchema()[i]);
        }
    } catch (Exception ex) {
        ex.printStackTrace();
        throw new RuntimeException(ex);
    }
}

Example 33 with FrameBlock

use of org.apache.sysml.runtime.matrix.data.FrameBlock in project incubator-systemml by apache.

the class FrameSerializationTest method runFrameSerializeTest.

/**
 * @param sparseM1
 * @param sparseM2
 * @param instType
 */
private void runFrameSerializeTest(ValueType[] schema, SerType stype) {
    try {
        // data generation
        double[][] A = getRandomMatrix(rows, schema.length, -10, 10, 0.9, 8234);
        // init data frame
        FrameBlock frame = new FrameBlock(schema);
        // init data frame
        Object[] row = new Object[schema.length];
        for (int i = 0; i < rows; i++) {
            for (int j = 0; j < schema.length; j++) A[i][j] = UtilFunctions.objectToDouble(schema[j], row[j] = UtilFunctions.doubleToObject(schema[j], A[i][j]));
            frame.appendRow(row);
        }
        // core serialization and deserialization
        if (stype == SerType.WRITABLE_SER) {
            // serialization
            ByteArrayOutputStream bos = new ByteArrayOutputStream();
            DataOutputStream dos = new DataOutputStream(bos);
            frame.write(dos);
            // deserialization
            ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
            DataInputStream dis = new DataInputStream(bis);
            frame = new FrameBlock();
            frame.readFields(dis);
        } else if (stype == SerType.JAVA_SER) {
            // serialization
            ByteArrayOutputStream bos = new ByteArrayOutputStream();
            ObjectOutputStream oos = new ObjectOutputStream(bos);
            oos.writeObject(frame);
            // deserialization
            ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
            ObjectInputStream ois = new ObjectInputStream(bis);
            frame = (FrameBlock) ois.readObject();
        }
        // check basic meta data
        if (frame.getNumRows() != rows)
            Assert.fail("Wrong number of rows: " + frame.getNumRows() + ", expected: " + rows);
        // check correct values
        for (int i = 0; i < rows; i++) for (int j = 0; j < schema.length; j++) {
            double tmp = UtilFunctions.objectToDouble(schema[j], frame.get(i, j));
            if (tmp != A[i][j])
                Assert.fail("Wrong get value for cell (" + i + "," + j + "): " + tmp + ", expected: " + A[i][j]);
        }
    } catch (Exception ex) {
        ex.printStackTrace();
        throw new RuntimeException(ex);
    }
}

Example 34 with FrameBlock

use of org.apache.sysml.runtime.matrix.data.FrameBlock in project incubator-systemml by apache.

the class ParforFrameIntermediateTest method runParforFrameIntermediatesTest.

private void runParforFrameIntermediatesTest(ExecType et) {
    RUNTIME_PLATFORM platformOld = rtplatform;
    switch(et) {
        case SPARK:
            rtplatform = RUNTIME_PLATFORM.SPARK;
            break;
        default:
            rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK;
            break;
    }
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    try {
        // setup testcase
        getAndLoadTestConfiguration(TEST_NAME);
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-explain", "-args", input("F") };
        // generate input data and write as frame
        double[][] A = getRandomMatrix(rows, cols, -10, 10, 0.9, 8362);
        FrameBlock fA = DataConverter.convertToFrameBlock(DataConverter.convertToMatrixBlock(A));
        FrameWriterFactory.createFrameWriter(OutputInfo.CSVOutputInfo).writeFrameToHDFS(fA, input("F"), rows, cols);
        // run test
        runTest(true, false, null, -1);
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    } finally {
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}

Example 35 with FrameBlock

use of org.apache.sysml.runtime.matrix.data.FrameBlock in project incubator-systemml by apache.

the class FrameCSVReadWriteTest method runCSVQuotesReadWriteTest.

/**
 * @param rt
 * @param ofmt
 * @param dataset
 */
private void runCSVQuotesReadWriteTest(RUNTIME_PLATFORM rt, String ofmt) {
    // set runtime platform
    RUNTIME_PLATFORM rtold = rtplatform;
    rtplatform = rt;
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    if (!ofmt.equals("csv"))
        throw new RuntimeException("Unsupported test output format");
    try {
        getAndLoadTestConfiguration(TEST_NAME1);
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME1 + ".dml";
        programArgs = new String[] { "-explain", "-args", HOME + "input/" + DATASET, output("R") };
        runTest(true, false, null, -1);
        // read input/output and compare
        FrameReader reader1 = FrameReaderFactory.createFrameReader(InputInfo.CSVInputInfo, new CSVFileFormatProperties(false, ",", false));
        FrameBlock fb1 = reader1.readFrameFromHDFS(HOME + "input/" + DATASET, -1L, -1L);
        FrameReader reader2 = FrameReaderFactory.createFrameReader(InputInfo.CSVInputInfo);
        FrameBlock fb2 = reader2.readFrameFromHDFS(output("R"), -1L, -1L);
        String[][] R1 = DataConverter.convertToStringFrame(fb1);
        String[][] R2 = DataConverter.convertToStringFrame(fb2);
        TestUtils.compareFrames(R1, R2, R1.length, R1[0].length);
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    } finally {
        rtplatform = rtold;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}