Examples with FrameBlock org.apache.sysml.runtime.matrix.data.FrameBlock used on opensource projects

Example 56 with FrameBlock

use of org.apache.sysml.runtime.matrix.data.FrameBlock in project incubator-systemml by apache.

the class FrameFunctionTest method runFrameFunctionTest.

/**
 * @param et
 */
private void runFrameFunctionTest(ExecType et, boolean IPA) {
    // rtplatform for MR
    RUNTIME_PLATFORM platformOld = rtplatform;
    switch(et) {
        case SPARK:
            rtplatform = RUNTIME_PLATFORM.SPARK;
            break;
        default:
            rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK;
            break;
    }
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    boolean oldIPA = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS;
    OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = IPA;
    try {
        // setup testcase
        getAndLoadTestConfiguration(TEST_NAME);
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-explain", "-args", input("F"), output("F2") };
        // generate input data and write as frame
        double[][] A = getRandomMatrix(rows, cols, -10, 10, 0.9, 8362);
        FrameBlock fA = DataConverter.convertToFrameBlock(DataConverter.convertToMatrixBlock(A));
        FrameWriterFactory.createFrameWriter(OutputInfo.CSVOutputInfo).writeFrameToHDFS(fA, input("F"), rows, cols);
        // run test
        runTest(true, false, null, -1);
        // read input/output and compare
        FrameBlock fB = FrameReaderFactory.createFrameReader(InputInfo.CSVInputInfo).readFrameFromHDFS(output("F2"), rows, cols);
        String[][] R1 = DataConverter.convertToStringFrame(fA);
        String[][] R2 = DataConverter.convertToStringFrame(fB);
        TestUtils.compareFrames(R1, R2, R1.length, R1[0].length);
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    } finally {
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
        OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = oldIPA;
    }
}

Example 57 with FrameBlock

use of org.apache.sysml.runtime.matrix.data.FrameBlock in project incubator-systemml by apache.

the class FrameGetSetTest method runFrameGetSetTest.

/**
 * @param sparseM1
 * @param sparseM2
 * @param instType
 */
private void runFrameGetSetTest(ValueType[] schema, InitType itype) {
    try {
        // data generation
        double[][] A = getRandomMatrix(rows, schema.length, -10, 10, 0.9, 8234);
        // init data frame
        FrameBlock frame = new FrameBlock(schema);
        // init data frame
        if (itype == InitType.COLUMN) {
            for (int j = 0; j < schema.length; j++) {
                ValueType vt = schema[j];
                switch(vt) {
                    case STRING:
                        String[] tmp1 = new String[rows];
                        for (int i = 0; i < rows; i++) tmp1[i] = (String) UtilFunctions.doubleToObject(vt, A[i][j]);
                        frame.appendColumn(tmp1);
                        break;
                    case BOOLEAN:
                        boolean[] tmp2 = new boolean[rows];
                        for (int i = 0; i < rows; i++) A[i][j] = (tmp2[i] = (Boolean) UtilFunctions.doubleToObject(vt, A[i][j], false)) ? 1 : 0;
                        frame.appendColumn(tmp2);
                        break;
                    case INT:
                        long[] tmp3 = new long[rows];
                        for (int i = 0; i < rows; i++) A[i][j] = tmp3[i] = (Long) UtilFunctions.doubleToObject(vt, A[i][j], false);
                        frame.appendColumn(tmp3);
                        break;
                    case DOUBLE:
                        double[] tmp4 = new double[rows];
                        for (int i = 0; i < rows; i++) tmp4[i] = (Double) UtilFunctions.doubleToObject(vt, A[i][j], false);
                        frame.appendColumn(tmp4);
                        break;
                    default:
                        throw new RuntimeException("Unsupported value type: " + vt);
                }
            }
        } else if (itype == InitType.ROW_OBJ) {
            Object[] row = new Object[schema.length];
            for (int i = 0; i < rows; i++) {
                for (int j = 0; j < schema.length; j++) A[i][j] = UtilFunctions.objectToDouble(schema[j], row[j] = UtilFunctions.doubleToObject(schema[j], A[i][j]));
                frame.appendRow(row);
            }
        } else if (itype == InitType.ROW_STRING) {
            String[] row = new String[schema.length];
            for (int i = 0; i < rows; i++) {
                for (int j = 0; j < schema.length; j++) {
                    Object obj = UtilFunctions.doubleToObject(schema[j], A[i][j]);
                    A[i][j] = UtilFunctions.objectToDouble(schema[j], obj);
                    row[j] = (obj != null) ? obj.toString() : null;
                }
                frame.appendRow(row);
            }
        }
        // some updates via set
        for (int i = 7; i < 13; i++) for (int j = 0; j <= 2; j++) {
            frame.set(i, j, UtilFunctions.doubleToObject(schema[j], (double) i * j));
            A[i][j] = (double) i * j;
        }
        // check basic meta data
        if (frame.getNumRows() != rows)
            Assert.fail("Wrong number of rows: " + frame.getNumRows() + ", expected: " + rows);
        // check correct values
        for (int i = 0; i < rows; i++) for (int j = 0; j < schema.length; j++) {
            double tmp = UtilFunctions.objectToDouble(schema[j], frame.get(i, j));
            if (tmp != A[i][j])
                Assert.fail("Wrong get value for cell (" + i + "," + j + "): " + tmp + ", expected: " + A[i][j]);
        }
    } catch (Exception ex) {
        ex.printStackTrace();
        throw new RuntimeException(ex);
    }
}

Example 58 with FrameBlock

use of org.apache.sysml.runtime.matrix.data.FrameBlock in project incubator-systemml by apache.

the class FrameMatrixCastingTest method writeMatrixOrFrameInput.

private static void writeMatrixOrFrameInput(String fname, double[][] A, int rows, int cols, DataType dt, ValueType vt) throws IOException {
    int blksize = ConfigurationManager.getBlocksize();
    // write input data
    if (dt == DataType.FRAME) {
        FrameBlock fb = DataConverter.convertToFrameBlock(DataConverter.convertToMatrixBlock(A), vt);
        FrameWriter writer = FrameWriterFactory.createFrameWriter(OutputInfo.BinaryBlockOutputInfo);
        writer.writeFrameToHDFS(fb, fname, rows, cols);
    } else {
        MatrixBlock mb = DataConverter.convertToMatrixBlock(A);
        MatrixWriter writer = MatrixWriterFactory.createMatrixWriter(OutputInfo.BinaryBlockOutputInfo);
        writer.writeMatrixToHDFS(mb, fname, (long) rows, (long) cols, blksize, blksize, -1);
    }
    // write meta data
    MatrixCharacteristics mc = new MatrixCharacteristics(rows, cols, blksize, blksize);
    MapReduceTool.writeMetaDataFile(fname + ".mtd", vt, null, dt, mc, OutputInfo.BinaryBlockOutputInfo);
}

Example 59 with FrameBlock

use of org.apache.sysml.runtime.matrix.data.FrameBlock in project incubator-systemml by apache.

the class FrameMatrixReblockTest method writeFrameInput.

private static void writeFrameInput(String fname, String ofmt, double[][] frame, int rows, int cols) throws IOException {
    MatrixBlock mb = DataConverter.convertToMatrixBlock(frame);
    FrameBlock fb = DataConverter.convertToFrameBlock(mb);
    // write input data
    FrameWriter writer = FrameWriterFactory.createFrameWriter(InputInfo.getMatchingOutputInfo(InputInfo.stringExternalToInputInfo(ofmt)));
    writer.writeFrameToHDFS(fb, fname, rows, cols);
}

Example 60 with FrameBlock

use of org.apache.sysml.runtime.matrix.data.FrameBlock in project incubator-systemml by apache.

the class FrameMatrixWriteTest method readFrameInput.

private static double[][] readFrameInput(String fname, String ofmt, int rows, int cols) throws IOException {
    // read input data
    FrameReader reader = FrameReaderFactory.createFrameReader(InputInfo.stringExternalToInputInfo(ofmt));
    FrameBlock fb = reader.readFrameFromHDFS(fname, rows, cols);
    MatrixBlock ret = DataConverter.convertToMatrixBlock(fb);
    return DataConverter.convertToDoubleMatrix(ret);
}