Examples with Connection org.apache.sysml.api.jmlc.Connection used on opensource projects

Example 71 with Connection

use of org.apache.sysml.api.jmlc.Connection in project systemml by apache.

the class JMLCInputStreamReadTest method runJMLCInputStreamReadTest.

private void runJMLCInputStreamReadTest(DataType dt, boolean sparse, String format, boolean metaData) throws IOException {
    TestConfiguration config = getTestConfiguration(TEST_NAME);
    loadTestConfiguration(config);
    // generate inputs
    OutputInfo oinfo = format.equals("csv") ? OutputInfo.CSVOutputInfo : OutputInfo.TextCellOutputInfo;
    double[][] data = TestUtils.round(getRandomMatrix(rows, cols, 0.51, 7.49, sparse ? sparsity2 : sparsity1, 7));
    Connection conn = new Connection();
    try {
        if (dt == DataType.MATRIX) {
            // write input matrix
            MatrixBlock mb = DataConverter.convertToMatrixBlock(data);
            MatrixWriter writer = MatrixWriterFactory.createMatrixWriter(oinfo);
            writer.writeMatrixToHDFS(mb, output("X"), rows, cols, -1, -1, -1);
            // read matrix from input stream
            FileInputStream fis = new FileInputStream(output("X"));
            double[][] data2 = conn.convertToDoubleMatrix(fis, rows, cols, format);
            fis.close();
            // compare matrix result
            TestUtils.compareMatrices(data, data2, rows, cols, 0);
        } else if (dt == DataType.FRAME) {
            // write input frame
            String[][] fdata = FrameTransformTest.createFrameData(data, "V");
            // test quoted tokens w/ inner quotes
            fdata[3][1] = "\"ab\"\"cdef\"";
            if (format.equals("csv"))
                // test delimiter and space tokens
                fdata[7][2] = "\"a,bc def\"";
            FrameBlock fb = DataConverter.convertToFrameBlock(fdata);
            if (metaData) {
                fb.setColumnNames(IntStream.range(0, cols).mapToObj(i -> "CC" + i).collect(Collectors.toList()).toArray(new String[0]));
            }
            FrameWriter writer = FrameWriterFactory.createFrameWriter(oinfo);
            writer.writeFrameToHDFS(fb, output("X"), rows, cols);
            // read frame from input stream
            FileInputStream fis = new FileInputStream(output("X"));
            String[][] fdata2 = conn.convertToStringFrame(fis, rows, cols, format);
            fis.close();
            // compare frame result
            TestUtils.compareFrames(fdata, fdata2, rows, cols);
        } else {
            throw new IOException("Unsupported data type: " + dt.name());
        }
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    } finally {
        MapReduceTool.deleteFileIfExistOnHDFS(output("X"));
        IOUtilFunctions.closeSilently(conn);
    }
}

Example 72 with Connection

use of org.apache.sysml.api.jmlc.Connection in project systemml by apache.

the class JMLCParfor2ForCompileTest method runJMLCParFor2ForTest.

private void runJMLCParFor2ForTest(boolean par) throws IOException {
    try {
        Connection conn = !par ? new Connection() : new Connection(ConfigType.PARALLEL_LOCAL_OR_REMOTE_PARFOR);
        String script = "  X = rand(rows=10, cols=10);" + "R = matrix(0, rows=10, cols=1)" + "parfor(i in 1:nrow(X))" + "  R[i,] = sum(X[i,])" + "print(sum(R))";
        DMLScript.STATISTICS = true;
        Statistics.reset();
        PreparedScript pscript = conn.prepareScript(script, new String[] {}, new String[] {}, false);
        pscript.executeScript();
        conn.close();
    } catch (Exception ex) {
        Assert.fail("JMLC parfor test failed: " + ex.getMessage());
    }
    // check for existing or non-existing parfor
    Assert.assertTrue(Statistics.getParforOptCount() == (par ? 1 : 0));
}

Example 73 with Connection

use of org.apache.sysml.api.jmlc.Connection in project systemml by apache.

the class MulticlassSVMScoreTest method execDMLScriptviaJMLC.

private static ArrayList<double[][]> execDMLScriptviaJMLC(ArrayList<double[][]> X, boolean flags) throws IOException {
    Timing time = new Timing(true);
    ArrayList<double[][]> ret = new ArrayList<double[][]>();
    // establish connection to SystemML
    Connection conn = !flags ? new Connection() : new Connection(ConfigType.PARALLEL_CP_MATRIX_OPERATIONS, ConfigType.PARALLEL_LOCAL_OR_REMOTE_PARFOR, ConfigType.ALLOW_DYN_RECOMPILATION);
    try {
        // For now, JMLC pipeline only allows dml
        boolean parsePyDML = false;
        // read and precompile script
        String script = conn.readScript(SCRIPT_DIR + TEST_DIR + TEST_NAME + ".dml");
        PreparedScript pstmt = conn.prepareScript(script, new String[] { "X", "W" }, new String[] { "predicted_y" }, parsePyDML);
        // read model
        String modelData = conn.readScript(SCRIPT_DIR + TEST_DIR + MODEL_FILE);
        double[][] W = conn.convertToDoubleMatrix(modelData, rows, cols);
        // execute script multiple times
        for (int i = 0; i < nRuns; i++) {
            // bind input parameters
            pstmt.setMatrix("W", W);
            pstmt.setMatrix("X", X.get(i));
            // execute script
            ResultVariables rs = pstmt.executeScript();
            // get output parameter
            double[][] Y = rs.getMatrix("predicted_y");
            // keep result for comparison
            ret.add(Y);
        }
    } catch (Exception ex) {
        ex.printStackTrace();
        throw new IOException(ex);
    } finally {
        if (conn != null)
            conn.close();
    }
    System.out.println("JMLC scoring w/ " + nRuns + " runs in " + time.stop() + "ms.");
    return ret;
}

Example 74 with Connection

use of org.apache.sysml.api.jmlc.Connection in project systemml by apache.

the class FrameCastingTest method execDMLScriptviaJMLC.

private static ArrayList<String[][]> execDMLScriptviaJMLC(String testname, String[][] F1, boolean modelReuse) throws IOException {
    Timing time = new Timing(true);
    ArrayList<String[][]> ret = new ArrayList<String[][]>();
    // establish connection to SystemML
    Connection conn = new Connection();
    try {
        // prepare input arguments
        HashMap<String, String> args = new HashMap<String, String>();
        args.put("$TRANSFORM_SPEC", "{ \"ids\": true ,\"recode\": [ 1, 2, 3] }");
        // read and precompile script
        String script = conn.readScript(SCRIPT_DIR + TEST_DIR + testname + ".dml");
        PreparedScript pstmt = conn.prepareScript(script, args, new String[] { "F1", "M" }, new String[] { "F2" }, false);
        if (modelReuse)
            pstmt.setFrame("F1", F1, true);
        // execute script multiple times
        for (int i = 0; i < nRuns; i++) {
            // bind input parameters
            if (!modelReuse)
                pstmt.setFrame("F1", F1);
            // execute script
            ResultVariables rs = pstmt.executeScript();
            // get output parameter
            String[][] Y = rs.getFrame("F2");
            // keep result for comparison
            ret.add(Y);
        }
    } catch (Exception ex) {
        ex.printStackTrace();
        throw new IOException(ex);
    } finally {
        IOUtilFunctions.closeSilently(conn);
    }
    System.out.println("JMLC scoring w/ " + nRuns + " runs in " + time.stop() + "ms.");
    return ret;
}

Example 75 with Connection

use of org.apache.sysml.api.jmlc.Connection in project systemml by apache.

the class FrameEncodeTest method execDMLScriptviaJMLC.

private static ArrayList<String[][]> execDMLScriptviaJMLC(String testname, String[][] F1, boolean modelReuse) throws IOException {
    Timing time = new Timing(true);
    ArrayList<String[][]> ret = new ArrayList<String[][]>();
    // establish connection to SystemML
    Connection conn = new Connection();
    try {
        // prepare input arguments
        HashMap<String, String> args = new HashMap<String, String>();
        args.put("$TRANSFORM_SPEC", "{ \"ids\": true ,\"recode\": [ 1, 2, 3] }");
        // read and precompile script
        String script = conn.readScript(SCRIPT_DIR + TEST_DIR + testname + ".dml");
        PreparedScript pstmt = conn.prepareScript(script, args, new String[] { "F1", "M" }, new String[] { "F2" }, false);
        if (modelReuse)
            pstmt.setFrame("F1", F1, true);
        // execute script multiple times
        for (int i = 0; i < nRuns; i++) {
            // bind input parameters
            if (!modelReuse)
                pstmt.setFrame("F1", F1);
            // execute script
            ResultVariables rs = pstmt.executeScript();
            // get output parameter
            String[][] Y = rs.getFrame("F2");
            // keep result for comparison
            ret.add(Y);
        }
    } catch (Exception ex) {
        ex.printStackTrace();
        throw new IOException(ex);
    } finally {
        IOUtilFunctions.closeSilently(conn);
    }
    System.out.println("JMLC scoring w/ " + nRuns + " runs in " + time.stop() + "ms.");
    return ret;
}