Examples with RUNTIME_PLATFORM org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM used on opensource projects

Example 46 with RUNTIME_PLATFORM

use of org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM in project incubator-systemml by apache.

the class FrameScalarCastingIntegratedTest method runFrameScalarCastingTest.

private void runFrameScalarCastingTest(ValueType vtIn, RUNTIME_PLATFORM et) {
    RUNTIME_PLATFORM platformOld = rtplatform;
    rtplatform = et;
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    try {
        getAndLoadTestConfiguration(TEST_NAME);
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-stats", "-args", input("V"), output("R") };
        // generate input data
        switch(vtIn) {
            case STRING:
                MapReduceTool.writeStringToHDFS("foo", input("V"));
                break;
            case INT:
                MapReduceTool.writeIntToHDFS(7, input("V"));
                break;
            case BOOLEAN:
                MapReduceTool.writeBooleanToHDFS(true, input("V"));
                break;
            case DOUBLE:
                MapReduceTool.writeDoubleToHDFS(7.3, input("V"));
                break;
            default:
                throw new RuntimeException("Unsupported type: " + vtIn);
        }
        MapReduceTool.writeScalarMetaDataFile(input("V") + ".mtd", vtIn);
        // run tests
        runTest(true, false, null, -1);
        // compare output
        Assert.assertEquals(readDMLMatrixFromHDFS("R").get(new CellIndex(1, 1)), Double.valueOf(1));
        if (et != RUNTIME_PLATFORM.SPARK) {
            Assert.assertTrue(Statistics.getNoOfCompiledSPInst() == 0);
            Assert.assertTrue(Statistics.getNoOfExecutedSPInst() == 0);
        }
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    } finally {
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}

Example 47 with RUNTIME_PLATFORM

use of org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM in project incubator-systemml by apache.

the class ParforFrameIntermediateTest method runParforFrameIntermediatesTest.

private void runParforFrameIntermediatesTest(ExecType et) {
    RUNTIME_PLATFORM platformOld = rtplatform;
    switch(et) {
        case SPARK:
            rtplatform = RUNTIME_PLATFORM.SPARK;
            break;
        default:
            rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK;
            break;
    }
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    try {
        // setup testcase
        getAndLoadTestConfiguration(TEST_NAME);
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-explain", "-args", input("F") };
        // generate input data and write as frame
        double[][] A = getRandomMatrix(rows, cols, -10, 10, 0.9, 8362);
        FrameBlock fA = DataConverter.convertToFrameBlock(DataConverter.convertToMatrixBlock(A));
        FrameWriterFactory.createFrameWriter(OutputInfo.CSVOutputInfo).writeFrameToHDFS(fA, input("F"), rows, cols);
        // run test
        runTest(true, false, null, -1);
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    } finally {
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}

Example 48 with RUNTIME_PLATFORM

use of org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM in project incubator-systemml by apache.

the class Jdk7IssueRightIndexingTest method runIndexingTest.

/**
 * @param sparse
 * @param et
 */
public void runIndexingTest(boolean sparse, ExecType et) {
    RUNTIME_PLATFORM oldRTP = rtplatform;
    try {
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        if (et == ExecType.SPARK) {
            rtplatform = RUNTIME_PLATFORM.SPARK;
        } else {
            rtplatform = (et == null) ? RUNTIME_PLATFORM.HYBRID : (et == ExecType.MR) ? RUNTIME_PLATFORM.HADOOP : RUNTIME_PLATFORM.SINGLE_NODE;
        }
        config.addVariable("rows", rows);
        config.addVariable("cols", cols);
        double sparsity = sparse ? sparsity2 : sparsity1;
        String TEST_CACHE_DIR = "";
        if (TEST_CACHE_ENABLED) {
            TEST_CACHE_DIR = sparsity + "/";
        }
        loadTestConfiguration(config, TEST_CACHE_DIR);
        String RI_HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = RI_HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-args", input("M"), output("R") };
        fullRScriptName = RI_HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
        // generate input data
        double[][] M = getRandomMatrix(rows, cols, 0, 1, sparsity, 7);
        writeInputMatrixWithMTD("M", M, true);
        // run test
        runTest(true, false, null, -1);
        runRScript(true);
        // compare results
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R");
        TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
    } finally {
        rtplatform = oldRTP;
    }
}

Example 49 with RUNTIME_PLATFORM

use of org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM in project incubator-systemml by apache.

the class LeftIndexingScalarTest method runLeftIndexingTest.

private void runLeftIndexingTest(ExecType instType) {
    // rtplatform for MR
    RUNTIME_PLATFORM platformOld = rtplatform;
    if (instType == ExecType.SPARK) {
        rtplatform = RUNTIME_PLATFORM.SPARK;
    } else {
        rtplatform = (instType == ExecType.MR) ? RUNTIME_PLATFORM.HADOOP : RUNTIME_PLATFORM.HYBRID;
    }
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    try {
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        loadTestConfiguration(config);
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-explain", "-args", input("A"), Long.toString(rows), Long.toString(cols), output("A") };
        fullRScriptName = HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
        double[][] A = getRandomMatrix(rows, cols, min, max, sparsity, System.currentTimeMillis());
        writeInputMatrix("A", A, true);
        runTest(true, false, null, -1);
        runRScript(true);
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("A");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("A");
        TestUtils.compareMatrices(dmlfile, rfile, epsilon, "A-DML", "A-R");
        checkDMLMetaDataFile("A", new MatrixCharacteristics(rows, cols, 1, 1));
    } finally {
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}

Example 50 with RUNTIME_PLATFORM

use of org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM in project incubator-systemml by apache.

the class LeftIndexingSparseSparseTest method runLeftIndexingSparseSparseTest.

/**
 * @param type
 */
public void runLeftIndexingSparseSparseTest(LixType type, ExecType et, LeftIndexingOp.LeftIndexingMethod indexingMethod) {
    // setup range (column lower/upper)
    int cl = -1;
    switch(type) {
        case LEFT_ALIGNED:
            cl = 1;
            break;
        case LEFT2_ALIGNED:
            cl = 2;
            break;
        case RIGHT_ALIGNED:
            cl = cols1 - cols2 + 1;
            break;
        case RIGHT2_ALIGNED:
            cl = cols1 - cols2;
            break;
        case CENTERED:
            cl = (cols1 - cols2) / 2;
            break;
    }
    int cu = cl + cols2 - 1;
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    RUNTIME_PLATFORM oldRTP = rtplatform;
    try {
        if (indexingMethod != null) {
            LeftIndexingOp.FORCED_LEFT_INDEXING = indexingMethod;
        }
        if (et == ExecType.SPARK) {
            rtplatform = RUNTIME_PLATFORM.SPARK;
        } else {
            // rtplatform = (et==ExecType.MR)? RUNTIME_PLATFORM.HADOOP : RUNTIME_PLATFORM.SINGLE_NODE;
            rtplatform = RUNTIME_PLATFORM.HYBRID;
        }
        if (rtplatform == RUNTIME_PLATFORM.SPARK)
            DMLScript.USE_LOCAL_SPARK_CONFIG = true;
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        String TEST_CACHE_DIR = "";
        if (TEST_CACHE_ENABLED) {
            TEST_CACHE_DIR = type.toString() + "/";
        }
        loadTestConfiguration(config, TEST_CACHE_DIR);
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-args", input("A"), input("B"), String.valueOf(cl), String.valueOf(cu), output("R") };
        fullRScriptName = HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + cl + " " + cu + " " + expectedDir();
        // generate input data sets
        double[][] A = getRandomMatrix(rows1, cols1, -1, 1, sparsity1, 1234);
        writeInputMatrixWithMTD("A", A, true);
        double[][] B = getRandomMatrix(rows2, cols2, -1, 1, sparsity2, 5678);
        writeInputMatrixWithMTD("B", B, true);
        // REBLOCK
        runTest(true, false, null, 1);
    } finally {
        rtplatform = oldRTP;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
        LeftIndexingOp.FORCED_LEFT_INDEXING = null;
    }
    runRScript(true);
    HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
    HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R");
    TestUtils.compareMatrices(dmlfile, rfile, 0, "DML", "R");
    checkDMLMetaDataFile("R", new MatrixCharacteristics(rows1, cols1, 1, 1));
}