Examples with RUNTIME_PLATFORM org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM used on opensource projects

Example 6 with RUNTIME_PLATFORM

use of org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM in project incubator-systemml by apache.

the class FullRowAggregateTest method runRowAggregateOperationTest.

/**
	 * 
	 * @param sparseM1
	 * @param sparseM2
	 * @param instType
	 */
private void runRowAggregateOperationTest(OpType type, boolean sparse, boolean vector, ExecType instType, boolean specialData, boolean rewrites) {
    RUNTIME_PLATFORM platformOld = rtplatform;
    switch(instType) {
        case MR:
            rtplatform = RUNTIME_PLATFORM.HADOOP;
            break;
        case SPARK:
            rtplatform = RUNTIME_PLATFORM.SPARK;
            break;
        default:
            rtplatform = RUNTIME_PLATFORM.HYBRID;
            break;
    }
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    boolean oldRewritesFlag = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION;
    OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewrites;
    try {
        String TEST_NAME = null;
        switch(type) {
            case ROW_SUMS:
                TEST_NAME = TEST_NAME1;
                break;
            case ROW_MEANS:
                TEST_NAME = TEST_NAME2;
                break;
            case ROW_MAX:
                TEST_NAME = TEST_NAME3;
                break;
            case ROW_MIN:
                TEST_NAME = TEST_NAME4;
                break;
            case ROW_INDEXMAX:
                TEST_NAME = TEST_NAME5;
                break;
            case ROW_INDEXMIN:
                TEST_NAME = TEST_NAME6;
                break;
        }
        int rows = (vector) ? rows1 : rows2;
        double sparsity = (sparse) ? sparsity1 : sparsity2;
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        //generate actual dataset
        double min, max;
        // in case of ROW_INDEXMIN, generate all positive data.
        if (type == OpType.ROW_INDEXMAX) {
            //special data: negative, 0 is actual max
            min = specialData ? -1 : -0.05;
            max = specialData ? -0.05 : 1;
        } else if (type == OpType.ROW_INDEXMIN) {
            //special data: positive, 0 is actual min
            min = specialData ? 0.05 : -1;
            max = specialData ? 1 : 0.05;
        } else {
            min = -0.05;
            max = 1;
        }
        String TEST_CACHE_DIR = "";
        if (TEST_CACHE_ENABLED) {
            TEST_CACHE_DIR = type.ordinal() + "_" + rows + "_" + specialData + "_" + sparsity + "/";
        }
        loadTestConfiguration(config, TEST_CACHE_DIR);
        /* This is for running the junit test the new way, i.e., construct the arguments directly */
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-explain", "-args", input("A"), Integer.toString(rows), Integer.toString(cols), output("B") };
        fullRScriptName = HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
        double[][] A = getRandomMatrix(rows, cols, min, max, sparsity, 7);
        writeInputMatrix("A", A, true);
        boolean exceptionExpected = false;
        runTest(true, exceptionExpected, null, -1);
        runRScript(true);
        //compare matrices 
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("B");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("B");
        TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
    } finally {
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
        OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = oldRewritesFlag;
    }
}

Example 7 with RUNTIME_PLATFORM

use of org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM in project incubator-systemml by apache.

the class UnivariateStatsBase method testScaleWithR.

/**
	 * Shared test driver for tests of univariate statistics over continuous,
	 * scaled, but not weighted, data
	 * 
	 * @param sz
	 *            size of primary input vector
	 * @param rng
	 *            range of randomly-generated values to use
	 * @param sp
	 *            sparsity of generated data
	 * @param rt
	 *            backend platform to test
	 */
protected void testScaleWithR(SIZE sz, RANGE rng, SPARSITY sp, RUNTIME_PLATFORM rt) {
    RUNTIME_PLATFORM oldrt = rtplatform;
    rtplatform = rt;
    try {
        TestConfiguration config = getTestConfiguration("Scale");
        config.addVariable("rows1", sz.size);
        config.addVariable("rows2", rows2);
        loadTestConfiguration(config);
        // This is for running the junit test the new way, i.e., construct
        // the arguments directly
        String S_HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = S_HOME + "Scale" + ".dml";
        programArgs = new String[] { "-args", input("vector"), Integer.toString(sz.size), input("prob"), Integer.toString(rows2), output("mean"), output("std"), output("se"), output("var"), output("cv"), output("min"), output("max"), output("rng"), output("g1"), output("se_g1"), output("g2"), output("se_g2"), output("median"), output("iqm"), output("out_minus"), output("out_plus"), output("quantile") };
        fullRScriptName = S_HOME + "Scale" + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
        double[][] vector = getRandomMatrix(sz.size, 1, rng.min, rng.max, sp.sparsity, 20160124165501L);
        double[][] prob = getRandomMatrix(rows2, 1, 0, 1, 1, 20160124165502L);
        writeInputMatrix("vector", vector, true);
        writeInputMatrix("prob", prob, true);
        runTest(true, false, null, -1);
        runRScript(true);
        for (String file : config.getOutputFiles()) {
            // NOte that some files do not contain matrix, but just a single
            // scalar value inside
            HashMap<CellIndex, Double> dmlfile;
            HashMap<CellIndex, Double> rfile;
            if (file.endsWith(".scalar")) {
                file = file.replace(".scalar", "");
                dmlfile = readDMLScalarFromHDFS(file);
                rfile = readRScalarFromFS(file);
            } else {
                dmlfile = readDMLMatrixFromHDFS(file);
                rfile = readRMatrixFromFS(file);
            }
            TestUtils.compareMatrices(dmlfile, rfile, epsilon, file + "-DML", file + "-R");
        }
    } finally {
        rtplatform = oldrt;
    }
}

Example 8 with RUNTIME_PLATFORM

use of org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM in project incubator-systemml by apache.

the class ParForNaiveBayesTest method runParForNaiveBayesTest.

/**
	 * 
	 * @param outer
	 * @param instType
	 * @param smallMem
	 * @param sparse
	 */
private void runParForNaiveBayesTest(PExecMode outer, ExecType instType, boolean smallMem, boolean sparse) {
    int cols = (instType == ExecType.MR) ? cols2 : cols1;
    //inst exec type, influenced via rows
    RUNTIME_PLATFORM oldPlatform = rtplatform;
    rtplatform = (instType == ExecType.MR) ? RUNTIME_PLATFORM.HADOOP : RUNTIME_PLATFORM.HYBRID;
    //determine the script
    int scriptNum = -1;
    if (//constrained opt
    outer == PExecMode.LOCAL)
        //constrained opt
        scriptNum = 1;
    else if (//constrained opt
    outer == PExecMode.REMOTE_MR)
        //constrained opt
        scriptNum = 2;
    else if (//constrained opt
    outer == PExecMode.REMOTE_MR_DP)
        //constrained opt
        scriptNum = 3;
    else
        //opt
        scriptNum = 4;
    //invocation arguments
    TestConfiguration config = getTestConfiguration(TEST_NAME);
    config.addVariable("rows", rows);
    config.addVariable("cols", cols);
    loadTestConfiguration(config);
    String HOME = SCRIPT_DIR + TEST_DIR;
    fullDMLScriptName = HOME + TEST_NAME + scriptNum + ".dml";
    programArgs = new String[] { "-args", input("D"), input("C"), Integer.toString((int) maxVal), output("class_prior"), output("class_conditionals") };
    fullRScriptName = HOME + TEST_NAME + ".R";
    rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + Integer.toString((int) maxVal) + " " + expectedDir();
    //input data
    double[][] D = getRandomMatrix(rows, cols, -1, 1, sparse ? sparsity2 : sparsity1, 7);
    double[][] C = TestUtils.round(getRandomMatrix(rows, 1, minVal, maxVal, 1.0, 3));
    MatrixCharacteristics mc1 = new MatrixCharacteristics(rows, cols, -1, -1);
    writeInputMatrixWithMTD("D", D, true, mc1);
    MatrixCharacteristics mc2 = new MatrixCharacteristics(rows, 1, -1, -1);
    writeInputMatrixWithMTD("C", C, true, mc2);
    //set memory budget (to test automatic opt for remote_mr_dp)
    long oldmem = InfrastructureAnalyzer.getLocalMaxMemory();
    if (smallMem) {
        long mem = 1024 * 1024 * 8;
        InfrastructureAnalyzer.setLocalMaxMemory(mem);
    }
    try {
        //run the testcase (DML and R)
        runTest(true, false, null, -1);
        runRScript(true);
        //compare output matrices
        for (String out : new String[] { "class_prior", "class_conditionals" }) {
            HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS(out);
            HashMap<CellIndex, Double> rfile = readRMatrixFromFS(out);
            TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
        }
    } finally {
        rtplatform = oldPlatform;
        InfrastructureAnalyzer.setLocalMaxMemory(oldmem);
    }
}

Example 9 with RUNTIME_PLATFORM

use of org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM in project incubator-systemml by apache.

the class CovarianceTest method runCovarianceTest.

/**
	 * 
	 * @param sparseM1
	 * @param sparseM2
	 * @param instType
	 */
private void runCovarianceTest(boolean sparse, ExecType et) {
    //rtplatform for MR
    RUNTIME_PLATFORM platformOld = rtplatform;
    switch(et) {
        case MR:
            rtplatform = RUNTIME_PLATFORM.HADOOP;
            break;
        case SPARK:
            rtplatform = RUNTIME_PLATFORM.SPARK;
            break;
        default:
            rtplatform = RUNTIME_PLATFORM.HYBRID;
            break;
    }
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    try {
        getAndLoadTestConfiguration(TEST_NAME);
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-args", input("A"), input("B"), output("R") };
        fullRScriptName = HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
        //generate actual dataset (always dense because values <=0 invalid)
        double sparsitya = sparse ? sparsity2 : sparsity1;
        double[][] A = getRandomMatrix(rows, 1, 1, maxVal, sparsitya, 7);
        writeInputMatrixWithMTD("A", A, true);
        double[][] B = getRandomMatrix(rows, 1, 1, maxVal, sparsitya, 823);
        writeInputMatrixWithMTD("B", B, true);
        runTest(true, false, null, -1);
        runRScript(true);
        //compare matrices 
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R");
        TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
    } finally {
        rtplatform = platformOld;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}

Example 10 with RUNTIME_PLATFORM

use of org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM in project incubator-systemml by apache.

the class DiagMatrixMultiplicationTest method runDiagMatrixMultiplicationTest.

/**
	 * 
	 * @param sparseM1
	 * @param sparseM2
	 * @param instType
	 */
private void runDiagMatrixMultiplicationTest(boolean sparseM1, boolean sparseM2, boolean rightTranspose, boolean rightVector, ExecType instType, boolean simplify) {
    //rtplatform for MR
    RUNTIME_PLATFORM platformOld = rtplatform;
    switch(instType) {
        case MR:
            rtplatform = RUNTIME_PLATFORM.HADOOP;
            break;
        case SPARK:
            rtplatform = RUNTIME_PLATFORM.SPARK;
            break;
        default:
            rtplatform = RUNTIME_PLATFORM.HYBRID;
            break;
    }
    boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
    if (rtplatform == RUNTIME_PLATFORM.SPARK)
        DMLScript.USE_LOCAL_SPARK_CONFIG = true;
    String TEST_NAME = rightTranspose ? TEST_NAME2 : TEST_NAME1;
    boolean oldFlagSimplify = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION;
    try {
        TestConfiguration config = getTestConfiguration(TEST_NAME);
        OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = simplify;
        double sparsityM1 = sparseM1 ? sparsity2 : sparsity1;
        double sparsityM2 = sparseM2 ? sparsity2 : sparsity1;
        String TEST_CACHE_DIR = "";
        if (TEST_CACHE_ENABLED) {
            TEST_CACHE_DIR = sparsityM1 + "_" + sparsityM2 + "_" + rightTranspose + "_" + rightVector + "/";
        }
        loadTestConfiguration(config, TEST_CACHE_DIR);
        /* This is for running the junit test the new way, i.e., construct the arguments directly */
        String HOME = SCRIPT_DIR + TEST_DIR;
        fullDMLScriptName = HOME + TEST_NAME + ".dml";
        programArgs = new String[] { "-explain", "-args", input("A"), Integer.toString(rowsA), Integer.toString(colsA), input("B"), Integer.toString(rowsB), Integer.toString(rightVector ? 1 : colsB), output("C") };
        fullRScriptName = HOME + TEST_NAME + ".R";
        rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
        //generate actual dataset
        double[][] A = getRandomMatrix(rowsA, colsA, 0, 1, sparsityM1, 7);
        writeInputMatrix("A", A, true);
        double[][] B = getRandomMatrix(rowsB, rightVector ? 1 : colsB, 0, 1, sparsityM2, 3);
        writeInputMatrix("B", B, true);
        boolean exceptionExpected = false;
        runTest(true, exceptionExpected, null, -1);
        runRScript(true);
        //compare matrices 
        HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("C");
        HashMap<CellIndex, Double> rfile = readRMatrixFromFS("C");
        TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R");
    } finally {
        rtplatform = platformOld;
        OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = oldFlagSimplify;
        DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
    }
}