Examples with MLContext org.apache.sysml.api.mlcontext.MLContext used on opensource projects

Example 11 with MLContext

use of org.apache.sysml.api.mlcontext.MLContext in project incubator-systemml by apache.

the class MLContextTestBase method setUpClass.

@BeforeClass
public static void setUpClass() {
    spark = createSystemMLSparkSession("SystemML MLContext Test", "local");
    ml = new MLContext(spark);
    sc = MLContextUtil.getJavaSparkContext(ml);
}

Example 12 with MLContext

use of org.apache.sysml.api.mlcontext.MLContext in project incubator-systemml by apache.

the class MLContextMultipleScriptsTest method runMLContextTestMultipleScript.

private static void runMLContextTestMultipleScript(RUNTIME_PLATFORM platform, boolean wRead) {
    RUNTIME_PLATFORM oldplatform = DMLScript.rtplatform;
    DMLScript.rtplatform = platform;
    // create mlcontext
    SparkSession spark = createSystemMLSparkSession("MLContextMultipleScriptsTest", "local");
    MLContext ml = new MLContext(spark);
    ml.setExplain(true);
    String dml1 = baseDirectory + File.separator + "MultiScript1.dml";
    String dml2 = baseDirectory + File.separator + (wRead ? "MultiScript2b.dml" : "MultiScript2.dml");
    String dml3 = baseDirectory + File.separator + (wRead ? "MultiScript3b.dml" : "MultiScript3.dml");
    try {
        // run script 1
        Script script1 = dmlFromFile(dml1).in("$rows", rows).in("$cols", cols).out("X");
        Matrix X = ml.execute(script1).getMatrix("X");
        Script script2 = dmlFromFile(dml2).in("X", X).out("Y");
        Matrix Y = ml.execute(script2).getMatrix("Y");
        Script script3 = dmlFromFile(dml3).in("X", X).in("Y", Y).out("z");
        String z = ml.execute(script3).getString("z");
        System.out.println(z);
    } finally {
        DMLScript.rtplatform = oldplatform;
        // stop underlying spark context to allow single jvm tests (otherwise the
        // next test that tries to create a SparkContext would fail)
        spark.stop();
        // clear status mlcontext and spark exec context
        ml.close();
    }
}

Example 13 with MLContext

use of org.apache.sysml.api.mlcontext.MLContext in project incubator-systemml by apache.

the class APICodegenTest method runMLContextParforDatasetTest.

private void runMLContextParforDatasetTest(boolean jmlc) {
    try {
        double[][] X = getRandomMatrix(rows, cols, -10, 10, sparsity, 76543);
        MatrixBlock mX = DataConverter.convertToMatrixBlock(X);
        String s = "X = read(\"/tmp\");" + "R = colSums(X/rowSums(X));" + "write(R, \"tmp2\")";
        // execute scripts
        if (jmlc) {
            DMLScript.STATISTICS = true;
            Connection conn = new Connection(ConfigType.CODEGEN_ENABLED, ConfigType.ALLOW_DYN_RECOMPILATION);
            PreparedScript pscript = conn.prepareScript(s, new String[] { "X" }, new String[] { "R" }, false);
            pscript.setMatrix("X", mX, false);
            pscript.executeScript();
            conn.close();
            System.out.println(Statistics.display());
        } else {
            SparkConf conf = SparkExecutionContext.createSystemMLSparkConf().setAppName("MLContextTest").setMaster("local");
            JavaSparkContext sc = new JavaSparkContext(conf);
            MLContext ml = new MLContext(sc);
            ml.setConfigProperty(DMLConfig.CODEGEN, "true");
            ml.setStatistics(true);
            Script script = dml(s).in("X", mX).out("R");
            ml.execute(script);
            ml.resetConfig();
            sc.stop();
            ml.close();
        }
        // check for generated operator
        Assert.assertTrue(heavyHittersContainsSubString("spoofRA"));
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    }
}

Example 14 with MLContext

use of org.apache.sysml.api.mlcontext.MLContext in project systemml by apache.

the class GenerateClassesForMLContext method addConvenienceMethodsToMLContext.

/**
 * Add methods to MLContext to allow tab-completion to folders/packages
 * (such as {@code ml.scripts()} and {@code ml.nn()}).
 *
 * @param source
 *            path to source directory (typically, the scripts directory)
 * @param fullDirClassName
 *            the full name of the class representing the source (scripts)
 *            directory
 */
public static void addConvenienceMethodsToMLContext(String source, String fullDirClassName) {
    try {
        ClassPool pool = ClassPool.getDefault();
        CtClass ctMLContext = pool.get(MLContext.class.getName());
        CtClass dirClass = pool.get(fullDirClassName);
        String methodName = convertFullClassNameToConvenienceMethodName(fullDirClassName);
        System.out.println("Adding " + methodName + "() to " + ctMLContext.getName());
        String methodBody = "{ " + fullDirClassName + " z = new " + fullDirClassName + "(); return z; }";
        CtMethod ctMethod = CtNewMethod.make(Modifier.PUBLIC, dirClass, methodName, null, null, methodBody, ctMLContext);
        ctMLContext.addMethod(ctMethod);
        addPackageConvenienceMethodsToMLContext(source, ctMLContext);
        ctMLContext.writeFile(destination);
    } catch (FileNotFoundException e) {
        e.printStackTrace();
    } catch (IOException e) {
        e.printStackTrace();
    } catch (RuntimeException e) {
        e.printStackTrace();
    } catch (NotFoundException e) {
        e.printStackTrace();
    } catch (CannotCompileException e) {
        e.printStackTrace();
    }
}

Example 15 with MLContext

use of org.apache.sysml.api.mlcontext.MLContext in project systemml by apache.

the class GPUTests method runOnGPU.

/**
 * Runs a program on the GPU
 *
 * @param spark     a valid {@link SparkSession}
 * @param scriptStr the script to run (as a string)
 * @param inputs    map of input variables names in the scriptStr (of variable_name -> object)
 * @param outStrs   list of variable names needed as output from the scriptStr
 * @return list of output objects in order of outStrs
 */
protected List<Object> runOnGPU(SparkSession spark, String scriptStr, Map<String, Object> inputs, List<String> outStrs) {
    // and other side effects.
    synchronized (GPUTests.class) {
        MLContext gpuMLC = new MLContext(spark);
        gpuMLC.setConfigProperty("sysml.floating.point.precision", FLOATING_POINT_PRECISION);
        if (IGNORE_CLEAR_MEMORY_BUG)
            gpuMLC.setConfigProperty("sysml.gpu.eager.cudaFree", "true");
        gpuMLC.setGPU(true);
        gpuMLC.setForceGPU(true);
        gpuMLC.setStatistics(true);
        List<Object> outputs = new ArrayList<>();
        Script script = ScriptFactory.dmlFromString(scriptStr).in(inputs).out(outStrs);
        MLResults res = gpuMLC.execute(script);
        for (String outStr : outStrs) {
            Object output = res.get(outStr);
            outputs.add(output);
        }
        gpuMLC.close();
        return outputs;
    }
}