use of org.apache.sysml.api.mlcontext.Script in project incubator-systemml by apache.
the class MLContextTest method testInputBinaryBlockMatrixDML.
@Test
public void testInputBinaryBlockMatrixDML() {
System.out.println("MLContextTest - input BinaryBlockMatrix DML");
List<String> list = new ArrayList<String>();
list.add("10,20,30");
list.add("40,50,60");
list.add("70,80,90");
JavaRDD<String> javaRddString = sc.parallelize(list);
JavaRDD<Row> javaRddRow = javaRddString.map(new CommaSeparatedValueStringToRow());
List<StructField> fields = new ArrayList<StructField>();
fields.add(DataTypes.createStructField("C1", DataTypes.StringType, true));
fields.add(DataTypes.createStructField("C2", DataTypes.StringType, true));
fields.add(DataTypes.createStructField("C3", DataTypes.StringType, true));
StructType schema = DataTypes.createStructType(fields);
Dataset<Row> dataFrame = spark.createDataFrame(javaRddRow, schema);
BinaryBlockMatrix binaryBlockMatrix = new BinaryBlockMatrix(dataFrame);
Script script = dml("avg = avg(M);").in("M", binaryBlockMatrix).out("avg");
double avg = ml.execute(script).getDouble("avg");
Assert.assertEquals(50.0, avg, 0.0);
}
use of org.apache.sysml.api.mlcontext.Script in project incubator-systemml by apache.
the class MLContextTest method testInputBinaryBlockMatrixPYDML.
@Test
public void testInputBinaryBlockMatrixPYDML() {
System.out.println("MLContextTest - input BinaryBlockMatrix PYDML");
List<String> list = new ArrayList<String>();
list.add("10,20,30");
list.add("40,50,60");
list.add("70,80,90");
JavaRDD<String> javaRddString = sc.parallelize(list);
JavaRDD<Row> javaRddRow = javaRddString.map(new CommaSeparatedValueStringToRow());
List<StructField> fields = new ArrayList<StructField>();
fields.add(DataTypes.createStructField("C1", DataTypes.StringType, true));
fields.add(DataTypes.createStructField("C2", DataTypes.StringType, true));
fields.add(DataTypes.createStructField("C3", DataTypes.StringType, true));
StructType schema = DataTypes.createStructType(fields);
Dataset<Row> dataFrame = spark.createDataFrame(javaRddRow, schema);
BinaryBlockMatrix binaryBlockMatrix = new BinaryBlockMatrix(dataFrame);
Script script = pydml("avg = avg(M)").in("M", binaryBlockMatrix).out("avg");
double avg = ml.execute(script).getDouble("avg");
Assert.assertEquals(50.0, avg, 0.0);
}
use of org.apache.sysml.api.mlcontext.Script in project incubator-systemml by apache.
the class GPUTests method runOnGPU.
/**
* Runs a program on the GPU
*
* @param spark a valid {@link SparkSession}
* @param scriptStr the script to run (as a string)
* @param inputs map of input variables names in the scriptStr (of variable_name -> object)
* @param outStrs list of variable names needed as output from the scriptStr
* @return list of output objects in order of outStrs
*/
protected List<Object> runOnGPU(SparkSession spark, String scriptStr, Map<String, Object> inputs, List<String> outStrs) {
// and other side effects.
synchronized (GPUTests.class) {
MLContext gpuMLC = new MLContext(spark);
gpuMLC.setConfigProperty("sysml.floating.point.precision", FLOATING_POINT_PRECISION);
if (IGNORE_CLEAR_MEMORY_BUG)
gpuMLC.setConfigProperty("sysml.gpu.eager.cudaFree", "true");
gpuMLC.setGPU(true);
gpuMLC.setForceGPU(true);
gpuMLC.setStatistics(true);
List<Object> outputs = new ArrayList<>();
Script script = ScriptFactory.dmlFromString(scriptStr).in(inputs).out(outStrs);
MLResults res = gpuMLC.execute(script);
for (String outStr : outStrs) {
Object output = res.get(outStr);
outputs.add(output);
}
gpuMLC.close();
return outputs;
}
}
use of org.apache.sysml.api.mlcontext.Script in project incubator-systemml by apache.
the class GPUTests method runOnCPU.
/**
* Runs a program on the CPU
*
* @param spark a valid {@link SparkSession}
* @param scriptStr the script to run (as a string)
* @param inputs map of input variables names in the scriptStr (of variable_name -> object)
* @param outStrs list of variable names needed as output from the scriptStr
* @return list of output objects in order of outStrs
*/
protected List<Object> runOnCPU(SparkSession spark, String scriptStr, Map<String, Object> inputs, List<String> outStrs) {
MLContext cpuMLC = new MLContext(spark);
List<Object> outputs = new ArrayList<>();
Script script = ScriptFactory.dmlFromString(scriptStr).in(inputs).out(outStrs);
MLResults res = cpuMLC.execute(script);
for (String outStr : outStrs) {
Object output = res.get(outStr);
outputs.add(output);
}
cpuMLC.close();
return outputs;
}
use of org.apache.sysml.api.mlcontext.Script in project incubator-systemml by apache.
the class GPUTests method generateInputMatrix.
/**
* Generates a random input matrix with a given size and sparsity
*
* @param spark valid instance of {@link SparkSession}
* @param m number of rows
* @param n number of columns
* @param sparsity sparsity (1 = completely dense, 0 = completely sparse)
* @return a random matrix with given size and sparsity
*/
protected Matrix generateInputMatrix(SparkSession spark, int m, int n, double sparsity, int seed) {
// Generate a random matrix of size m * n
MLContext genMLC = new MLContext(spark);
String scriptStr;
if (sparsity == 0.0) {
scriptStr = "in1 = matrix(0, rows=" + m + ", cols=" + n + ")";
} else {
scriptStr = "in1 = rand(rows=" + m + ", cols=" + n + ", sparsity = " + sparsity + ", seed= " + seed + ", min=-1.0, max=1.0)";
}
Script generateScript = ScriptFactory.dmlFromString(scriptStr).out("in1");
Matrix in1 = genMLC.execute(generateScript).getMatrix("in1");
genMLC.close();
return in1;
}
Aggregations