use of org.apache.sysml.api.mlcontext.Matrix in project incubator-systemml by apache.
the class BinaryOpTests method runSolveTest.
/**
* Runs the test for solve (Ax = b) for input with given dimensions and sparsities
* A can be overdetermined (rows in A > columns in A)
*
* @param sparsity sparsity for the block A and b
* @param m rows in A
* @param n columns in A
*/
protected void runSolveTest(double sparsity, int m, int n) {
String scriptStr = "x = solve(A, b)";
System.out.println("In solve, A[" + m + ", " + n + "], b[" + m + ", 1]");
Matrix A = generateInputMatrix(spark, m, n, sparsity, seed);
Matrix b = generateInputMatrix(spark, m, 1, sparsity, seed);
HashMap<String, Object> inputs = new HashMap<>();
inputs.put("A", A);
inputs.put("b", b);
List<Object> outCPU = runOnCPU(spark, scriptStr, inputs, Arrays.asList("x"));
List<Object> outGPU = runOnGPU(spark, scriptStr, inputs, Arrays.asList("x"));
assertHeavyHitterPresent("gpu_solve");
assertEqualObjects(outCPU.get(0), outGPU.get(0));
}
use of org.apache.sysml.api.mlcontext.Matrix in project incubator-systemml by apache.
the class MatrixMatrixElementWiseOpTests method runMatrixColumnVectorTest.
/**
* Run O = X op Y where X is a matrix, Y is a column vector
*
* @param scriptStr the script string
* @param matrixInput name of the matrix input variable in the script string
* @param vectorInput name of the vector input variable in the script string
* @param output name of the output variable in the script string
* @param heavyHitterOpcode the string printed for the unary op heavy hitter when executed on gpu
*/
private void runMatrixColumnVectorTest(String scriptStr, String matrixInput, String vectorInput, String output, String heavyHitterOpcode) {
int[] rows = new int[] { 64, 130, 1024, 2049 };
int[] cols = new int[] { 64, 130, 1024, 2049 };
for (int i = 0; i < rows.length; i++) {
for (int j = 0; j < cols.length; j++) {
for (int k = 0; k < sparsities.length; k++) {
int m = rows[i];
int n = cols[j];
double sparsity = sparsities[k];
Matrix X = generateInputMatrix(spark, m, n, sparsity, seed);
Matrix Y = generateInputMatrix(spark, m, 1, sparsity, seed);
HashMap<String, Object> inputs = new HashMap<>();
inputs.put(matrixInput, X);
inputs.put(vectorInput, Y);
System.out.println("Vector[" + m + ", 1] op Matrix[" + m + ", " + n + "], sparsity = " + sparsity);
List<Object> cpuOut = runOnCPU(spark, scriptStr, inputs, Arrays.asList(output));
List<Object> gpuOut = runOnGPU(spark, scriptStr, inputs, Arrays.asList(output));
//assertHeavyHitterPresent(heavyHitterOpcode);
assertEqualObjects(cpuOut.get(0), gpuOut.get(0));
}
}
}
}
use of org.apache.sysml.api.mlcontext.Matrix in project incubator-systemml by apache.
the class GNMFTest method testGNMFWithRDMLAndJava.
@Test
public void testGNMFWithRDMLAndJava() throws IOException, DMLException, ParseException {
System.out.println("------------ BEGIN " + TEST_NAME + " TEST {" + numRegisteredInputs + ", " + numRegisteredOutputs + "} ------------");
this.scriptType = ScriptType.DML;
int m = 2000;
int n = 1500;
int k = 50;
int maxiter = 2;
double Eps = Math.pow(10, -8);
getAndLoadTestConfiguration(TEST_NAME);
List<String> proArgs = new ArrayList<String>();
proArgs.add(input("v"));
proArgs.add(input("w"));
proArgs.add(input("h"));
proArgs.add(Integer.toString(maxiter));
proArgs.add(output("w"));
proArgs.add(output("h"));
programArgs = proArgs.toArray(new String[proArgs.size()]);
fullDMLScriptName = getScript();
rCmd = getRCmd(inputDir(), Integer.toString(maxiter), expectedDir());
double[][] v = getRandomMatrix(m, n, 1, 5, 0.2, System.currentTimeMillis());
double[][] w = getRandomMatrix(m, k, 0, 1, 1, System.currentTimeMillis());
double[][] h = getRandomMatrix(k, n, 0, 1, 1, System.currentTimeMillis());
writeInputMatrixWithMTD("v", v, true);
writeInputMatrixWithMTD("w", w, true);
writeInputMatrixWithMTD("h", h, true);
for (int i = 0; i < maxiter; i++) {
double[][] tW = TestUtils.performTranspose(w);
double[][] tWV = TestUtils.performMatrixMultiplication(tW, v);
double[][] tWW = TestUtils.performMatrixMultiplication(tW, w);
double[][] tWWH = TestUtils.performMatrixMultiplication(tWW, h);
for (int j = 0; j < k; j++) {
for (int l = 0; l < n; l++) {
h[j][l] = h[j][l] * (tWV[j][l] / (tWWH[j][l] + Eps));
}
}
double[][] tH = TestUtils.performTranspose(h);
double[][] vTH = TestUtils.performMatrixMultiplication(v, tH);
double[][] hTH = TestUtils.performMatrixMultiplication(h, tH);
double[][] wHTH = TestUtils.performMatrixMultiplication(w, hTH);
for (int j = 0; j < m; j++) {
for (int l = 0; l < k; l++) {
w[j][l] = w[j][l] * (vTH[j][l] / (wHTH[j][l] + Eps));
}
}
}
boolean oldConfig = DMLScript.USE_LOCAL_SPARK_CONFIG;
DMLScript.USE_LOCAL_SPARK_CONFIG = true;
RUNTIME_PLATFORM oldRT = DMLScript.rtplatform;
try {
DMLScript.rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK;
Script script = ScriptFactory.dmlFromFile(fullDMLScriptName);
// set positional argument values
for (int argNum = 1; argNum <= proArgs.size(); argNum++) {
script.in("$" + argNum, proArgs.get(argNum - 1));
}
// Read two matrices through RDD and one through HDFS
if (numRegisteredInputs >= 1) {
JavaRDD<String> vIn = sc.sc().textFile(input("v"), 2).toJavaRDD();
MatrixMetadata mm = new MatrixMetadata(MatrixFormat.IJV, m, n);
script.in("V", vIn, mm);
}
if (numRegisteredInputs >= 2) {
JavaRDD<String> wIn = sc.sc().textFile(input("w"), 2).toJavaRDD();
MatrixMetadata mm = new MatrixMetadata(MatrixFormat.IJV, m, k);
script.in("W", wIn, mm);
}
if (numRegisteredInputs >= 3) {
JavaRDD<String> hIn = sc.sc().textFile(input("h"), 2).toJavaRDD();
MatrixMetadata mm = new MatrixMetadata(MatrixFormat.IJV, k, n);
script.in("H", hIn, mm);
}
// Output one matrix to HDFS and get one as RDD
if (numRegisteredOutputs >= 1) {
script.out("H");
}
if (numRegisteredOutputs >= 2) {
script.out("W");
ml.setConfigProperty("cp.parallel.matrixmult", "false");
}
MLResults results = ml.execute(script);
if (numRegisteredOutputs >= 2) {
String configStr = ConfigurationManager.getDMLConfig().getConfigInfo();
if (configStr.contains("cp.parallel.matrixmult: true"))
Assert.fail("Configuration not updated via setConfig");
}
if (numRegisteredOutputs >= 1) {
RDD<String> hOut = results.getRDDStringIJV("H");
String fName = output("h");
try {
MapReduceTool.deleteFileIfExistOnHDFS(fName);
} catch (IOException e) {
throw new DMLRuntimeException("Error: While deleting file on HDFS");
}
hOut.saveAsTextFile(fName);
}
if (numRegisteredOutputs >= 2) {
JavaRDD<String> javaRDDStringIJV = results.getJavaRDDStringIJV("W");
JavaRDD<MatrixEntry> matRDD = javaRDDStringIJV.map(new StringToMatrixEntry());
Matrix matrix = results.getMatrix("W");
MatrixCharacteristics mcW = matrix.getMatrixMetadata().asMatrixCharacteristics();
CoordinateMatrix coordinateMatrix = new CoordinateMatrix(matRDD.rdd(), mcW.getRows(), mcW.getCols());
JavaPairRDD<MatrixIndexes, MatrixBlock> binaryRDD = RDDConverterUtilsExt.coordinateMatrixToBinaryBlock(sc, coordinateMatrix, mcW, true);
JavaRDD<String> wOut = RDDConverterUtils.binaryBlockToTextCell(binaryRDD, mcW);
String fName = output("w");
try {
MapReduceTool.deleteFileIfExistOnHDFS(fName);
} catch (IOException e) {
throw new DMLRuntimeException("Error: While deleting file on HDFS");
}
wOut.saveAsTextFile(fName);
}
runRScript(true);
//compare matrices
HashMap<CellIndex, Double> hmWDML = readDMLMatrixFromHDFS("w");
HashMap<CellIndex, Double> hmHDML = readDMLMatrixFromHDFS("h");
HashMap<CellIndex, Double> hmWR = readRMatrixFromFS("w");
HashMap<CellIndex, Double> hmHR = readRMatrixFromFS("h");
TestUtils.compareMatrices(hmWDML, hmWR, 0.000001, "hmWDML", "hmWR");
TestUtils.compareMatrices(hmHDML, hmHR, 0.000001, "hmHDML", "hmHR");
} finally {
DMLScript.rtplatform = oldRT;
DMLScript.USE_LOCAL_SPARK_CONFIG = oldConfig;
}
}
use of org.apache.sysml.api.mlcontext.Matrix in project incubator-systemml by apache.
the class DataFrameVectorScriptTest method testDataFrameScriptInput.
private void testDataFrameScriptInput(ValueType[] schema, boolean containsID, boolean dense, boolean unknownDims) {
//TODO fix inconsistency ml context vs jmlc register Xf
try {
//generate input data and setup metadata
int cols = schema.length + colsVector - 1;
double sparsity = dense ? sparsity1 : sparsity2;
double[][] A = TestUtils.round(getRandomMatrix(rows1, cols, -10, 1000, sparsity, 2373));
MatrixBlock mbA = DataConverter.convertToMatrixBlock(A);
int blksz = ConfigurationManager.getBlocksize();
MatrixCharacteristics mc1 = new MatrixCharacteristics(rows1, cols, blksz, blksz, mbA.getNonZeros());
MatrixCharacteristics mc2 = unknownDims ? new MatrixCharacteristics() : new MatrixCharacteristics(mc1);
//create input data frame
Dataset<Row> df = createDataFrame(spark, mbA, containsID, schema);
// Create full frame metadata, and empty frame metadata
FrameMetadata meta = new FrameMetadata(containsID ? FrameFormat.DF_WITH_INDEX : FrameFormat.DF, mc2.getRows(), mc2.getCols());
FrameMetadata metaEmpty = new FrameMetadata();
//run scripts and obtain result
Script script1 = dml("Xm = as.matrix(Xf);").in("Xf", df, meta).out("Xm");
Script script2 = dml("Xm = as.matrix(Xf);").in("Xf", df, metaEmpty).out(// empty metadata
"Xm");
Matrix Xm1 = ml.execute(script1).getMatrix("Xm");
Matrix Xm2 = ml.execute(script2).getMatrix("Xm");
MatrixBlock mbB1 = Xm1.toBinaryBlockMatrix().getMatrixBlock();
MatrixBlock mbB2 = Xm2.toBinaryBlockMatrix().getMatrixBlock();
//compare frame blocks
double[][] B1 = DataConverter.convertToDoubleMatrix(mbB1);
double[][] B2 = DataConverter.convertToDoubleMatrix(mbB2);
TestUtils.compareMatrices(A, B1, rows1, cols, eps);
TestUtils.compareMatrices(A, B2, rows1, cols, eps);
} catch (Exception ex) {
ex.printStackTrace();
throw new RuntimeException(ex);
}
}
use of org.apache.sysml.api.mlcontext.Matrix in project incubator-systemml by apache.
the class MLContextScratchCleanupTest method runMLContextTestMultipleScript.
/**
*
* @param platform
*/
private void runMLContextTestMultipleScript(RUNTIME_PLATFORM platform, boolean wRead) {
RUNTIME_PLATFORM oldplatform = DMLScript.rtplatform;
DMLScript.rtplatform = platform;
//create mlcontext
SparkSession spark = createSystemMLSparkSession("MLContextScratchCleanupTest", "local");
MLContext ml = new MLContext(spark);
ml.setExplain(true);
String dml1 = baseDirectory + File.separator + "ScratchCleanup1.dml";
String dml2 = baseDirectory + File.separator + (wRead ? "ScratchCleanup2b.dml" : "ScratchCleanup2.dml");
try {
Script script1 = dmlFromFile(dml1).in("$rows", rows).in("$cols", cols).out("X");
Matrix X = ml.execute(script1).getMatrix("X");
//clear in-memory/cached data to emulate on-disk storage
X.toMatrixObject().clearData();
Script script2 = dmlFromFile(dml2).in("X", X).out("z");
String z = ml.execute(script2).getString("z");
System.out.println(z);
} catch (Exception ex) {
throw new RuntimeException(ex);
} finally {
DMLScript.rtplatform = oldplatform;
// stop underlying spark context to allow single jvm tests (otherwise the
// next test that tries to create a SparkContext would fail)
spark.stop();
// clear status mlcontext and spark exec context
ml.close();
}
}
Aggregations