Examples with MatrixIndexes org.apache.sysml.runtime.matrix.data.MatrixIndexes used on opensource projects

Example 81 with MatrixIndexes

use of org.apache.sysml.runtime.matrix.data.MatrixIndexes in project incubator-systemml by apache.

the class MMCJMRReducerWithAggregator method processJoin.

private void processJoin(int tag, long inIndex, MatrixValue inValue) throws IOException {
    try {
        if (// for the cached matrix
        tag == 0) {
            cache.put(inIndex, inValue);
        } else // for the probing matrix
        {
            for (int i = 0; i < cache.getCacheSize(); i++) {
                Pair<MatrixIndexes, MatrixValue> tmp = cache.get(i);
                if (// left cached
                tagForLeft == 0) {
                    // perform matrix multiplication
                    indexesbuffer.setIndexes(tmp.getKey().getRowIndex(), inIndex);
                    OperationsOnMatrixValues.performAggregateBinaryIgnoreIndexes((MatrixBlock) tmp.getValue(), (MatrixBlock) inValue, (MatrixBlock) valueBuffer, (AggregateBinaryOperator) aggBinInstruction.getOperator());
                } else // right cached
                {
                    // perform matrix multiplication
                    indexesbuffer.setIndexes(inIndex, tmp.getKey().getColumnIndex());
                    OperationsOnMatrixValues.performAggregateBinaryIgnoreIndexes((MatrixBlock) inValue, (MatrixBlock) tmp.getValue(), (MatrixBlock) valueBuffer, (AggregateBinaryOperator) aggBinInstruction.getOperator());
                }
                // aggregate block to output buffer or direct output
                if (aggBinInstruction.getMMCJType() == MMCJType.AGG) {
                    aggregator.aggregateToBuffer(indexesbuffer, valueBuffer, tagForLeft == 0);
                } else {
                    // MMCJType.NO_AGG
                    collectFinalMultipleOutputs.collectOutput(indexesbuffer, valueBuffer, 0, cachedReporter);
                    resultsNonZeros[0] += valueBuffer.getNonZeros();
                }
            }
        }
    } catch (Exception ex) {
        throw new IOException(ex);
    }
}

Example 82 with MatrixIndexes

use of org.apache.sysml.runtime.matrix.data.MatrixIndexes in project incubator-systemml by apache.

the class MLContextTest method testOutputBinaryBlocksPYDML.

@Test
public void testOutputBinaryBlocksPYDML() {
    System.out.println("MLContextTest - output binary blocks PYDML");
    String s = "M = full('1 2 3 4', rows=2, cols=2);";
    MLResults results = ml.execute(pydml(s).out("M"));
    Matrix m = results.getMatrix("M");
    JavaPairRDD<MatrixIndexes, MatrixBlock> binaryBlocks = m.toBinaryBlocks();
    MatrixMetadata mm = m.getMatrixMetadata();
    MatrixCharacteristics mc = mm.asMatrixCharacteristics();
    JavaRDD<String> javaRDDStringIJV = RDDConverterUtils.binaryBlockToTextCell(binaryBlocks, mc);
    List<String> lines = javaRDDStringIJV.collect();
    Assert.assertEquals("1 1 1.0", lines.get(0));
    Assert.assertEquals("1 2 2.0", lines.get(1));
    Assert.assertEquals("2 1 3.0", lines.get(2));
    Assert.assertEquals("2 2 4.0", lines.get(3));
}

Example 83 with MatrixIndexes

use of org.apache.sysml.runtime.matrix.data.MatrixIndexes in project incubator-systemml by apache.

the class MLContextTest method testDataFrameToBinaryBlocks.

@Test
public void testDataFrameToBinaryBlocks() {
    System.out.println("MLContextTest - DataFrame to binary blocks");
    List<String> list = new ArrayList<String>();
    list.add("1,2,3");
    list.add("4,5,6");
    list.add("7,8,9");
    JavaRDD<String> javaRddString = sc.parallelize(list);
    JavaRDD<Row> javaRddRow = javaRddString.map(new CommaSeparatedValueStringToDoubleArrayRow());
    List<StructField> fields = new ArrayList<StructField>();
    fields.add(DataTypes.createStructField("C1", DataTypes.DoubleType, true));
    fields.add(DataTypes.createStructField("C2", DataTypes.DoubleType, true));
    fields.add(DataTypes.createStructField("C3", DataTypes.DoubleType, true));
    StructType schema = DataTypes.createStructType(fields);
    Dataset<Row> dataFrame = spark.createDataFrame(javaRddRow, schema);
    JavaPairRDD<MatrixIndexes, MatrixBlock> binaryBlocks = MLContextConversionUtil.dataFrameToMatrixBinaryBlocks(dataFrame);
    Tuple2<MatrixIndexes, MatrixBlock> first = binaryBlocks.first();
    MatrixBlock mb = first._2();
    double[][] matrix = DataConverter.convertToDoubleMatrix(mb);
    Assert.assertArrayEquals(new double[] { 1.0, 2.0, 3.0 }, matrix[0], 0.0);
    Assert.assertArrayEquals(new double[] { 4.0, 5.0, 6.0 }, matrix[1], 0.0);
    Assert.assertArrayEquals(new double[] { 7.0, 8.0, 9.0 }, matrix[2], 0.0);
}

Example 84 with MatrixIndexes

use of org.apache.sysml.runtime.matrix.data.MatrixIndexes in project incubator-systemml by apache.

the class MLContextOutputBlocksizeTest method runMLContextOutputBlocksizeTest.

private void runMLContextOutputBlocksizeTest(String format) {
    try {
        double[][] A = getRandomMatrix(rows, cols, -10, 10, sparsity, 76543);
        MatrixBlock mbA = DataConverter.convertToMatrixBlock(A);
        int blksz = ConfigurationManager.getBlocksize();
        MatrixCharacteristics mc = new MatrixCharacteristics(rows, cols, blksz, blksz, mbA.getNonZeros());
        // create input dataset
        JavaPairRDD<MatrixIndexes, MatrixBlock> in = SparkExecutionContext.toMatrixJavaPairRDD(sc, mbA, blksz, blksz);
        Matrix m = new Matrix(in, new MatrixMetadata(mc));
        ml.setExplain(true);
        ml.setExplainLevel(ExplainLevel.HOPS);
        // execute script
        String s = "if( sum(X) > 0 )" + "   X = X/2;" + "R = X;" + "write(R, \"/tmp\", format=\"" + format + "\");";
        Script script = dml(s).in("X", m).out("R");
        MLResults results = ml.execute(script);
        // compare output matrix characteristics
        MatrixCharacteristics mcOut = results.getMatrix("R").getMatrixMetadata().asMatrixCharacteristics();
        Assert.assertEquals(blksz, mcOut.getRowsPerBlock());
        Assert.assertEquals(blksz, mcOut.getColsPerBlock());
    } catch (Exception ex) {
        ex.printStackTrace();
        throw new RuntimeException(ex);
    }
}

Example 85 with MatrixIndexes

use of org.apache.sysml.runtime.matrix.data.MatrixIndexes in project incubator-systemml by apache.

the class MLContextParforDatasetTest method runMLContextParforDatasetTest.

private void runMLContextParforDatasetTest(boolean vector, boolean unknownDims, boolean multiInputs) {
    // modify memory budget to trigger fused datapartition-execute
    long oldmem = InfrastructureAnalyzer.getLocalMaxMemory();
    // 1MB
    InfrastructureAnalyzer.setLocalMaxMemory(1 * 1024 * 1024);
    try {
        double[][] A = getRandomMatrix(rows, cols, -10, 10, sparsity, 76543);
        MatrixBlock mbA = DataConverter.convertToMatrixBlock(A);
        int blksz = ConfigurationManager.getBlocksize();
        MatrixCharacteristics mc1 = new MatrixCharacteristics(rows, cols, blksz, blksz, mbA.getNonZeros());
        MatrixCharacteristics mc2 = unknownDims ? new MatrixCharacteristics() : new MatrixCharacteristics(mc1);
        // create input dataset
        SparkSession sparkSession = SparkSession.builder().sparkContext(sc.sc()).getOrCreate();
        JavaPairRDD<MatrixIndexes, MatrixBlock> in = SparkExecutionContext.toMatrixJavaPairRDD(sc, mbA, blksz, blksz);
        Dataset<Row> df = RDDConverterUtils.binaryBlockToDataFrame(sparkSession, in, mc1, vector);
        MatrixMetadata mm = new MatrixMetadata(vector ? MatrixFormat.DF_VECTOR_WITH_INDEX : MatrixFormat.DF_DOUBLES_WITH_INDEX);
        mm.setMatrixCharacteristics(mc2);
        String s1 = "v = matrix(0, rows=nrow(X), cols=1)" + "parfor(i in 1:nrow(X), log=DEBUG) {" + "   v[i, ] = sum(X[i, ]);" + "}" + "r = sum(v);";
        String s2 = "v = matrix(0, rows=nrow(X), cols=1)" + "Y = X;" + "parfor(i in 1:nrow(X), log=DEBUG) {" + "   v[i, ] = sum(X[i, ]+Y[i, ]);" + "}" + "r = sum(v);";
        String s = multiInputs ? s2 : s1;
        ml.setExplain(true);
        ml.setExplainLevel(ExplainLevel.RUNTIME);
        ml.setStatistics(true);
        Script script = dml(s).in("X", df, mm).out("r");
        MLResults results = ml.execute(script);
        // compare aggregation results
        double sum1 = results.getDouble("r");
        double sum2 = mbA.sum() * (multiInputs ? 2 : 1);
        TestUtils.compareScalars(sum2, sum1, 0.000001);
    } catch (Exception ex) {
        ex.printStackTrace();
        throw new RuntimeException(ex);
    } finally {
        InfrastructureAnalyzer.setLocalMaxMemory(oldmem);
    }
}