Examples with MatrixIndexes org.apache.sysml.runtime.matrix.data.MatrixIndexes used on opensource projects

Example 31 with MatrixIndexes

use of org.apache.sysml.runtime.matrix.data.MatrixIndexes in project incubator-systemml by apache.

the class CopyBinaryCellFunction method call.

@Override
public Tuple2<MatrixIndexes, MatrixCell> call(Tuple2<MatrixIndexes, MatrixCell> arg0) throws Exception {
    MatrixIndexes ix = new MatrixIndexes(arg0._1());
    MatrixCell cell = new MatrixCell();
    cell.copy(arg0._2());
    return new Tuple2<>(ix, cell);
}

Example 32 with MatrixIndexes

use of org.apache.sysml.runtime.matrix.data.MatrixIndexes in project incubator-systemml by apache.

the class ExtractGroupNWeights method call.

@Override
public Iterator<Tuple2<MatrixIndexes, WeightedCell>> call(Tuple2<MatrixIndexes, Tuple2<Tuple2<MatrixBlock, MatrixBlock>, MatrixBlock>> arg) throws Exception {
    MatrixBlock group = arg._2._1._1;
    MatrixBlock target = arg._2._1._2;
    MatrixBlock weight = arg._2._2;
    // sanity check matching block dimensions
    if (group.getNumRows() != target.getNumRows() || group.getNumRows() != target.getNumRows()) {
        throw new Exception("The blocksize for group/target/weight blocks are mismatched: " + group.getNumRows() + ", " + target.getNumRows() + ", " + weight.getNumRows());
    }
    // output weighted cells
    ArrayList<Tuple2<MatrixIndexes, WeightedCell>> groupValuePairs = new ArrayList<>();
    for (int i = 0; i < group.getNumRows(); i++) {
        WeightedCell weightedCell = new WeightedCell();
        weightedCell.setValue(target.quickGetValue(i, 0));
        weightedCell.setWeight(weight.quickGetValue(i, 0));
        long groupVal = UtilFunctions.toLong(group.quickGetValue(i, 0));
        if (groupVal < 1) {
            throw new Exception("Expected group values to be greater than equal to 1 but found " + groupVal);
        }
        MatrixIndexes ix = new MatrixIndexes(groupVal, 1);
        groupValuePairs.add(new Tuple2<>(ix, weightedCell));
    }
    return groupValuePairs.iterator();
}

Example 33 with MatrixIndexes

use of org.apache.sysml.runtime.matrix.data.MatrixIndexes in project incubator-systemml by apache.

the class GetMIMBFromRow method call.

@Override
public Tuple2<MatrixIndexes, MatrixBlock> call(Row row) throws Exception {
    MatrixIndexes indx = (MatrixIndexes) row.apply(0);
    MatrixBlock blk = (MatrixBlock) row.apply(1);
    return new Tuple2<>(indx, blk);
}

Example 34 with MatrixIndexes

use of org.apache.sysml.runtime.matrix.data.MatrixIndexes in project incubator-systemml by apache.

the class ReorgMapFunction method call.

@Override
public Tuple2<MatrixIndexes, MatrixBlock> call(Tuple2<MatrixIndexes, MatrixBlock> arg0) throws Exception {
    MatrixIndexes ixIn = arg0._1();
    MatrixBlock blkIn = arg0._2();
    // swap the matrix indexes
    MatrixIndexes ixOut = new MatrixIndexes(ixIn);
    _indexFnObject.execute(ixIn, ixOut);
    // swap the matrix block data
    MatrixBlock blkOut = (MatrixBlock) blkIn.reorgOperations(_reorgOp, new MatrixBlock(), -1, -1, -1);
    // output new tuple
    return new Tuple2<>(ixOut, blkOut);
}

Example 35 with MatrixIndexes

use of org.apache.sysml.runtime.matrix.data.MatrixIndexes in project incubator-systemml by apache.

the class RemoteDPParForSpark method getPartitionedInput.

@SuppressWarnings("unchecked")
private static JavaPairRDD<Long, Writable> getPartitionedInput(SparkExecutionContext sec, String matrixvar, OutputInfo oi, PartitionFormat dpf) {
    InputInfo ii = InputInfo.BinaryBlockInputInfo;
    MatrixObject mo = sec.getMatrixObject(matrixvar);
    MatrixCharacteristics mc = mo.getMatrixCharacteristics();
    // NOTE: there will always be a checkpoint rdd on top of the input rdd and the dataset
    if (hasInputDataSet(dpf, mo)) {
        DatasetObject dsObj = (DatasetObject) mo.getRDDHandle().getLineageChilds().get(0).getLineageChilds().get(0);
        Dataset<Row> in = dsObj.getDataset();
        // construct or reuse row ids
        JavaPairRDD<Row, Long> prepinput = dsObj.containsID() ? in.javaRDD().mapToPair(new DataFrameExtractIDFunction(in.schema().fieldIndex(RDDConverterUtils.DF_ID_COLUMN))) : // zip row index
        in.javaRDD().zipWithIndex();
        // convert row to row in matrix block format
        return prepinput.mapToPair(new DataFrameToRowBinaryBlockFunction(mc.getCols(), dsObj.isVectorBased(), dsObj.containsID()));
    } else // binary block input rdd without grouping
    if (!requiresGrouping(dpf, mo)) {
        // get input rdd and data partitioning
        JavaPairRDD<MatrixIndexes, MatrixBlock> in = sec.getBinaryBlockRDDHandleForVariable(matrixvar);
        DataPartitionerRemoteSparkMapper dpfun = new DataPartitionerRemoteSparkMapper(mc, ii, oi, dpf._dpf, dpf._N);
        return in.flatMapToPair(dpfun);
    } else // default binary block input rdd with grouping
    {
        // get input rdd, avoid unnecessary caching if input is checkpoint and not cached yet
        // to reduce memory pressure for shuffle and subsequent
        JavaPairRDD<MatrixIndexes, MatrixBlock> in = sec.getBinaryBlockRDDHandleForVariable(matrixvar);
        if (mo.getRDDHandle().isCheckpointRDD() && !sec.isRDDCached(in.id()))
            in = (JavaPairRDD<MatrixIndexes, MatrixBlock>) ((RDDObject) mo.getRDDHandle().getLineageChilds().get(0)).getRDD();
        // data partitioning of input rdd
        DataPartitionerRemoteSparkMapper dpfun = new DataPartitionerRemoteSparkMapper(mc, ii, oi, dpf._dpf, dpf._N);
        return in.flatMapToPair(dpfun);
    }
}