Examples with ShiftMatrix org.apache.sysml.runtime.instructions.spark.AppendGSPInstruction.ShiftMatrix used on opensource projects

Example 1 with ShiftMatrix

use of org.apache.sysml.runtime.instructions.spark.AppendGSPInstruction.ShiftMatrix in project systemml by apache.

the class BuiltinNarySPInstruction method processInstruction.

@Override
public void processInstruction(ExecutionContext ec) {
    SparkExecutionContext sec = (SparkExecutionContext) ec;
    boolean cbind = getOpcode().equals("cbind");
    // compute output characteristics
    MatrixCharacteristics mcOut = computeOutputMatrixCharacteristics(sec, inputs, cbind);
    // get consolidated input via union over shifted and padded inputs
    MatrixCharacteristics off = new MatrixCharacteristics(0, 0, mcOut.getRowsPerBlock(), mcOut.getColsPerBlock(), 0);
    JavaPairRDD<MatrixIndexes, MatrixBlock> out = null;
    for (CPOperand input : inputs) {
        MatrixCharacteristics mcIn = sec.getMatrixCharacteristics(input.getName());
        JavaPairRDD<MatrixIndexes, MatrixBlock> in = sec.getBinaryBlockRDDHandleForVariable(input.getName()).flatMapToPair(new ShiftMatrix(off, mcIn, cbind)).mapToPair(// just padding
        new PadBlocksFunction(mcOut));
        out = (out != null) ? out.union(in) : in;
        updateMatrixCharacteristics(mcIn, off, cbind);
    }
    // aggregate partially overlapping blocks w/ single shuffle
    int numPartOut = SparkUtils.getNumPreferredPartitions(mcOut);
    out = RDDAggregateUtils.mergeByKey(out, numPartOut, false);
    // set output RDD and add lineage
    sec.getMatrixCharacteristics(output.getName()).set(mcOut);
    sec.setRDDHandleForVariable(output.getName(), out);
    for (CPOperand input : inputs) sec.addLineageRDD(output.getName(), input.getName());
}

Example 2 with ShiftMatrix

use of org.apache.sysml.runtime.instructions.spark.AppendGSPInstruction.ShiftMatrix in project incubator-systemml by apache.

the class BuiltinNarySPInstruction method processInstruction.

@Override
public void processInstruction(ExecutionContext ec) {
    SparkExecutionContext sec = (SparkExecutionContext) ec;
    boolean cbind = getOpcode().equals("cbind");
    // compute output characteristics
    MatrixCharacteristics mcOut = computeOutputMatrixCharacteristics(sec, inputs, cbind);
    // get consolidated input via union over shifted and padded inputs
    MatrixCharacteristics off = new MatrixCharacteristics(0, 0, mcOut.getRowsPerBlock(), mcOut.getColsPerBlock(), 0);
    JavaPairRDD<MatrixIndexes, MatrixBlock> out = null;
    for (CPOperand input : inputs) {
        MatrixCharacteristics mcIn = sec.getMatrixCharacteristics(input.getName());
        JavaPairRDD<MatrixIndexes, MatrixBlock> in = sec.getBinaryBlockRDDHandleForVariable(input.getName()).flatMapToPair(new ShiftMatrix(off, mcIn, cbind)).mapToPair(// just padding
        new PadBlocksFunction(mcOut));
        out = (out != null) ? out.union(in) : in;
        updateMatrixCharacteristics(mcIn, off, cbind);
    }
    // aggregate partially overlapping blocks w/ single shuffle
    int numPartOut = SparkUtils.getNumPreferredPartitions(mcOut);
    out = RDDAggregateUtils.mergeByKey(out, numPartOut, false);
    // set output RDD and add lineage
    sec.getMatrixCharacteristics(output.getName()).set(mcOut);
    sec.setRDDHandleForVariable(output.getName(), out);
    for (CPOperand input : inputs) sec.addLineageRDD(output.getName(), input.getName());
}