use of org.apache.sysml.runtime.instructions.spark.functions.MatrixMatrixBinaryOpFunction in project incubator-systemml by apache.
the class BinarySPInstruction method processMatrixMatrixBinaryInstruction.
/**
* Common binary matrix-matrix process instruction
*
* @param ec execution context
*/
protected void processMatrixMatrixBinaryInstruction(ExecutionContext ec) {
SparkExecutionContext sec = (SparkExecutionContext) ec;
// sanity check dimensions
checkMatrixMatrixBinaryCharacteristics(sec);
updateBinaryOutputMatrixCharacteristics(sec);
// Get input RDDs
JavaPairRDD<MatrixIndexes, MatrixBlock> in1 = sec.getBinaryBlockRDDHandleForVariable(input1.getName());
JavaPairRDD<MatrixIndexes, MatrixBlock> in2 = sec.getBinaryBlockRDDHandleForVariable(input2.getName());
MatrixCharacteristics mc1 = sec.getMatrixCharacteristics(input1.getName());
MatrixCharacteristics mc2 = sec.getMatrixCharacteristics(input2.getName());
MatrixCharacteristics mcOut = sec.getMatrixCharacteristics(output.getName());
BinaryOperator bop = (BinaryOperator) _optr;
// vector replication if required (mv or outer operations)
boolean rowvector = (mc2.getRows() == 1 && mc1.getRows() > 1);
long numRepLeft = getNumReplicas(mc1, mc2, true);
long numRepRight = getNumReplicas(mc1, mc2, false);
if (numRepLeft > 1)
in1 = in1.flatMapToPair(new ReplicateVectorFunction(false, numRepLeft));
if (numRepRight > 1)
in2 = in2.flatMapToPair(new ReplicateVectorFunction(rowvector, numRepRight));
int numPrefPart = SparkUtils.isHashPartitioned(in1) ? in1.getNumPartitions() : SparkUtils.isHashPartitioned(in2) ? in2.getNumPartitions() : Math.min(in1.getNumPartitions() + in2.getNumPartitions(), 2 * SparkUtils.getNumPreferredPartitions(mcOut));
// execute binary operation
JavaPairRDD<MatrixIndexes, MatrixBlock> out = in1.join(in2, numPrefPart).mapValues(new MatrixMatrixBinaryOpFunction(bop));
// set output RDD
sec.setRDDHandleForVariable(output.getName(), out);
sec.addLineageRDD(output.getName(), input1.getName());
sec.addLineageRDD(output.getName(), input2.getName());
}
use of org.apache.sysml.runtime.instructions.spark.functions.MatrixMatrixBinaryOpFunction in project systemml by apache.
the class BinarySPInstruction method processMatrixMatrixBinaryInstruction.
/**
* Common binary matrix-matrix process instruction
*
* @param ec execution context
*/
protected void processMatrixMatrixBinaryInstruction(ExecutionContext ec) {
SparkExecutionContext sec = (SparkExecutionContext) ec;
// sanity check dimensions
checkMatrixMatrixBinaryCharacteristics(sec);
updateBinaryOutputMatrixCharacteristics(sec);
// Get input RDDs
JavaPairRDD<MatrixIndexes, MatrixBlock> in1 = sec.getBinaryBlockRDDHandleForVariable(input1.getName());
JavaPairRDD<MatrixIndexes, MatrixBlock> in2 = sec.getBinaryBlockRDDHandleForVariable(input2.getName());
MatrixCharacteristics mc1 = sec.getMatrixCharacteristics(input1.getName());
MatrixCharacteristics mc2 = sec.getMatrixCharacteristics(input2.getName());
MatrixCharacteristics mcOut = sec.getMatrixCharacteristics(output.getName());
BinaryOperator bop = (BinaryOperator) _optr;
// vector replication if required (mv or outer operations)
boolean rowvector = (mc2.getRows() == 1 && mc1.getRows() > 1);
long numRepLeft = getNumReplicas(mc1, mc2, true);
long numRepRight = getNumReplicas(mc1, mc2, false);
if (numRepLeft > 1)
in1 = in1.flatMapToPair(new ReplicateVectorFunction(false, numRepLeft));
if (numRepRight > 1)
in2 = in2.flatMapToPair(new ReplicateVectorFunction(rowvector, numRepRight));
int numPrefPart = SparkUtils.isHashPartitioned(in1) ? in1.getNumPartitions() : SparkUtils.isHashPartitioned(in2) ? in2.getNumPartitions() : Math.min(in1.getNumPartitions() + in2.getNumPartitions(), 2 * SparkUtils.getNumPreferredPartitions(mcOut));
// execute binary operation
JavaPairRDD<MatrixIndexes, MatrixBlock> out = in1.join(in2, numPrefPart).mapValues(new MatrixMatrixBinaryOpFunction(bop));
// set output RDD
sec.setRDDHandleForVariable(output.getName(), out);
sec.addLineageRDD(output.getName(), input1.getName());
sec.addLineageRDD(output.getName(), input2.getName());
}
Aggregations